{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":86142,"databundleVersionId":9786425,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-10T09:41:26.410534Z","iopub.execute_input":"2024-10-10T09:41:26.410914Z","iopub.status.idle":"2024-10-10T09:41:27.444521Z","shell.execute_reply.started":"2024-10-10T09:41:26.410875Z","shell.execute_reply":"2024-10-10T09:41:27.443367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\n# Define the correct base directory path\nbase_dir = '/kaggle/input/iitg-ai-overnight-hackathon-2024/dataset/dataset'","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:41:27.447047Z","iopub.execute_input":"2024-10-10T09:41:27.447786Z","iopub.status.idle":"2024-10-10T09:41:27.453334Z","shell.execute_reply.started":"2024-10-10T09:41:27.447729Z","shell.execute_reply":"2024-10-10T09:41:27.451925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\ndef explore_dataset(base_dir):\n    print(f\"Exploring dataset in: {base_dir}\")\n    print(\"-\" * 50)\n\n    # List main directories\n    main_dirs = [d for d in os.listdir(base_dir) if os.path.isdir(os.path.join(base_dir, d))]\n    print(\"Main directories:\")\n    for dir in main_dirs:\n        print(f\"- {dir}\")\n    print()\n\n    # Explore each main directory\n    for dir in main_dirs:\n        dir_path = os.path.join(base_dir, dir)\n        print(f\"Contents of {dir}:\")\n        \n        # List subdirectories and files (limit to first 5)\n        contents = os.listdir(dir_path)[:5]\n        for item in contents:\n            item_path = os.path.join(dir_path, item)\n            if os.path.isdir(item_path):\n                print(f\"  📁 {item}\")\n                # Show first 3 items in subdirectory\n                subcontents = os.listdir(item_path)[:3]\n                for subitem in subcontents:\n                    print(f\"    - {subitem}\")\n            else:\n                print(f\"  📄 {item}\")\n        \n        if len(os.listdir(dir_path)) > 5:\n            print(\"  ...\")\n        print()\n\n# Usage\nbase_dir = '/kaggle/input/iitg-ai-overnight-hackathon-2024/dataset/dataset'\nexplore_dataset(base_dir)","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:41:27.454814Z","iopub.execute_input":"2024-10-10T09:41:27.455268Z","iopub.status.idle":"2024-10-10T09:41:27.592555Z","shell.execute_reply.started":"2024-10-10T09:41:27.455221Z","shell.execute_reply":"2024-10-10T09:41:27.591329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nfrom collections import defaultdict\nimport random\nimport shutil\n\ndef map_and_visualize_dataset(base_dir, output_dir, num_samples=5):\n    train_dir = os.path.join(base_dir, 'train')\n    labels_dir = os.path.join(base_dir, 'labels')\n\n    # Create output directory structure\n    os.makedirs(os.path.join(output_dir, 'images'), exist_ok=True)\n    os.makedirs(os.path.join(output_dir, 'labels', 'polygons'), exist_ok=True)\n    os.makedirs(os.path.join(output_dir, 'labels', 'labelColors'), exist_ok=True)\n    os.makedirs(os.path.join(output_dir, 'labels', 'labellevel3Ids'), exist_ok=True)\n\n    def get_frame_number(filename):\n        return filename.split('_')[0].replace('frame', '')\n\n    # Collect all images and map labels\n    images = defaultdict(dict)\n    for subdir in os.listdir(train_dir):\n        subdir_path = os.path.join(train_dir, subdir)\n        if os.path.isdir(subdir_path):\n            for img_file in os.listdir(subdir_path):\n                if img_file.endswith('_leftImg8bit.jpg'):\n                    frame_num = get_frame_number(img_file)\n                    images[frame_num]['image'] = os.path.join(subdir_path, img_file)\n                    images[frame_num]['subdir'] = subdir\n\n    for subdir in os.listdir(labels_dir):\n        subdir_path = os.path.join(labels_dir, subdir)\n        if os.path.isdir(subdir_path):\n            for label_file in os.listdir(subdir_path):\n                frame_num = get_frame_number(label_file)\n                if frame_num in images and images[frame_num]['subdir'] == subdir:\n                    if label_file.endswith('_gtFine_polygons.json'):\n                        images[frame_num]['polygons'] = os.path.join(subdir_path, label_file)\n                    elif label_file.endswith('_gtFine_labelColors.png'):\n                        images[frame_num]['labelColors'] = os.path.join(subdir_path, label_file)\n                    elif label_file.endswith('_gtFine_labellevel3Ids.png'):\n                        images[frame_num]['labellevel3Ids'] = os.path.join(subdir_path, label_file)\n\n    # Filter out incomplete mappings and create final dataset\n    complete_dataset = []\n    for frame_num, data in images.items():\n        if all(key in data for key in ['image', 'polygons', 'labelColors', 'labellevel3Ids']):\n            # Copy files to organized structure\n            new_image_path = os.path.join(output_dir, 'images', f\"{frame_num}.jpg\")\n            new_polygons_path = os.path.join(output_dir, 'labels', 'polygons', f\"{frame_num}.json\")\n            new_labelColors_path = os.path.join(output_dir, 'labels', 'labelColors', f\"{frame_num}.png\")\n            new_labellevel3Ids_path = os.path.join(output_dir, 'labels', 'labellevel3Ids', f\"{frame_num}.png\")\n\n            shutil.copy(data['image'], new_image_path)\n            shutil.copy(data['polygons'], new_polygons_path)\n            shutil.copy(data['labelColors'], new_labelColors_path)\n            shutil.copy(data['labellevel3Ids'], new_labellevel3Ids_path)\n\n            complete_dataset.append({\n                'image': new_image_path,\n                'polygons': new_polygons_path,\n                'labelColors': new_labelColors_path,\n                'labellevel3Ids': new_labellevel3Ids_path\n            })\n\n    # Randomly sample entries if we have more than requested\n    if len(complete_dataset) > num_samples:\n        sampled_dataset = random.sample(complete_dataset, num_samples)\n    else:\n        sampled_dataset = complete_dataset\n\n    # Visualize the sampled entries\n    print(f\"Displaying {len(sampled_dataset)} randomly sampled complete image-label pairs:\")\n    for i, entry in enumerate(sampled_dataset, 1):\n        print(f\"\\nSample {i}:\")\n        print(f\"Image: {entry['image']}\")\n        print(f\"Polygons: {entry['polygons']}\")\n        print(f\"Label Colors: {entry['labelColors']}\")\n        print(f\"Label Level 3 IDs: {entry['labellevel3Ids']}\")\n\n    # Save metadata\n    metadata = {\n        'total_samples': len(complete_dataset),\n        'organized_data_path': output_dir\n    }\n    with open(os.path.join(output_dir, 'metadata.json'), 'w') as f:\n        json.dump(metadata, f, indent=4)\n\n    return complete_dataset\n\n# Usage\nbase_dir = '/kaggle/input/iitg-ai-overnight-hackathon-2024/dataset/dataset'\noutput_dir = '/kaggle/working/organized_dataset'\nlabeled_dataset = map_and_visualize_dataset(base_dir, output_dir, num_samples=5)\nprint(f\"\\nTotal complete image-label pairs in the dataset: {len(labeled_dataset)}\")\nprint(f\"Organized data saved to: {output_dir}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:41:27.595017Z","iopub.execute_input":"2024-10-10T09:41:27.595371Z","iopub.status.idle":"2024-10-10T09:45:19.089464Z","shell.execute_reply.started":"2024-10-10T09:41:27.595335Z","shell.execute_reply":"2024-10-10T09:45:19.087605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport random\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport json\nimport numpy as np\n\ndef display_random_mapped_images(organized_data_path, num_samples=5):\n    # Load metadata\n    with open(os.path.join(organized_data_path, 'metadata.json'), 'r') as f:\n        metadata = json.load(f)\n    \n    # Get all image files\n    image_dir = os.path.join(organized_data_path, 'images')\n    all_images = [f for f in os.listdir(image_dir) if f.endswith('.jpg')]\n    \n    # Randomly select images\n    selected_images = random.sample(all_images, min(num_samples, len(all_images)))\n    \n    for img_file in selected_images:\n        frame_num = os.path.splitext(img_file)[0]\n        \n        # Load image and labels\n        img_path = os.path.join(image_dir, img_file)\n        label_colors_path = os.path.join(organized_data_path, 'labels', 'labelColors', f\"{frame_num}.png\")\n        label_level_3ID_path = os.path.join(organized_data_path, 'labels', 'labellevel3Ids', f\"{frame_num}.png\")\n        polygons_path = os.path.join(organized_data_path, 'labels', 'polygons', f\"{frame_num}.json\")\n        \n        img = Image.open(img_path)\n        label_colors = Image.open(label_colors_path)\n        label_level_3ID = Image.open(label_level_3ID_path)\n        \n        with open(polygons_path, 'r') as f:\n            polygons_data = json.load(f)\n        \n        # Create a figure with subplots\n        fig, (ax1, ax2, ax3, ax4) = plt.subplots(1, 4, figsize=(25, 7))\n        fig.suptitle(f\"Frame: {frame_num}\", fontsize=16)\n        \n        # Display original image\n        ax1.imshow(img)\n        ax1.set_title('Original Image')\n        ax1.axis('off')\n        \n        # Display label colors\n        ax2.imshow(label_colors)\n        ax2.set_title('Label Colors')\n        ax2.axis('off')\n        \n        # Display label level 3 IDs\n        ax3.imshow(label_level_3ID)\n        ax3.set_title('Label Level 3 IDs')\n        ax3.axis('off')\n        \n        # Display original image with polygon overlay\n        ax4.imshow(img)\n        for obj in polygons_data['objects']:\n            polygon = np.array(obj['polygon'])\n            ax4.plot(polygon[:, 0], polygon[:, 1], linewidth=2)\n        ax4.set_title('Polygons Overlay')\n        ax4.axis('off')\n        \n        plt.tight_layout()\n        plt.show()\n        \n        # Print file paths for reference\n        print(f\"Image path: {img_path}\")\n        print(f\"Label Colors path: {label_colors_path}\")\n        print(f\"Label Level 3 IDs path: {label_level_3ID_path}\")\n        print(f\"Polygons path: {polygons_path}\")\n        print(\"=\" * 50)\n\n# Usage\norganized_data_path = '/kaggle/working/organized_dataset'\ndisplay_random_mapped_images(organized_data_path, num_samples=5)","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:45:19.091889Z","iopub.execute_input":"2024-10-10T09:45:19.092584Z","iopub.status.idle":"2024-10-10T09:45:28.743247Z","shell.execute_reply.started":"2024-10-10T09:45:19.092522Z","shell.execute_reply":"2024-10-10T09:45:28.742092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nfrom collections import Counter\n\ndef count_classes_in_new_structure(organized_data_path):\n    polygons_dir = os.path.join(organized_data_path, 'labels', 'polygons')\n    class_counts = Counter()\n\n    for root, _, files in os.walk(polygons_dir):\n        for file in files:\n            if file.endswith('.json'):\n                with open(os.path.join(root, file), 'r') as f:\n                    data = json.load(f)\n                    for obj in data.get('objects', []):\n                        class_counts[obj['label']] += 1\n\n    return class_counts\n\n# Usage\norganized_data_path = '/kaggle/working/organized_dataset'\nclass_counts = count_classes_in_new_structure(organized_data_path)\n\nprint(\"Class distribution:\")\nfor class_name, count in class_counts.most_common():\n    print(f\"{class_name}: {count}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:45:28.744951Z","iopub.execute_input":"2024-10-10T09:45:28.74539Z","iopub.status.idle":"2024-10-10T09:46:14.821253Z","shell.execute_reply.started":"2024-10-10T09:45:28.745343Z","shell.execute_reply":"2024-10-10T09:46:14.82Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom collections import Counter\n\n# Step 1: Function to count classes in the new dataset structure\ndef count_classes_in_new_structure(organized_data_path):\n    polygons_dir = os.path.join(organized_data_path, 'labels', 'polygons')\n    class_counts = Counter()\n\n    for root, _, files in os.walk(polygons_dir):\n        for file in files:\n            if file.endswith('.json'):\n                with open(os.path.join(root, file), 'r') as f:\n                    data = json.load(f)\n                    for obj in data.get('objects', []):\n                        class_counts[obj['label']] += 1\n\n    return class_counts\n\n# Step 2: Get class counts from the new dataset structure\norganized_data_path = '/kaggle/working/organized_dataset'\nclass_counts = count_classes_in_new_structure(organized_data_path)\n\n# Step 3: Analyze and visualize the class distribution\ntotal_objects = sum(class_counts.values())\n\n# Create a DataFrame for easier analysis\ndf = pd.DataFrame(list(class_counts.items()), columns=['Class', 'Count'])\ndf['Frequency'] = df['Count'] / total_objects\ndf = df.sort_values('Count', ascending=False).reset_index(drop=True)\n\n# Calculate imbalance ratios\ndf['Imbalance_Ratio'] = df['Count'].max() / df['Count']\n\n# Print summary\nprint(\"Class Distribution Summary:\")\nprint(df)\n\n# Identify underrepresented classes (e.g., classes with less than 1% of total objects)\nunderrepresented = df[df['Frequency'] < 0.01]\nprint(\"\\nUnderrepresented Classes (< 1% of total objects):\")\nprint(underrepresented)\n\n# Calculate and print imbalance metrics\nimbalance_ratio = df['Count'].max() / df['Count'].min()\nprint(f\"\\nOverall Imbalance Ratio (Most common / Least common): {imbalance_ratio:.2f}\")\n\ngini_coefficient = 1 - ((df['Frequency']**2).sum())\nprint(f\"Gini Coefficient of Inequality: {gini_coefficient:.4f}\")\n\n# Visualize class distribution\nplt.figure(figsize=(15, 8))\nplt.bar(df['Class'], df['Count'])\nplt.xticks(rotation=90)\nplt.xlabel('Class')\nplt.ylabel('Count')\nplt.title('Class Distribution in the Dataset')\nplt.tight_layout()\nplt.show()\n\n# Visualize class distribution (log scale)\nplt.figure(figsize=(15, 8))\nplt.bar(df['Class'], df['Count'])\nplt.yscale('log')\nplt.xticks(rotation=90)\nplt.xlabel('Class')\nplt.ylabel('Count (log scale)')\nplt.title('Class Distribution in the Dataset (Log Scale)')\nplt.tight_layout()\nplt.show()\n\n# Visualize imbalance ratios\nplt.figure(figsize=(15, 8))\nplt.bar(df['Class'], df['Imbalance_Ratio'])\nplt.yscale('log')\nplt.xticks(rotation=90)\nplt.xlabel('Class')\nplt.ylabel('Imbalance Ratio (log scale)')\nplt.title('Class Imbalance Ratios')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:46:14.822826Z","iopub.execute_input":"2024-10-10T09:46:14.823324Z","iopub.status.idle":"2024-10-10T09:46:58.952282Z","shell.execute_reply.started":"2024-10-10T09:46:14.823266Z","shell.execute_reply":"2024-10-10T09:46:58.951221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nfrom PIL import Image\nimport numpy as np\nfrom tqdm import tqdm\n\ndef preprocess_images(input_dir, target_size=(224, 224)):\n    # Create preprocessed images directory\n    preprocessed_dir = os.path.join(input_dir, 'preprocessed_images')\n    os.makedirs(preprocessed_dir, exist_ok=True)\n\n    # Load metadata\n    with open(os.path.join(input_dir, 'metadata.json'), 'r') as f:\n        metadata = json.load(f)\n\n    # Get the list of image files\n    image_dir = os.path.join(input_dir, 'images')\n    image_files = [f for f in os.listdir(image_dir) if f.endswith('.jpg')]\n\n    # Process each image\n    for image_file in tqdm(image_files, desc=\"Preprocessing images\"):\n        # Load image\n        img_path = os.path.join(image_dir, image_file)\n        with Image.open(img_path) as img:\n            # Resize image\n            img_resized = img.resize(target_size, Image.LANCZOS)\n            \n            # Convert to numpy array and normalize\n            img_array = np.array(img_resized).astype(np.float32) / 255.0\n\n        # Save preprocessed image\n        output_filename = f\"preprocessed_{image_file.split('.')[0]}.npy\"\n        output_path = os.path.join(preprocessed_dir, output_filename)\n        np.save(output_path, img_array)\n\n    # Update metadata\n    metadata['preprocessed_images_path'] = preprocessed_dir\n    metadata['target_size'] = target_size\n    with open(os.path.join(input_dir, 'updated_metadata.json'), 'w') as f:\n        json.dump(metadata, f, indent=4)\n\n    print(f\"Preprocessing complete. Preprocessed images saved to: {preprocessed_dir}\")\n\n# Usage\ninput_dir = '/kaggle/working/organized_dataset'\npreprocess_images(input_dir)","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:46:58.953712Z","iopub.execute_input":"2024-10-10T09:46:58.954061Z","iopub.status.idle":"2024-10-10T09:51:42.28282Z","shell.execute_reply.started":"2024-10-10T09:46:58.954024Z","shell.execute_reply":"2024-10-10T09:51:42.281558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install torch torchvision albumentations scikit-learn pillow numpy","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:53:22.827376Z","iopub.status.idle":"2024-10-10T09:53:22.827849Z","shell.execute_reply.started":"2024-10-10T09:53:22.827641Z","shell.execute_reply":"2024-10-10T09:53:22.827663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nimport numpy as np\nfrom collections import defaultdict\nfrom PIL import Image\n\ndef analyze_dataset_structure(input_dir):\n    meta = {\n        \"dataset_path\": input_dir,\n        \"directories\": {},\n        \"files\": defaultdict(list),\n        \"image_info\": {},\n        \"numpy_info\": {},\n        \"json_info\": {}\n    }\n\n    for root, dirs, files in os.walk(input_dir):\n        rel_path = os.path.relpath(root, input_dir)\n        if rel_path == '.':\n            meta[\"directories\"][\"root\"] = dirs\n        else:\n            meta[\"directories\"][rel_path] = dirs\n        \n        for file in files:\n            file_path = os.path.join(root, file)\n            file_type = os.path.splitext(file)[1]\n            meta[\"files\"][file_type].append(os.path.relpath(file_path, input_dir))\n\n            if file_type in ['.jpg', '.png', '.jpeg']:\n                analyze_image(file_path, meta[\"image_info\"])\n            elif file_type == '.npy':\n                analyze_numpy(file_path, meta[\"numpy_info\"])\n            elif file_type == '.json':\n                analyze_json(file_path, meta[\"json_info\"])\n\n    # Convert defaultdict to regular dict for JSON serialization\n    meta[\"files\"] = dict(meta[\"files\"])\n\n    # Write the meta information to a JSON file\n    output_path = os.path.join(input_dir, 'comprehensive_meta.json')\n    with open(output_path, 'w') as f:\n        json.dump(meta, f, indent=2)\n\n    print(f\"Comprehensive meta.json file has been created at: {output_path}\")\n\ndef analyze_image(file_path, image_info):\n    try:\n        with Image.open(file_path) as img:\n            rel_path = os.path.relpath(file_path, input_dir)\n            image_info[rel_path] = {\n                \"size\": img.size,\n                \"mode\": img.mode\n            }\n    except Exception as e:\n        print(f\"Error analyzing image {file_path}: {str(e)}\")\n\ndef analyze_numpy(file_path, numpy_info):\n    try:\n        data = np.load(file_path)\n        rel_path = os.path.relpath(file_path, input_dir)\n        numpy_info[rel_path] = {\n            \"shape\": data.shape,\n            \"dtype\": str(data.dtype),\n            \"min\": float(np.min(data)),\n            \"max\": float(np.max(data)),\n            \"mean\": float(np.mean(data)),\n            \"unique_values\": len(np.unique(data))\n        }\n    except Exception as e:\n        print(f\"Error analyzing numpy file {file_path}: {str(e)}\")\n\ndef analyze_json(file_path, json_info):\n    try:\n        with open(file_path, 'r') as f:\n            data = json.load(f)\n        rel_path = os.path.relpath(file_path, input_dir)\n        json_info[rel_path] = {\n            \"keys\": list(data.keys()) if isinstance(data, dict) else \"Not a dictionary\",\n            \"length\": len(data) if isinstance(data, (dict, list)) else \"Not a container type\"\n        }\n    except Exception as e:\n        print(f\"Error analyzing JSON file {file_path}: {str(e)}\")\n\n# Usage\ninput_dir = '/kaggle/working/organized_dataset'\nanalyze_dataset_structure(input_dir)","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:51:42.284643Z","iopub.execute_input":"2024-10-10T09:51:42.285026Z","iopub.status.idle":"2024-10-10T09:53:14.820195Z","shell.execute_reply.started":"2024-10-10T09:51:42.284987Z","shell.execute_reply":"2024-10-10T09:53:14.818974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nimport os\nfrom pprint import pprint\n\ndef display_comprehensive_metadata(input_dir):\n    metadata_path = os.path.join(input_dir, 'comprehensive_meta.json')\n    \n    if not os.path.exists(metadata_path):\n        print(f\"Error: Comprehensive metadata file not found at {metadata_path}\")\n        return\n\n    with open(metadata_path, 'r') as f:\n        metadata = json.load(f)\n\n    print(\"=== Comprehensive Metadata Contents ===\\n\")\n\n    print(\"Dataset Path:\")\n    print(metadata['dataset_path'])\n    print()\n\n    print(\"Directory Structure:\")\n    for dir_name, subdirs in metadata['directories'].items():\n        print(f\"  {dir_name}/\")\n        for subdir in subdirs:\n            print(f\"    {subdir}/\")\n    print()\n\n    print(\"File Types:\")\n    for file_type, files in metadata['files'].items():\n        print(f\"  {file_type}: {len(files)} files\")\n    print()\n\n    print(\"Sample Image Information:\")\n    for i, (image_path, info) in enumerate(metadata['image_info'].items()):\n        print(f\"  {image_path}:\")\n        print(f\"    Size: {info['size']}\")\n        print(f\"    Mode: {info['mode']}\")\n        if i == 2:  # Display info for up to 3 images\n            print(\"  ...\")\n            break\n    print()\n\n    print(\"Sample Numpy Array Information:\")\n    for i, (numpy_path, info) in enumerate(metadata['numpy_info'].items()):\n        print(f\"  {numpy_path}:\")\n        print(f\"    Shape: {info['shape']}\")\n        print(f\"    Data Type: {info['dtype']}\")\n        print(f\"    Min: {info['min']:.2f}, Max: {info['max']:.2f}, Mean: {info['mean']:.2f}\")\n        print(f\"    Unique Values: {info['unique_values']}\")\n        if i == 2:  # Display info for up to 3 numpy files\n            print(\"  ...\")\n            break\n    print()\n\n    print(\"Sample JSON File Information:\")\n    for i, (json_path, info) in enumerate(metadata['json_info'].items()):\n        print(f\"  {json_path}:\")\n        print(f\"    Keys: {info['keys']}\")\n        print(f\"    Length: {info['length']}\")\n        if i == 2:  # Display info for up to 3 JSON files\n            print(\"  ...\")\n            break\n    print()\n\n    print(\"Note: For brevity, only a sample of each type of information is displayed.\")\n    print(\"To see full details, please refer to the comprehensive_meta.json file.\")\n\n# Usage\ninput_dir = '/kaggle/working/organized_dataset'\ndisplay_comprehensive_metadata(input_dir)","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:53:14.823477Z","iopub.execute_input":"2024-10-10T09:53:14.823856Z","iopub.status.idle":"2024-10-10T09:53:14.989001Z","shell.execute_reply.started":"2024-10-10T09:53:14.823819Z","shell.execute_reply":"2024-10-10T09:53:14.98788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install -U albumentations","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:53:42.531655Z","iopub.execute_input":"2024-10-10T09:53:42.532114Z","iopub.status.idle":"2024-10-10T09:53:56.279692Z","shell.execute_reply.started":"2024-10-10T09:53:42.532073Z","shell.execute_reply":"2024-10-10T09:53:56.278203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile utils.py\nimport os\nimport json\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\ndef prepare_data(input_dir, val_split=0.2, random_state=42):\n    preprocessed_dir = os.path.join(input_dir, 'preprocessed_images')\n    labels_dir = os.path.join(input_dir, 'labels')\n    image_files = [f for f in os.listdir(preprocessed_dir) if f.endswith('.npy')]\n    train_files, val_files = train_test_split(image_files, test_size=val_split, random_state=random_state)\n\n    def get_filenames(files, dir_name, extension):\n        return [os.path.join(dir_name, f.replace('preprocessed_', '').replace('.npy', extension)) for f in files]\n\n    train_data = {\n        'image_filenames': [os.path.join(preprocessed_dir, f) for f in train_files],\n        'label3_filenames': get_filenames(train_files, os.path.join(labels_dir, 'labellevel3Ids'), '.png'),\n        'labelcolor_filenames': get_filenames(train_files, os.path.join(labels_dir, 'labelColors'), '.png'),\n        'polygon_filenames': get_filenames(train_files, os.path.join(labels_dir, 'polygons'), '.json')\n    }\n\n    val_data = {\n        'image_filenames': [os.path.join(preprocessed_dir, f) for f in val_files],\n        'label3_filenames': get_filenames(val_files, os.path.join(labels_dir, 'labellevel3Ids'), '.png'),\n        'labelcolor_filenames': get_filenames(val_files, os.path.join(labels_dir, 'labelColors'), '.png'),\n        'polygon_filenames': get_filenames(val_files, os.path.join(labels_dir, 'polygons'), '.json')\n    }\n\n    return train_data, val_data\n\ndef get_transform(is_train=True):\n    if is_train:\n        return A.Compose([\n            A.HorizontalFlip(p=0.5),\n            A.RandomBrightnessContrast(p=0.2),\n            A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n            ToTensorV2(),\n        ], additional_targets={'labelcolor': 'image'})\n    else:\n        return A.Compose([\n            A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n            ToTensorV2(),\n        ], additional_targets={'labelcolor': 'image'})","metadata":{"execution":{"iopub.status.busy":"2024-10-10T10:18:36.775002Z","iopub.execute_input":"2024-10-10T10:18:36.775509Z","iopub.status.idle":"2024-10-10T10:18:36.785003Z","shell.execute_reply.started":"2024-10-10T10:18:36.775465Z","shell.execute_reply":"2024-10-10T10:18:36.783817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile dataset.py\nimport torch\nfrom torch.utils.data import Dataset\nfrom PIL import Image\nimport numpy as np\nimport json\n\nclass SelfDrivingCarDataset(Dataset):\n    def __init__(self, data_paths, transform=None):\n        self.image_filenames = data_paths['image_filenames']\n        self.label3_filenames = data_paths['label3_filenames']\n        self.labelcolor_filenames = data_paths['labelcolor_filenames']\n        self.polygon_filenames = data_paths['polygon_filenames']\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_filenames)\n\n    def __getitem__(self, idx):\n        image = np.load(self.image_filenames[idx])\n        label3 = np.array(Image.open(self.label3_filenames[idx]))\n        labelcolor = np.array(Image.open(self.labelcolor_filenames[idx]))\n\n        with open(self.polygon_filenames[idx], 'r') as f:\n            polygon_data = json.load(f)\n\n        if self.transform:\n            augmented = self.transform(image=image, mask=label3, labelcolor=labelcolor)\n            image = augmented['image']\n            label3 = augmented['mask']\n            labelcolor = augmented['labelcolor']\n\n        return {\n            'image': image,\n            'label3': torch.from_numpy(label3).long(),\n            'labelcolor': torch.from_numpy(labelcolor).permute(2, 0, 1).float(),\n            'polygons': polygon_data\n        }\n","metadata":{"execution":{"iopub.status.busy":"2024-10-10T10:18:44.40742Z","iopub.execute_input":"2024-10-10T10:18:44.407883Z","iopub.status.idle":"2024-10-10T10:18:44.416187Z","shell.execute_reply.started":"2024-10-10T10:18:44.407842Z","shell.execute_reply":"2024-10-10T10:18:44.414869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile model.py\nimport torch\nimport torch.nn as nn\nimport torchvision.models as models\n\nclass DeepLabV3Plus(nn.Module):\n    def __init__(self, num_classes=10):\n        super(DeepLabV3Plus, self).__init__()\n        self.backbone = models.resnet50(pretrained=True)\n        self.aspp = ASPP(2048, 256)\n        self.decoder = Decoder(256, num_classes)\n\n    def forward(self, x):\n        x = self.backbone.conv1(x)\n        x = self.backbone.bn1(x)\n        x = self.backbone.relu(x)\n        x = self.backbone.maxpool(x)\n\n        x = self.backbone.layer1(x)\n        low_level_feat = x\n        x = self.backbone.layer2(x)\n        x = self.backbone.layer3(x)\n        x = self.backbone.layer4(x)\n\n        x = self.aspp(x)\n        x = self.decoder(x, low_level_feat)\n        \n        return x\n\nclass ASPP(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        super(ASPP, self).__init__()\n        self.conv1 = nn.Conv2d(in_channels, out_channels, 1, bias=False)\n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.relu = nn.ReLU(inplace=True)\n        # Add more ASPP components as needed\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        return x\n\nclass Decoder(nn.Module):\n    def __init__(self, in_channels, num_classes):\n        super(Decoder, self).__init__()\n        self.conv1 = nn.Conv2d(in_channels, 256, 3, padding=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(256)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = nn.Conv2d(256, num_classes, 1)\n\n    def forward(self, x, low_level_feat):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.conv2(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-10-10T10:18:50.573315Z","iopub.execute_input":"2024-10-10T10:18:50.573723Z","iopub.status.idle":"2024-10-10T10:18:50.582108Z","shell.execute_reply.started":"2024-10-10T10:18:50.573687Z","shell.execute_reply":"2024-10-10T10:18:50.580684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import DataLoader\nfrom utils import prepare_data, get_transform\nfrom dataset import SelfDrivingCarDataset\nfrom model import DeepLabV3Plus\n\ndef train_model(model, train_loader, val_loader, num_epochs=100, device='cuda'):\n    model = model.to(device)\n    optimizer = torch.optim.Adam(model.parameters())\n    criterion = torch.nn.CrossEntropyLoss()\n\n    for epoch in range(num_epochs):\n        model.train()\n        train_loss = 0.0\n\n        for batch in train_loader:\n            images = batch['image'].to(device)\n            labels = batch['label3'].to(device)\n\n            optimizer.zero_grad()\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n\n            train_loss += loss.item()\n\n        # Validation\n        model.eval()\n        val_loss = 0.0\n        with torch.no_grad():\n            for batch in val_loader:\n                images = batch['image'].to(device)\n                labels = batch['label3'].to(device)\n                outputs = model(images)\n                loss = criterion(outputs, labels)\n                val_loss += loss.item()\n\n        print(f\"Epoch {epoch+1}/{num_epochs}, \"\n              f\"Train Loss: {train_loss/len(train_loader):.4f}, \"\n              f\"Val Loss: {val_loss/len(val_loader):.4f}\")\n\n    torch.save(model.state_dict(), 'final_model.pth')\n\nif __name__ == \"__main__\":\n    input_dir = '/kaggle/working/organized_dataset' \n    train_data, val_data = prepare_data(input_dir)\n\n    train_dataset = SelfDrivingCarDataset(train_data, transform=get_transform(is_train=True))\n    val_dataset = SelfDrivingCarDataset(val_data, transform=get_transform(is_train=False))\n\n    train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True, num_workers=4)\n    val_loader = DataLoader(val_dataset, batch_size=32, shuffle=False, num_workers=4)\n\n    model = DeepLabV3Plus(num_classes=10)  # Adjust num_classes as needed\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    train_model(model, train_loader, val_loader, num_epochs=100, device=device)","metadata":{"execution":{"iopub.status.busy":"2024-10-10T10:28:40.188152Z","iopub.execute_input":"2024-10-10T10:28:40.188753Z","iopub.status.idle":"2024-10-10T10:28:43.533315Z","shell.execute_reply.started":"2024-10-10T10:28:40.188702Z","shell.execute_reply":"2024-10-10T10:28:43.530667Z"},"trusted":true},"execution_count":null,"outputs":[]}]}