{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport shutil\nimport pandas as pd\n\ndef prepare_dataset(csv_path, search_dirs, output_dir, malignant_only=0):\n    \"\"\"\n    Prepares a unified dataset compatible with the original unmodified Swin-MDEP notebook.\n    \n    1. Scans search_dirs recursively for all images.\n    2. Standardizes columns (image/image_name/isic_id -> isic_id, MEL/target -> target).\n    3. Filters for malignant cases only if malignant_only == 1.\n    4. Creates a single directory output_dir/unified_images/ and symlinks (or copies) all found images there (APPEND MODE).\n    5. Appends to metadata.csv if it already exists, removing duplicates by isic_id.\n    \n    Args:\n        csv_path (str): Path to the input metadata CSV file.\n        search_dirs (list): List of directories containing image files.\n        output_dir (str): Working directory where output folder and CSV will be written.\n        malignant_only (int): 1 to only extract/keep malignant cases (target == 1), \n                              0 to keep all cases (benign and malignant). Default is 0.\n    \"\"\"\n    images_output_dir = os.path.join(output_dir, \"unified_images\")\n    os.makedirs(images_output_dir, exist_ok=True)\n    \n    # 1. Scan for available images\n    valid_extensions = {'.jpg', '.jpeg', '.png', '.JPG', '.JPEG', '.PNG'}\n    print(\"Scanning directories for images recursively...\")\n    image_map = {}\n    for s_dir in search_dirs:\n        if not os.path.exists(s_dir):\n            print(f\"[WARNING] Directory not found, skipping: {s_dir}\")\n            continue\n        print(f\"Scanning: {s_dir}\")\n        for root, _, files in os.walk(s_dir):\n            for file in files:\n                ext = os.path.splitext(file)[1]\n                if ext in valid_extensions:\n                    isic_id = os.path.splitext(file)[0]\n                    image_map[isic_id] = os.path.join(root, file)\n                    \n    print(f\"Scan complete. Found {len(image_map)} total unique images.\")\n    \n    # 2. Load dataset metadata\n    print(f\"\\nLoading metadata from {csv_path}...\")\n    df = pd.read_csv(csv_path)\n    \n    # Standardize column: isic_id\n    if 'isic_id' not in df.columns:\n        if 'image' in df.columns:\n            print(\"[INFO] Mapping 'image' column to 'isic_id'...\")\n            df['isic_id'] = df['image'].apply(lambda x: os.path.splitext(str(x))[0])\n        elif 'image_name' in df.columns:\n            print(\"[INFO] Mapping 'image_name' column to 'isic_id'...\")\n            df['isic_id'] = df['image_name'].apply(lambda x: os.path.splitext(str(x))[0])\n        else:\n            raise KeyError(\"Metadata CSV must contain 'isic_id', 'image', or 'image_name' to identify images.\")\n    \n    # Standardize column: target\n    if 'target' not in df.columns:\n        if 'MEL' in df.columns:\n            print(\"[INFO] Mapping 'MEL' column to 'target'...\")\n            df['target'] = df['MEL'].apply(lambda x: 1 if float(x) == 1.0 else 0)\n        else:\n            raise KeyError(\"Metadata CSV must contain 'target' or 'MEL' as the class label.\")\n            \n    # Standardize column: patient_id\n    if 'patient_id' not in df.columns:\n        print(\"[WARNING] 'patient_id' column not found. Generating dummy patient_ids from isic_id...\")\n        df['patient_id'] = df['isic_id']\n    else:\n        df['patient_id'] = df['patient_id'].fillna(df['isic_id'])\n        \n    # 3. Filter for malignant only if specified\n    if malignant_only == 1:\n        print(\"[INFO] Filtering for malignant cases only (target == 1)...\")\n        df = df[df['target'] == 1].reset_index(drop=True)\n        print(f\"Filtered metadata size: {len(df)} rows.\")\n        \n    # Map image paths to filter available rows\n    df['file_path'] = df['isic_id'].map(image_map)\n    df_available = df.dropna(subset=['file_path']).reset_index(drop=True)\n    print(f\"Metadata rows with available images: {len(df_available)} / {len(df)}\")\n    \n    if len(df_available) == 0:\n        print(\"[ERROR] No images found. Cannot prepare dataset.\")\n        return\n        \n    # 4. Create symbolic links (or fallback to copying if symlink fails/Windows local)\n    print(f\"\\nCreating symbolic links in {images_output_dir}...\")\n    link_count = 0\n    copy_count = 0\n    \n    for idx, row in df_available.iterrows():\n        src_path = row['file_path']\n        isic_id = row['isic_id']\n        dest_path = os.path.join(images_output_dir, f\"{isic_id}.jpg\")\n        \n        if not os.path.exists(dest_path):\n            try:\n                os.symlink(src_path, dest_path)\n                link_count += 1\n            except (AttributeError, OSError):\n                # Fallback to copy if symlinks are not supported (e.g. non-admin Windows)\n                shutil.copy(src_path, dest_path)\n                copy_count += 1\n        else:\n            link_count += 1\n            \n    print(f\"Dataset preparation complete: {link_count} symlinks created, {copy_count} files copied.\")\n    \n    # 5. Save/Append metadata CSV file\n    output_csv = os.path.join(output_dir, \"metadata.csv\")\n    df_save = df_available.drop(columns=['file_path'])\n    \n    if os.path.exists(output_csv):\n        print(f\"\\n[INFO] Existing metadata.csv found at {output_csv}. Appending new records...\")\n        df_existing = pd.read_csv(output_csv)\n        \n        # Merge columns safely (in case columns are not identical, fill missing with NaN)\n        df_combined = pd.concat([df_existing, df_save], ignore_index=True)\n        \n        # Remove duplicates by isic_id (keep the newest/last one)\n        total_before = len(df_combined)\n        df_combined = df_combined.drop_duplicates(subset=['isic_id'], keep='last').reset_index(drop=True)\n        duplicates_removed = total_before - len(df_combined)\n        \n        df_combined.to_csv(output_csv, index=False)\n        print(f\"Appended records to existing metadata.csv.\")\n        print(f\"  - Duplicates removed: {duplicates_removed}\")\n        print(f\"  - Total records now: {len(df_combined)}\")\n    else:\n        df_save.to_csv(output_csv, index=False)\n        print(f\"Created new metadata.csv with {len(df_save)} records.\")\n        \n    print(f\"\\nPoint your notebook's csv_path to: {output_csv}\")\n    print(f\"Point your notebook's image_dir to: {images_output_dir}\")\n\nif __name__ == \"__main__\":\n    # Example local/Kaggle config\n    csv_path = \"/kaggle/input/isic-2024-challenge/train-metadata.csv\"\n    search_directories = [\"/kaggle/input/isic-2024-challenge/train-image\"]\n    output_directory = \"/kaggle/working/\"\n    \n    if os.path.exists(csv_path):\n        prepare_dataset(csv_path, search_directories, output_directory, malignant_only=0)\n    else:\n        print(\"[INFO] Run inside Kaggle by importing this file and calling prepare_dataset().\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-15T05:27:46.290376Z","iopub.execute_input":"2026-06-15T05:27:46.290708Z","iopub.status.idle":"2026-06-15T05:27:46.369811Z","shell.execute_reply.started":"2026-06-15T05:27:46.290681Z","shell.execute_reply":"2026-06-15T05:27:46.368832Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Chạy gom dữ liệu cho một dataset (ví dụ ISIC 2020)\nprepare_dataset(\n    csv_path=\"/kaggle/input/competitions/siim-isic-melanoma-classification/train.csv\",\n    search_dirs=[\"/kaggle/input/competitions/siim-isic-melanoma-classification/jpeg/train\"],\n    output_dir=\"/kaggle/working/\",\n    malignant_only=1\n)\nprepare_dataset(\n    csv_path=\"/kaggle/input/competitions/isic-2024-challenge/train-metadata.csv\",\n    search_dirs=[\"/kaggle/input/competitions/isic-2024-challenge/train-image\"],\n    output_dir=\"/kaggle/working/\",\n    malignant_only=0\n)\nprepare_dataset(\n    csv_path=\"/kaggle/input/datasets/andrewmvd/isic-2019/ISIC_2019_Training_GroundTruth.csv\",\n    search_dirs=[\"/kaggle/input/datasets/andrewmvd/isic-2019/ISIC_2019_Training_Input/ISIC_2019_Training_Input\"],\n    output_dir=\"/kaggle/working/\",\n    malignant_only=1\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-15T05:27:53.003176Z","iopub.execute_input":"2026-06-15T05:27:53.004017Z","iopub.status.idle":"2026-06-15T05:29:19.426490Z","shell.execute_reply.started":"2026-06-15T05:27:53.003982Z","shell.execute_reply":"2026-06-15T05:29:19.425198Z"}},"outputs":[],"execution_count":null}]}