{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport sys\nimport platform\n\nprint(\"Python:\", sys.version)\nprint(\"Platform:\", platform.platform())\nprint(\"Working directory:\", os.getcwd())","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nprint(\"Isi /kaggle/input:\")\nfor item in os.listdir(\"/kaggle/input\"):\n    print(\" -\", item)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport glob\nimport json\nimport random\nimport hashlib\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\n\nprint(\"NumPy:\", np.__version__)\nprint(\"Pandas:\", pd.__version__)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from PIL import Image\n\nprint(\"PIL berhasil di-load\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pathlib import Path\n\ndef show_directory_tree(root, max_depth=2):\n    root = Path(root)\n\n    for path in sorted(root.rglob(\"*\")):\n        try:\n            relative = path.relative_to(root)\n            depth = len(relative.parts) - 1\n\n            if depth <= max_depth:\n                prefix = \"    \" * depth\n                print(f\"{prefix}{relative.name}\")\n        except Exception:\n            pass","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"INPUT_DIR = Path(\"/kaggle/input\")\n\nfor dataset_dir in sorted(INPUT_DIR.iterdir()):\n    if dataset_dir.is_dir():\n        print(\"\\n\" + \"=\" * 60)\n        print(\"DATASET:\", dataset_dir.name)\n        print(\"=\" * 60)\n\n        show_directory_tree(dataset_dir, max_depth=2)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"IMAGE_EXTENSIONS = {\n    \".jpg\",\n    \".jpeg\",\n    \".png\",\n    \".bmp\",\n    \".tif\",\n    \".tiff\",\n    \".webp\"\n}\n\ndef find_images(root):\n    root = Path(root)\n\n    images = []\n\n    for path in root.rglob(\"*\"):\n        if path.is_file() and path.suffix.lower() in IMAGE_EXTENSIONS:\n            images.append(path)\n\n    return images","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataset_inventory = []\n\nfor dataset_dir in sorted(INPUT_DIR.iterdir()):\n    if not dataset_dir.is_dir():\n        continue\n\n    images = find_images(dataset_dir)\n\n    dataset_inventory.append({\n        \"dataset\": dataset_dir.name,\n        \"num_images\": len(images)\n    })\n\ninventory_df = pd.DataFrame(dataset_inventory)\n\ninventory_df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"APTOS_DIR = Path(\"/kaggle/input/aptos2019-blindness-detection\")\n\nprint(APTOS_DIR.exists())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for item in APTOS_DIR.iterdir():\n    print(item)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"aptos_train_csv = APTOS_DIR / \"train.csv\"\n\naptos_df = pd.read_csv(aptos_train_csv)\n\nprint(\"Shape:\", aptos_df.shape)\n\naptos_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(aptos_df.columns.tolist())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\n    aptos_df[\"diagnosis\"]\n    .value_counts()\n    .sort_index()\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\n    aptos_df[\"diagnosis\"]\n    .value_counts(normalize=True)\n    .sort_index() * 100\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nclass_counts = aptos_df[\"diagnosis\"].value_counts().sort_index()\n\nplt.figure(figsize=(8, 5))\nclass_counts.plot(kind=\"bar\")\nplt.xlabel(\"DR Grade\")\nplt.ylabel(\"Number of Images\")\nplt.title(\"APTOS Class Distribution\")\nplt.xticks(rotation=0)\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"APTOS_IMAGE_DIR = APTOS_DIR / \"train_images\"\n\naptos_images = find_images(APTOS_IMAGE_DIR)\n\nprint(\"Jumlah image:\", len(aptos_images))\nprint(\"Contoh:\")\n\nfor image_path in aptos_images[:10]:\n    print(image_path)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"image_map = {\n    path.stem: path\n    for path in aptos_images\n}\n\naptos_df[\"image_path\"] = aptos_df[\"id_code\"].map(image_map)\n\nprint(\"Total records:\", len(aptos_df))\nprint(\"Image ditemukan:\", aptos_df[\"image_path\"].notna().sum())\nprint(\"Image hilang:\", aptos_df[\"image_path\"].isna().sum())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_image_info(path):\n    try:\n        with Image.open(path) as img:\n            return {\n                \"width\": img.width,\n                \"height\": img.height,\n                \"mode\": img.mode,\n                \"format\": img.format\n            }\n    except Exception as e:\n        return {\n            \"width\": None,\n            \"height\": None,\n            \"mode\": None,\n            \"format\": None\n        }","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_paths = aptos_df[\"image_path\"].dropna().sample(\n    min(100, aptos_df[\"image_path\"].notna().sum()),\n    random_state=42\n)\n\nimage_info = []\n\nfor path in sample_paths:\n    info = get_image_info(path)\n    info[\"image_path\"] = str(path)\n    image_info.append(info)\n\nimage_info_df = pd.DataFrame(image_info)\n\nimage_info_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Width:\")\nprint(image_info_df[\"width\"].describe())\n\nprint(\"\\nHeight:\")\nprint(image_info_df[\"height\"].describe())\nprint(\"\\nFormats:\")\nprint(image_info_df[\"format\"].value_counts())\n\nprint(\"\\nColor modes:\")\nprint(image_info_df[\"mode\"].value_counts())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(2, 4, figsize=(16, 8))\n\nsample_paths = aptos_df[\"image_path\"].dropna().sample(\n    8,\n    random_state=42\n)\n\nfor ax, path in zip(axes.ravel(), sample_paths):\n    img = Image.open(path)\n\n    ax.imshow(img)\n    ax.set_title(Path(path).name)\n    ax.axis(\"off\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"aptos_standard = pd.DataFrame({\n    \"image_id\": aptos_df[\"id_code\"].astype(str),\n    \"dataset\": \"APTOS\",\n    \"patient_id\": pd.NA,\n    \"eye\": pd.NA,\n    \"dr_grade\": aptos_df[\"diagnosis\"],\n    \"image_quality\": pd.NA,\n    \"image_path\": aptos_df[\"image_path\"].astype(str)\n})\n\naptos_standard.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Unique DR labels:\")\nprint(sorted(aptos_standard[\"dr_grade\"].dropna().unique()))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"aptos_standard.isna().sum()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"duplicate_ids = aptos_standard[\n    aptos_standard[\"image_id\"].duplicated(keep=False)\n]\n\nprint(\"Duplicate image IDs:\", len(duplicate_ids))\n\nduplicate_ids.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"aptos_standard[\"file_exists\"] = aptos_standard[\"image_path\"].apply(\n    lambda x: Path(x).exists()\n)\n\nprint(\n    aptos_standard[\"file_exists\"].value_counts()\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def md5_hash(path, chunk_size=1024 * 1024):\n    hash_md5 = hashlib.md5()\n\n    with open(path, \"rb\") as f:\n        for chunk in iter(lambda: f.read(chunk_size), b\"\"):\n            hash_md5.update(chunk)\n\n    return hash_md5.hexdigest()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"hash_sample = aptos_standard.sample(\n    min(500, len(aptos_standard)),\n    random_state=42\n).copy()\n\nhash_sample[\"md5\"] = hash_sample[\"image_path\"].apply(md5_hash)\n\nprint(\"Total sample:\", len(hash_sample))\nprint(\"Unique hashes:\", hash_sample[\"md5\"].nunique())\nprint(\n    \"Possible duplicates:\",\n    len(hash_sample) - hash_sample[\"md5\"].nunique()\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"OUTPUT_DIR = Path(\"/kaggle/working/dr_audit\")\n\nOUTPUT_DIR.mkdir(\n    parents=True,\n    exist_ok=True\n)\n\nprint(\"Output directory:\", OUTPUT_DIR)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"aptos_standard.to_csv(\n    OUTPUT_DIR / \"aptos_metadata.csv\",\n    index=False\n)\n\nprint(\"Saved:\")\nprint(OUTPUT_DIR / \"aptos_metadata.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"audit_summary = {\n    \"dataset\": \"APTOS\",\n    \"num_records\": int(len(aptos_standard)),\n    \"num_images_found\": int(aptos_standard[\"file_exists\"].sum()),\n    \"num_missing_images\": int((~aptos_standard[\"file_exists\"]).sum()),\n    \"num_unique_image_ids\": int(aptos_standard[\"image_id\"].nunique()),\n    \"num_classes\": int(aptos_standard[\"dr_grade\"].nunique())\n}\n\nwith open(\n    OUTPUT_DIR / \"aptos_audit_summary.json\",\n    \"w\"\n) as f:\n    json.dump(audit_summary, f, indent=4)\n\naudit_summary","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}