{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --upgrade scikit-learn\n!pip install --upgrade imbalanced-learn","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T21:43:39.717714Z","iopub.execute_input":"2025-10-18T21:43:39.718207Z","iopub.status.idle":"2025-10-18T21:43:46.132134Z","shell.execute_reply.started":"2025-10-18T21:43:39.718182Z","shell.execute_reply":"2025-10-18T21:43:46.131389Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nfrom PIL import Image, ImageDraw\n\nimport matplotlib.pyplot as plt\nfrom scipy.ndimage import gaussian_filter\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\n\n# Paths\ndata_dir = '/kaggle/input/aptos2019-blindness-detection'\ntrain_csv_path = os.path.join(data_dir, 'train.csv')\nimg_dir = os.path.join(data_dir, 'train_images')\nprocessed_dir = '/kaggle/working/processed_images'\nos.makedirs(processed_dir, exist_ok=True)\n\n# Step 1: Visualize Initial Data\ndef visualize_initial_data(num_samples_per_class=2):\n    train_csv = pd.read_csv(train_csv_path)\n    fig, axs = plt.subplots(5, num_samples_per_class, figsize=(10, 20))\n    for class_label in range(5):\n        class_samples = train_csv[train_csv['diagnosis'] == class_label].sample(num_samples_per_class, random_state=42)\n        for i, (_, row) in enumerate(class_samples.iterrows()):\n            img_path = os.path.join(img_dir, f\"{row['id_code']}.png\")\n            img = Image.open(img_path)\n            axs[class_label, i].imshow(img)\n            axs[class_label, i].set_title(f\"Class {class_label}: {row['id_code']}\")\n            axs[class_label, i].axis('off')\n    plt.tight_layout()\n    plt.show()\n\n# Call\nvisualize_initial_data()\n\n\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-18T21:43:46.133702Z","iopub.execute_input":"2025-10-18T21:43:46.133953Z","iopub.status.idle":"2025-10-18T21:43:56.338988Z","shell.execute_reply.started":"2025-10-18T21:43:46.133927Z","shell.execute_reply":"2025-10-18T21:43:56.337702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 2: Image Processing Functions\ndef ben_graham_process(img, sigmaX=10):\n    # Convert PIL to NumPy (RGB)\n    img_array = np.array(img).astype(np.float32)\n    blurred = np.zeros_like(img_array)\n    for c in range(3):  # Process each RGB channel\n        blurred[:, :, c] = gaussian_filter(img_array[:, :, c], sigma=sigmaX)\n    # Ben Graham: 4*original - 4*blurred + 128\n    enhanced = 4 * img_array - 4 * blurred + 128\n    enhanced = np.clip(enhanced, 0, 255).astype(np.uint8)\n    return Image.fromarray(enhanced)\n\ndef circular_crop(img, threshold=10):\n    # Use mean RGB for thresholding to keep color\n    img_array = np.array(img)\n    mean_rgb = img_array.mean(axis=2)\n    mask = mean_rgb > threshold\n    if not np.any(mask):\n        return img\n    \n    # Find bounding box\n    rows = np.any(mask, axis=1)\n    cols = np.any(mask, axis=0)\n    min_row, max_row = np.where(rows)[0][[0, -1]]\n    min_col, max_col = np.where(cols)[0][[0, -1]]\n    \n    # Compute center and radius for circular mask\n    width, height = max_col - min_col + 1, max_row - min_row + 1\n    x_center, y_center = (min_col + max_col) // 2, (min_row + max_row) // 2\n    radius = int(min(width, height) / 2 * 0.95)  # 95% to avoid edge clipping\n    \n    # Create circular mask\n    mask_img = Image.new('L', img.size, 0)\n    draw = ImageDraw.Draw(mask_img)\n    draw.ellipse((x_center - radius, y_center - radius, x_center + radius, y_center + radius), fill=255)\n    \n    # Apply mask to RGB image\n    img_array = np.array(img)\n    mask_array = np.array(mask_img)[:, :, np.newaxis]\n    masked = np.where(mask_array == 255, img_array, 0).astype(np.uint8)\n    img = Image.fromarray(masked)\n    \n    # Crop to square bounding box of circle\n    img = img.crop((x_center - radius, y_center - radius, x_center + radius, y_center + radius))\n    return img\n\ndef preprocess_image(img_path, target_size=512):\n    img = Image.open(img_path).convert('RGB')\n    # Order: Crop -> Resize -> Ben Graham (per Kaggle load_ben_color)\n    img = circular_crop(img)\n    img = img.resize((target_size, target_size), Image.BILINEAR)\n    img = ben_graham_process(img, sigmaX=10)\n    return img\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T21:43:56.340060Z","iopub.execute_input":"2025-10-18T21:43:56.340333Z","iopub.status.idle":"2025-10-18T21:43:56.515089Z","shell.execute_reply.started":"2025-10-18T21:43:56.340308Z","shell.execute_reply":"2025-10-18T21:43:56.514427Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n# Step 3: Visualize Processed Data\ndef visualize_processed_data(num_samples_per_class=2):\n    train_csv = pd.read_csv(train_csv_path)\n    fig, axs = plt.subplots(5, 2 * num_samples_per_class, figsize=(20, 20))\n    for class_label in range(5):\n        class_samples = train_csv[train_csv['diagnosis'] == class_label].sample(num_samples_per_class, random_state=42)\n        for i, (_, row) in enumerate(class_samples.iterrows()):\n            img_path = os.path.join(img_dir, f\"{row['id_code']}.png\")\n            original = Image.open(img_path).convert('RGB')\n            processed = preprocess_image(img_path)\n            \n            axs[class_label, 2*i].imshow(original)\n            axs[class_label, 2*i].set_title(f\"Original Class {class_label}: {row['id_code']}\")\n            axs[class_label, 2*i].axis('off')\n            \n            axs[class_label, 2*i + 1].imshow(processed)\n            axs[class_label, 2*i + 1].set_title(f\"Processed Class {class_label}: {row['id_code']}\")\n            axs[class_label, 2*i + 1].axis('off')\n    plt.tight_layout()\n    plt.show()\n\n# Call\nvisualize_processed_data()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T21:43:56.516765Z","iopub.execute_input":"2025-10-18T21:43:56.517003Z","iopub.status.idle":"2025-10-18T21:44:09.448651Z","shell.execute_reply.started":"2025-10-18T21:43:56.516985Z","shell.execute_reply":"2025-10-18T21:44:09.447462Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 4: Save Processed Images\ndef save_processed_images():\n    train_csv = pd.read_csv(train_csv_path)\n    for _, row in train_csv.iterrows():\n        img_path = os.path.join(img_dir, f\"{row['id_code']}.png\")\n        processed_img = preprocess_image(img_path)\n        save_path = os.path.join(processed_dir, f\"{row['id_code']}.png\")\n        processed_img.save(save_path)\n    print(f\"Processed and saved {len(train_csv)} images to {processed_dir}\")\n\n# Call\nsave_processed_images()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T21:44:09.449506Z","iopub.execute_input":"2025-10-18T21:44:09.449841Z","iopub.status.idle":"2025-10-18T22:10:32.327357Z","shell.execute_reply.started":"2025-10-18T21:44:09.449811Z","shell.execute_reply":"2025-10-18T22:10:32.326423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.model_selection import train_test_split\n\n# Paths\ndata_dir = '/kaggle/input/aptos2019-blindness-detection'\ntrain_csv_path = os.path.join(data_dir, 'train.csv')\nprocessed_dir = '/kaggle/working/processed_images'\n\n# Step 1: Stratified Train/Val/Test Split\ndef get_stratified_split(csv_path, train_size=0.8, val_size=0.1):\n    df = pd.read_csv(csv_path)\n    # Train + Val vs Test\n    train_val_df, test_df = train_test_split(\n        df, test_size=0.1, stratify=df['diagnosis'], random_state=42\n    )\n    # Train vs Val\n    relative_val_size = val_size / (train_size + val_size)\n    train_df, val_df = train_test_split(\n        train_val_df, test_size=relative_val_size, stratify=train_val_df['diagnosis'], random_state=42\n    )\n    return train_df, val_df, test_df\n\n# Save splits (optional, for reference)\ntrain_df, val_df, test_df = get_stratified_split(train_csv_path)\ntrain_df.to_csv(os.path.join(processed_dir, 'train_split.csv'), index=False)\nval_df.to_csv(os.path.join(processed_dir, 'val_split.csv'), index=False)\ntest_df.to_csv(os.path.join(processed_dir, 'test_split.csv'), index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T22:10:32.328305Z","iopub.execute_input":"2025-10-18T22:10:32.328857Z","iopub.status.idle":"2025-10-18T22:10:32.701597Z","shell.execute_reply.started":"2025-10-18T22:10:32.328826Z","shell.execute_reply":"2025-10-18T22:10:32.700976Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nimport os\n\n# Paths\ndata_dir = '/kaggle/input/aptos2019-blindness-detection'\ntrain_csv_path = os.path.join(data_dir, 'train.csv')\nprocessed_dir = '/kaggle/working/processed_images'\nfeature_dir = '/kaggle/working/features'\nos.makedirs(feature_dir, exist_ok=True)\n\n# Load stratified splits\ntrain_df = pd.read_csv(os.path.join(processed_dir, 'train_split.csv'))\nval_df = pd.read_csv(os.path.join(processed_dir, 'val_split.csv'))\ntest_df = pd.read_csv(os.path.join(processed_dir, 'test_split.csv'))\n\n# Compute class weights for sampling and loss\ndef get_class_weights(df):\n    class_counts = df['diagnosis'].value_counts().sort_index()\n    total_samples = len(df)\n    num_classes = 5\n    weights = [total_samples / (num_classes * class_counts[i]) if i in class_counts else 0 for i in range(num_classes)]\n    weights = np.array(weights)\n    weights = weights / weights.sum()\n    return weights\n\nclass_weights = get_class_weights(train_df)\nsample_weights = [class_weights[row['diagnosis']] for _, row in train_df.iterrows()]\nsample_weights = torch.tensor(sample_weights, dtype=torch.float)\nloss_weights = torch.tensor(class_weights, dtype=torch.float).to(torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\"))\nprint(f\"Class weights for sampling/loss: {class_weights}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T22:10:32.702384Z","iopub.execute_input":"2025-10-18T22:10:32.702679Z","iopub.status.idle":"2025-10-18T22:10:32.938946Z","shell.execute_reply.started":"2025-10-18T22:10:32.702658Z","shell.execute_reply":"2025-10-18T22:10:32.938180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom PIL import Image\nimport torch\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\n\n# Custom Dataset\nclass APTOSDataset(Dataset):\n    def __init__(self, df, processed_dir, transform=None):\n        self.data = df\n        self.processed_dir = processed_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        row = self.data.iloc[idx]\n        img_name = f\"{row['id_code']}.png\"\n        img_path = os.path.join(self.processed_dir, img_name)\n        image = Image.open(img_path).convert('RGB')\n        label = torch.tensor(row['diagnosis'], dtype=torch.long)\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        return image, label\n\n# Transforms\ntrain_transforms = transforms.Compose([\n    transforms.RandomHorizontalFlip(p=0.5),\n    transforms.RandomVerticalFlip(p=0.5),\n    transforms.RandomRotation(degrees=15),\n    transforms.RandomAffine(degrees=0, scale=(0.9, 1.1), shear=5),\n    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.05),\n    transforms.RandomApply([transforms.Lambda(lambda img: Image.fromarray(\n        np.clip(np.array(img) ** np.random.uniform(0.8, 1.2), 0, 255).astype(np.uint8)\n    ))], p=0.5),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\n\nval_test_transforms = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T22:10:32.939895Z","iopub.execute_input":"2025-10-18T22:10:32.940191Z","iopub.status.idle":"2025-10-18T22:10:32.949607Z","shell.execute_reply.started":"2025-10-18T22:10:32.940166Z","shell.execute_reply":"2025-10-18T22:10:32.948982Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 4: Visualize Augmented Images\ndef visualize_augmented_data(df, processed_dir, transform, num_samples_per_class=2):\n    dataset = APTOSDataset(df, processed_dir, transform=transform)\n    fig, axs = plt.subplots(5, num_samples_per_class, figsize=(15, 25))\n    for class_label in range(5):\n        class_samples = df[df['diagnosis'] == class_label].sample(num_samples_per_class, random_state=42)\n        for i, (_, row) in enumerate(class_samples.iterrows()):\n            img_name = f\"{row['id_code']}.png\"\n            img_path = os.path.join(processed_dir, img_name)\n            img = Image.open(img_path).convert('RGB')\n            # Apply transforms\n            img_tensor = transform(img)\n            # Convert back to PIL for display (denormalize)\n            img_array = img_tensor.permute(1, 2, 0).numpy()\n            img_array = img_array * np.array([0.229, 0.224, 0.225]) + np.array([0.485, 0.456, 0.406])\n            img_array = np.clip(img_array, 0, 1)\n            axs[class_label, i].imshow(img_array)\n            axs[class_label, i].set_title(f\"Augmented Class {class_label}: {row['id_code']}\")\n            axs[class_label, i].axis('off')\n    plt.tight_layout()\n    plt.show()\n\n# Call\nvisualize_augmented_data(train_df, processed_dir, train_transforms)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T22:10:32.950210Z","iopub.execute_input":"2025-10-18T22:10:32.950412Z","iopub.status.idle":"2025-10-18T22:10:35.831380Z","shell.execute_reply.started":"2025-10-18T22:10:32.950398Z","shell.execute_reply":"2025-10-18T22:10:35.830104Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch.utils.data import DataLoader, WeightedRandomSampler\n\n# DataLoaders\ntrain_dataset = APTOSDataset(train_df, processed_dir, transform=train_transforms)\nval_dataset = APTOSDataset(val_df, processed_dir, transform=val_test_transforms)\ntest_dataset = APTOSDataset(test_df, processed_dir, transform=val_test_transforms)\n\nsampler = WeightedRandomSampler(weights=sample_weights, num_samples=len(train_df), replacement=True)\ntrain_loader = DataLoader(train_dataset, batch_size=16, sampler=sampler, num_workers=4)\nval_loader = DataLoader(val_dataset, batch_size=16, shuffle=False, num_workers=4)\ntest_loader = DataLoader(test_dataset, batch_size=16, shuffle=False, num_workers=4)\n\n# Test batch\nfor images, labels in train_loader:\n    print(f\"Train batch: images shape {images.shape}, labels {labels}\")\n    break\nfor images, labels in val_loader:\n    print(f\"Val batch: images shape {images.shape}, labels {labels}\")\n    break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T22:10:35.833638Z","iopub.execute_input":"2025-10-18T22:10:35.833889Z","iopub.status.idle":"2025-10-18T22:10:39.315254Z","shell.execute_reply.started":"2025-10-18T22:10:35.833858Z","shell.execute_reply":"2025-10-18T22:10:39.314475Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torchvision import models\nfrom tqdm.notebook import tqdm\n\n# Initialize ResNet50\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = models.resnet50(pretrained=True)\nmodel.fc = nn.Linear(model.fc.in_features, 5)  # 5 classes\nmodel = model.to(device)\n\n# Freeze all layers except layer4 and fc\nfor name, param in model.named_parameters():\n    if not ('layer4' in name or 'fc' in name):\n        param.requires_grad = False\n\n# Optimizer and loss\noptimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-4)\ncriterion = nn.CrossEntropyLoss(weight=loss_weights)\n\n# Fine-tuning loop\nmodel.train()\nnum_epochs = 20\nfor epoch in tqdm(range(num_epochs), desc=\"Fine-tuning Epochs\"):\n    running_loss = 0.0\n    correct = 0\n    total = 0\n    for images, labels in tqdm(train_loader, desc=f\"Epoch {epoch+1}\", leave=False):\n        images, labels = images.to(device), labels.to(device)\n        optimizer.zero_grad()\n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        running_loss += loss.item()\n        _, predicted = torch.max(outputs, 1)\n        total += labels.size(0)\n        correct += (predicted == labels).sum().item()\n    print(f\"Epoch {epoch+1}, Loss: {running_loss/len(train_loader):.4f}, Accuracy: {100*correct/total:.2f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T22:10:39.316858Z","iopub.execute_input":"2025-10-18T22:10:39.317088Z","iopub.status.idle":"2025-10-18T22:29:47.871035Z","shell.execute_reply.started":"2025-10-18T22:10:39.317064Z","shell.execute_reply":"2025-10-18T22:29:47.870019Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport numpy as np\n\n# Extract features\nmodel.eval()\ndef extract_features(loader, model, save_path):\n    features, labels = [], []\n    with torch.no_grad():\n        for images, lbls in loader:\n            images = images.to(device)\n            feats = model(images).cpu().numpy().reshape(len(images), -1)\n            features.append(feats)\n            labels.append(lbls.numpy())\n    features = np.concatenate(features)\n    labels = np.concatenate(labels)\n    np.savez(save_path, features=features, labels=labels)\n    return features, labels\n\n# Extract and save\ntrain_features, train_labels = extract_features(train_loader, model, os.path.join(feature_dir, 'train_features.npz'))\nval_features, val_labels = extract_features(val_loader, model, os.path.join(feature_dir, 'val_features.npz'))\ntest_features, test_labels = extract_features(test_loader, model, os.path.join(feature_dir, 'test_features.npz'))\nprint(f\"Train features shape: {train_features.shape}, Val features shape: {val_features.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T22:29:47.872519Z","iopub.execute_input":"2025-10-18T22:29:47.873175Z","iopub.status.idle":"2025-10-18T22:30:51.938426Z","shell.execute_reply.started":"2025-10-18T22:29:47.873146Z","shell.execute_reply":"2025-10-18T22:30:51.937511Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torchvision import models\nfrom tqdm.notebook import tqdm\nimport numpy as np\nfrom sklearn.preprocessing import StandardScaler\n\n# === Device setup ===\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# === Initialize DenseNet121 ===\nmodel = models.densenet121(weights=models.DenseNet121_Weights.IMAGENET1K_V1)\nnum_features = model.classifier.in_features\nmodel.classifier = nn.Linear(num_features, 5)  # 5 classes\nmodel = model.to(device)\n\n# === Freeze earlier layers (fine-tuning strategy) ===\nfor name, param in model.features.named_parameters():\n    param.requires_grad = False  # freeze backbone\n\n# Unfreeze the last dense block for fine-tuning\nfor name, param in model.features.denseblock4.named_parameters():\n    param.requires_grad = True\n\n# === Define optimizer and loss ===\noptimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-4)\ncriterion = nn.CrossEntropyLoss(weight=loss_weights.to(device))  # assume you defined loss_weights\n\n# === Training loop ===\nnum_epochs = 100\nmodel.train()\nfor epoch in tqdm(range(num_epochs), desc=\"Fine-tuning Epochs\"):\n    running_loss, correct, total = 0.0, 0, 0\n    for images, labels in tqdm(train_loader, desc=f\"Epoch {epoch+1}\", leave=False):\n        images, labels = images.to(device), labels.to(device)\n        optimizer.zero_grad()\n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        running_loss += loss.item()\n        _, predicted = torch.max(outputs, 1)\n        total += labels.size(0)\n        correct += (predicted == labels).sum().item()\n    acc = 100 * correct / total\n    print(f\"Epoch {epoch+1}, Loss: {running_loss/len(train_loader):.4f}, Accuracy: {acc:.2f}%\")\n\n# === Feature extraction model (remove final classifier) ===\nfeature_extractor = models.densenet121(weights=models.DenseNet121_Weights.IMAGENET1K_V1)\nfeature_extractor.classifier = nn.Identity()  # remove classifier layer\nfeature_extractor.load_state_dict(model.state_dict(), strict=False)\nfeature_extractor = feature_extractor.to(device)\nfeature_extractor.eval()\n\n# === Feature extraction function ===\ndef extract_features(loader, model, save_path):\n    all_features, all_labels = [], []\n    with torch.no_grad():\n        for images, lbls in tqdm(loader, desc=f\"Extracting {os.path.basename(save_path)}\"):\n            images = images.to(device)\n            feats = model(images).cpu().numpy()\n            all_features.append(feats)\n            all_labels.append(lbls.numpy())\n    features = np.concatenate(all_features)\n    labels = np.concatenate(all_labels)\n\n    # ⚠️ DO NOT SCALE HERE — leave it raw for downstream SMOTE + scaling\n    np.savez(save_path, features=features, labels=labels)\n    return features, labels\n\n\n# === Save directory ===\nfeature_dir = \"features\"\nos.makedirs(feature_dir, exist_ok=True)\n\n# === Extract and save features ===\ntrain_features, train_labels = extract_features(train_loader, feature_extractor, os.path.join(feature_dir, 'train_features.npz'))\nval_features, val_labels = extract_features(val_loader, feature_extractor, os.path.join(feature_dir, 'val_features.npz'))\ntest_features, test_labels = extract_features(test_loader, feature_extractor, os.path.join(feature_dir, 'test_features.npz'))\n\nprint(f\"✅ Feature extraction complete!\")\nprint(f\"Train features shape: {train_features.shape}, Val features shape: {val_features.shape}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(train_features)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import StandardScaler, label_binarize\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom sklearn.metrics import f1_score, roc_curve, auc\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.neighbors import KNeighborsClassifier, NearestCentroid\nfrom sklearn.linear_model import (\n    LogisticRegression, PassiveAggressiveClassifier, Perceptron,\n    RidgeClassifier, SGDClassifier\n)\nfrom sklearn.neural_network import MLPClassifier\nfrom scipy.stats import randint, loguniform\nfrom imblearn.combine import SMOTEENN\nfrom itertools import cycle\nimport joblib\n\n# === Plot style ===\nplt.style.use('seaborn-v0_8')\nsns.set(font_scale=1.2)\n\n# === Paths ===\nfeature_dir = '/kaggle/working/features'\noutput_dir = \"tuned_classifiers\"\nroc_dir = os.path.join(\"images\", \"tunedModel\")\nos.makedirs(output_dir, exist_ok=True)\nos.makedirs(roc_dir, exist_ok=True)\n\n# === Load data ===\ntrain_data = np.load(os.path.join(feature_dir, 'train_features.npz'))\nval_data = np.load(os.path.join(feature_dir, 'val_features.npz'))\ntest_data = np.load(os.path.join(feature_dir, 'test_features.npz'))\n\ntrain_features, train_labels = train_data['features'], train_data['labels']\nprint('Len' + str(len(train_features)))\nval_features, val_labels = val_data['features'], val_data['labels']\ntest_features, test_labels = test_data['features'], test_data['labels']\n\n# === Apply SMOTEENN for hybrid resampling ===\nprint(\"⚖️ Balancing training data using SMOTEENN...\")\nsmote_enn = SMOTEENN(random_state=42)\ntrain_features_res, train_labels_res = smote_enn.fit_resample(train_features, train_labels)\nprint(f\"Resampled training set size: {train_features_res.shape[0]} samples\")\n\n# === Standardize features ===\nscaler = StandardScaler()\ntrain_features_scaled = scaler.fit_transform(train_features_res)\nval_features_scaled = scaler.transform(val_features)\ntest_features_scaled = scaler.transform(test_features)\njoblib.dump(scaler, os.path.join(output_dir, 'scaler.pkl'))\nprint('Len' + str(len(train_features_scaled)))\n# === Define classifiers ===\nclassifiers = {\n    'dt': DecisionTreeClassifier(random_state=42),\n    'rf': RandomForestClassifier(random_state=42, n_jobs=-1),\n    'svm': SVC(probability=True, random_state=42),\n    'knn': KNeighborsClassifier(n_jobs=-1),\n    'lr': LogisticRegression(max_iter=1000, random_state=42, n_jobs=-1),\n    'et': ExtraTreesClassifier(random_state=42, n_jobs=-1),\n    'mlp': MLPClassifier(max_iter=1000, random_state=42),\n    'nc': NearestCentroid(),\n    'pa': PassiveAggressiveClassifier(max_iter=1000, random_state=42),\n    'perceptron': Perceptron(random_state=42),\n    'ridge': RidgeClassifier(random_state=42),\n    'sgd': SGDClassifier(loss='log_loss', max_iter=1000, random_state=42)\n}\n\n# === Parameter distributions for RandomizedSearch ===\nparam_distributions = {\n    'dt': {'max_depth': [5, 10, 20, None],\n           'min_samples_split': randint(2, 10),\n           'min_samples_leaf': randint(1, 5)},\n    'rf': {'n_estimators': randint(100, 300),\n           'max_depth': [10, 20, None],\n           'min_samples_split': randint(2, 10),\n           'min_samples_leaf': randint(1, 5)},\n    'svm': {'C': loguniform(0.1, 10),\n            'kernel': ['rbf', 'linear'],\n            'gamma': ['scale', 'auto']},\n    'knn': {'n_neighbors': randint(3, 10),\n            'weights': ['uniform', 'distance'],\n            'p': [1, 2]},\n    'lr': {'C': loguniform(0.01, 10),\n           'solver': ['lbfgs', 'liblinear'],\n           'penalty': ['l2']},\n    'et': {'n_estimators': randint(100, 300),\n           'max_depth': [10, 20, None],\n           'min_samples_split': randint(2, 10),\n           'min_samples_leaf': randint(1, 5)},\n    'mlp': {'hidden_layer_sizes': [(64,), (128,), (128, 64), (256, 128)],\n            'alpha': loguniform(1e-5, 1e-2),\n            'learning_rate': ['constant', 'adaptive']},\n    'nc': {},\n    'pa': {'C': loguniform(0.01, 10),\n           'loss': ['hinge', 'squared_hinge']},\n    'perceptron': {'penalty': [None, 'l2', 'l1'],\n                   'alpha': loguniform(1e-5, 1e-2),\n                   'eta0': [0.1, 1.0]},\n    'ridge': {'alpha': loguniform(0.1, 100)},\n    'sgd': {'alpha': loguniform(1e-5, 1e-2),\n            'learning_rate': ['constant', 'optimal', 'adaptive'],\n            'eta0': [0.001, 0.01, 0.1]}\n}\n\n# === ROC plotting ===\ndef plot_roc(model_name, model, X, y, save_dir):\n    \"\"\"Plot one-vs-rest ROC for multiclass classification.\"\"\"\n    if not hasattr(model, \"predict_proba\"):\n        print(f\"⚠️ {model_name} does not support predict_proba, skipping ROC.\")\n        return\n\n    classes = np.unique(y)\n    y_bin = label_binarize(y, classes=classes)\n    y_score = model.predict_proba(X)\n\n    plt.figure(figsize=(7, 6))\n    colors = cycle(plt.cm.tab10.colors)\n\n    for i, color in zip(range(len(classes)), colors):\n        fpr, tpr, _ = roc_curve(y_bin[:, i], y_score[:, i])\n        roc_auc = auc(fpr, tpr)\n        plt.plot(fpr, tpr, color=color, lw=2,\n                 label=f'Class {classes[i]} (AUC={roc_auc:.2f})')\n\n    plt.plot([0, 1], [0, 1], 'k--', lw=2)\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title(f'ROC Curve - {model_name}')\n    plt.legend(loc=\"lower right\")\n    plt.tight_layout()\n    plt.savefig(os.path.join(save_dir, f\"{model_name}_roc.png\"))\n    plt.close()\n\n# === Training & Evaluation ===\nresults = []\n\nfor name, clf in classifiers.items():\n    print(f\"\\n🔹 Tuning {name}...\")\n\n    params = param_distributions[name]\n    if params:\n        search = RandomizedSearchCV(\n            clf,\n            param_distributions=params,\n            n_iter=20,\n            cv=5,\n            scoring='f1_weighted',\n            random_state=42,\n            n_jobs=-1,\n            verbose=1\n        )\n        search.fit(train_features_scaled, train_labels_res)\n        best_model = search.best_estimator_\n        print(f\"✅ Best Params: {search.best_params_}\")\n    else:\n        best_model = clf.fit(train_features_scaled, train_labels_res)\n\n    # Save model\n    joblib.dump(best_model, os.path.join(output_dir, f\"{name}_tuned_model.pkl\"))\n\n    # Evaluate\n    val_pred = best_model.predict(val_features_scaled)\n    test_pred = best_model.predict(test_features_scaled)\n    val_f1 = f1_score(val_labels, val_pred, average='weighted')\n    test_f1 = f1_score(test_labels, test_pred, average='weighted')\n\n    print(f\"Validation F1: {val_f1:.4f} | Test F1: {test_f1:.4f}\")\n    results.append((name, val_f1, test_f1))\n\n    # ROC\n    plot_roc(name, best_model, test_features_scaled, test_labels, roc_dir)\n\n# === Summary ===\nprint(\"\\n📊 Final F1-scores:\")\nfor name, val_f1, test_f1 in results:\n    print(f\"{name:10s} | Val F1: {val_f1:.4f} | Test F1: {test_f1:.4f}\")\n\nprint(f\"\\n✅ All ROC curves saved to: {roc_dir}\")\nprint(f\"✅ Tuned models saved to: {output_dir}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nimport joblib\n\n# === Paths ===\nfeature_dir = '/kaggle/working/features'  # or your feature directory\noutput_dir = \"tuned_classifiers\"\nmeta_dir = \"meta_features\"\nos.makedirs(meta_dir, exist_ok=True)\n\n# === Load training features & labels ===\ntrain_data = np.load(os.path.join(feature_dir, 'train_features.npz'))\ntrain_features = train_data['features']\ntrain_labels = train_data['labels']\n\n# === Load scaler (used in base models) and scale train features ===\nscaler = joblib.load(os.path.join(output_dir, 'scaler.pkl'))\ntrain_features_scaled = scaler.transform(train_features)\n\n# === Load tuned base models ===\nclassifiers = {\n    'dt': joblib.load(os.path.join(output_dir, 'dt_tuned_model.pkl')),\n    'rf': joblib.load(os.path.join(output_dir, 'rf_tuned_model.pkl')),\n    'svm': joblib.load(os.path.join(output_dir, 'svm_tuned_model.pkl')),\n    'knn': joblib.load(os.path.join(output_dir, 'knn_tuned_model.pkl')),\n    'lr': joblib.load(os.path.join(output_dir, 'lr_tuned_model.pkl')),\n    'et': joblib.load(os.path.join(output_dir, 'et_tuned_model.pkl')),\n    'mlp': joblib.load(os.path.join(output_dir, 'mlp_tuned_model.pkl')),\n    'nc': joblib.load(os.path.join(output_dir, 'nc_tuned_model.pkl')),\n    'pa': joblib.load(os.path.join(output_dir, 'pa_tuned_model.pkl')),\n    'perceptron': joblib.load(os.path.join(output_dir, 'perceptron_tuned_model.pkl')),\n    'ridge': joblib.load(os.path.join(output_dir, 'ridge_tuned_model.pkl')),\n    'sgd': joblib.load(os.path.join(output_dir, 'sgd_tuned_model.pkl'))\n}\n\n# === Base models that support predict_proba ===\nproba_classifiers = ['dt', 'rf', 'svm', 'knn', 'lr', 'et', 'mlp', 'sgd']\n\n# === Prepare one-hot encoder for models without predict_proba ===\nunique_labels = np.unique(train_labels)\nonehot_encoder = OneHotEncoder(sparse_output=False, categories=[unique_labels])\n\n# === Generate predictions (meta-features) ===\nmeta_features = []\nfor name, clf in classifiers.items():\n    print(f\"Generating training meta-features for {name}...\")\n    \n    if name in proba_classifiers:\n        preds = clf.predict_proba(train_features_scaled)\n    else:\n        pred_labels = clf.predict(train_features_scaled)\n        preds = onehot_encoder.fit_transform(pred_labels.reshape(-1, 1))\n    \n    np.save(os.path.join(meta_dir, f\"{name}_train_predictions.npy\"), preds)\n    meta_features.append(preds)\n\n# === Stack meta-features from all base models ===\nmeta_features = np.hstack(meta_features)\n\n# === Scale meta-features ===\nmeta_scaler = StandardScaler()\nmeta_features_scaled = meta_scaler.fit_transform(meta_features)\njoblib.dump(meta_scaler, os.path.join(meta_dir, 'meta_scaler.pkl'))\n\n# === Save meta-features and labels ===\nnp.save(os.path.join(meta_dir, 'meta_features_train.npy'), meta_features_scaled)\nnp.save(os.path.join(meta_dir, 'train_labels.npy'), train_labels)\n\nprint(\"✅ Meta-features for TRAIN set generated and saved successfully.\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nimport joblib\n\n# === Configuration ===\nfeature_dir = '/kaggle/working/features'   # Path to feature files\noutput_dir = \"tuned_classifiers\"           # Folder with trained base models\nmeta_dir = \"meta_features\"                 # Folder to store meta features\nos.makedirs(meta_dir, exist_ok=True)\n\n# === Load validation data ===\nval_data = np.load(os.path.join(feature_dir, 'val_features.npz'))\ntrain_data = np.load(os.path.join(feature_dir, 'train_features.npz'))\n\nval_features = val_data['features']\nval_labels = val_data['labels']\ntrain_labels = train_data['labels']\n\n# === Standardize validation features using saved scaler ===\nscaler = joblib.load(os.path.join(output_dir, 'scaler.pkl'))\nval_features_scaled = scaler.transform(val_features)\n\n# === Load tuned models ===\nclassifiers = {\n    'dt': joblib.load(os.path.join(output_dir, 'dt_tuned_model.pkl')),\n    'rf': joblib.load(os.path.join(output_dir, 'rf_tuned_model.pkl')),\n    'svm': joblib.load(os.path.join(output_dir, 'svm_tuned_model.pkl')),\n    'knn': joblib.load(os.path.join(output_dir, 'knn_tuned_model.pkl')),\n    'lr': joblib.load(os.path.join(output_dir, 'lr_tuned_model.pkl')),\n    'et': joblib.load(os.path.join(output_dir, 'et_tuned_model.pkl')),\n    'mlp': joblib.load(os.path.join(output_dir, 'mlp_tuned_model.pkl')),\n    'nc': joblib.load(os.path.join(output_dir, 'nc_tuned_model.pkl')),\n    'pa': joblib.load(os.path.join(output_dir, 'pa_tuned_model.pkl')),\n    'perceptron': joblib.load(os.path.join(output_dir, 'perceptron_tuned_model.pkl')),\n    'ridge': joblib.load(os.path.join(output_dir, 'ridge_tuned_model.pkl')),\n    'sgd': joblib.load(os.path.join(output_dir, 'sgd_tuned_model.pkl'))\n}\n\n# Models that support predict_proba\nproba_classifiers = ['dt', 'rf', 'svm', 'knn', 'lr', 'et', 'mlp', 'sgd']\n\n# === Prepare meta features ===\nmeta_features = []\nunique_labels = np.unique(train_labels)\nonehot_encoder = OneHotEncoder(sparse_output=False, categories=[unique_labels.reshape(-1,)])\n\nprint(\"\\nGenerating meta-features for stacking...\\n\")\n\nfor name, clf in classifiers.items():\n    print(f\"Processing {name}...\")\n    # Use probabilities if available; otherwise use one-hot encoded predictions\n    if name in proba_classifiers:\n        preds = clf.predict_proba(val_features_scaled)\n    else:\n        pred_labels = clf.predict(val_features_scaled)\n        preds = onehot_encoder.fit_transform(pred_labels.reshape(-1, 1))\n    \n    # Save individual model predictions (for debugging or analysis)\n    np.save(os.path.join(meta_dir, f\"{name}_val_predictions.npy\"), preds)\n    meta_features.append(preds)\n\n# === Combine and standardize meta features ===\nmeta_features = np.hstack(meta_features)\nmeta_scaler = StandardScaler()\nmeta_features_scaled = meta_scaler.fit_transform(meta_features)\n\n# === Save meta data for next (stacking) stage ===\njoblib.dump(meta_scaler, os.path.join(meta_dir, 'meta_scaler.pkl'))\nnp.save(os.path.join(meta_dir, 'meta_features_val.npy'), meta_features_scaled)\nnp.save(os.path.join(meta_dir, 'val_labels.npy'), val_labels)\n\nprint(\"\\n✅ Meta-feature generation complete.\")\nprint(f\"Saved stacked features to: {meta_dir}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nimport joblib\n\n# === Paths ===\nfeature_dir = '/kaggle/working/features'\noutput_dir = \"tuned_classifiers\"\nmeta_dir = \"meta_features\"\nos.makedirs(meta_dir, exist_ok=True)\n\n# === Load test data ===\ntest_features = np.load(os.path.join(feature_dir, 'test_features.npz'))['features']\ntest_labels = np.load(os.path.join(feature_dir, 'test_features.npz'))['labels']\ntrain_labels = np.load(os.path.join(feature_dir, 'train_features.npz'))['labels']\n\n# === Load scaler and standardize test features ===\nscaler = joblib.load(os.path.join(output_dir, 'scaler.pkl'))\ntest_features_scaled = scaler.transform(test_features)\n\n# === Load tuned base models ===\nclassifiers = {\n    'dt': joblib.load(os.path.join(output_dir, 'dt_tuned_model.pkl')),\n    'rf': joblib.load(os.path.join(output_dir, 'rf_tuned_model.pkl')),\n    'svm': joblib.load(os.path.join(output_dir, 'svm_tuned_model.pkl')),\n    'knn': joblib.load(os.path.join(output_dir, 'knn_tuned_model.pkl')),\n    'lr': joblib.load(os.path.join(output_dir, 'lr_tuned_model.pkl')),\n    'et': joblib.load(os.path.join(output_dir, 'et_tuned_model.pkl')),\n    'mlp': joblib.load(os.path.join(output_dir, 'mlp_tuned_model.pkl')),\n    'nc': joblib.load(os.path.join(output_dir, 'nc_tuned_model.pkl')),\n    'pa': joblib.load(os.path.join(output_dir, 'pa_tuned_model.pkl')),\n    'perceptron': joblib.load(os.path.join(output_dir, 'perceptron_tuned_model.pkl')),\n    'ridge': joblib.load(os.path.join(output_dir, 'ridge_tuned_model.pkl')),\n    'sgd': joblib.load(os.path.join(output_dir, 'sgd_tuned_model.pkl'))\n}\n\nproba_classifiers = ['dt', 'rf', 'svm', 'knn', 'lr', 'et', 'mlp', 'sgd']\nunique_labels = np.unique(train_labels)\nonehot_encoder = OneHotEncoder(sparse_output=False, categories=[unique_labels])\n\n# === Generate meta features for test ===\nmeta_features_test = []\nfor name, clf in classifiers.items():\n    print(f\"Generating test predictions for {name}...\")\n    if name in proba_classifiers:\n        pred = clf.predict_proba(test_features_scaled)\n    else:\n        pred_labels = clf.predict(test_features_scaled)\n        pred = onehot_encoder.fit_transform(pred_labels.reshape(-1, 1))\n    meta_features_test.append(pred)\n    np.save(os.path.join(meta_dir, f\"{name}_test_predictions.npy\"), pred)\n\n# === Stack and standardize meta features ===\nmeta_features_test = np.hstack(meta_features_test)\nmeta_scaler = joblib.load(os.path.join(meta_dir, 'meta_scaler.pkl'))  # same scaler from val\nmeta_features_test_scaled = meta_scaler.transform(meta_features_test)\n\n# === Save for next step ===\nnp.save(os.path.join(meta_dir, 'meta_features_test.npy'), meta_features_test_scaled)\nnp.save(os.path.join(meta_dir, 'test_labels.npy'), test_labels)\n\nprint(\"Meta-features for test set generated and saved in\", meta_dir)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\nimport joblib\n\n# === Paths ===\nmeta_dir = \"meta_features\"\nos.makedirs(\"images/stacking\", exist_ok=True)\n\n# === Load meta-feature datasets ===\nmeta_features_train = np.load(os.path.join(meta_dir, 'meta_features_train.npy'))\ntrain_labels = np.load(os.path.join(meta_dir, 'train_labels.npy'))\n\nmeta_features_val = np.load(os.path.join(meta_dir, 'meta_features_val.npy'))\nval_labels = np.load(os.path.join(meta_dir, 'val_labels.npy'))\n\nmeta_features_test = np.load(os.path.join(meta_dir, 'meta_features_test.npy'))\ntest_labels = np.load(os.path.join(meta_dir, 'test_labels.npy'))\n\n# === Define parameter grid for RandomizedSearch ===\nparam_distributions = {\n    'hidden_layer_sizes': [(64,), (128,), (128, 64), (256, 128), (256, 128, 64)],\n    'activation': ['relu', 'tanh'],\n    'solver': ['adam', 'sgd'],\n    'alpha': [1e-5, 1e-4, 1e-3, 1e-2],\n    'learning_rate': ['constant', 'adaptive'],\n    'batch_size': [32, 64, 128, 256],\n}\n\n# === Base MLP model ===\nbase_mlp = MLPClassifier(\n    max_iter=1000,\n    random_state=42,\n    early_stopping=True,\n    validation_fraction=0.1,\n    n_iter_no_change=20,\n)\n\n# === Randomized Search ===\nprint(\"🔍 Running Randomized Search for Meta MLP...\")\nrandom_search = RandomizedSearchCV(\n    estimator=base_mlp,\n    param_distributions=param_distributions,\n    n_iter=20,  # Try 20 random combinations\n    scoring='f1_weighted',\n    cv=3,\n    verbose=2,\n    n_jobs=-1,\n    random_state=42\n)\n\nrandom_search.fit(meta_features_train, train_labels)\nprint(f\"\\n✅ Best Parameters: {random_search.best_params_}\")\nprint(f\"🏆 Best CV Score: {random_search.best_score_:.4f}\")\n\n# === Train the best model on full training meta-features ===\nbest_mlp = random_search.best_estimator_\nbest_mlp.fit(meta_features_train, train_labels)\n\n# === Save tuned meta model ===\njoblib.dump(best_mlp, os.path.join(meta_dir, 'meta_mlp_tuned_model.pkl'))\n\n# === Evaluate on VALIDATION set ===\nval_preds = best_mlp.predict(meta_features_val)\nprint(\"\\n=== Meta MLP Model Evaluation (Validation Set) ===\")\nprint(classification_report(val_labels, val_preds, digits=4))\n\n# === Confusion Matrix (Validation) ===\ncm = confusion_matrix(val_labels, val_preds)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm)\ndisp.plot(cmap=\"Blues\")\nplt.title(\"Meta MLP Confusion Matrix (Validation Set)\")\nplt.tight_layout()\nplt.savefig(\"images/stacking/meta_mlp_confusion_matrix_val.png\")\nplt.show()\n\n# === Evaluate on TEST set ===\ntest_preds = best_mlp.predict(meta_features_test)\nprint(\"\\n=== Meta MLP Model Final Test Evaluation ===\")\nprint(classification_report(test_labels, test_preds, digits=4))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Ensemble Val Accuracy: {accuracy_score(val_labels, val_preds):.4f}\")\nprint(f\"Ensemble Val Kappa: {cohen_kappa_score(test_labels, test_preds, weights='quadratic'):.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.metrics import confusion_matrix, f1_score\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport joblib\n\n# Set plot style\nplt.style.use('seaborn')\nsns.set(font_scale=1.2)\n\n# Define paths\nfeature_dir = '/kaggle/working/features'  # Update with your Kaggle path\noutput_dir = \"tuned_classifiers\"\nmeta_dir = \"meta_features\"\nplots_dir = \"plots\"\nos.makedirs(meta_dir, exist_ok=True)\nos.makedirs(plots_dir, exist_ok=True)\n\n# Load data\nval_features = np.load(os.path.join(feature_dir, 'val_features.npz'))['features']\nval_labels = np.load(os.path.join(feature_dir, 'val_features.npz'))['labels']\ntrain_labels = np.load(os.path.join(feature_dir, 'train_features.npz'))['labels']\n\n# Load scaler and standardize validation features\nscaler = joblib.load(os.path.join(output_dir, 'scaler.pkl'))\nval_features_scaled = scaler.transform(val_features)\n\n# Load tuned models\nclassifiers = {\n    'dt': joblib.load(os.path.join(output_dir, 'dt_tuned_model.pkl')),\n    'rf': joblib.load(os.path.join(output_dir, 'rf_tuned_model.pkl')),\n    'svm': joblib.load(os.path.join(output_dir, 'svm_tuned_model.pkl')),\n    'knn': joblib.load(os.path.join(output_dir, 'knn_tuned_model.pkl')),\n    'lr': joblib.load(os.path.join(output_dir, 'lr_tuned_model.pkl')),\n    'et': joblib.load(os.path.join(output_dir, 'et_tuned_model.pkl')),\n    'mlp': joblib.load(os.path.join(output_dir, 'mlp_tuned_model.pkl')),\n    'nc': joblib.load(os.path.join(output_dir, 'nc_tuned_model.pkl')),\n    'pa': joblib.load(os.path.join(output_dir, 'pa_tuned_model.pkl')),\n    'perceptron': joblib.load(os.path.join(output_dir, 'perceptron_tuned_model.pkl')),\n    'ridge': joblib.load(os.path.join(output_dir, 'ridge_tuned_model.pkl')),\n    'sgd': joblib.load(os.path.join(output_dir, 'sgd_tuned_model.pkl'))\n}\n\n# Classifiers that support predict_proba\nproba_classifiers = ['dt', 'rf', 'svm', 'knn', 'lr', 'et', 'mlp', 'sgd']\nunique_labels = np.unique(train_labels)\nclass_names = [str(i) for i in unique_labels]  # Replace with actual class names if available\n\n# Generate predictions and visualizations\nmeta_features = []\nf1_scores = {}\nonehot_encoder = OneHotEncoder(sparse_output=False, categories=[unique_labels])\nfor name, clf in classifiers.items():\n    print(f\"Generating predictions for {name}...\")\n    if name in proba_classifiers:\n        pred = clf.predict_proba(val_features_scaled)\n        pred_labels = clf.predict(val_features_scaled)  # For confusion matrix\n    else:\n        pred_labels = clf.predict(val_features_scaled)\n        pred = onehot_encoder.fit_transform(pred_labels.reshape(-1, 1))\n    meta_features.append(pred)\n    np.save(os.path.join(meta_dir, f\"{name}_val_predictions.npy\"), pred)\n    \n    # Compute F1 score\n    f1 = f1_score(val_labels, pred_labels, average='weighted')\n    f1_scores[name] = f1\n    \n    # Plot confusion matrix\n    cm = confusion_matrix(val_labels, pred_labels, labels=unique_labels)\n    plt.figure(figsize=(8, 6))\n    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=class_names, yticklabels=class_names)\n    plt.title(f'Confusion Matrix for {name.upper()}')\n    plt.xlabel('Predicted')\n    plt.ylabel('True')\n    plt.tight_layout()\n    plt.savefig(os.path.join(plots_dir, f'{name}_confusion_matrix.png'))\n    plt.close()\n\n# Stack and standardize meta-features\nmeta_features = np.hstack(meta_features)\nmeta_scaler = StandardScaler()\nmeta_features_scaled = meta_scaler.fit_transform(meta_features)\njoblib.dump(meta_scaler, os.path.join(meta_dir, 'meta_scaler.pkl'))\nnp.save(os.path.join(meta_dir, 'meta_features_val.npy'), meta_features_scaled)\nnp.save(os.path.join(meta_dir, 'val_labels.npy'), val_labels)\n\n# Plot F1 scores\nplt.figure(figsize=(12, 6))\nsns.barplot(x=list(f1_scores.keys()), y=list(f1_scores.values()))\nplt.title('Validation F1 Scores for Base Classifiers')\nplt.xlabel('Classifier')\nplt.ylabel('F1 Score (Weighted)')\nplt.xticks(rotation=45)\nplt.tight_layout()\nplt.savefig(os.path.join(plots_dir, 'base_classifiers_val_f1_scores.png'))\nplt.close()\n\n\nprint(\"Meta-features generation complete. Plots saved in\", plots_dir)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport joblib\n\n# Set plot style\nplt.style.use('seaborn')\nsns.set(font_scale=1.2)\n\n# Define paths\n# Define paths\nfeature_dir = '/kaggle/working/features'\noutput_dir = \"tuned_classifiers\"\nmeta_dir = \"meta_features\"\nmodel_dir = \"meta_model\"\nsubmissions_dir = \"submissions\"\nplots_dir = \"plots\"\nos.makedirs(submission_dir, exist_ok=True)\nos.makedirs(plots_dir, exist_ok=True)\n\n# Load test data\ntest_data = np.load(os.path.join(feature_dir, 'test_features.npz'))\ntest_features = test_data['features']\nif 'ids' in test_data:\n    test_ids = test_data['ids']\nelse:\n    test_ids = np.arange(len(test_features))\n\n# Load scaler and standardize test features\nscaler = joblib.load(os.path.join(output_dir, 'scaler.pkl'))\ntest_features_scaled = scaler.transform(test_features)\n\n# Load tuned models\nclassifiers = {\n    'dt': joblib.load(os.path.join(output_dir, 'dt_tuned_model.pkl')),\n    'rf': joblib.load(os.path.join(output_dir, 'rf_tuned_model.pkl')),\n    'svm': joblib.load(os.path.join(output_dir, 'svm_tuned_model.pkl')),\n    'knn': joblib.load(os.path.join(output_dir, 'knn_tuned_model.pkl')),\n    'lr': joblib.load(os.path.join(output_dir, 'lr_tuned_model.pkl')),\n    'et': joblib.load(os.path.join(output_dir, 'et_tuned_model.pkl')),\n    'mlp': joblib.load(os.path.join(output_dir, 'mlp_tuned_model.pkl')),\n    'nc': joblib.load(os.path.join(output_dir, 'nc_tuned_model.pkl')),\n    'pa': joblib.load(os.path.join(output_dir, 'pa_tuned_model.pkl')),\n    'perceptron': joblib.load(os.path.join(output_dir, 'perceptron_tuned_model.pkl')),\n    'ridge': joblib.load(os.path.join(output_dir, 'ridge_tuned_model.pkl')),\n    'sgd': joblib.load(os.path.join(output_dir, 'sgd_tuned_model.pkl'))\n}\n\n# Classifiers that support predict_proba\nproba_classifiers = ['dt', 'rf', 'svm', 'knn', 'lr', 'et', 'mlp', 'sgd']\ntrain_labels = np.load(os.path.join(feature_dir, 'train_features.npz'))['labels']\nunique_labels = np.unique(train_labels)\nclass_names = [str(i) for i in unique_labels]  # Replace with actual class names if available\nonehot_encoder = OneHotEncoder(sparse_output=False, categories=[unique_labels])\n\n# Generate test predictions\ntest_meta_features = []\nfor name, clf in classifiers.items():\n    print(f\"Generating test predictions for {name}...\")\n    if name in proba_classifiers:\n        pred = clf.predict_proba(test_features_scaled)\n    else:\n        pred = clf.predict(test_features_scaled).reshape(-1, 1)\n        pred = onehot_encoder.fit_transform(pred)\n    test_meta_features.append(pred)\n\n# Stack and standardize test meta-features\ntest_meta_features = np.hstack(test_meta_features)\nmeta_scaler = joblib.load(os.path.join(meta_dir, 'meta_scaler.pkl'))\ntest_meta_features_scaled = meta_scaler.transform(test_meta_features)\n\n# Load meta-model and predict\nmeta_model = joblib.load(os.path.join(model_dir, 'mlp_meta_model.pkl'))\ntest_predictions = meta_model.predict(test_meta_features_scaled)\n\n# Plot prediction distribution\nplt.figure(figsize=(10, 6))\nsns.histplot(test_predictions, bins=len(unique_labels), stat='count')\nplt.title('Distribution of Predicted Classes in Test Set')\nplt.xlabel('Predicted Class')\nplt.ylabel('Count')\nplt.xticks(ticks=unique_labels, labels=class_names)\nplt.tight_layout()\nplt.savefig(os.path.join(plots_dir, 'test_prediction_distribution.png'))\nplt.close()\n\n# Optional: Plot average probability scores (if meta-model supports predict_proba)\ntest_probabilities = meta_model.predict_proba(test_meta_features_scaled)\navg_probs = np.mean(test_probabilities, axis=0)\nplt.figure(figsize=(8, 6))\nsns.heatmap(avg_probs.reshape(1, -1), annot=True, fmt='.4f', cmap='Blues', xticklabels=class_names)\nplt.title('Average Predicted Probabilities per Class')\nplt.xlabel('Class')\nplt.ylabel('Average Probability')\nplt.tight_layout()\nplt.savefig(os.path.join(plots_dir, 'test_avg_probabilities.png'))\nplt.close()\n\n# Create submission DataFrame\nsubmission = pd.DataFrame({\n    'id': test_ids,\n    'class': test_predictions\n})\nsubmission_file = os.path.join(submission_dir, 'submission.csv')\nsubmission.to_csv(submission_file, index=False)\nprint(f\"Submission saved to {submission_file}. Plots saved in {plots_dir}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T22:33:31.875924Z","iopub.status.idle":"2025-10-18T22:33:31.876155Z","shell.execute_reply.started":"2025-10-18T22:33:31.876046Z","shell.execute_reply":"2025-10-18T22:33:31.876056Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}