{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":132732,"databundleVersionId":16583342,"isSourceIdPinned":false},{"sourceType":"modelInstanceVersion","sourceId":842731,"databundleVersionId":16837690,"modelInstanceId":641090,"modelId":653116,"isSourceIdPinned":false},{"sourceType":"modelInstanceVersion","sourceId":839946,"databundleVersionId":16794647,"modelInstanceId":638969,"modelId":650955,"isSourceIdPinned":false}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ============ ENSEMBLE: EFFICIENTNET (0.30) + CONVNEXT (0.70) ============\nprint(\"ENSEMBLE: EFFICIENTNET + CONVNEXT\")\nprint(\"=\"*60)\n\nimport os\nimport pandas as pd\nimport numpy as np\nfrom PIL import Image\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nimport torchvision.transforms.functional as TF\nfrom tqdm import tqdm\nimport timm\nfrom sklearn.model_selection import train_test_split  # ADD THIS IMPORT\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Clear memory\ntorch.cuda.empty_cache()\ngc.collect()\n\n# ============ CONFIGURATION ============\nclass Config:\n    TRAIN_PATH = \"/kaggle/input/competitions/dlmmdd-workshop-synthetic-source-attribution-challenge/Data/Data/Training\"\n    TEST_PATH = \"/kaggle/input/competitions/dlmmdd-workshop-synthetic-source-attribution-challenge/Data/Data/Test\"\n    TRAIN_CSV = \"/kaggle/input/competitions/dlmmdd-workshop-synthetic-source-attribution-challenge/Data/Data/training.csv\"\n    TEST_CSV = \"/kaggle/input/competitions/dlmmdd-workshop-synthetic-source-attribution-challenge/Data/Data/test.csv\"\n    \n    IMG_SIZE = 384\n    BATCH_SIZE = 32\n    NUM_CLASSES = 10\n    \n    # Ensemble weights\n    CNN_WEIGHT = 0.30\n    CONVNEXT_WEIGHT = 0.70\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\n\n# ============ TRANSFORMS ============\nclass TestTransform:\n    def __init__(self, img_size=384):\n        self.img_size = img_size\n        self.normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n    \n    def __call__(self, image):\n        image = TF.to_tensor(image)\n        image = TF.resize(image, [self.img_size, self.img_size])\n        image = self.normalize(image)\n        return image\n\n# ============ DATASET ============\nclass ImageDataset(Dataset):\n    def __init__(self, df, transform=None, is_train=True):\n        self.df = df\n        self.transform = transform\n        self.is_train = is_train\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        img_path = self.df.iloc[idx]['full_path']\n        image = Image.open(img_path).convert('RGB')\n        if self.transform:\n            image = self.transform(image)\n        if self.is_train:\n            return image, self.df.iloc[idx]['y']\n        else:\n            return image, self.df.iloc[idx]['ID']\n\n# ============ LOAD DATA ============\nprint(\"\\n1. LOADING DATA\")\ntrain_df = pd.read_csv(Config.TRAIN_CSV)\ntest_df = pd.read_csv(Config.TEST_CSV)\n\ntrain_df['full_path'] = train_df['path'].apply(lambda x: os.path.join(Config.TRAIN_PATH, os.path.basename(x)))\ntest_df['full_path'] = test_df['path'].apply(lambda x: os.path.join(Config.TEST_PATH, os.path.basename(x)))\n\ntrain_data, val_data = train_test_split(train_df, test_size=0.2, stratify=train_df['y'], random_state=42)\nprint(f\"Training: {len(train_data)}, Validation: {len(val_data)}\")\n\n# ============ CREATE DATALOADERS ============\ntransform = TestTransform(Config.IMG_SIZE)\nval_dataset = ImageDataset(val_data, transform=transform, is_train=True)\ntest_dataset = ImageDataset(test_df, transform=transform, is_train=False)\n\nval_loader = DataLoader(val_dataset, batch_size=Config.BATCH_SIZE, shuffle=False, num_workers=4)\ntest_loader = DataLoader(test_dataset, batch_size=Config.BATCH_SIZE, shuffle=False, num_workers=4)\n\n# ============ DEFINE MODELS ============\nclass EfficientNetModel(nn.Module):\n    def __init__(self, num_classes=10, dropout_rate=0.3):\n        super().__init__()\n        self.backbone = timm.create_model('efficientnet_b4', pretrained=False, num_classes=0)\n        num_features = self.backbone.num_features\n        self.classifier = nn.Sequential(\n            nn.Dropout(dropout_rate),\n            nn.Linear(num_features, 512),\n            nn.ReLU(inplace=True),\n            nn.Dropout(dropout_rate),\n            nn.Linear(512, num_classes)\n        )\n    \n    def forward(self, x):\n        return self.classifier(self.backbone(x))\n\nclass ConvNeXtModel(nn.Module):\n    def __init__(self, num_classes=10, dropout_rate=0.3):\n        super().__init__()\n        self.backbone = timm.create_model('convnext_tiny', pretrained=False, num_classes=0)\n        num_features = self.backbone.num_features\n        self.classifier = nn.Sequential(\n            nn.Dropout(dropout_rate),\n            nn.Linear(num_features, 512),\n            nn.ReLU(inplace=True),\n            nn.Dropout(dropout_rate),\n            nn.Linear(512, num_classes)\n        )\n    \n    def forward(self, x):\n        return self.classifier(self.backbone(x))\n\n# ============ LOAD EFFICIENTNET MODEL ============\nprint(\"\\n2. LOADING EFFICIENTNET MODEL\")\nprint(\"-\"*40)\n\neff_model_path = \"/kaggle/input/models/nicolequilang/efficientnet-synthetic-attribution/pytorch/default/1/best_model.pth\"\n\nif os.path.exists(eff_model_path):\n    print(f\"Found EfficientNet at: {eff_model_path}\")\n    efficientnet_model = EfficientNetModel().to(device)\n    eff_checkpoint = torch.load(eff_model_path, map_location=device, weights_only=False)\n    \n    if 'model_state_dict' in eff_checkpoint:\n        efficientnet_model.load_state_dict(eff_checkpoint['model_state_dict'])\n        print(f\"Loaded EfficientNet from epoch {eff_checkpoint.get('epoch', 'unknown')}\")\n        val_acc = eff_checkpoint.get('val_acc', 0)\n        if isinstance(val_acc, torch.Tensor):\n            val_acc = val_acc.item()\n        print(f\"Validation accuracy: {val_acc*100:.2f}%\")\n    else:\n        efficientnet_model.load_state_dict(eff_checkpoint)\n        print(\"Loaded EfficientNet model\")\n    efficientnet_model.eval()\nelse:\n    print(f\"ERROR: EfficientNet model not found at {eff_model_path}\")\n    raise FileNotFoundError(f\"Model not found: {eff_model_path}\")\n\n# ============ LOAD CONVNEXT MODEL ============\nprint(\"\\n3. LOADING CONVNEXT MODEL\")\nprint(\"-\"*40)\n\ncnx_model_path = \"/kaggle/input/models/nicolequilang/best-covnext-model/pytorch/default/1/best_convnext_model (1).pth\"\n\nif os.path.exists(cnx_model_path):\n    print(f\"Found ConvNeXt at: {cnx_model_path}\")\n    convnext_model = ConvNeXtModel().to(device)\n    cnx_checkpoint = torch.load(cnx_model_path, map_location=device, weights_only=False)\n    \n    if 'model_state_dict' in cnx_checkpoint:\n        convnext_model.load_state_dict(cnx_checkpoint['model_state_dict'])\n        print(f\"Loaded ConvNeXt from epoch {cnx_checkpoint.get('epoch', 'unknown')}\")\n        val_acc = cnx_checkpoint.get('val_acc', 0)\n        if isinstance(val_acc, torch.Tensor):\n            val_acc = val_acc.item()\n        print(f\"Validation accuracy: {val_acc*100:.2f}%\")\n    else:\n        convnext_model.load_state_dict(cnx_checkpoint)\n        print(\"Loaded ConvNeXt model\")\n    convnext_model.eval()\nelse:\n    print(f\"ERROR: ConvNeXt model not found at {cnx_model_path}\")\n    raise FileNotFoundError(f\"Model not found: {cnx_model_path}\")\n\n# ============ TTA PREDICTION FUNCTION ============\ndef predict_with_tta(model, images, device):\n    with torch.no_grad():\n        outputs = model(images)\n        outputs_flip = model(torch.flip(images, dims=[3]))\n        return (outputs + outputs_flip) / 2\n\n# ============ VALIDATE ENSEMBLE ============\nprint(\"\\n4. VALIDATING ENSEMBLE\")\nprint(\"-\"*40)\n\ncorrect = 0\ntotal = 0\n\nwith torch.no_grad():\n    for images, labels in tqdm(val_loader, desc=\"Validating ensemble\"):\n        images, labels = images.to(device), labels.to(device)\n        \n        eff_out = predict_with_tta(efficientnet_model, images, device)\n        cnx_out = predict_with_tta(convnext_model, images, device)\n        \n        ensemble_out = Config.CNN_WEIGHT * eff_out + Config.CONVNEXT_WEIGHT * cnx_out\n        preds = ensemble_out.argmax(dim=1)\n        correct += (preds == labels).sum().item()\n        total += labels.size(0)\n\nensemble_val_acc = correct / total\nprint(f\"Ensemble Validation Accuracy: {ensemble_val_acc:.4f} ({ensemble_val_acc*100:.2f}%)\")\nprint(f\"Using weights: EfficientNet={Config.CNN_WEIGHT:.2f}, ConvNeXt={Config.CONVNEXT_WEIGHT:.2f}\")\n\n# ============ GENERATE TEST PREDICTIONS ============\nprint(\"\\n5. GENERATING TEST PREDICTIONS\")\nprint(\"-\"*40)\n\npredictions = []\nimage_ids = []\n\nwith torch.no_grad():\n    for images, ids in tqdm(test_loader, desc=\"Ensemble predicting\"):\n        images = images.to(device)\n        \n        eff_out = predict_with_tta(efficientnet_model, images, device)\n        cnx_out = predict_with_tta(convnext_model, images, device)\n        \n        ensemble_out = Config.CNN_WEIGHT * eff_out + Config.CONVNEXT_WEIGHT * cnx_out\n        preds = ensemble_out.argmax(dim=1).cpu().numpy()\n        \n        predictions.extend(preds)\n        image_ids.extend(ids.numpy())\n\n# ============ CREATE SUBMISSION ============\nsubmission = pd.DataFrame({'ID': image_ids, 'TARGET': predictions})\nsubmission = submission.sort_values('ID')\nsubmission.to_csv('ensemble_submission.csv', index=False)\n\nprint(f\"\\nEnsemble submission saved: ensemble_submission.csv\")\nprint(f\"Total predictions: {len(submission)}\")\n\n# ============ SAVE ENSEMBLE MODEL ============\nprint(\"\\n6. SAVING ENSEMBLE MODEL\")\nprint(\"-\"*40)\n\nclass EnsembleModel(nn.Module):\n    def __init__(self, cnn_model, convnext_model, cnn_weight=0.30, convnext_weight=0.70):\n        super().__init__()\n        self.cnn = cnn_model\n        self.convnext = convnext_model\n        self.cnn_weight = cnn_weight\n        self.convnext_weight = convnext_weight\n    \n    def forward(self, x):\n        cnn_out = self.cnn(x)\n        convnext_out = self.convnext(x)\n        return self.cnn_weight * cnn_out + self.convnext_weight * convnext_out\n\n# Create and save ensemble model\nensemble_model = EnsembleModel(efficientnet_model, convnext_model, Config.CNN_WEIGHT, Config.CONVNEXT_WEIGHT)\ntorch.save({\n    'cnn_weight': Config.CNN_WEIGHT,\n    'convnext_weight': Config.CONVNEXT_WEIGHT,\n    'cnn_state_dict': efficientnet_model.state_dict(),\n    'convnext_state_dict': convnext_model.state_dict(),\n    'ensemble_val_acc': ensemble_val_acc\n}, 'ensemble_model.pth')\n\nprint(f\"Ensemble model saved: ensemble_model.pth\")\nprint(f\"Ensemble validation accuracy: {ensemble_val_acc*100:.2f}%\")\n\n# ============ PREDICTION DISTRIBUTION ============\nprint(\"\\n7. PREDICTION DISTRIBUTION\")\nprint(\"-\"*40)\npred_dist = submission['TARGET'].value_counts().sort_index()\nfor i in range(10):\n    count = pred_dist.get(i, 0)\n    percentage = count/len(submission)*100\n    bar = '*' * int(percentage/2)\n    print(f\"   Class {i}: {count:4d} ({percentage:5.2f}%) {bar}\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"ENSEMBLE COMPLETE\")\nprint(\"=\"*60)\nprint(\"\\nFiles created:\")\nprint(\"   - ensemble_submission.csv (upload to Kaggle)\")\nprint(\"   - ensemble_model.pth (ensemble model weights)\")\nprint(f\"\\nEnsemble weights: EfficientNet={Config.CNN_WEIGHT:.2f}, ConvNeXt={Config.CONVNEXT_WEIGHT:.2f}\")\nprint(f\"Ensemble validation accuracy: {ensemble_val_acc*100:.2f}%\")\nprint(\"\\nUpload ensemble_submission.csv to Kaggle\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}