{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":132732,"databundleVersionId":16583342}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\nfrom torchvision import transforms, models\nfrom scipy.linalg import svdvals\nfrom skimage.feature import local_binary_pattern\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import StratifiedKFold\nimport xgboost as xgb\nimport cv2\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings('ignore')\n\nclass CFG:\n    root = \"/kaggle/input/competitions/dlmmdd-workshop-synthetic-source-attribution-challenge/Data/Data\"\n    train_csv = os.path.join(root, \"training.csv\")\n    test_csv = os.path.join(root, \"test.csv\")\n    train_dir = os.path.join(root, \"Training\")\n    test_dir = os.path.join(root, \"Test\")\n    \n    # EffNet Config\n    device = \"cpu\"\n    batch_size = 16\n    epochs = 16\n    lr = 3e-4\n    num_classes = 10\n    img_size = 224\n    \n    # XGBoost Config\n    svd_components = 150\n    xgb_weight = 0.40 # 20% weight to XGBoost, 80% to EffNet\n\ndef resolve_path(p):\n    for c in [\n        os.path.join(CFG.root, p),\n        os.path.join(CFG.train_dir, os.path.basename(p)),\n        os.path.join(CFG.test_dir, os.path.basename(p)),\n        p\n    ]:\n        if os.path.exists(c):\n            return c\n    raise FileNotFoundError(p)\n\nprint(\"✅ Configurations loaded. Running on:\", CFG.device)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- DATASETS ---\nclass TrainDataset(Dataset):\n    def __init__(self, df, tfm):\n        self.df = df\n        self.tfm = tfm\n\n    def __len__(self): return len(self.df)\n\n    def __getitem__(self, i):\n        r = self.df.iloc[i]\n        img = Image.open(resolve_path(r[\"path\"])).convert(\"RGB\").resize((CFG.img_size, CFG.img_size))\n        return self.tfm(img), torch.tensor(r[\"y\"]).long()\n\nclass TestDataset(Dataset):\n    def __init__(self, df, tfm):\n        self.df = df\n        self.tfm = tfm\n\n    def __len__(self): return len(self.df)\n\n    def __getitem__(self, i):\n        r = self.df.iloc[i]\n        img = Image.open(resolve_path(r[\"path\"])).convert(\"RGB\").resize((CFG.img_size, CFG.img_size))\n        return self.tfm(img), r[\"ID\"]\n\ntfm_train = transforms.Compose([\n    transforms.RandomResizedCrop(224, scale=(0.7, 1.0)),\n    transforms.RandomHorizontalFlip(),\n    transforms.ColorJitter(0.2,0.2,0.2,0.1),\n    transforms.ToTensor(),\n    transforms.Normalize([0.5]*3,[0.5]*3)\n])\n\ntfm_test = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize([0.5]*3,[0.5]*3)\n])\n\n# --- MODEL & EMA ---\nclass Model(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.backbone = models.efficientnet_b0(weights=models.EfficientNet_B0_Weights.DEFAULT)\n        self.backbone.classifier = nn.Sequential(\n            nn.Dropout(0.3),\n            nn.Linear(1280, CFG.num_classes)\n        )\n\n    def forward(self, x): return self.backbone(x)\n\nclass EMA:\n    def __init__(self, model, decay=0.999):\n        self.model = model\n        self.decay = decay\n        self.shadow = {k:v.clone().detach() for k,v in model.state_dict().items()}\n\n    def update(self):\n        for k,v in self.model.state_dict().items():\n            self.shadow[k] = self.decay*self.shadow[k] + (1-self.decay)*v\n\n    def apply(self):\n        self.backup = self.model.state_dict()\n        self.model.load_state_dict(self.shadow)\n\n    def restore(self):\n        self.model.load_state_dict(self.backup)\n\n# --- TRAINING ---\ndef train_effnet():\n    print(\"\\n🚀 Starting EfficientNet Training (CPU)...\")\n    train_df = pd.read_csv(CFG.train_csv)\n    train_loader = DataLoader(TrainDataset(train_df, tfm_train), batch_size=CFG.batch_size, shuffle=True, num_workers=0)\n\n    model = Model().to(CFG.device)\n    opt = optim.AdamW(model.parameters(), lr=CFG.lr)\n    ema = EMA(model)\n    criterion = nn.CrossEntropyLoss(label_smoothing=0.1)\n\n    for epoch in range(CFG.epochs):\n        model.train()\n        total_loss = 0\n        for x, y in tqdm(train_loader, desc=f\"Epoch {epoch+1}/{CFG.epochs}\"):\n            x, y = x.to(CFG.device), y.to(CFG.device)\n            opt.zero_grad()\n            loss = criterion(model(x), y)\n            loss.backward()\n            opt.step()\n            ema.update()\n            total_loss += loss.item()\n        print(f\"Epoch {epoch+1} Loss: {total_loss/len(train_loader):.4f}\")\n\n    # Apply EMA weights for inference\n    ema.apply()\n    \n    # --- INFERENCE ---\n    print(\"\\n🚀 Running EffNet Inference with TTA...\")\n    test_df = pd.read_csv(CFG.test_csv)\n    test_loader = DataLoader(TestDataset(test_df, tfm_test), batch_size=CFG.batch_size, shuffle=False, num_workers=0)\n    \n    model.eval()\n    all_probs = []\n    all_ids = []\n    \n    with torch.no_grad():\n        for x, i in tqdm(test_loader, desc=\"EffNet TTA\"):\n            x = x.to(CFG.device)\n            # TTA: Average original and flipped logits, then softmax\n            logits = (model(x) + model(torch.flip(x, [3]))) / 2.0\n            probs = torch.softmax(logits, dim=1).cpu().numpy()\n            all_probs.append(probs)\n            all_ids.extend(i.numpy())\n            \n    effnet_probs = np.concatenate(all_probs, axis=0)\n    np.save(\"effnet_probs.npy\", effnet_probs)\n    print(\"✅ Saved EffNet probabilities to effnet_probs.npy\")\n    \n    ema.restore()\n    return effnet_probs, all_ids\n\neffnet_probs, test_ids = train_effnet()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_advanced_features(image_path):\n    try:\n        img = cv2.imread(image_path)\n        if img is None: return None\n        img = cv2.resize(img, (256, 256))\n        img_ycbcr = cv2.cvtColor(img, cv2.COLOR_BGR2YCrCb)\n        Y, Cb, Cr = cv2.split(img_ycbcr)\n        \n        features = {}\n        \n        # A. NOISE RESIDUALS\n        Y_noise = Y.astype(np.float32) - cv2.medianBlur(Y, 5).astype(np.float32)\n        Cb_noise = Cb.astype(np.float32) - cv2.medianBlur(Cb, 5).astype(np.float32)\n        Cr_noise = Cr.astype(np.float32) - cv2.medianBlur(Cr, 5).astype(np.float32)\n        \n        # B. SVD SPECTRAL PROFILE\n        for ch_name, ch_noise in [('Y_noise', Y_noise), ('Cb_noise', Cb_noise), ('Cr_noise', Cr_noise)]:\n            s_values = svdvals(ch_noise)\n            top_k = 30 # Slightly reduced for CPU speed\n            s_top = s_values[:top_k]\n            for i in range(min(top_k, len(s_top))):\n                features[f'{ch_name}_svd_{i}'] = s_top[i]\n            s_diff = np.diff(s_top)\n            features[f'{ch_name}_svd_diff_mean'] = np.mean(s_diff)\n            features[f'{ch_name}_svd_diff_var'] = np.var(s_diff)\n            \n        # C. LOCAL BINARY PATTERNS\n        radius = 3\n        n_points = 8 * radius\n        for ch_name, ch in [('Y', Y)]:\n            lbp = local_binary_pattern(ch, n_points, radius, method='uniform')\n            hist, _ = np.histogram(lbp, bins=np.arange(0, n_points + 3), density=True)\n            for i, val in enumerate(hist):\n                features[f'{ch_name}_lbp_{i}'] = val\n                \n        # D. CROSS-CHANNEL CORRELATIONS\n        Y_flat, Cb_flat, Cr_flat = Y_noise.ravel(), Cb_noise.ravel(), Cr_noise.ravel()\n        features['corr_Y_Cb'] = np.corrcoef(Y_flat, Cb_flat)[0, 1]\n        features['corr_Y_Cr'] = np.corrcoef(Y_flat, Cr_flat)[0, 1]\n        features['corr_Cb_Cr'] = np.corrcoef(Cb_flat, Cr_flat)[0, 1]\n        \n        # E. DCT SCORE\n        dct = cv2.dct(np.float32(Y) / 255.0)\n        features['dct_ac_var'] = np.var(dct[5:, 5:])\n        \n        return features\n    except Exception as e:\n        return None\n\ndef run_xgboost_pipeline():\n    # If features are already in memory, skip extraction!\n    if 'X_train_raw' not in globals():\n        print(\"\\n🚀 Starting SVD-XGBoost Feature Extraction...\")\n        train_df = pd.read_csv(CFG.train_csv)\n        test_df = pd.read_csv(CFG.test_csv)\n        \n        # Extract Train\n        X_train_raw, y_train = [], []\n        for idx, row in tqdm(train_df.iterrows(), total=len(train_df), desc=\"Train Features\"):\n            feats = extract_advanced_features(resolve_path(row[\"path\"]))\n            if feats:\n                X_train_raw.append(feats)\n                y_train.append(row['y'])\n        X_train_raw = pd.DataFrame(X_train_raw).fillna(0)\n        y_train = np.array(y_train)\n\n        # Extract Test\n        X_test_raw, test_ids_xgb = [], []\n        for idx, row in tqdm(test_df.iterrows(), total=len(test_df), desc=\"Test Features\"):\n            feats = extract_advanced_features(resolve_path(row[\"path\"]))\n            if feats:\n                X_test_raw.append(feats)\n                test_ids_xgb.append(row['ID'])\n        X_test_raw = pd.DataFrame(X_test_raw).fillna(0)\n        X_test_raw = X_test_raw.reindex(columns=X_train_raw.columns, fill_value=0)\n    else:\n        print(\"\\n✅ Features already extracted! Skipping to scaling...\")\n\n    # Scale & SVD Compression\n    print(f\"Original Feature Dim: {X_train_raw.shape[1]}\")\n    scaler = StandardScaler()\n    X_train_scaled = scaler.fit_transform(X_train_raw)\n    X_test_scaled = scaler.transform(X_test_raw)\n\n    # FIX: Cap n_components to be at most (number of features - 1)\n    n_comp = min(CFG.svd_components, X_train_scaled.shape[1] - 1)\n    \n    svd = TruncatedSVD(n_components=n_comp, random_state=42)\n    X_train_compressed = svd.fit_transform(X_train_scaled)\n    X_test_compressed = svd.transform(X_test_scaled)\n    print(f\"SVD Compressed Dim: {X_train_compressed.shape[1]} | Explained Var: {svd.explained_variance_ratio_.sum():.3f}\")\n\n    # Train XGBoost\n    print(\"Training XGBoost...\")\n    test_preds = np.zeros((len(X_test_compressed), 10))\n    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\n    for fold, (train_idx, val_idx) in enumerate(skf.split(X_train_compressed, y_train)):\n        X_tr, X_val = X_train_compressed[train_idx], X_train_compressed[val_idx]\n        y_tr, y_val = y_train[train_idx], y_train[val_idx]\n        \n        model = xgb.XGBClassifier(\n            objective='multi:softprob', num_class=10, n_estimators=500, learning_rate=0.05,\n            max_depth=6, subsample=0.8, colsample_bytree=0.8, tree_method='hist',\n            eval_metric='mlogloss', early_stopping_rounds=20, verbosity=0\n        )\n        model.fit(X_tr, y_tr, eval_set=[(X_val, y_val)], verbose=False)\n        test_preds += model.predict_proba(X_test_compressed) / 5.0\n\n    np.save(\"xgb_probs.npy\", test_preds)\n    print(\"✅ Saved XGBoost probabilities to xgb_probs.npy\")\n    return test_preds\n\nxgb_probs = run_xgboost_pipeline()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n🚀 Blending EffNet + XGBoost...\")\n\n# Load probabilities just in case\neffnet_probs = np.load(\"effnet_probs.npy\")\nxgb_probs = np.load(\"xgb_probs.npy\")\n\n# The Magic Blend: 80% Deep Learning, 20% Forensic Math\nfinal_probs = (0.8 * effnet_probs) + (CFG.xgb_weight * xgb_probs)\nfinal_predictions = final_probs.argmax(axis=1)\n\n# Create submission\ntest_df = pd.read_csv(CFG.test_csv)\nsubmission = pd.DataFrame({\n    \"ID\": test_df[\"ID\"],\n    \"TARGET\": final_predictions\n})\n\nsubmission.to_csv(\"submission.csv\", index=False)\n\nprint(\"\\n🎉🎉🎉 MEGA ENSEMBLE SUBMISSION SAVED! 🎉🎉🎉\")\nprint(submission.head())\nprint(f\"\\nEffNet Predictions Distribution:\\n{np.unique(effnet_probs.argmax(1), return_counts=True)[1]}\")\nprint(f\"Final Ensemble Distribution:\\n{np.unique(final_predictions, return_counts=True)[1]}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}