{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Phase 2 — Malware Classification: Train & Predict\n\n**Depends on:** Run `malware-convert.ipynb` (Phase 1) first for both train and test modes.\n\n**Pipeline:**\n1. Load features + PNG images from Phase 1 output\n2. Monte Carlo CV: train ResNet18 × N runs → predict test set each run → average\n3. Monte Carlo CV: train LightGBM × N runs → predict test set each run → average\n4. Train Meta-NN on combined ResNet + LightGBM meta-features\n5. Compare all models on test set → metrics, confusion matrices, visualizations\n\n**Run:** Enable GPU (Settings → Accelerator → GPU T4), then **Run All**.","metadata":{}},{"cell_type":"markdown","source":"## Step 1 — Configuration","metadata":{}},{"cell_type":"code","source":"# ============================================================\n# PATHS — must match Phase 1 (malware-convert.ipynb)\n# ============================================================\n\nCOMP_DIR   = \"/kaggle/input/competitions/malware-classification\"\nPHASE1_DIR = \"/kaggle/input/datasets/hieuhuynhvan/malware-images-dataset\"\n\n# ============================================================\n# HYPERPARAMETERS\n# ============================================================\nIMG_SIZE   = 224\n\n# Monte Carlo: number of random stratified splits\nMC_RUNS      = 3\nTRAIN_RATIO  = 0.80   # fraction of train data per MC split\n\n# ResNet18\nRN_EPOCHS    = 5\nRN_BATCH     = 64\nRN_LR        = 1e-4\nRN_PATIENCE  = 2\n\n# LightGBM\nLGB_LEAVES   = 63\nLGB_ROUNDS   = 600\nLGB_EARLY    = 50\n\n# Meta-NN\nMETA_EPOCHS  = 30\nMETA_BATCH   = 256\nMETA_LR      = 1e-3\n\nBASE_SEED = 42\nSKIP_TEST = True  # set False if test features/images are available\n\nprint(f\"Monte Carlo runs  : {MC_RUNS}\")\nprint(f\"Train ratio/split : {TRAIN_RATIO}\")\nprint(f\"ResNet epochs     : {RN_EPOCHS}\")\nprint(f\"LightGBM rounds   : {LGB_ROUNDS}\")\nprint(f\"Meta-NN epochs    : {META_EPOCHS}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T13:24:15.973866Z","iopub.execute_input":"2026-07-24T13:24:15.97469Z","iopub.status.idle":"2026-07-24T13:24:15.981304Z","shell.execute_reply.started":"2026-07-24T13:24:15.974658Z","shell.execute_reply":"2026-07-24T13:24:15.980589Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 2 — Imports & Reproducibility","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport copy\nimport time\nimport random\nimport warnings\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models, transforms\n\nimport lightgbm as lgb\n\nfrom sklearn.metrics import (\n    accuracy_score,\n    precision_score,\n    recall_score,\n    f1_score,\n    log_loss,\n    classification_report,\n    confusion_matrix,\n)\n\nfrom sklearn.model_selection import train_test_split\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nwarnings.filterwarnings('ignore')\nsns.set_style(\"whitegrid\")\nplt.rcParams[\"figure.dpi\"] = 100\nplt.rcParams[\"font.size\"] = 10\n\nos.environ[\"CUBLAS_WORKSPACE_CONFIG\"] = \":4096:8\"\nos.environ[\"OMP_NUM_THREADS\"] = \"1\"\nos.environ[\"MKL_NUM_THREADS\"] = \"1\"\n\ndef _seed_all(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark     = False\n    torch.use_deterministic_algorithms(True, warn_only=True)\n\n_seed_all(BASE_SEED)\n\nDEVICE    = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nOUT_DIR   = Path(\"/kaggle/working\")\nIMG_DIR   = Path(PHASE1_DIR) / \"images\"\n\nCLASS_NAMES = [\n    \"Ramnit\", \"Lollipop\", \"Kelihos_ver3\", \"Vundo\", \"Simda\",\n    \"Tracur\", \"Kelihos_ver1\", \"Obfuscator.ACY\", \"Gatak\"\n]\nN_CLASSES = len(CLASS_NAMES)\n\nprint(f\"Device   : {DEVICE}\")\nprint(f\"PyTorch  : {torch.__version__}\")\nprint(f\"Classes  : {N_CLASSES}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T13:24:15.982765Z","iopub.execute_input":"2026-07-24T13:24:15.983113Z","iopub.status.idle":"2026-07-24T13:24:16.010203Z","shell.execute_reply.started":"2026-07-24T13:24:15.983092Z","shell.execute_reply":"2026-07-24T13:24:16.009349Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 3 — Load Train & Test Data","metadata":{}},{"cell_type":"code","source":"# ============================================================\n# TRAINING FEATURES\n# ============================================================\nfeat_csv = OUT_DIR / \"train_tabular_features.csv\"\nif feat_csv.exists():\n    train_feat_df = pd.read_csv(feat_csv, dtype={\"Id\": str})\n    print(f\"Loaded merged CSV: {len(train_feat_df):,} rows\")\nelse:\n    batch_dir   = Path(PHASE1_DIR) / \"table_data\"\n    batch_files = sorted(batch_dir.glob(\"batch_*.csv\"))\n    train_feat_df = pd.concat(\n        [pd.read_csv(p, dtype={\"Id\": str}) for p in batch_files],\n        ignore_index=True, sort=False\n    )\n    train_feat_df[\"Id\"] = train_feat_df[\"Id\"].astype(str)\n    print(f\"Loaded {len(batch_files)} batches: {len(train_feat_df):,} rows\")\n\nlabels_df = pd.read_csv(f\"{COMP_DIR}/trainLabels.csv\", dtype={\"Id\": str})\nlabels_df[\"Id\"] = labels_df[\"Id\"].astype(str)\n\ntrain_feat_df = train_feat_df.drop_duplicates(\"Id\", keep=\"last\")\ndata_df = train_feat_df.merge(labels_df, on=\"Id\", how=\"inner\", validate=\"one_to_one\")\ndata_df = data_df.sort_values(\"Id\").reset_index(drop=True)\n\nif len(data_df) == 0:\n    raise RuntimeError(\"Training dataframe is empty — check Phase 1 output.\")\n\ntrain_ids = data_df[\"Id\"].astype(str).tolist()\nN_TRAIN   = len(train_ids)\n\n# Class labels 0-indexed\ndata_df[\"Class\"] = data_df[\"Class\"] - 1\nlabels = data_df[\"Class\"].values.astype(np.int64)\n\n# Feature columns\nfeat_cols = [c for c in data_df.columns if c not in [\"Id\", \"Class\"]]\nfor col in [\"has_bytes\", \"has_asm\"]:\n    if col in feat_cols:\n        feat_cols.remove(col)\n\nfor col in feat_cols:\n    data_df[col] = pd.to_numeric(data_df[col], errors=\"coerce\")\ndata_df[feat_cols] = data_df[feat_cols].replace([np.inf, -np.inf], np.nan)\n\nnan_mask = data_df[feat_cols].isna().any(axis=1)\nif nan_mask.any():\n    print(f\"Dropping {nan_mask.sum()} samples with NaN features\")\n    data_df  = data_df[~nan_mask].reset_index(drop=True)\n    train_ids = data_df[\"Id\"].astype(str).tolist()\n    labels    = data_df[\"Class\"].values.astype(np.int64)\n    N_TRAIN   = len(train_ids)\n\nconst_mask = data_df[feat_cols].std(axis=0) == 0\nconst_cols = [feat_cols[i] for i in range(len(feat_cols)) if const_mask.iloc[i]]\nif const_cols:\n    feat_cols = [c for c in feat_cols if c not in const_cols]\n    print(f\"Removed {len(const_cols)} constant columns\")\n\ntabular = data_df[feat_cols].values.astype(np.float32)\nprint(f\"Train samples : {N_TRAIN}\")\nprint(f\"Features     : {len(feat_cols)}\")\n\n# ============================================================\n# TRAINING IMAGES\n# ============================================================\nmissing = []\nimages  = np.zeros((N_TRAIN, IMG_SIZE, IMG_SIZE), dtype=np.uint8)\nfor idx, fid in enumerate(train_ids):\n    p = IMG_DIR / f\"{fid}.png\"\n    if p.is_file() and p.stat().st_size > 0:\n        try:\n            img = Image.open(p)\n            images[idx] = np.array(img.convert(\"L\"), dtype=np.uint8)\n        except Exception:\n            missing.append(fid)\n    else:\n        missing.append(fid)\n\nif missing:\n    raise RuntimeError(f\"{len(missing)} missing train images: {missing[:5]}\")\n\nprint(f\"Train images : {images.shape}\")\nprint(\"Class distribution:\")\nfor u, c in zip(*np.unique(labels, return_counts=True)):\n    print(f\"  {u} ({CLASS_NAMES[u]}): {c:,}\")\n\n# ============================================================\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T13:24:16.011293Z","iopub.execute_input":"2026-07-24T13:24:16.011603Z","iopub.status.idle":"2026-07-24T13:25:50.428811Z","shell.execute_reply.started":"2026-07-24T13:24:16.011573Z","shell.execute_reply":"2026-07-24T13:25:50.42806Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if not SKIP_TEST:\n    test_sub = pd.read_csv(f\"{COMP_DIR}/sampleSubmission.csv\")\n    test_ids = test_sub[\"Id\"].astype(str).tolist()\n    N_TEST   = len(test_ids)\n    print(f\"\\nTest samples : {N_TEST:,}\")\n\n    test_feat_path = OUT_DIR / \"test_features.csv\"\n    if not test_feat_path.exists():\n        raise FileNotFoundError(\n            f\"{test_feat_path} not found. Run Phase 1 in test mode first.\"\n        )\n\n    test_feat_df = pd.read_csv(test_feat_path, dtype={\"Id\": str})\n    test_feat_df = test_feat_df.set_index(\"Id\").loc[test_ids].reset_index()\n    for col in feat_cols:\n        if col not in test_feat_df.columns:\n            test_feat_df[col] = 0.0\n    test_tabular = test_feat_df[feat_cols].values.astype(np.float32)\n\n    missing_test = []\n    test_images  = np.zeros((N_TEST, IMG_SIZE, IMG_SIZE), dtype=np.uint8)\n    for idx, fid in enumerate(test_ids):\n        p = IMG_DIR / f\"{fid}.png\"\n        if p.is_file() and p.stat().st_size > 0:\n            try:\n                img = Image.open(p)\n                test_images[idx] = np.array(img.convert(\"L\"), dtype=np.uint8)\n            except Exception:\n                missing_test.append(fid)\n        else:\n            missing_test.append(fid)\n\n    if missing_test:\n        raise RuntimeError(f\"{len(missing_test)} missing test images: {missing_test[:5]}\")\n\n    print(f\"Test images  : {test_images.shape}\")\n    print(f\"Test tabular : {test_tabular.shape}\")\nelse:\n    N_TEST = 0\n    test_tabular = np.empty((0, len(feat_cols)), dtype=np.float32)\n    test_images  = np.empty((0, IMG_SIZE, IMG_SIZE), dtype=np.uint8)\n    print(\"\\nTest set skipped (SKIP_TEST=True). Using 80/20 split for evaluation.\")\n","metadata":{"jupyter":{"source_hidden":false},"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T13:25:50.430594Z","iopub.execute_input":"2026-07-24T13:25:50.430815Z","iopub.status.idle":"2026-07-24T13:25:50.43996Z","shell.execute_reply.started":"2026-07-24T13:25:50.430794Z","shell.execute_reply":"2026-07-24T13:25:50.439252Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 4 — Dataset, Model Definitions & Metrics","metadata":{}},{"cell_type":"code","source":"# ============================================================\n# PYTORCH DATASETS\n# ============================================================\nclass ImageDataset(Dataset):\n    def __init__(self, X, y=None, augment=True):\n        self.X = torch.from_numpy(X).unsqueeze(1).float() / 255.0\n        self.y = torch.from_numpy(y).long() if y is not None else None\n        self.augment = augment\n        self.aug = transforms.Compose([\n            transforms.RandomHorizontalFlip(),\n            transforms.RandomRotation(10),\n        ])\n\n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, idx):\n        img = self.X[idx].repeat(3, 1, 1)\n        if self.augment and self.y is not None:\n            img = self.aug(img)\n        if self.y is not None:\n            return img, self.y[idx]\n        return img\n\n\nclass ImageDatasetInf(Dataset):\n    def __init__(self, X):\n        self.X = torch.from_numpy(X).unsqueeze(1).float() / 255.0\n\n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, idx):\n        return self.X[idx].repeat(3, 1, 1)\n\n\n# ============================================================\n# MODELS\n# ============================================================\ndef build_resnet(n_classes):\n    model = models.resnet18(weights=models.ResNet18_Weights.IMAGENET1K_V1)\n    model.fc = nn.Sequential(\n        nn.Dropout(0.3),\n        nn.Linear(model.fc.in_features, n_classes)\n    )\n    return model\n\n\nclass MetaNN(nn.Module):\n    def __init__(self, n_classes=9):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(n_classes * 2, 32),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(32, n_classes)\n        )\n\n    def forward(self, x):\n        return self.net(x)\n\n\n# ============================================================\n# SOFTMAX HELPER\n# ============================================================\ndef softmax_np(arr):\n    e = np.exp(arr - arr.max(axis=1, keepdims=True))\n    return e / e.sum(axis=1, keepdims=True)\n\n\n# ============================================================\n# METRICS\n# ============================================================\ndef compute_metrics(y_true, y_prob, name=\"\"):\n    y_pred = np.argmax(y_prob, axis=1)\n    return {\n        \"model\":        name,\n        \"accuracy\":     accuracy_score(y_true, y_pred),\n        \"macro_f1\":    f1_score(y_true, y_pred, average=\"macro\",    zero_division=0),\n        \"weighted_f1\":  f1_score(y_true, y_pred, average=\"weighted\", zero_division=0),\n        \"macro_prec\":   precision_score(y_true, y_pred, average=\"macro\",    zero_division=0),\n        \"macro_recall\": recall_score(y_true, y_pred, average=\"macro\",    zero_division=0),\n        \"logloss\":      log_loss(y_true, y_prob, labels=list(range(N_CLASSES))),\n    }\n\n\ndef per_class_metrics(y_true, y_prob):\n    y_pred = np.argmax(y_prob, axis=1)\n    return pd.DataFrame({\n        \"class\":    CLASS_NAMES,\n        \"precision\": precision_score(y_true, y_pred, average=None, zero_division=0),\n        \"recall\":    recall_score(y_true, y_pred, average=None, zero_division=0),\n        \"f1\":        f1_score(y_true, y_pred, average=None, zero_division=0),\n        \"support\":   np.bincount(y_true, minlength=N_CLASSES),\n    })\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T13:25:50.441803Z","iopub.execute_input":"2026-07-24T13:25:50.442159Z","iopub.status.idle":"2026-07-24T13:25:50.4656Z","shell.execute_reply.started":"2026-07-24T13:25:50.442138Z","shell.execute_reply":"2026-07-24T13:25:50.465049Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 5 — ResNet18: Monte Carlo Training","metadata":{}},{"cell_type":"code","source":"# ============================================================\n# TRAIN ONE RESNET18 RUN\n# ============================================================\ndef run_resnet(run_idx, X_tr, y_tr, X_va, y_va, X_test):\n    \"\"\"Train one ResNet18 on a split. Return val metrics + test probs.\"\"\"\n    _seed_all(BASE_SEED + run_idx * 1000)\n\n    train_ds = ImageDataset(X_tr, y_tr, augment=True)\n    valid_ds = ImageDataset(X_va, y_va, augment=False)\n    train_dl = DataLoader(train_ds, batch_size=RN_BATCH, shuffle=True, num_workers=2, pin_memory=True)\n    valid_dl = DataLoader(valid_ds, batch_size=RN_BATCH*2, shuffle=False, num_workers=2, pin_memory=True)\n\n    model    = build_resnet(N_CLASSES).to(DEVICE)\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.AdamW(model.parameters(), lr=RN_LR, weight_decay=1e-4)\n    scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=RN_EPOCHS)\n\n    best_state, best_vl, patience = None, float(\"inf\"), 0\n\n    for epoch in range(RN_EPOCHS):\n        model.train()\n        for imgs, targets in train_dl:\n            imgs, targets = imgs.to(DEVICE), targets.to(DEVICE)\n            optimizer.zero_grad()\n            criterion(model(imgs), targets).backward()\n            optimizer.step()\n        scheduler.step()\n\n        model.eval()\n        vl_list, tgt_list = [], []\n        with torch.no_grad():\n            for imgs, targets in valid_dl:\n                vl_list.append(model(imgs.to(DEVICE)).cpu().numpy())\n                tgt_list.append(targets.numpy())\n        vl = softmax_np(np.concatenate(vl_list))\n        vl_ll = log_loss(np.concatenate(tgt_list), vl, labels=list(range(N_CLASSES)))\n        vl_acc = (vl.argmax(axis=1) == np.concatenate(tgt_list)).mean()\n        print(f\"      epoch {epoch+1}/{RN_EPOCHS}  loss={vl_ll:.4f}  acc={vl_acc:.4f}\")\n\n        if vl_ll < best_vl:\n            best_state = copy.deepcopy(model.state_dict())\n            best_vl    = vl_ll\n            patience    = 0\n        else:\n            patience += 1\n            if patience >= RN_PATIENCE:\n                break\n\n    # Predict on test set with best model\n    model.load_state_dict(best_state)\n    model.eval()\n    if X_test.shape[0] > 0:\n        with torch.no_grad():\n            test_dl = DataLoader(ImageDatasetInf(X_test), batch_size=RN_BATCH*2, shuffle=False, num_workers=2)\n            test_logits = np.concatenate([\n                model(imgs.to(DEVICE)).cpu().numpy()\n                for imgs in test_dl\n            ])\n        test_probs = softmax_np(test_logits)\n    else:\n        test_probs = np.empty((0, N_CLASSES), dtype=np.float32)\n\n    del model, optimizer, scheduler\n    gc.collect()\n    torch.cuda.empty_cache()\n\n    return compute_metrics(y_va, softmax_np(np.concatenate(vl_list))), test_probs, best_state\n\n\n# ============================================================\n# MONTE CARLO LOOP\n# ============================================================\nprint(\"\\n\" + \"=\"*64)\nprint(\"RESNET18 — MONTE CARLO TRAINING\")\nprint(\"=\"*64)\n\nrn_val_list   = []\nrn_test_probs = []\nt0 = time.time()\n\nfor run in range(MC_RUNS):\n    print(f\"\\n--- Run {run+1}/{MC_RUNS} ---\")\n    t_run = time.time()\n\n    tr_idx, va_idx = train_test_split(\n        np.arange(N_TRAIN),\n        test_size=1 - TRAIN_RATIO,\n        stratify=labels,\n        random_state=BASE_SEED + run\n    )\n\n    val_m, test_p, rn_best_state = run_resnet(\n        run,\n        images[tr_idx], labels[tr_idx],\n        images[va_idx], labels[va_idx],\n        test_images\n    )\n    rn_val_list.append(val_m)\n    if not SKIP_TEST:\n        rn_test_probs.append(test_p)\n    # Keep last run's best state for pseudo-test inference\n    last_rn_state = rn_best_state\n\n    print(f\"  Val  acc={val_m['accuracy']:.4f}  macro_f1={val_m['macro_f1']:.4f}  logloss={val_m['logloss']:.4f}\")\n    print(f\"  Time : {time.time()-t_run:.0f}s\")\n\n# Average test predictions\n    if not SKIP_TEST:\n        rn_test_avg = np.mean(rn_test_probs, axis=0)\n\nprint(f\"\\nResNet18 total: {time.time()-t0:.0f}s\")\nprint(\"\\nValidation summary across MC runs:\")\nprint(pd.DataFrame(rn_val_list)[[\"accuracy\",\"macro_f1\",\"weighted_f1\",\"logloss\"]].describe().round(4))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T13:25:50.466551Z","iopub.execute_input":"2026-07-24T13:25:50.466836Z","iopub.status.idle":"2026-07-24T13:33:32.378419Z","shell.execute_reply.started":"2026-07-24T13:25:50.466809Z","shell.execute_reply":"2026-07-24T13:33:32.377692Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 6 — LightGBM: Monte Carlo Training","metadata":{}},{"cell_type":"code","source":"# ============================================================\n# TRAIN ONE LIGHTGBM RUN\n# ============================================================\ndef run_lgbm(run_idx, X_tr, y_tr, X_va, y_va, X_test):\n    \"\"\"Train one LightGBM per-class on a split. Return val metrics + test probs.\"\"\"\n    seed = BASE_SEED + run_idx * 1000\n\n    va_preds   = np.zeros((len(y_va), N_CLASSES), dtype=np.float64)\n    if X_test.shape[0] > 0:\n        test_preds = np.zeros((X_test.shape[0], N_CLASSES), dtype=np.float64)\n    else:\n        test_preds = np.empty((0, N_CLASSES), dtype=np.float64)\n\n    models = []\n    for cls in range(N_CLASSES):\n        y_tr_bin = (y_tr == cls).astype(int)\n        y_va_bin = (y_va == cls).astype(int)\n        dtrain   = lgb.Dataset(X_tr, label=y_tr_bin)\n        dvalid   = lgb.Dataset(X_va, label=y_va_bin, reference=dtrain)\n        params   = {\n            \"objective\": \"binary\", \"metric\": \"binary_logloss\",\n            \"num_leaves\": LGB_LEAVES, \"learning_rate\": 0.05,\n            \"feature_fraction\": 0.8, \"bagging_fraction\": 0.8,\n            \"bagging_freq\": 5, \"min_child_samples\": 20,\n            \"verbose\": -1, \"seed\": seed, \"device\": \"cpu\",\n        }\n        model = lgb.train(\n            params, dtrain,\n            num_boost_round=LGB_ROUNDS,\n            valid_sets=[dvalid],\n            callbacks=[lgb.early_stopping(LGB_EARLY, verbose=False)],\n        )\n        models.append(model)\n        va_preds[:, cls]  = model.predict(X_va)\n        test_preds[:, cls] = model.predict(X_test)\n\n    va_preds   = va_preds   / va_preds.sum(axis=1, keepdims=True)\n    if test_preds.shape[0] > 0:\n        test_preds = test_preds / test_preds.sum(axis=1, keepdims=True)\n\n    return compute_metrics(y_va, va_preds), test_preds, models\n\n\n# ============================================================\n# MONTE CARLO LOOP\n# ============================================================\nprint(\"\\n\" + \"=\"*64)\nprint(\"LIGHTGBM — MONTE CARLO TRAINING\")\nprint(\"=\"*64)\n\nlgb_val_list   = []\nlgb_test_probs = []\nt0 = time.time()\n\nfor run in range(MC_RUNS):\n    print(f\"\\n--- Run {run+1}/{MC_RUNS} ---\")\n    t_run = time.time()\n\n    tr_idx, va_idx = train_test_split(\n        np.arange(N_TRAIN),\n        test_size=1 - TRAIN_RATIO,\n        stratify=labels,\n        random_state=BASE_SEED + run\n    )\n\n    val_m, test_p, lgb_models = run_lgbm(\n        run,\n        tabular[tr_idx], labels[tr_idx],\n        tabular[va_idx], labels[va_idx],\n        test_tabular\n    )\n    lgb_val_list.append(val_m)\n    if not SKIP_TEST:\n        lgb_test_probs.append(test_p)\n    last_lgb_models = lgb_models\n\n    print(f\"  Val  acc={val_m['accuracy']:.4f}  macro_f1={val_m['macro_f1']:.4f}  logloss={val_m['logloss']:.4f}\")\n    print(f\"  Time : {time.time()-t_run:.0f}s\")\n\n# Average test predictions\n    if not SKIP_TEST:\n        lgb_test_avg = np.mean(lgb_test_probs, axis=0)\n\nprint(f\"\\nLightGBM total: {time.time()-t0:.0f}s\")\nprint(\"\\nValidation summary across MC runs:\")\nprint(pd.DataFrame(lgb_val_list)[[\"accuracy\",\"macro_f1\",\"weighted_f1\",\"logloss\"]].describe().round(4))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T13:33:32.379611Z","iopub.execute_input":"2026-07-24T13:33:32.379974Z","iopub.status.idle":"2026-07-24T13:38:39.618795Z","shell.execute_reply.started":"2026-07-24T13:33:32.379944Z","shell.execute_reply":"2026-07-24T13:38:39.618189Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 7 — Meta-NN: Stacking Ensemble","metadata":{}},{"cell_type":"code","source":"# ============================================================\n# GENERATE META-FEATURES FROM MONTE CARLO SPLITS\n# For each MC run: train ResNet + LightGBM → get val predictions → meta-features\n# ============================================================\nprint(\"\\n\" + \"=\"*64)\nprint(\"META-NN — BUILDING META-FEATURES\")\nprint(\"=\"*64)\n\nmeta_features_list = []\nmeta_labels_list   = []\nt0 = time.time()\n\nfor run in range(MC_RUNS):\n    tr_idx, va_idx = train_test_split(\n        np.arange(N_TRAIN),\n        test_size=1 - TRAIN_RATIO,\n        stratify=labels,\n        random_state=BASE_SEED + run\n    )\n\n    X_tr_i, X_va_i = images[tr_idx], images[va_idx]\n    y_tr_i, y_va_i = labels[tr_idx], labels[va_idx]\n    T_tr_i, T_va_i = tabular[tr_idx], tabular[va_idx]\n\n    # --- ResNet val probs ---\n    _seed_all(BASE_SEED + run * 1000)\n    train_ds = ImageDataset(X_tr_i, y_tr_i, augment=True)\n    valid_ds = ImageDataset(X_va_i, y_va_i, augment=False)\n    train_dl = DataLoader(train_ds, batch_size=RN_BATCH, shuffle=True, num_workers=2, pin_memory=True)\n    valid_dl = DataLoader(valid_ds, batch_size=RN_BATCH*2, shuffle=False, num_workers=2, pin_memory=True)\n\n    model    = build_resnet(N_CLASSES).to(DEVICE)\n    optimizer = optim.AdamW(model.parameters(), lr=RN_LR, weight_decay=1e-4)\n    scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=RN_EPOCHS)\n    best_state, best_vl = None, float(\"inf\")\n\n    for epoch in range(RN_EPOCHS):\n        model.train()\n        for imgs, targets in train_dl:\n            imgs, targets = imgs.to(DEVICE), targets.to(DEVICE)\n            optimizer.zero_grad()\n            nn.CrossEntropyLoss()(model(imgs), targets).backward()\n            optimizer.step()\n        scheduler.step()\n        model.eval()\n        vl_list, tgt_list = [], []\n        with torch.no_grad():\n            for imgs, targets in valid_dl:\n                vl_list.append(model(imgs.to(DEVICE)).cpu().numpy())\n                tgt_list.append(targets.numpy())\n        vl_ll = log_loss(np.concatenate(tgt_list), softmax_np(np.concatenate(vl_list)), labels=list(range(N_CLASSES)))\n        vl_acc = (softmax_np(np.concatenate(vl_list)).argmax(axis=1) == np.concatenate(tgt_list)).mean()\n        print(f\"      epoch {epoch+1}/{RN_EPOCHS}  loss={vl_ll:.4f}  acc={vl_acc:.4f}\")\n        if vl_ll < best_vl:\n            best_state = copy.deepcopy(model.state_dict())\n            best_vl    = vl_ll\n\n    model.load_state_dict(best_state)\n    model.eval()\n    with torch.no_grad():\n        rn_probs = softmax_np(np.concatenate([\n            model(imgs.to(DEVICE)).cpu().numpy() for imgs, _ in valid_dl\n        ]))\n    del model, optimizer, scheduler\n    gc.collect()\n    torch.cuda.empty_cache()\n\n    # --- LightGBM val probs ---\n    lgbm_probs = np.zeros((len(va_idx), N_CLASSES), dtype=np.float64)\n    for cls in range(N_CLASSES):\n        dtrain = lgb.Dataset(T_tr_i, label=(y_tr_i == cls).astype(int))\n        dvalid = lgb.Dataset(T_va_i, label=(y_va_i == cls).astype(int), reference=dtrain)\n        params = {\"objective\":\"binary\",\"metric\":\"binary_logloss\",\n                  \"num_leaves\":LGB_LEAVES,\"learning_rate\":0.05,\n                  \"feature_fraction\":0.8,\"bagging_fraction\":0.8,\n                  \"bagging_freq\":5,\"min_child_samples\":20,\n                  \"verbose\":-1,\"seed\":BASE_SEED+run*1000,\"device\":\"cpu\"}\n        m = lgb.train(params, dtrain, num_boost_round=LGB_ROUNDS,\n                      valid_sets=[dvalid],\n                      callbacks=[lgb.early_stopping(LGB_EARLY, verbose=False)])\n        lgbm_probs[:, cls] = m.predict(T_va_i)\n    lgbm_probs = lgbm_probs / lgbm_probs.sum(axis=1, keepdims=True)\n\n    # Concatenate → meta-features\n    meta_feat = np.concatenate([rn_probs, lgbm_probs], axis=1)\n    meta_features_list.append(meta_feat)\n    meta_labels_list.append(y_va_i)\n\n    print(f\"  Run {run+1}: meta-features = {meta_feat.shape}\")\n\n# Combine all meta-features\nmeta_X = np.concatenate(meta_features_list, axis=0)\nmeta_y = np.concatenate(meta_labels_list, axis=0)\nprint(f\"\\nTotal meta-features: {meta_X.shape}\")\n\n# Split meta-features for Meta-NN training/validation\nmeta_tr_idx, meta_va_idx = train_test_split(\n    np.arange(len(meta_y)),\n    test_size=0.2, stratify=meta_y, random_state=BASE_SEED\n)\nX_mt = torch.from_numpy(meta_X[meta_tr_idx]).float().to(DEVICE)\ny_mt = torch.from_numpy(meta_y[meta_tr_idx]).long().to(DEVICE)\nX_mv = torch.from_numpy(meta_X[meta_va_idx]).float().to(DEVICE)\ny_mv = torch.from_numpy(meta_y[meta_va_idx]).long().to(DEVICE)\n\n# Test meta-features: ResNet MC-averaged + LightGBM MC-averaged\nif not SKIP_TEST:\n    test_meta_X = np.concatenate([rn_test_avg, lgb_test_avg], axis=1).astype(np.float32)\n    X_mtest = torch.from_numpy(test_meta_X).float().to(DEVICE)\n    print(f\"Test meta-features: {test_meta_X.shape}\")\nelse:\n    X_mtest = None\n\n# Train Meta-NN\nprint(\"\\nTraining Meta-NN...\")\nmeta_model  = MetaNN(N_CLASSES).to(DEVICE)\ncriterion   = nn.CrossEntropyLoss()\noptimizer   = optim.Adam(meta_model.parameters(), lr=META_LR)\nbest_state, best_vl = None, float(\"inf\")\n\nfor epoch in range(META_EPOCHS):\n    meta_model.train()\n    perm = torch.randperm(len(X_mt))\n    for i in range(0, len(X_mt), META_BATCH):\n        idx_b = perm[i : i + META_BATCH]\n        optimizer.zero_grad()\n        criterion(meta_model(X_mt[idx_b]), y_mt[idx_b]).backward()\n        optimizer.step()\n\n    meta_model.eval()\n    with torch.no_grad():\n        va_probs = F.softmax(meta_model(X_mv), dim=1).cpu().numpy()\n    vl = log_loss(y_mv.cpu().numpy(), va_probs, labels=list(range(N_CLASSES)))\n    if vl < best_vl:\n        best_state = copy.deepcopy(meta_model.state_dict())\n        best_vl    = vl\n\n    if (epoch + 1) % 5 == 0 or epoch == 0:\n        va_acc = (va_probs.argmax(axis=1) == y_mv.cpu().numpy()).mean()\n        print(f\"      epoch {epoch+1}/{META_EPOCHS}  loss={vl:.4f}  acc={va_acc:.4f}\")\n\n# Predict on test\nmeta_model.load_state_dict(best_state)\nmeta_model.eval()\nif X_mtest is not None:\n    with torch.no_grad():\n        meta_test_probs = F.softmax(meta_model(X_mtest), dim=1).cpu().numpy()\nelse:\n    meta_test_probs = np.empty((0, N_CLASSES), dtype=np.float32)\n\ndel meta_model, optimizer\ngc.collect()\ntorch.cuda.empty_cache()\n\nprint(f\"\\nMeta-NN done in {time.time()-t0:.0f}s\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T13:38:39.619905Z","iopub.execute_input":"2026-07-24T13:38:39.620272Z","iopub.status.idle":"2026-07-24T13:51:50.734328Z","shell.execute_reply.started":"2026-07-24T13:38:39.620248Z","shell.execute_reply":"2026-07-24T13:51:50.733464Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 8— Visualizations","metadata":{}},{"cell_type":"code","source":"import os\n\nOUT_DIR = Path(\"/kaggle/working\")\nos.makedirs(OUT_DIR, exist_ok=True)\n\nMODEL_COLORS = {\"ResNet18\": \"#2196F3\", \"LightGBM\": \"#4CAF50\", \"Meta-NN\": \"#FF9800\"}\n\n# ============================================================\n# 1. CONFUSION MATRICES\n# ============================================================\nfor name, probs in pseudo_preds.items():\n    cm = confusion_matrix(y_pseudo, np.argmax(probs, axis=1))\n    fig, ax = plt.subplots(figsize=(10, 9))\n    sns.heatmap(cm, ax=ax, annot=True, fmt=\"d\", cmap=\"Blues\",\n                xticklabels=CLASS_NAMES, yticklabels=CLASS_NAMES)\n    ax.set_title(\n        f\"{name} — Confusion Matrix\\n\"\n        f\"Accuracy={metrics_df.loc[name, 'accuracy']:.4f} | \"\n        f\"Macro-F1={metrics_df.loc[name, 'macro_f1']:.4f}\"\n    )\n    ax.set_xlabel(\"Predicted\")\n    ax.set_ylabel(\"Actual\")\n    ax.tick_params(axis=\"x\", rotation=45)\n    fig.tight_layout()\n    fig.savefig(OUT_DIR / f\"fig_confusion_matrix_{name.replace(' ', '_')}.png\", dpi=160, bbox_inches=\"tight\")\n    plt.show()\n    plt.close(fig)\n\n\n# ============================================================\n# 3. METRIC BAR COMPARISON (4 decimal places)\n# ============================================================\nMETRIC_KEYS   = [\"accuracy\", \"macro_f1\", \"weighted_f1\", \"macro_prec\", \"macro_recall\"]\nMETRIC_LABELS = [\"Accuracy\", \"Macro F1\", \"Weighted F1\", \"Macro Precision\", \"Macro Recall\"]\n\nfig, ax = plt.subplots(figsize=(12, 6))\nx = np.arange(len(METRIC_KEYS))\nbw = 0.25\nfor i, name in enumerate(pseudo_preds.keys()):\n    vals = [metrics_df.loc[name, k] for k in METRIC_KEYS]\n    bars = ax.bar(x + (i - 1) * bw, vals, bw, label=name, color=MODEL_COLORS[name], alpha=0.85)\n    for bar, val in zip(bars, vals):\n        ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.005,\n                f\"{val:.4f}\", ha=\"center\", va=\"bottom\", fontsize=8)\nax.set_xlabel(\"Metric\")\nax.set_ylabel(\"Score\")\nax.set_title(\"Model Comparison — Key Metrics (Test Set)\", fontsize=13, fontweight=\"bold\")\nax.set_xticks(x)\nax.set_xticklabels(METRIC_LABELS)\nax.legend(title=\"Model\")\nax.set_ylim(0, 1.15)\nax.grid(axis=\"y\", alpha=0.3)\nfig.tight_layout()\nfig.savefig(OUT_DIR / \"fig_metric_comparison.png\", dpi=160, bbox_inches=\"tight\")\nplt.show()\nplt.close(fig)\n\n\n\n# ============================================================\n# 5. LOGLOSS COMPARISON\n# ============================================================\nfig, ax = plt.subplots(figsize=(8, 5))\nnames_ll = list(pseudo_preds.keys())\nll_vals  = [metrics_df.loc[n, \"logloss\"] for n in names_ll]\ncolors   = [MODEL_COLORS[n] for n in names_ll]\nbars = ax.bar(names_ll, ll_vals, color=colors, alpha=0.85)\nfor bar, val in zip(bars, ll_vals):\n    ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.002,\n            f\"{val:.4f}\", ha=\"center\", va=\"bottom\", fontsize=11)\nax.set_ylabel(\"Log Loss\")\nax.set_title(\"Log Loss — Lower is Better\", fontsize=12, fontweight=\"bold\")\nax.grid(axis=\"y\", alpha=0.3)\nax.set_ylim(0, max(ll_vals) * 1.2)\nfig.tight_layout()\nfig.savefig(OUT_DIR / \"fig_logloss.png\", dpi=160, bbox_inches=\"tight\")\nplt.show()\nplt.close(fig)\n\nprint(f\"\\nAll visualizations saved to: {OUT_DIR}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-24T13:53:27.863589Z","iopub.execute_input":"2026-07-24T13:53:27.86415Z","iopub.status.idle":"2026-07-24T13:53:30.370358Z","shell.execute_reply.started":"2026-07-24T13:53:27.864119Z","shell.execute_reply":"2026-07-24T13:53:30.369658Z"}},"outputs":[],"execution_count":null}]}