{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":4117,"databundleVersionId":46665},{"sourceType":"datasetVersion","sourceId":15588068,"datasetId":9973608,"databundleVersionId":16520332},{"sourceType":"datasetVersion","sourceId":15603415,"datasetId":9984844,"databundleVersionId":16536697},{"sourceType":"datasetVersion","sourceId":15605231,"datasetId":9986188,"databundleVersionId":16538649},{"sourceType":"datasetVersion","sourceId":15610660,"datasetId":9989983,"databundleVersionId":16544376}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\"\"\"\n╔══════════════════════════════════════════════════════════════════════╗\n║        MS Malware Classification — Ultimate Pipeline v5              ║\n║                                                                      ║\n║  Kiến trúc: Multi-branch CNN (Gray + RGB + Bigram) + 5-Fold CV       ║\n║            + SPP (Spatial Pyramid Pooling)                           ║\n║            + XGBoost trên ALL 3 branch embeddings (1536-d)           ║\n║            + Optimal OOF Weighted Ensemble                           ║\n╚══════════════════════════════════════════════════════════════════════╝\n\"\"\"\n\nimport os\nimport cv2\nimport joblib\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport xgboost as xgb\nfrom scipy.optimize import minimize\nfrom sklearn.model_selection import StratifiedKFold\nfrom tensorflow.keras.layers import (\n    Input, Conv2D, MaxPooling2D, GlobalAveragePooling2D,\n    Dense, Dropout, BatchNormalization, Concatenate, Lambda\n)\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.applications import ResNet50, EfficientNetB0\nfrom tensorflow.keras.regularizers import l2\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\n\n# ═══════════════════════════════════════════════════════════\n# 1. CẤU HÌNH\n# ═══════════════════════════════════════════════════════════\nCFG = {\n    'labels_path': '/kaggle/input/competitions/malware-classification/trainLabels.csv',\n    'train_dirs': [\n        '/kaggle/input/datasets/habao2603/malware-processed-images-v1',\n        '/kaggle/input/datasets/habao2603/malware-processed-images-part-1',\n        '/kaggle/input/datasets/habao2603/malware-processed-images-part-2',\n        '/kaggle/input/datasets/habao2603/malware-processed-images-part-3',\n    ],\n    'n_folds':       5,           # CẢI TIẾN 5: Kích hoạt 5-Fold CV\n    'batch_size':    8,\n    'img_h':         512,\n    'img_w':         256,\n    'n_classes':     9,\n    'lr_phase1':     1e-4,\n    'epochs_phase1': 8,\n    'lr_phase2':     1e-5,\n    'epochs_phase2': 15,          \n    'patience':      4,           # Giảm nhẹ patience để tiết kiệm thời gian chạy Fold\n    'output_dir':    '/kaggle/working',\n}\n\n# ═══════════════════════════════════════════════════════════\n# 2. ĐỌC VÀ LỌC DỮ LIỆU (Giờ trả về toàn bộ dữ liệu)\n# ═══════════════════════════════════════════════════════════\ndef prepare_dataframe(cfg):\n    df_labels = pd.read_csv(cfg['labels_path'])\n    records = []\n    for directory in cfg['train_dirs']:\n        gray_dir = os.path.join(directory, 'gray')\n        if not os.path.exists(gray_dir):\n            continue\n        for f in os.listdir(gray_dir):\n            if f.endswith('.png'):\n                records.append({'Id': f.split('.')[0], 'BaseDir': directory})\n\n    df_available = pd.DataFrame(records)\n    df = pd.merge(df_labels, df_available, on='Id', how='inner')\n\n    if df['Class'].min() == 1:\n        df['Class'] = df['Class'] - 1\n\n    print(f\"Tổng số mẫu dữ liệu: {len(df)}\")\n    return df.reset_index(drop=True)\n\n# ═══════════════════════════════════════════════════════════\n# 3. DATASET PIPELINE\n# ═══════════════════════════════════════════════════════════\ndef _load_img(path, channels):\n    img = tf.io.read_file(path)\n    img = tf.image.decode_png(img, channels=channels)\n    return tf.cast(img, tf.float32) / 255.0\n\ndef _parse(paths, label, is_training, cfg):\n    gray_path, rgb_path, bigram_path = paths\n\n    img_gray   = _load_img(gray_path,   channels=1)\n    img_rgb    = _load_img(rgb_path,    channels=3)\n    img_bigram = _load_img(bigram_path, channels=1)\n\n    H, W = cfg['img_h'], cfg['img_w']\n    img_gray   = tf.image.resize(img_gray,   [H, W],     method='nearest')\n    img_rgb    = tf.image.resize(img_rgb,    [H, W],     method='nearest')\n    img_bigram = tf.image.resize(img_bigram, [256, 256], method='nearest')\n\n    if is_training:\n        img_rgb = tf.image.random_brightness(img_rgb, max_delta=0.15)\n        img_rgb = tf.image.random_contrast(img_rgb, lower=0.85, upper=1.15)\n        img_rgb = tf.clip_by_value(img_rgb, 0.0, 1.0)\n        img_gray = tf.clip_by_value(\n            img_gray + tf.random.normal(tf.shape(img_gray), stddev=0.02),\n            0.0, 1.0\n        )\n\n    inputs = {\n        'input_gray':   img_gray,\n        'input_rgb':    img_rgb,\n        'input_bigram': img_bigram,\n    }\n    return inputs, tf.one_hot(label, depth=cfg['n_classes'])\n\ndef make_dataset(df, cfg, is_training=True):\n    ids       = df['Id'].values\n    base_dirs = df['BaseDir'].values\n    labels    = df['Class'].values.astype(np.int32)\n\n    gray_paths   = [os.path.join(b, 'gray',   f\"{i}.png\") for b, i in zip(base_dirs, ids)]\n    rgb_paths    = [os.path.join(b, 'rgb',    f\"{i}.png\") for b, i in zip(base_dirs, ids)]\n    bigram_paths = [os.path.join(b, 'bigram', f\"{i}.png\") for b, i in zip(base_dirs, ids)]\n\n    ds = tf.data.Dataset.from_tensor_slices(((gray_paths, rgb_paths, bigram_paths), labels))\n    ds = ds.map(lambda p, l: _parse(p, l, is_training, cfg), num_parallel_calls=tf.data.AUTOTUNE)\n\n    if is_training:\n        ds = ds.shuffle(buffer_size=1000, seed=42)\n\n    return ds.batch(cfg['batch_size']).prefetch(tf.data.AUTOTUNE)\n\n# ═══════════════════════════════════════════════════════════\n# 4. KIẾN TRÚC MÔ HÌNH (CÓ SPP)\n# ═══════════════════════════════════════════════════════════\ndef SPP(x, name_prefix='spp_'):\n    \"\"\" CẢI TIẾN 2: SPP giữ lại phân bố không gian thay vì cào bằng \"\"\"\n    p1 = GlobalAveragePooling2D(name=f'{name_prefix}gap_1')(x)\n    p2 = MaxPooling2D(pool_size=(2,2), padding='same', name=f'{name_prefix}maxpool_2')(x)\n    p2 = GlobalAveragePooling2D(name=f'{name_prefix}gap_2')(p2)\n    p4 = MaxPooling2D(pool_size=(4,4), padding='same', name=f'{name_prefix}maxpool_4')(x)\n    p4 = GlobalAveragePooling2D(name=f'{name_prefix}gap_4')(p4)\n    return Concatenate(name=f'{name_prefix}concat')([p1, p2, p4])\n\ndef _build_bigram_branch(input_bi):\n    x = Conv2D(32,  3, padding='same')(input_bi)\n    x = BatchNormalization()(x)\n    x = tf.keras.layers.Activation('relu')(x)\n    x = MaxPooling2D(2)(x)\n    x = Conv2D(64,  3, padding='same')(x)\n    x = BatchNormalization()(x)\n    x = tf.keras.layers.Activation('relu')(x)\n    x = MaxPooling2D(2)(x)\n    x = Conv2D(128, 3, padding='same')(x)\n    x = BatchNormalization()(x)\n    x = tf.keras.layers.Activation('relu')(x)\n    x = MaxPooling2D(2)(x)\n    x = Conv2D(256, 3, padding='same')(x)\n    x = BatchNormalization()(x)\n    x = tf.keras.layers.Activation('relu')(x)\n    x = GlobalAveragePooling2D()(x)\n    return Dense(512, activation='relu', name='bigram_embed')(x)\n\ndef build_model(cfg, freeze_backbones=True):\n    H, W = cfg['img_h'], cfg['img_w']\n\n    # ── Nhánh Gray ──────────────────────────────\n    input_gray = Input(shape=(H, W, 1), name='input_gray')\n    x_gray = Lambda(lambda x: tf.tile(x, [1, 1, 1, 3]), name='gray_tile')(input_gray)\n\n    # ĐÃ SỬA: Xóa name='resnet50_gray' để Keras tải đúng weights\n    resnet = ResNet50(weights='imagenet', include_top=False)\n    resnet.trainable = not freeze_backbones\n    feat_gray = resnet(x_gray, training=False)\n    vec_gray  = SPP(feat_gray, name_prefix='spp_gray_')\n    vec_gray  = Dense(512, activation='relu', name='gray_embed')(vec_gray)\n\n    # ── Nhánh RGB ─────────────────────────\n    input_rgb = Input(shape=(H, W, 3), name='input_rgb')\n    \n    # ĐÃ SỬA: Xóa name='effnet_rgb' để Keras tải đúng file efficientnetb0_notop.h5\n    effnet = EfficientNetB0(weights='imagenet', include_top=False)\n    effnet.trainable = not freeze_backbones\n    feat_rgb = effnet(input_rgb, training=False)\n    vec_rgb  = SPP(feat_rgb, name_prefix='spp_rgb_')\n    vec_rgb  = Dense(512, activation='relu', name='rgb_embed')(vec_rgb)\n\n    # ── Nhánh Bigram ─────────────────────\n    input_bi = Input(shape=(256, 256, 1), name='input_bigram')\n    vec_bi   = _build_bigram_branch(input_bi)\n\n    vec_gray = Dropout(0.25, name='drop_gray')(vec_gray)\n    vec_rgb  = Dropout(0.25, name='drop_rgb')(vec_rgb)\n    vec_bi   = Dropout(0.25, name='drop_bi')(vec_bi)\n\n    # ── Fusion ─────────────────────────────────────────────\n    merged = Concatenate(name='fusion')([vec_gray, vec_rgb, vec_bi])\n    x = BatchNormalization(name='bn_fusion')(merged)\n    x = Dense(512, activation='relu', kernel_regularizer=l2(1e-4), name='fc1')(x)\n    x = Dropout(0.4, name='drop_head')(x)\n    x = Dense(128, activation='relu', kernel_regularizer=l2(1e-4), name='fc2')(x)\n    output = Dense(cfg['n_classes'], activation='softmax', name='output')(x)\n\n    model = Model(inputs=[input_gray, input_rgb, input_bi], outputs=output)\n    return model, resnet, effnet\n\ndef compile_model(model, lr):\n    model.compile(\n        optimizer=tf.keras.optimizers.AdamW(learning_rate=lr, weight_decay=1e-4),\n        loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.1),\n        metrics=['accuracy']\n    )\n\n# ═══════════════════════════════════════════════════════════\n# 5. 2-PHASE TRAINING & EXTRACTOR\n# ═══════════════════════════════════════════════════════════\ndef train_phase1(model, train_ds, val_ds, cfg, fold_num):\n    compile_model(model, lr=cfg['lr_phase1'])\n    ckpt_path = os.path.join(cfg['output_dir'], f'phase1_fold{fold_num}_best.keras')\n    callbacks = [\n        ModelCheckpoint(ckpt_path, monitor='val_accuracy', save_best_only=True, mode='max', verbose=1),\n        EarlyStopping(monitor='val_accuracy', patience=cfg['patience'], restore_best_weights=True, verbose=1),\n    ]\n    return model.fit(train_ds, validation_data=val_ds, epochs=cfg['epochs_phase1'], callbacks=callbacks, verbose=1)\n\ndef train_phase2(model, resnet, effnet, train_ds, val_ds, cfg, fold_num):\n    for backbone in [resnet, effnet]:\n        backbone.trainable = True\n        n = len(backbone.layers)\n        unfreeze_from = int(n * 0.70)\n        \n        # CẢI TIẾN 1: Khóa cứng BatchNormalization khi fine-tune\n        for i, layer in enumerate(backbone.layers):\n            if isinstance(layer, tf.keras.layers.BatchNormalization):\n                layer.trainable = False\n            else:\n                layer.trainable = (i >= unfreeze_from)\n\n    compile_model(model, lr=cfg['lr_phase2'])\n    ckpt_path = os.path.join(cfg['output_dir'], f'best_cnn_fold{fold_num}.keras')\n    callbacks = [\n        ModelCheckpoint(ckpt_path, monitor='val_accuracy', save_best_only=True, mode='max', verbose=1),\n        EarlyStopping(monitor='val_accuracy', patience=cfg['patience'], restore_best_weights=True, verbose=1),\n        ReduceLROnPlateau(monitor='val_accuracy', factor=0.5, patience=2, min_lr=1e-7, verbose=1),\n    ]\n    return model.fit(train_ds, validation_data=val_ds, epochs=cfg['epochs_phase2'], callbacks=callbacks, verbose=1)\n\ndef build_all_extractors(cnn_model, fold_num):\n    \"\"\" CẢI TIẾN 4: Trích xuất cả 3 nhánh \"\"\"\n    ext_gray = Model(inputs=cnn_model.get_layer('input_gray').output, outputs=cnn_model.get_layer('gray_embed').output, name=f'ext_g_f{fold_num}')\n    ext_rgb = Model(inputs=cnn_model.get_layer('input_rgb').output, outputs=cnn_model.get_layer('rgb_embed').output, name=f'ext_r_f{fold_num}')\n    ext_bigram = Model(inputs=cnn_model.get_layer('input_bigram').output, outputs=cnn_model.get_layer('bigram_embed').output, name=f'ext_b_f{fold_num}')\n    return ext_gray, ext_rgb, ext_bigram\n\ndef extract_all_embeddings(extractors, df, cfg, batch_size=32):\n    ext_gray, ext_rgb, ext_bigram = extractors\n    H, W = cfg['img_h'], cfg['img_w']\n    all_emb_gray, all_emb_rgb, all_emb_bigram = [], [], []\n    n = len(df)\n\n    for start in range(0, n, batch_size):\n        batch_rows = df.iloc[start : start + batch_size]\n        batch_g, batch_r, batch_b = [], [], []\n\n        for _, row in batch_rows.iterrows():\n            base, sid = row['BaseDir'], row['Id']\n            # Gray\n            img_g = cv2.resize(cv2.imread(os.path.join(base, 'gray', f\"{sid}.png\"), cv2.IMREAD_GRAYSCALE), (W, H), interpolation=cv2.INTER_NEAREST)\n            batch_g.append(img_g.astype(np.float32) / 255.0)\n            # RGB\n            img_r = cv2.resize(cv2.imread(os.path.join(base, 'rgb', f\"{sid}.png\"), cv2.IMREAD_COLOR), (W, H), interpolation=cv2.INTER_NEAREST)[:, :, ::-1]\n            batch_r.append(img_r.astype(np.float32) / 255.0)\n            # Bigram\n            img_b = cv2.resize(cv2.imread(os.path.join(base, 'bigram', f\"{sid}.png\"), cv2.IMREAD_GRAYSCALE), (256, 256), interpolation=cv2.INTER_NEAREST)\n            batch_b.append(img_b.astype(np.float32) / 255.0)\n\n        arr_g = np.expand_dims(np.array(batch_g), -1)\n        arr_r = np.array(batch_r)\n        arr_b = np.expand_dims(np.array(batch_b), -1)\n\n        all_emb_gray.append(ext_gray.predict(arr_g, verbose=0))\n        all_emb_rgb.append(ext_rgb.predict(arr_r, verbose=0))\n        all_emb_bigram.append(ext_bigram.predict(arr_b, verbose=0))\n\n    return np.concatenate([np.vstack(all_emb_gray), np.vstack(all_emb_rgb), np.vstack(all_emb_bigram)], axis=1)\n\n# ═══════════════════════════════════════════════════════════\n# 6. FULL PIPELINE (5-FOLD OOF)\n# ═══════════════════════════════════════════════════════════\nif __name__ == \"__main__\":\n    df = prepare_dataframe(CFG)\n    \n    # Ma trận lưu trữ dự đoán OOF\n    oof_cnn_probs = np.zeros((len(df), CFG['n_classes']))\n    oof_xgb_probs = np.zeros((len(df), CFG['n_classes']))\n    \n    # CẢI TIẾN 5: Bắt đầu 5-Fold Stratified\n    skf = StratifiedKFold(n_splits=CFG['n_folds'], shuffle=True, random_state=42)\n\n    for fold, (train_idx, val_idx) in enumerate(skf.split(df, df['Class'])):\n        print(\"\\n\" + \"█\"*55)\n        print(f\"███ FOLD {fold + 1}/{CFG['n_folds']} ███\")\n        print(\"█\"*55)\n        \n        # Xóa GPU Session để giải phóng RAM/VRAM\n        tf.keras.backend.clear_session()\n        \n        train_fold_df = df.iloc[train_idx].reset_index(drop=True)\n        val_fold_df   = df.iloc[val_idx].reset_index(drop=True)\n        \n        train_ds = make_dataset(train_fold_df, CFG, is_training=True)\n        val_ds   = make_dataset(val_fold_df, CFG, is_training=False)\n\n        # 1. Train CNN\n        model, resnet, effnet = build_model(CFG, freeze_backbones=True)\n        train_phase1(model, train_ds, val_ds, CFG, fold + 1)\n        train_phase2(model, resnet, effnet, train_ds, val_ds, CFG, fold + 1)\n        \n        # 2. Predict OOF CNN\n        cnn_probs_fold = []\n        for batch_inputs, _ in val_ds:\n            cnn_probs_fold.append(model.predict(batch_inputs, verbose=0))\n        oof_cnn_probs[val_idx] = np.vstack(cnn_probs_fold)\n\n        # 3. Train XGBoost trên 1536-d\n        extractors = build_all_extractors(model, fold + 1)\n        print(f\"\\n[Fold {fold+1}] Extracting Train Embeddings (1536-d)...\")\n        X_train_emb = extract_all_embeddings(extractors, train_fold_df, CFG)\n        print(f\"[Fold {fold+1}] Extracting Val Embeddings (1536-d)...\")\n        X_val_emb   = extract_all_embeddings(extractors, val_fold_df, CFG)\n        \n        xgb_model = xgb.XGBClassifier(\n            n_estimators=500, max_depth=7, learning_rate=0.05,\n            subsample=0.8, colsample_bytree=0.8, min_child_weight=3,\n            gamma=0.1, reg_alpha=0.1, reg_lambda=1.0,\n            tree_method='hist', device='cuda', objective='multi:softprob',\n            num_class=CFG['n_classes'], eval_metric='mlogloss', early_stopping_rounds=30, verbosity=0\n        )\n        xgb_model.fit(X_train_emb, train_fold_df['Class'].values, eval_set=[(X_val_emb, val_fold_df['Class'].values)], verbose=100)\n        \n        # 4. Predict OOF XGB\n        oof_xgb_probs[val_idx] = xgb_model.predict_proba(X_val_emb)\n        xgb_model.save_model(os.path.join(CFG['output_dir'], f'best_xgb_fold{fold+1}.json'))\n\n    # ── TÌM ALPHA TỐI ƯU DỰA TRÊN 100% OOF DỮ LIỆU ────────\n    def logloss(alpha):\n        alpha = float(np.clip(alpha, 0.0, 1.0))\n        mixed = alpha * oof_cnn_probs + (1 - alpha) * oof_xgb_probs\n        mixed = np.clip(mixed, 1e-7, 1.0)\n        return -np.mean([np.log(mixed[i, df['Class'].values[i]]) for i in range(len(df))])\n\n    print(\"\\n\" + \"═\"*55)\n    print(\"TỐI ƯU HÓA ENSEMBLE TỪ OUT-OF-FOLD (OOF)\")\n    print(\"═\"*55)\n    result = minimize(logloss, x0=[0.5], bounds=[(0.0, 1.0)], method='L-BFGS-B')\n    best_alpha = float(result.x[0])\n    \n    final_oof_probs = best_alpha * oof_cnn_probs + (1 - best_alpha) * oof_xgb_probs\n    final_oof_preds = np.argmax(final_oof_probs, axis=1)\n    ensemble_acc = (final_oof_preds == df['Class'].values).mean()\n\n    print(f\"OOF CNN Accuracy      : {(np.argmax(oof_cnn_probs, axis=1) == df['Class'].values).mean()*100:.2f}%\")\n    print(f\"OOF XGBoost Accuracy  : {(np.argmax(oof_xgb_probs, axis=1) == df['Class'].values).mean()*100:.2f}%\")\n    print(f\"OOF Ensemble Accuracy : {ensemble_acc*100:.2f}%\")\n    print(f\"Trọng số tối ưu       : CNN={best_alpha:.3f} | XGB={1-best_alpha:.3f}\")\n    \n    joblib.dump(best_alpha, os.path.join(CFG['output_dir'], 'ensemble_alpha_v5.joblib'))\n    print(f\"\\nĐã hoàn thành xuất sắc 5-Fold. Toàn bộ trọng số được lưu tại {CFG['output_dir']}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T06:42:03.437878Z","iopub.execute_input":"2026-04-13T06:42:03.438502Z"}},"outputs":[],"execution_count":null}]}