{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.12"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"databundleVersionId":46665,"sourceId":4117,"sourceType":"competition"},{"databundleVersionId":16663061,"datasetId":10073093,"sourceId":15722285,"sourceType":"datasetVersion"}],"dockerImageVersionId":31328,"isGpuEnabled":true,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"papermill":{"default_parameters":{},"duration":1873.893491,"end_time":"2026-04-14T05:52:07.320383+00:00","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2026-04-14T05:20:53.426892+00:00","version":"2.7.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"e9764e8e","cell_type":"markdown","source":"# Improved Training Notebook (v2 — Merged Data)\n\n## Input dataset\nNotebook này đọc **9 files đã gộp** từ `merge-parts.ipynb`:\n```\nX_train_tab.npy      X_test_tab.npy\nX_train_ng.npz       X_test_ng.npz\nX_train_opseq.pkl    X_test_opseq.pkl\nX_train_pixel.npy    X_test_pixel.npy\ny_train.npy\n```\n\n## Pipeline\n```\n[tab ~75] + [byte_ng 3K] + [opcode_ng 2K] + [pixel 1024] = ~6099 features\n        ↓\n  Level-0: XGBoost · LightGBM · ExtraTrees (5-fold OOF)\n        ↓  27 OOF proba cols  +  pseudo-labels\n  Level-1: Logistic Regression\n        ↓\n  (Optional) Extended Level-1: LR output + raw features → XGBoost shallow\n        ↓\n  submission.csv\n```","metadata":{"papermill":{"duration":0.003469,"end_time":"2026-04-14T05:20:55.945838+00:00","exception":false,"start_time":"2026-04-14T05:20:55.942369+00:00","status":"completed"},"tags":[]}},{"id":"7f87fc19","cell_type":"code","source":"# ============================================================\n# CELL 1 — IMPORTS\n# ============================================================\nimport os\nimport gc\nimport time\nimport pickle\nimport joblib\nimport numpy as np\nimport pandas as pd\nimport scipy.sparse as sp\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import log_loss, accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_selection import SelectKBest, mutual_info_classif\n\nprint(\"Import xong!\")","metadata":{"execution":{"iopub.execute_input":"2026-04-14T05:20:55.952119Z","iopub.status.busy":"2026-04-14T05:20:55.951847Z","iopub.status.idle":"2026-04-14T05:21:04.000410Z","shell.execute_reply":"2026-04-14T05:21:03.999582Z"},"papermill":{"duration":8.053323,"end_time":"2026-04-14T05:21:04.001907+00:00","exception":false,"start_time":"2026-04-14T05:20:55.948584+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"6a8b7b13","cell_type":"code","source":"# ============================================================\n# CELL 2 — CẤU HÌNH ĐƯỜNG DẪN\n# ============================================================\n\nDATA_IN_DIR   = '/kaggle/input/datasets/trankimhuu/data-ml-big-2015-ver-2'\n\nWORKING_DIR   = '/kaggle/working'\nMODEL_DIR     = os.path.join(WORKING_DIR, 'models')\nCKPT_DIR      = os.path.join(WORKING_DIR, 'checkpoints')\n\nos.makedirs(MODEL_DIR, exist_ok=True)\nos.makedirs(CKPT_DIR,  exist_ok=True)\n\nSUBMISSION_CSV = '/kaggle/input/competitions/malware-classification/sampleSubmission.csv'\nNUM_CLASSES    = 9\nN_FOLDS        = 5\nRANDOM_STATE   = 42\n\nprint(f\"📂 Input : {DATA_IN_DIR}\")\nprint(f\"📂 Models: {MODEL_DIR}\")","metadata":{"execution":{"iopub.execute_input":"2026-04-14T05:21:04.009199Z","iopub.status.busy":"2026-04-14T05:21:04.008594Z","iopub.status.idle":"2026-04-14T05:21:04.015521Z","shell.execute_reply":"2026-04-14T05:21:04.014573Z"},"papermill":{"duration":0.01203,"end_time":"2026-04-14T05:21:04.016919+00:00","exception":false,"start_time":"2026-04-14T05:21:04.004889+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"800711d8","cell_type":"code","source":"# ============================================================\n# CELL 3 — LOAD DỮ LIỆU\n# ============================================================\n# THAY ĐỔI QUAN TRỌNG: KHÔNG load X_test_ng ở đây.\n# X_train_ng (3 GB) + X_test_ng (3 GB) = 6 GB cùng lúc.\n# Thêm bất kỳ thứ gì (HashVec output, opseq) → vượt 13 GB.\n#\n# Strategy mới:\n#   Cell 3: Load train_ng + tất cả non-ng data\n#   Cell 4: Xử lý train (opcode + ng selection) → lưu file → del train_ng\n#   Cell 5: Load test_ng → xử lý test → del test_ng\n#   Cell 6: Ghép features từ file đã lưu\n\nprint(\"=\" * 55)\nprint(\"BƯỚC 1: LOAD DỮ LIỆU (train_ng riêng, test_ng sau)\")\nprint(\"=\" * 55)\n\ndef load_file(filename):\n    fp = os.path.join(DATA_IN_DIR, filename)\n    size_mb = os.path.getsize(fp) / 1e6\n    print(f\"  {filename:<30} ({size_mb:.0f} MB)...\", end=\"\", flush=True)\n    return fp\n\n# ── TRAIN (tất cả) ──────────────────────────────────────────\nfp = load_file(\"X_train_tab.npy\");   X_train_tab   = np.load(fp);          print(f\" {X_train_tab.shape}\")\nfp = load_file(\"X_train_ng.npz\");    X_train_ng    = sp.load_npz(fp);       print(f\" {X_train_ng.shape}\")\nfp = load_file(\"X_train_pixel.npy\"); X_train_pixel = np.load(fp);           print(f\" {X_train_pixel.shape}\")\nfp = load_file(\"X_train_opseq.pkl\")\nwith open(fp, \"rb\") as f: train_opseq = pickle.load(f)\nprint(f\" n={len(train_opseq):,}\")\nfp = load_file(\"y_train.npy\");       y_train = np.load(fp);                 print(f\" {y_train.shape}\")\n\n# ── TEST (chỉ load non-ng, test_ng load sau ở Cell 5) ────────\nfp = load_file(\"X_test_tab.npy\");    X_test_tab    = np.load(fp);           print(f\" {X_test_tab.shape}\")\nfp = load_file(\"X_test_pixel.npy\");  X_test_pixel  = np.load(fp);           print(f\" {X_test_pixel.shape}\")\nfp = load_file(\"X_test_opseq.pkl\")\nwith open(fp, \"rb\") as f: test_opseq = pickle.load(f)\nprint(f\" n={len(test_opseq):,}\")\n# X_test_ng KHÔNG load ở đây — sẽ load ở Cell 5\n\nprint(f\"\\nTrain={X_train_tab.shape[0]:,} | Test={X_test_tab.shape[0]:,}\")\nprint(f\"RAM hiện tại: X_train_ng ({X_train_ng.data.nbytes/1e9:.1f} GB) + các mảng nhỏ\")\nprint(\"X_test_ng chưa load — sẽ xử lý ở Cell 5 sau khi del X_train_ng\")\n","metadata":{"execution":{"iopub.execute_input":"2026-04-14T05:21:04.023555Z","iopub.status.busy":"2026-04-14T05:21:04.023069Z","iopub.status.idle":"2026-04-14T05:22:08.591813Z","shell.execute_reply":"2026-04-14T05:22:08.591074Z"},"papermill":{"duration":64.573769,"end_time":"2026-04-14T05:22:08.593401+00:00","exception":false,"start_time":"2026-04-14T05:21:04.019632+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"01a9caa2","cell_type":"code","source":"# ============================================================\n# CELL 4 — XỬ LÝ TRAIN: Opcode N-gram + Byte N-gram selection\n# ============================================================\n# Xử lý toàn bộ TRAIN xong → del X_train_ng → Cell 5 mới load X_test_ng.\n# Peak RAM cell này: X_train_ng (3 GB) + X_train_opng (0.6 GB) = 3.6 GB ✓\n\nUSE_OPCODE = True   # Đổi False nếu muốn bỏ opcode\n\nK_BYTE_NG   = 3000\nK_OPCODE_NG = 2000\nCHUNK_SIZE  = 500   # Mỗi chunk densify 500×131072 = ~250 MB → an toàn\n\nprint(\"=\" * 55)\nprint(\"BƯỚC 2+3 TRAIN: Opcode + Byte N-gram\")\nprint(\"=\" * 55)\n\n# ── 2a. Opcode N-gram (TRAIN) ────────────────────────────────\nif USE_OPCODE:\n    from sklearn.feature_extraction.text import HashingVectorizer as _HashVec\n    lengths = [len(s.split()) for s in train_opseq]\n    avg_len = sum(lengths) / max(len(lengths), 1)\n    print(f\"  Avg opcode tokens/seq: {avg_len:.0f}\")\n\n    opcode_hash_vec = _HashVec(\n        ngram_range    = (2, 3),\n        n_features     = 2**14,\n        analyzer       = \"word\",\n        norm           = None,\n        alternate_sign = False,\n    )\n    print(\"  Vectorizing train opcode...\", end=\"\", flush=True)\n    t0 = time.time()\n    X_train_opng = opcode_hash_vec.transform(train_opseq)\n    print(f\" ({time.time()-t0:.1f}s) {X_train_opng.shape}\")\n    joblib.dump(opcode_hash_vec, os.path.join(MODEL_DIR, \"opcode_hash_vec.pkl\"))\nelse:\n    X_train_opng = None\n\ndel train_opseq\ngc.collect()\n\n# ── 2b. Opcode chi2 selection (TRAIN) ────────────────────────\nif USE_OPCODE and X_train_opng is not None:\n    from sklearn.feature_selection import chi2, SelectKBest\n    k_op = min(K_OPCODE_NG, X_train_opng.shape[1])\n    print(f\"  Opcode chi2 {X_train_opng.shape[1]:,}→{k_op}...\", end=\"\", flush=True)\n    t0 = time.time()\n    sel_opcode = SelectKBest(chi2, k=k_op)\n    X_train_opng_sel = sel_opcode.fit_transform(X_train_opng, y_train).toarray().astype(np.float32)\n    del X_train_opng; gc.collect()\n    print(f\" ({time.time()-t0:.1f}s) {X_train_opng_sel.shape}\")\n    joblib.dump(sel_opcode, os.path.join(MODEL_DIR, \"selector_opcode.pkl\"))\nelse:\n    X_train_opng_sel = np.zeros((X_train_tab.shape[0], 0), dtype=np.float32)\n\n# ── 3. Byte N-gram chi2 (TRAIN, chunking) ────────────────────\nprint(f\"  Byte N-gram chi2 train {X_train_ng.shape[1]:,}→{K_BYTE_NG} (chunking)...\", end=\"\", flush=True)\nt0 = time.time()\nn_samples, n_features = X_train_ng.shape\nn_classes = len(np.unique(y_train))\n\nY = np.zeros((n_samples, n_classes), dtype=np.float32)\nY[np.arange(n_samples), y_train] = 1.0\n\n# Tính chi2 scores theo chunk — không densify toàn bộ cùng lúc\nobserved     = np.zeros((n_classes, n_features), dtype=np.float64)\nfeature_sum  = np.zeros(n_features, dtype=np.float64)\n\nfor start in range(0, n_samples, CHUNK_SIZE):\n    end      = min(start + CHUNK_SIZE, n_samples)\n    X_chunk  = X_train_ng[start:end]          # vẫn sparse\n    observed += (X_chunk.T.dot(Y[start:end])).T\n    feature_sum += np.asarray(X_chunk.sum(axis=0)).ravel()\n\nclass_prob = Y.mean(axis=0)\nexpected   = np.outer(class_prob, feature_sum)\nnp.clip(expected, 1e-9, None, out=expected)\nchi2_scores = np.sum((observed - expected) ** 2 / expected, axis=0)\ntop_k_idx   = np.argsort(chi2_scores)[-K_BYTE_NG:]\njoblib.dump(top_k_idx, os.path.join(MODEL_DIR, \"top_k_byte_idx.pkl\"))\n\n# Densify train theo chunk chỉ top-k cols\nX_train_byteng = np.zeros((n_samples, K_BYTE_NG), dtype=np.float32)\nfor start in range(0, n_samples, CHUNK_SIZE):\n    end = min(start + CHUNK_SIZE, n_samples)\n    X_train_byteng[start:end] = X_train_ng[start:end].toarray()[:, top_k_idx]\n\ndel X_train_ng, Y, observed, feature_sum\ngc.collect()\nprint(f\" ({time.time()-t0:.1f}s) {X_train_byteng.shape}\")\n\n# ── Ghép + lưu X_train final ─────────────────────────────────\nX_train = np.hstack([\n    X_train_tab.astype(np.float32),\n    X_train_byteng,\n    X_train_opng_sel,\n    X_train_pixel.astype(np.float32),\n]).astype(np.float32)\n\ndel X_train_tab, X_train_byteng, X_train_opng_sel, X_train_pixel\ngc.collect()\n\nnp.save(os.path.join(CKPT_DIR, \"X_train.npy\"), X_train)\nprint(f\"\\nX_train saved: {X_train.shape}  ({X_train.nbytes/1e9:.2f} GB)\")\nprint(\"X_train_ng đã del. Sẵn sàng load X_test_ng ở Cell 5.\")\n","metadata":{"execution":{"iopub.execute_input":"2026-04-14T05:22:08.601436Z","iopub.status.busy":"2026-04-14T05:22:08.600949Z","iopub.status.idle":"2026-04-14T05:24:08.714087Z","shell.execute_reply":"2026-04-14T05:24:08.713201Z"},"papermill":{"duration":120.119038,"end_time":"2026-04-14T05:24:08.715758+00:00","exception":false,"start_time":"2026-04-14T05:22:08.596720+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"8d43968f","cell_type":"code","source":"# ============================================================\n# CELL 5 — XỬ LÝ TEST: Load X_test_ng → select → del\n# ============================================================\n# Lúc này X_train_ng đã del → chỉ có X_train (~180 MB) trong RAM.\n# Load X_test_ng (3 GB) an toàn vì tổng = ~3.2 GB << 13 GB.\n\nprint(\"=\" * 55)\nprint(\"BƯỚC 2+3 TEST: Load X_test_ng → select → del\")\nprint(\"=\" * 55)\n\n# Load top_k_idx và sel_opcode đã fit ở Cell 4\ntop_k_idx  = joblib.load(os.path.join(MODEL_DIR, \"top_k_byte_idx.pkl\"))\nsel_opcode = joblib.load(os.path.join(MODEL_DIR, \"selector_opcode.pkl\")) if USE_OPCODE else None\n\n# ── Opcode (TEST) ─────────────────────────────────────────────\nif USE_OPCODE:\n    opcode_hash_vec = joblib.load(os.path.join(MODEL_DIR, \"opcode_hash_vec.pkl\"))\n    print(\"  Vectorizing test opcode...\", end=\"\", flush=True)\n    t0 = time.time()\n    X_test_opng = opcode_hash_vec.transform(test_opseq)\n    X_test_opng_sel = sel_opcode.transform(X_test_opng).toarray().astype(np.float32)\n    del X_test_opng; gc.collect()\n    print(f\" ({time.time()-t0:.1f}s) {X_test_opng_sel.shape}\")\nelse:\n    X_test_opng_sel = np.zeros((X_test_tab.shape[0], 0), dtype=np.float32)\n\ndel test_opseq; gc.collect()\n\n# ── Byte N-gram (TEST) — load ngay, select, del ───────────────\nfp = os.path.join(DATA_IN_DIR, \"X_test_ng.npz\")\nsize_mb = os.path.getsize(fp) / 1e6\nprint(f\"  Loading X_test_ng.npz ({size_mb:.0f} MB)...\", end=\"\", flush=True)\nX_test_ng = sp.load_npz(fp)\nprint(f\" {X_test_ng.shape}\")\n\nCHUNK_SIZE = 500\nn_test = X_test_ng.shape[0]\nX_test_byteng = np.zeros((n_test, len(top_k_idx)), dtype=np.float32)\nprint(f\"  Densify top-{len(top_k_idx)} cols (chunking)...\", end=\"\", flush=True)\nt0 = time.time()\nfor start in range(0, n_test, CHUNK_SIZE):\n    end = min(start + CHUNK_SIZE, n_test)\n    X_test_byteng[start:end] = X_test_ng[start:end].toarray()[:, top_k_idx]\ndel X_test_ng; gc.collect()\nprint(f\" ({time.time()-t0:.1f}s) {X_test_byteng.shape}\")\n\n# ── Ghép + lưu X_test final ──────────────────────────────────\nX_test = np.hstack([\n    X_test_tab.astype(np.float32),\n    X_test_byteng,\n    X_test_opng_sel,\n    X_test_pixel.astype(np.float32),\n]).astype(np.float32)\n\ndel X_test_tab, X_test_byteng, X_test_opng_sel, X_test_pixel\ngc.collect()\n\nnp.save(os.path.join(CKPT_DIR, \"X_test.npy\"), X_test)\nprint(f\"\\nX_test saved: {X_test.shape}  ({X_test.nbytes/1e9:.2f} GB)\")\nprint(\"X_test_ng đã del. RAM đã sạch.\")\n","metadata":{"execution":{"iopub.execute_input":"2026-04-14T05:24:08.724549Z","iopub.status.busy":"2026-04-14T05:24:08.723714Z","iopub.status.idle":"2026-04-14T05:26:35.191948Z","shell.execute_reply":"2026-04-14T05:26:35.190969Z"},"papermill":{"duration":146.47429,"end_time":"2026-04-14T05:26:35.193605+00:00","exception":false,"start_time":"2026-04-14T05:24:08.719315+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"5ef124bc","cell_type":"code","source":"# ============================================================\n# CELL 6 — LOAD FEATURES TỪ CHECKPOINT\n# ============================================================\n# X_train và X_test đã được lưu ở Cell 4 và 5.\n# Cell này chỉ load lại để sẵn sàng cho training.\n\nprint(\"=\" * 55)\nprint(\"BƯỚC 4: LOAD FEATURES TỪ CHECKPOINT\")\nprint(\"=\" * 55)\n\nX_train = np.load(os.path.join(CKPT_DIR, \"X_train.npy\"))\nX_test  = np.load(os.path.join(CKPT_DIR, \"X_test.npy\"))\nnp.save(os.path.join(CKPT_DIR, \"y_train.npy\"), y_train)\n\nprint(f\"  X_train: {X_train.shape}  ({X_train.nbytes/1e9:.2f} GB)\")\nprint(f\"  X_test : {X_test.shape}  ({X_test.nbytes/1e9:.2f} GB)\")\n\n# Breakdown features\nn_tab   = X_train_tab.shape[1] if \"X_train_tab\" in dir() else 107\nn_byte  = 3000\nn_opng  = X_train_opng_sel.shape[1] if \"X_train_opng_sel\" in dir() else (2000 if USE_OPCODE else 0)\nn_pixel = 1024\nprint(f\"\\n  Breakdown: {n_tab} tab + {n_byte} byte_ng + {n_opng} opcode_ng + {n_pixel} pixel = {X_train.shape[1]} total\")\n","metadata":{"execution":{"iopub.execute_input":"2026-04-14T05:26:35.202268Z","iopub.status.busy":"2026-04-14T05:26:35.201729Z","iopub.status.idle":"2026-04-14T05:26:35.380727Z","shell.execute_reply":"2026-04-14T05:26:35.379819Z"},"papermill":{"duration":0.184959,"end_time":"2026-04-14T05:26:35.382347+00:00","exception":false,"start_time":"2026-04-14T05:26:35.197388+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"a634734b","cell_type":"code","source":"# ============================================================\n# CELL 7 — ĐỊNH NGHĨA BASE MODELS (LEVEL-0)\n# ============================================================\n# THAY ĐỔI: ExtraTrees → CatBoost\n#\n# ExtraTrees lưu toàn bộ 500 cây trong RAM:\n#   500 cây × 4K features × depth 20 ≈ 5-7 GB → OOM khi kết hợp\n#   với X_train (180 MB) và X_test (180 MB) đang còn trong RAM.\n#\n# CatBoost (GPU):\n#   - Xử lý trên GPU, RAM CPU thấp hơn nhiều (~0.5-1 GB)\n#   - Tốc độ nhanh hơn ExtraTrees trên dataset này\n#   - Thường cho kết quả tốt hơn ExtraTrees trong stacking\n#\n# XGBoost và LightGBM giảm n_estimators để giảm thời gian mỗi fold:\n#   - 5 folds × 3 models, mỗi fold train + predict → tổng ~15 lần\n#   - Dùng early stopping thay vì fixed n_estimators\n\nprint(\"=\" * 55)\nprint(\"BƯỚC 5: CẤU HÌNH BASE MODELS\")\nprint(\"=\" * 55)\n\nclasses = np.unique(y_train)\nweights = compute_class_weight(\"balanced\", classes=classes, y=y_train)\nclass_weights_dict = dict(zip(classes.astype(int), weights))\n\nprint(\"Class distribution:\")\nfor c, w in class_weights_dict.items():\n    cnt = (y_train == c).sum()\n    print(f\"  Class {c+1}: {cnt:5d}  weight={w:.3f}\")\n\n# ── XGBoost ─────────────────────────────────────────────────\nxgb_model = xgb.XGBClassifier(\n    objective        = \"multi:softprob\",\n    num_class        = NUM_CLASSES,\n    max_depth        = 7,\n    n_estimators     = 500,\n    learning_rate    = 0.05,\n    subsample        = 0.8,\n    colsample_bytree = 0.7,\n    min_child_weight = 3,\n    tree_method      = \"hist\",\n    device           = \"cuda\",\n    eval_metric      = \"mlogloss\",\n    random_state     = RANDOM_STATE,\n)\n\n# ── LightGBM ────────────────────────────────────────────────\nlgb_model = lgb.LGBMClassifier(\n    objective         = \"multiclass\",\n    num_class         = NUM_CLASSES,\n    class_weight      = class_weights_dict,\n    n_estimators      = 1000,\n    learning_rate     = 0.05,\n    num_leaves        = 127,\n    min_child_samples = 10,\n    feature_fraction  = 0.7,\n    bagging_fraction  = 0.8,\n    bagging_freq      = 5,\n    lambda_l1         = 0.1,\n    lambda_l2         = 0.1,\n    device            = \"gpu\",\n    random_state      = RANDOM_STATE,\n    verbose           = -1,\n)\n\n# ── CatBoost (thay ExtraTrees) ───────────────────────────────\ntry:\n    from catboost import CatBoostClassifier\n    cb_model = CatBoostClassifier(\n        iterations       = 500,\n        learning_rate    = 0.05,\n        depth            = 6,\n        loss_function    = \"MultiClass\",\n        eval_metric      = \"MultiClass\",\n        task_type        = \"GPU\",\n        class_weights    = list(weights),\n        random_seed      = RANDOM_STATE,\n        verbose          = 0,\n    )\n    USE_CATBOOST = True\n    print(\"\\nCatBoost GPU: OK\")\nexcept ImportError:\n    # Fallback: RandomForest với n_estimators nhỏ hơn\n    from sklearn.ensemble import RandomForestClassifier\n    cb_model = RandomForestClassifier(\n        n_estimators = 200,       # giảm từ 500 → 200\n        max_depth    = 15,        # giảm từ 20 → 15\n        max_features = \"sqrt\",\n        class_weight = \"balanced\",\n        n_jobs       = -1,\n        random_state = RANDOM_STATE,\n    )\n    USE_CATBOOST = False\n    print(\"\\nCatBoost không có, dùng RandomForest(200 trees)\")\n\nBASE_MODELS = {\n    \"xgb\": xgb_model,\n    \"lgb\": lgb_model,\n    \"cat\": cb_model,\n}\n\nskf = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=2026)\nprint(f\"\\n3 base models + {N_FOLDS}-fold CV sẵn sàng\")\n","metadata":{"execution":{"iopub.execute_input":"2026-04-14T05:26:35.391575Z","iopub.status.busy":"2026-04-14T05:26:35.390842Z","iopub.status.idle":"2026-04-14T05:26:36.251001Z","shell.execute_reply":"2026-04-14T05:26:36.250073Z"},"papermill":{"duration":0.866421,"end_time":"2026-04-14T05:26:36.252657+00:00","exception":false,"start_time":"2026-04-14T05:26:35.386236+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"5b2039d5","cell_type":"code","source":"# ============================================================\n# CELL 8 — LEVEL-0: OOF META-FEATURES (RAM-SAFE)\n# ============================================================\n# RAM-SAFE changes:\n# 1. Sau mỗi fold: gọi gc.collect() để giải phóng RAM fold model\n# 2. Lưu OOF train/test ngay sau mỗi model xong → checkpoint\n# 3. Nếu kernel crash giữa chừng: load checkpoint và resume\n# 4. Predict X_test theo từng fold rồi average → không giữ\n#    oof_test_folds (N_FOLDS × N_test × 9) trong RAM cùng lúc\n\nprint(\"=\" * 55)\nprint(\"BƯỚC 6: LEVEL-0 (OOF META-FEATURES)\")\nprint(\"=\" * 55)\n\nmeta_train_list = []\nmeta_test_list  = []\n\nfor model_name, model in BASE_MODELS.items():\n    ckpt_train = os.path.join(CKPT_DIR, f\"oof_train_{model_name}.npy\")\n    ckpt_test  = os.path.join(CKPT_DIR, f\"oof_test_{model_name}.npy\")\n\n    # Resume nếu đã có checkpoint\n    if os.path.exists(ckpt_train) and os.path.exists(ckpt_test):\n        print(f\"[{model_name.upper()}] Checkpoint found, loading...\")\n        meta_train_list.append(np.load(ckpt_train))\n        meta_test_list.append(np.load(ckpt_test))\n        continue\n\n    print(f\"\\n{'─'*45}\")\n    print(f\"[{model_name.upper()}]\")\n\n    oof_train    = np.zeros((X_train.shape[0], NUM_CLASSES), dtype=np.float32)\n    oof_test_avg = np.zeros((X_test.shape[0],  NUM_CLASSES), dtype=np.float32)\n\n    for fold, (tr_idx, val_idx) in enumerate(skf.split(X_train, y_train)):\n        print(f\"  Fold {fold+1}/{N_FOLDS}...\", end=\"\", flush=True)\n        t0 = time.time()\n\n        # Fit model trên fold\n        model.fit(X_train[tr_idx], y_train[tr_idx])\n\n        # Predict OOF\n        oof_train[val_idx] = model.predict_proba(X_train[val_idx])\n\n        # Predict test và cộng dồn (tránh lưu cả 5 bản cùng lúc)\n        oof_test_avg += model.predict_proba(X_test) / N_FOLDS\n\n        fold_ll = log_loss(y_train[val_idx], oof_train[val_idx])\n        print(f\" ll={fold_ll:.5f}  ({time.time()-t0:.1f}s)\")\n\n        # Giải phóng RAM fold model ngay\n        gc.collect()\n\n    overall_ll = log_loss(y_train, oof_train)\n    print(f\"  OOF logloss: {overall_ll:.5f}\")\n\n    # Checkpoint ngay sau khi xong 1 model\n    np.save(ckpt_train, oof_train)\n    np.save(ckpt_test,  oof_test_avg)\n    print(f\"  Checkpoint saved\")\n\n    meta_train_list.append(oof_train)\n    meta_test_list.append(oof_test_avg)\n\n    # Full model\n    print(f\"  Full model...\", end=\"\", flush=True)\n    t0 = time.time()\n    model.fit(X_train, y_train)\n    joblib.dump(model, os.path.join(MODEL_DIR, f\"{model_name}_full.pkl\"))\n    print(f\" ({time.time()-t0:.1f}s)\")\n\n    # Giải phóng RAM full model sau khi lưu\n    BASE_MODELS[model_name] = None\n    del model\n    gc.collect()\n\nX_train_L1 = np.hstack(meta_train_list).astype(np.float32)\nX_test_L1  = np.hstack(meta_test_list).astype(np.float32)\n\nprint(f\"\\nLevel-0 done! X_train_L1={X_train_L1.shape}  X_test_L1={X_test_L1.shape}\")\nnp.save(os.path.join(CKPT_DIR, \"X_train_L1.npy\"), X_train_L1)\nnp.save(os.path.join(CKPT_DIR, \"X_test_L1.npy\"),  X_test_L1)\n\n# Giải phóng X_train, X_test sau Level-0 (không cần nữa cho đến Cell 11)\ndel X_train\n# del X_test\ngc.collect()\n# print(\"X_train/X_test released from RAM\")\nprint(\"X_train released from RAM\")","metadata":{"execution":{"iopub.execute_input":"2026-04-14T05:26:36.261807Z","iopub.status.busy":"2026-04-14T05:26:36.261129Z","iopub.status.idle":"2026-04-14T05:51:14.985595Z","shell.execute_reply":"2026-04-14T05:51:14.984836Z"},"papermill":{"duration":1478.738675,"end_time":"2026-04-14T05:51:14.995127+00:00","exception":false,"start_time":"2026-04-14T05:26:36.256452+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"e6fa454e","cell_type":"code","source":"# ============================================================\n# CELL 9 — PSEUDO-LABELING (fixed)\n# ============================================================\n# FIX: Không dùng reshape(-1, 3, 9) vì phụ thuộc thứ tự models.\n# Thay bằng: lấy trực tiếp prediction từ từng full model đã lưu.\n\nprint(\"=\" * 55)\nprint(\"BƯỚC 7: PSEUDO-LABELING (threshold=0.99)\")\nprint(\"=\" * 55)\n\nCONFIDENCE_THRESHOLD = 0.99\n\n# Load 3 full models và predict trực tiếp trên X_test\n# → không phụ thuộc vào cấu trúc X_test_L1\nprint(\"  Loading full models để tạo pseudo-labels...\")\nproba_list = []\nfor model_name in BASE_MODELS.keys():\n    m = joblib.load(os.path.join(MODEL_DIR, f\"{model_name}_full.pkl\"))\n    proba_list.append(m.predict_proba(X_test))\n    del m; gc.collect()\n\n# Average xác suất từ 3 models\navg_test_proba = np.mean(proba_list, axis=0)   # (N_test, 9)\ndel proba_list; gc.collect()\n\nmax_conf      = avg_test_proba.max(axis=1)\nconfident_idx = np.where(max_conf >= CONFIDENCE_THRESHOLD)[0]\npseudo_labels = avg_test_proba.argmax(axis=1)[confident_idx]\n\nprint(f\"  Tổng test  : {len(avg_test_proba):,}\")\nprint(f\"  Confident  : {len(confident_idx):,} ({100*len(confident_idx)/len(avg_test_proba):.1f}%)\")\nprint(\"\\n  Phân bố pseudo-labels:\")\nfor c in range(NUM_CLASSES):\n    cnt = (pseudo_labels == c).sum()\n    print(f\"    Class {c+1}: {cnt:5d}\")\n\nX_train_L1_ext = np.vstack([X_train_L1, X_test_L1[confident_idx]])\ny_train_ext    = np.concatenate([y_train, pseudo_labels])\nprint(f\"\\n  X_train_L1_ext: {X_train_L1_ext.shape}\")\n","metadata":{"execution":{"iopub.execute_input":"2026-04-14T05:51:15.014481Z","iopub.status.busy":"2026-04-14T05:51:15.014200Z","iopub.status.idle":"2026-04-14T05:51:18.901720Z","shell.execute_reply":"2026-04-14T05:51:18.900902Z"},"papermill":{"duration":3.899146,"end_time":"2026-04-14T05:51:18.903218+00:00","exception":false,"start_time":"2026-04-14T05:51:15.004072+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"1d2dbf10","cell_type":"code","source":"# ============================================================\n# CELL 10 — LEVEL-1: META-LEARNER (Logistic Regression)\n# ============================================================\n# LR với C=0.1 (regularization mạnh) trên 27 features:\n# - Tránh overfit hơn XGBoost 500 trees\n# - Đúng về lý thuyết: tổ hợp tuyến tính các probability predictions\n# - Nhanh hơn nhiều\n\nprint(\"=\" * 55)\nprint(\"BƯỚC 8: LEVEL-1 META-LEARNER (Logistic Regression)\")\nprint(\"=\" * 55)\n\n# StandardScaler vì LR nhạy cảm với scale\nscaler = StandardScaler()\nX_train_L1_ext_s = scaler.fit_transform(X_train_L1_ext)\nX_test_L1_s      = scaler.transform(X_test_L1)\njoblib.dump(scaler, os.path.join(MODEL_DIR, 'meta_scaler.pkl'))\n\nmeta_lr = LogisticRegression(\n    C            = 0.1,\n    solver       = 'lbfgs',\n    max_iter     = 2000,\n    multi_class  = 'multinomial',\n    random_state = RANDOM_STATE,\n)\n\nprint(\"Training...\", end='', flush=True)\nt0 = time.time()\nmeta_lr.fit(X_train_L1_ext_s, y_train_ext)\nprint(f\" ✓ ({time.time()-t0:.1f}s)\")\n\n# OOF eval (không có pseudo-labels để đánh giá sạch)\nX_train_L1_s = scaler.transform(X_train_L1)\npreds_lr     = meta_lr.predict_proba(X_train_L1_s)\nll_lr        = log_loss(y_train, preds_lr)\nacc_lr       = accuracy_score(y_train, preds_lr.argmax(axis=1))\nprint(f\"  OOF logloss : {ll_lr:.5f}\")\nprint(f\"  OOF accuracy: {acc_lr:.4f}\")\n\njoblib.dump(meta_lr, os.path.join(MODEL_DIR, 'meta_lr.pkl'))\nprint(\"meta_lr.pkl saved\")","metadata":{"execution":{"iopub.execute_input":"2026-04-14T05:51:18.915853Z","iopub.status.busy":"2026-04-14T05:51:18.915316Z","iopub.status.idle":"2026-04-14T05:51:19.345594Z","shell.execute_reply":"2026-04-14T05:51:19.342979Z"},"papermill":{"duration":0.442568,"end_time":"2026-04-14T05:51:19.351635+00:00","exception":false,"start_time":"2026-04-14T05:51:18.909067+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"bc40725d","cell_type":"code","source":"# ============================================================\n# CELL 11 — EXTENDED META-LEARNER (OOF + RAW FEATURES)\n# ============================================================\n# Bổ sung features gốc vào Level-1 để meta-learner có thêm context.\n# Dùng XGBoost shallow (depth=3) vì input giờ có nhiều chiều hơn.\n\nprint(\"=\" * 55)\nprint(\"BƯỚC 9: EXTENDED META-LEARNER (27 OOF + raw features)\")\nprint(\"=\" * 55)\n\n# Load lại X_train / X_test từ checkpoint\nX_train_raw = np.load(os.path.join(CKPT_DIR, 'X_train.npy'))\nX_test_raw  = np.load(os.path.join(CKPT_DIR, 'X_test.npy'))\nprint(f\"  X_train_raw: {X_train_raw.shape}\")\n\n# Ghép [27 OOF proba] + [~6099 raw features]\nX_train_L1_full = np.hstack([X_train_L1, X_train_raw]).astype(np.float32)\nX_test_L1_full  = np.hstack([X_test_L1,  X_test_raw]).astype(np.float32)\n\nX_train_L1_full_ext = np.vstack([\n    X_train_L1_full,\n    X_test_L1_full[confident_idx]\n])\n\nscaler_full = StandardScaler()\nX_train_L1_full_ext_s = scaler_full.fit_transform(X_train_L1_full_ext)\nX_test_L1_full_s      = scaler_full.transform(X_test_L1_full)\nX_train_L1_full_s     = scaler_full.transform(X_train_L1_full)\njoblib.dump(scaler_full, os.path.join(MODEL_DIR, 'meta_scaler_full.pkl'))\n\n# XGBoost shallow (depth=3 là intentional)\nmeta_xgb = xgb.XGBClassifier(\n    objective    = 'multi:softprob',\n    num_class    = NUM_CLASSES,\n    max_depth    = 3,    # shallow để tránh overfit\n    n_estimators = 200,  # ít hơn nhiều so với cũ (500)\n    learning_rate= 0.05,\n    subsample    = 0.8,\n    tree_method  = 'hist',\n    device       = 'cuda',\n    random_state = 99,\n)\n\nprint(\"Training extended XGB meta-learner...\", end='', flush=True)\nt0 = time.time()\nmeta_xgb.fit(X_train_L1_full_ext_s, y_train_ext)\nprint(f\" ✓ ({time.time()-t0:.1f}s)\")\n\npreds_xgb = meta_xgb.predict_proba(X_train_L1_full_s)\nll_xgb    = log_loss(y_train, preds_xgb)\nacc_xgb   = accuracy_score(y_train, preds_xgb.argmax(axis=1))\nprint(f\"  OOF logloss (extended): {ll_xgb:.5f}\")\nprint(f\"  OOF accuracy (extended): {acc_xgb:.4f}\")\n\njoblib.dump(meta_xgb, os.path.join(MODEL_DIR, 'meta_xgb_full.pkl'))\nprint(\"meta_xgb_full.pkl saved\")\n\ndel X_train_raw, X_test_raw\ngc.collect()","metadata":{"execution":{"iopub.execute_input":"2026-04-14T05:51:19.416337Z","iopub.status.busy":"2026-04-14T05:51:19.413281Z","iopub.status.idle":"2026-04-14T05:52:02.740722Z","shell.execute_reply":"2026-04-14T05:52:02.740012Z"},"papermill":{"duration":43.361708,"end_time":"2026-04-14T05:52:02.742057+00:00","exception":false,"start_time":"2026-04-14T05:51:19.380349+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"d6bb083d","cell_type":"code","source":"# ============================================================\n# CELL 12 — TẠO SUBMISSION\n# ============================================================\n# FORMAT ĐÚNG: Kaggle BIG2015 dùng multiclass log loss.\n# sampleSubmission.csv có cột: Id, Prediction1..Prediction9\n# (mỗi cột là xác suất của 1 class → submit probability, không phải class label)\n#\n# Nếu sampleSubmission chỉ có cột \"Id\" và \"Class\":\n#   → df_sub[\"Class\"] = argmax + 1  (submit class label)\n# Kiểm tra format bằng cách print df_sub.columns ở dưới.\n\nprint(\"=\" * 55)\nprint(\"BƯỚC 10: TẠO SUBMISSION\")\nprint(\"=\" * 55)\n\ndf_sub = pd.read_csv(SUBMISSION_CSV)\nprint(f\"sampleSubmission columns: {list(df_sub.columns)}\")\nprint(f\"sampleSubmission shape  : {df_sub.shape}\")\n\n# Predictions từ 3 variants\nproba_lr  = meta_lr.predict_proba(X_test_L1_s)\nproba_xgb = meta_xgb.predict_proba(X_test_L1_full_s)\nproba_ens = (proba_lr + proba_xgb) / 2\n\n# ── Tự động detect format ────────────────────────────────────\nsub_cols = list(df_sub.columns)\nhas_pred_cols = any(c.startswith(\"Prediction\") for c in sub_cols)\n\nresults = {\n    \"xgb_full\"  : proba_xgb,\n    \"lr_only\"   : proba_lr,\n    \"ensemble\"  : proba_ens,\n}\n\nfor name, proba in results.items():\n    df_out = df_sub[[\"Id\"]].copy()\n\n    if has_pred_cols:\n        # Format: Id, Prediction1, ..., Prediction9  → submit probabilities\n        pred_cols = [f\"Prediction{i}\" for i in range(1, NUM_CLASSES + 1)]\n        for j, col in enumerate(pred_cols):\n            df_out[col] = proba[:, j]\n    else:\n        # Format: Id, Class  → submit class label (1-indexed)\n        df_out[\"Class\"] = proba.argmax(axis=1) + 1\n\n    path = os.path.join(WORKING_DIR, f\"submission_{name}.csv\")\n    df_out.to_csv(path, index=False)\n\n    # OOF logloss report\n    if name == \"xgb_full\":   ll_name = f\"OOF logloss: {ll_xgb:.5f}\"\n    elif name == \"lr_only\":  ll_name = f\"OOF logloss: {ll_lr:.5f}\"\n    else:                    ll_name = \"ensemble (average)\"\n\n    print(f\"\\n  {path}  [{ll_name}]\")\n    pred_classes = proba.argmax(axis=1) + 1\n    for c in range(1, 10):\n        cnt = (pred_classes == c).sum()\n        print(f\"    Class {c}: {cnt:5d} ({100*cnt/len(pred_classes):.1f}%)\")\n\nprint(\"\\n✅ HOÀN TẤT!\")\nprint(\"   Khuyên dùng: submission_xgb_full.csv (base model tốt nhất)\")\nprint(\"   Sau đó thử: submission_ensemble.csv\")\n","metadata":{"execution":{"iopub.execute_input":"2026-04-14T05:52:02.756391Z","iopub.status.busy":"2026-04-14T05:52:02.755963Z","iopub.status.idle":"2026-04-14T05:52:04.203848Z","shell.execute_reply":"2026-04-14T05:52:04.202946Z"},"papermill":{"duration":1.456821,"end_time":"2026-04-14T05:52:04.205462+00:00","exception":false,"start_time":"2026-04-14T05:52:02.748641+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"50ad3f21","cell_type":"code","source":"# # ============================================================\n# # CELL 13 — (TÙY CHỌN) OPTUNA TUNING CHO LightGBM\n# # ============================================================\n# # Bật ENABLE_OPTUNA=True nếu muốn tune LightGBM (~1-2h, 50 trials).\n# # Best params dùng để override lgb_model ở Cell 7 rồi chạy lại.\n\n# ENABLE_OPTUNA = False  # ← Đổi thành True để chạy\n\n# if ENABLE_OPTUNA:\n#     import optuna\n#     optuna.logging.set_verbosity(optuna.logging.WARNING)\n\n#     def lgb_objective(trial):\n#         params = {\n#             'objective'         : 'multiclass',\n#             'num_class'         : NUM_CLASSES,\n#             'metric'            : 'multi_logloss',\n#             'verbosity'         : -1,\n#             'device'            : 'gpu',\n#             'n_estimators'      : 1000,\n#             'learning_rate'     : trial.suggest_float('lr',         0.02, 0.1,  log=True),\n#             'num_leaves'        : trial.suggest_int('num_leaves',   63,   255),\n#             'min_child_samples' : trial.suggest_int('min_child',    5,    50),\n#             'feature_fraction'  : trial.suggest_float('feat_frac',  0.5,  1.0),\n#             'bagging_fraction'  : trial.suggest_float('bag_frac',   0.6,  1.0),\n#             'bagging_freq'      : trial.suggest_int('bag_freq',     1,    7),\n#             'lambda_l1'         : trial.suggest_float('l1',  1e-3, 10.0, log=True),\n#             'lambda_l2'         : trial.suggest_float('l2',  1e-3, 10.0, log=True),\n#             'max_depth'         : trial.suggest_int('max_depth',    6,    15),\n#         }\n#         dtrain = lgb.Dataset(X_train, label=y_train)\n#         cv_res = lgb.cv(\n#             params, dtrain,\n#             nfold=3, stratified=True,\n#             num_boost_round=500,\n#             callbacks=[lgb.early_stopping(30, verbose=False)],\n#         )\n#         return min(cv_res['valid multi_logloss-mean'])\n\n#     study = optuna.create_study(direction='minimize')\n#     study.optimize(lgb_objective, n_trials=50, show_progress_bar=True)\n\n#     print(f\"\\nBest logloss: {study.best_value:.5f}\")\n#     print(\"Best params (copy vào Cell 7):\")\n#     for k, v in study.best_params.items():\n#         print(f\"  '{k}': {v},\")\n\n#     joblib.dump(study, os.path.join(MODEL_DIR, 'optuna_study.pkl'))\n#     print(\"optuna_study.pkl saved\")\n# else:\n#     print(\"⏸  Optuna tắt. Đổi ENABLE_OPTUNA=True để chạy.\")","metadata":{"execution":{"iopub.execute_input":"2026-04-14T05:52:04.219225Z","iopub.status.busy":"2026-04-14T05:52:04.218539Z","iopub.status.idle":"2026-04-14T05:52:04.222736Z","shell.execute_reply":"2026-04-14T05:52:04.222141Z"},"papermill":{"duration":0.012544,"end_time":"2026-04-14T05:52:04.224110+00:00","exception":false,"start_time":"2026-04-14T05:52:04.211566+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}