{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":21651,"databundleVersionId":1595136,"sourceType":"competition"},{"sourceId":12202869,"sourceType":"datasetVersion","datasetId":7686788},{"sourceId":12203401,"sourceType":"datasetVersion","datasetId":7687172},{"sourceId":12212111,"sourceType":"datasetVersion","datasetId":7693107},{"sourceId":12212419,"sourceType":"datasetVersion","datasetId":7693334},{"sourceId":438620,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":357852,"modelId":379193}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-19T06:19:23.978680Z","iopub.execute_input":"2025-06-19T06:19:23.979027Z","iopub.status.idle":"2025-06-19T06:19:25.998143Z","shell.execute_reply.started":"2025-06-19T06:19:23.978997Z","shell.execute_reply":"2025-06-19T06:19:25.997378Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# A. LSTM+Attention 模型训练（改进：多轮训练、Dropout、LR Scheduler、EarlyStopping；已确保序列为 list）\nimport json, torch, torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# 1) 读入并映射\nwith open('/kaggle/input/clean-student-logs/clean_student_logs.json','r') as f:\n    raw = json.load(f)\nall_qids = sorted({int(e['question_id']) for e in raw})\nqid2idx = {qid:i+1 for i,qid in enumerate(all_qids)}\nnum_questions = len(qid2idx)\n\n# 2) 按 student 构建 list 序列\nseqs = {}\nfor e in raw:\n    sid = e['student_id']\n    q = qid2idx[int(e['question_id'])]\n    c = int(e['is_correct'])\n    seqs.setdefault(sid, []).append((q,c))\n\ndata = []\nfor seq in seqs.values():\n    if len(seq) < 2: continue\n    qs = [q for q,_ in seq]\n    cs = [c for _,c in seq]\n    data.append((qs[:-1], cs[:-1], cs[1:]))\n\ntrain_data, val_data = train_test_split(data, test_size=0.1, random_state=42)\n\n# 3) Dataset\nclass SeqDataset(Dataset):\n    def __init__(self, data, max_len=100):\n        self.data, self.max_len = data, max_len\n    def __len__(self): return len(self.data)\n    def __getitem__(self, i):\n        qs, cs, tar = self.data[i]\n        L = len(qs)\n        pad_q = [0]*(self.max_len-L) + qs[-self.max_len:]\n        pad_c = [0]*(self.max_len-L) + cs[-self.max_len:]\n        pad_t = [0]*(self.max_len-L) + tar[-self.max_len:]\n        return torch.LongTensor(pad_q), torch.FloatTensor(pad_c), torch.FloatTensor(pad_t)\n\nbatch_size = 128\ntrain_loader = DataLoader(SeqDataset(train_data), batch_size=batch_size, shuffle=True)\nval_loader   = DataLoader(SeqDataset(val_data),   batch_size=batch_size)\n\n# 4) 模型 + EarlyStopping\nclass LSTMAttnModel(nn.Module):\n    def __init__(self, num_q, emb_dim=64, hid_dim=128, dp=0.2):\n        super().__init__()\n        self.emb     = nn.Embedding(num_q+1, emb_dim, padding_idx=0)\n        self.lstm    = nn.LSTM(emb_dim+1, hid_dim, batch_first=True, dropout=dp)\n        self.attn    = nn.Linear(hid_dim, hid_dim)\n        self.dropout = nn.Dropout(dp)\n        self.fc      = nn.Linear(hid_dim,1)\n    def forward(self, q_seq, c_seq):\n        emb = self.emb(q_seq)\n        x   = torch.cat([emb, c_seq.unsqueeze(-1)], dim=-1)\n        out,(h,_) = self.lstm(x)\n        h_last    = h[-1]\n        scores    = torch.bmm(out, self.attn(h_last).unsqueeze(-1)).squeeze(-1)\n        α         = torch.softmax(scores, dim=1)\n        ctx       = (out * α.unsqueeze(-1)).sum(dim=1)\n        ctx       = self.dropout(ctx)\n        return torch.sigmoid(self.fc(ctx).squeeze(-1))\n\nclass EarlyStopping:\n    def __init__(self, patience=5, delta=1e-4, path='best_lstm.pth'):\n        self.patience,self.delta,self.path = patience,delta,path\n        self.best_loss,self.counter = None,0\n        self.early_stop = False\n    def __call__(self, val_loss, model):\n        if self.best_loss is None or val_loss < self.best_loss - self.delta:\n            self.best_loss, self.counter = val_loss, 0\n            torch.save(model.state_dict(), self.path)\n        else:\n            self.counter += 1\n            if self.counter >= self.patience:\n                self.early_stop = True\n\nmodel = LSTMAttnModel(num_questions).to(device)\nopt   = torch.optim.Adam(model.parameters(), lr=1e-3)\nsched = torch.optim.lr_scheduler.ReduceLROnPlateau(opt, mode='min', factor=0.5, patience=2, verbose=True)\ncrit  = nn.BCELoss()\nstopper = EarlyStopping(patience=5)\n\n# 5) 训练循环\nfor epoch in range(1, 51):\n    model.train()\n    tr_loss = 0\n    for q,c,t in train_loader:\n        q,c,t = q.to(device), c.to(device), t.to(device)\n        pred = model(q,c)\n        loss = crit(pred, t[:,-1])\n        opt.zero_grad(); loss.backward(); opt.step()\n        tr_loss += loss.item()\n    # 验证\n    model.eval()\n    val_loss = 0\n    with torch.no_grad():\n        for q,c,t in val_loader:\n            q,c,t = q.to(device), c.to(device), t.to(device)\n            val_loss += crit(model(q,c), t[:,-1]).item()\n    tr_loss /= len(train_loader)\n    val_loss /= len(val_loader)\n    sched.step(val_loss)\n    stopper(val_loss, model)\n    print(f\"LSTM Epoch {epoch}: train_loss={tr_loss:.4f}, val_loss={val_loss:.4f}\")\n    if stopper.early_stop:\n        print(\"Early stopping.\")\n        break\n\n# 加载最佳\nmodel.load_state_dict(torch.load('best_lstm.pth'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T06:19:25.999467Z","iopub.execute_input":"2025-06-19T06:19:26.000251Z","iopub.status.idle":"2025-06-19T06:19:44.728359Z","shell.execute_reply.started":"2025-06-19T06:19:26.000221Z","shell.execute_reply":"2025-06-19T06:19:44.727775Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# B. DKT 模型训练（同样改进；已确保序列为 list）\nimport pandas as pd, torch, torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\ndf = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', nrows=200_000)\ndf = df[df['content_type_id']==0]\nqs_unique = df['content_id'].unique()\nqid2idx2 = {q:i+1 for i,q in enumerate(qs_unique)}\nnum_q2 = len(qid2idx2)\n\ngrouped = df.groupby('user_id').apply(lambda d: list(zip(d['content_id'], d['answered_correctly'])))\ndata2 = []\nfor seq in grouped:\n    if len(seq) < 2: continue\n    qs = [qid2idx2[q] for q,_ in seq]\n    cs = [c for _,c in seq]\n    data2.append((qs[:-1], cs[:-1], cs[1:]))\n\ntrain2, val2 = train_test_split(data2, test_size=0.1, random_state=42)\n\nclass SeqDataset2(Dataset):\n    def __init__(self, data, max_len=100):\n        self.data, self.max_len = data, max_len\n    def __len__(self): return len(self.data)\n    def __getitem__(self, i):\n        qs, cs, tar = self.data[i]\n        L = len(qs)\n        pad_q = [0]*(self.max_len-L) + qs[-self.max_len:]\n        pad_c = [0]*(self.max_len-L) + cs[-self.max_len:]\n        pad_t = [0]*(self.max_len-L) + tar[-self.max_len:]\n        return torch.LongTensor(pad_q), torch.FloatTensor(pad_c), torch.FloatTensor(pad_t)\n\nbatch_size = 128\ntrain_loader2 = DataLoader(SeqDataset2(train2), batch_size=batch_size, shuffle=True)\nval_loader2   = DataLoader(SeqDataset2(val2),   batch_size=batch_size)\n\nclass DKTModel(nn.Module):\n    def __init__(self, num_q, emb_dim=64, hid_dim=128, dp=0.2):\n        super().__init__()\n        self.emb     = nn.Embedding(num_q+1, emb_dim, padding_idx=0)\n        self.lstm    = nn.LSTM(emb_dim+1, hid_dim, batch_first=True, dropout=dp)\n        self.dropout = nn.Dropout(dp)\n        self.fc      = nn.Linear(hid_dim,1)\n    def forward(self, q_seq, c_seq):\n        emb = self.emb(q_seq)\n        x   = torch.cat([emb, c_seq.unsqueeze(-1)], dim=-1)\n        out,(h,_) = self.lstm(x)\n        h_last    = self.dropout(h[-1])\n        return torch.sigmoid(self.fc(h_last).squeeze(-1))\n\nclass EarlyStopping2:\n    def __init__(self, patience=5, delta=1e-4, path='best_dkt.pth'):\n        self.patience,self.delta,self.path = patience,delta,path\n        self.best_loss,self.counter = None,0\n        self.early_stop = False\n    def __call__(self, val_loss, model):\n        if self.best_loss is None or val_loss < self.best_loss - self.delta:\n            self.best_loss, self.counter = val_loss,0\n            torch.save(model.state_dict(), self.path)\n        else:\n            self.counter += 1\n            if self.counter >= self.patience:\n                self.early_stop = True\n\nmodel2   = DKTModel(num_q2).to(device)\nopt2     = torch.optim.Adam(model2.parameters(), lr=1e-3)\nsched2   = torch.optim.lr_scheduler.ReduceLROnPlateau(opt2, mode='min', factor=0.5, patience=2, verbose=True)\ncrit2    = nn.BCELoss()\nstopper2 = EarlyStopping2(patience=5)\n\nfor epoch in range(1, 51):\n    model2.train()\n    trl=0\n    for q,c,t in train_loader2:\n        q,c,t = q.to(device),c.to(device),t.to(device)\n        loss = crit2(model2(q,c), t[:,-1])\n        opt2.zero_grad(); loss.backward(); opt2.step()\n        trl += loss.item()\n    model2.eval()\n    vl=0\n    with torch.no_grad():\n        for q,c,t in val_loader2:\n            q,c,t = q.to(device),c.to(device),t.to(device)\n            vl += crit2(model2(q,c), t[:,-1]).item()\n    trl /= len(train_loader2); vl /= len(val_loader2)\n    sched2.step(vl)\n    stopper2(vl, model2)\n    print(f\"DKT Epoch {epoch}: train_loss={trl:.4f}, val_loss={vl:.4f}\")\n    if stopper2.early_stop:\n        print(\"Early stopping.\")\n        break\n\nmodel2.load_state_dict(torch.load('best_dkt.pth'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T06:19:44.729142Z","iopub.execute_input":"2025-06-19T06:19:44.729506Z","iopub.status.idle":"2025-06-19T06:19:46.525215Z","shell.execute_reply.started":"2025-06-19T06:19:44.729480Z","shell.execute_reply":"2025-06-19T06:19:46.524613Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# C. Advanced Ensemble (Hierarchical Stacking + Mixture-of-Experts + Temporal Features + EarlyStopping)\nimport numpy as np\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.metrics import roc_auc_score\n\ndevice     = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nbatch_size = 128\n\n# ——— Helpers: train_fn & predict_fn ———\ndef train_fn(model, loader, optimizer, criterion, epochs=1):\n    model.train()\n    for _ in range(epochs):\n        for q_seq, c_seq, tar in loader:\n            q_seq, c_seq, tar = q_seq.to(device), c_seq.to(device), tar.to(device)\n            loss = criterion(model(q_seq, c_seq), tar[:, -1])\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\ndef predict_fn(model, loader):\n    model.eval()\n    preds = []\n    with torch.no_grad():\n        for q_seq, c_seq, _ in loader:\n            q_seq, c_seq = q_seq.to(device), c_seq.to(device)\n            preds.append(model(q_seq, c_seq).cpu().numpy())\n    return np.concatenate(preds)\n\n# ——— Assume train_data, val_data, SeqDataset, LSTMAttnModel, DKTModel, num_questions already defined ———\n\n# 1) Collect 10-fold OOF & test preds for Base Models A/B\noof_A, oof_B = np.zeros(len(train_data)), np.zeros(len(train_data))\ntest_preds_A, test_preds_B = [], []\ntest_loader = DataLoader(SeqDataset(val_data), batch_size=batch_size)\n\nkf = KFold(n_splits=10, shuffle=True, random_state=42)\nfor fold, (tr_idx, va_idx) in enumerate(kf.split(train_data), 1):\n    print(f\"Fold {fold}/10\")\n    tr_sub = [train_data[i] for i in tr_idx]\n    va_sub = [train_data[i] for i in va_idx]\n    tr_loader = DataLoader(SeqDataset(tr_sub), batch_size=batch_size, shuffle=True)\n    va_loader = DataLoader(SeqDataset(va_sub), batch_size=batch_size)\n\n    # Model A\n    mA  = LSTMAttnModel(num_questions).to(device)\n    optA = torch.optim.Adam(mA.parameters(), lr=1e-3)\n    train_fn(mA, tr_loader, optA, nn.BCELoss(), epochs=1)\n    oof_A[va_idx] = predict_fn(mA, va_loader)\n    test_preds_A.append(predict_fn(mA, test_loader))\n\n    # Model B (DKT on same qid mapping)\n    mB  = DKTModel(num_questions).to(device)\n    optB = torch.optim.Adam(mB.parameters(), lr=1e-3)\n    train_fn(mB, tr_loader, optB, nn.BCELoss(), epochs=1)\n    oof_B[va_idx] = predict_fn(mB, va_loader)\n    test_preds_B.append(predict_fn(mB, test_loader))\n\n# average test preds\ntest_A = np.mean(np.stack(test_preds_A, axis=1), axis=1)\ntest_B = np.mean(np.stack(test_preds_B, axis=1), axis=1)\n\n# 2) Temporal features\nrates      = np.array([np.mean(cs)      for _,cs,_ in train_data])\nlengths    = np.array([len(cs)         for _,cs,_ in train_data])\nrates_test = np.array([np.mean(cs)      for _,cs,_ in val_data])\nlens_test  = np.array([len(cs)         for _,cs,_ in val_data])\n\n# 3) GatingNet definition\nclass GatingNet(nn.Module):\n    def __init__(self, in_dim, hid=16, dp=0.2):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(in_dim, hid),\n            nn.ReLU(),\n            nn.Dropout(dp),\n            nn.Linear(hid, 2)\n        )\n    def forward(self, x):\n        return torch.softmax(self.net(x), dim=1)\n\n# 4) Prepare gating dataset\nX_meta = np.vstack([oof_A, oof_B, rates, lengths]).T\ny_meta = np.array([tar[-1] for _,_,tar in train_data])\n\nX_tr, X_val, y_tr, y_val = train_test_split(X_meta, y_meta, test_size=0.2, random_state=42)\n\nclass MetaDataset(Dataset):\n    def __init__(self, X, y):\n        self.X = torch.tensor(X, dtype=torch.float32)\n        self.y = torch.tensor(y, dtype=torch.float32)\n    def __len__(self): return len(self.y)\n    def __getitem__(self, i): return self.X[i], self.y[i]\n\ngd_train = DataLoader(MetaDataset(X_tr, y_tr), batch_size=64, shuffle=True)\ngd_val   = DataLoader(MetaDataset(X_val, y_val), batch_size=64)\n\n# 5) Train gating with EarlyStopping\ngating = GatingNet(in_dim=4).to(device)\nopt_g  = torch.optim.Adam(gating.parameters(), lr=1e-3)\nsch_g  = torch.optim.lr_scheduler.ReduceLROnPlateau(opt_g, mode='min', factor=0.5, patience=2, verbose=True)\ncrit   = nn.BCELoss()\n\nclass EarlyStop:\n    def __init__(self, patience=5, delta=1e-4, path='best_gating.pth'):\n        self.patience, self.delta, self.path = patience, delta, path\n        self.best, self.count = None, 0\n        self.stop = False\n    def __call__(self, loss, model):\n        if self.best is None or loss < self.best - self.delta:\n            self.best, self.count = loss, 0\n            torch.save(model.state_dict(), self.path)\n        else:\n            self.count += 1\n            if self.count >= self.patience:\n                self.stop = True\n\nstopper = EarlyStop(patience=5)\n\nfor epoch in range(1, 51):\n    gating.train()\n    t_loss = 0\n    for xb, yb in gd_train:\n        xb, yb = xb.to(device), yb.to(device)\n        w = gating(xb)\n        pred = w[:,0]*xb[:,0] + w[:,1]*xb[:,1]\n        loss = crit(pred, yb)\n        opt_g.zero_grad(); loss.backward(); opt_g.step()\n        t_loss += loss.item()\n    gating.eval()\n    v_loss = 0\n    with torch.no_grad():\n        for xb, yb in gd_val:\n            xb, yb = xb.to(device), yb.to(device)\n            w = gating(xb)\n            pred = w[:,0]*xb[:,0] + w[:,1]*xb[:,1]\n            v_loss += crit(pred, yb).item()\n    t_loss /= len(gd_train); v_loss /= len(gd_val)\n    sch_g.step(v_loss)\n    stopper(v_loss, gating)\n    print(f\"Gating Epoch {epoch}: train={t_loss:.4f}, val={v_loss:.4f}\")\n    if stopper.stop:\n        print(\"Early stopping on gating\")\n        break\n\ngating.load_state_dict(torch.load('best_gating.pth'))\n\n# 6) Final MoE prediction on val set\nX_test_meta = torch.tensor(\n    np.vstack([test_A, test_B, rates_test, lens_test]).T,\n    dtype=torch.float32, device=device\n)\nwith torch.no_grad():\n    w_test = gating(X_test_meta)\n    final_pred = (w_test[:,0]*X_test_meta[:,0] +\n                  w_test[:,1]*X_test_meta[:,1]).cpu().numpy()\n\ny_true = np.array([tar[-1] for _,_,tar in val_data])\nprint(\"Final MoE AUC:\", roc_auc_score(y_true, final_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T06:19:46.526799Z","iopub.execute_input":"2025-06-19T06:19:46.527111Z","iopub.status.idle":"2025-06-19T06:19:56.883792Z","shell.execute_reply.started":"2025-06-19T06:19:46.527093Z","shell.execute_reply":"2025-06-19T06:19:56.883181Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# D. 用 “2012data” 做纯 Hold-out 测试（修复 dtype mismatch）\n\nimport os\nimport pandas as pd\nimport numpy as np\nimport torch\nimport zipfile\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.metrics import roc_auc_score, log_loss\n\n# 1) 定位并加载 “2012data” 文件\nbase   = '/kaggle/input'\nfolder = [d for d in os.listdir(base) if '2012data' in d][0]\nfile0  = os.listdir(os.path.join(base, folder))[0]\npath0  = os.path.join(base, folder, file0)\n# 如果是 ZIP 就解压\nif zipfile.is_zipfile(path0):\n    with zipfile.ZipFile(path0, 'r') as z:\n        z.extractall('/kaggle/working/2012data')\n    csvs = [f for f in os.listdir('/kaggle/working/2012data') if f.lower().endswith('.csv')]\n    assert csvs, \"No CSV inside ZIP\"\n    path = os.path.join('/kaggle/working/2012data', csvs[0])\nelse:\n    path = path0\n\nprint(\"Loading hold-out data from:\", path)\ndf = pd.read_csv(path)\nprint(\"Total rows in hold-out:\", len(df))\n\n# 2) 构造序列 (qs, cs, tar)\nseqs = {}\nfor r in df.itertuples():\n    seqs.setdefault(r.user_id, []).append((r.problem_id, int(r.correct)))\n\nholdout = []\nfor logs in seqs.values():\n    if len(logs) < 2:\n        continue\n    qs, cs = zip(*logs)\n    holdout.append((list(qs[:-1]), list(cs[:-1]), list(cs[1:])))\nprint(\"Hold-out sequences:\", len(holdout))\n\n# 3) Dataset：unknown qids → 0\nclass HoldDataset(Dataset):\n    def __init__(self, data, max_len=100):\n        self.data, self.max_len = data, max_len\n    def __len__(self): \n        return len(self.data)\n    def __getitem__(self, i):\n        qs, cs, tar = self.data[i]\n        L = len(qs)\n        pad_q = [0]*self.max_len\n        pad_c = [0]*(self.max_len - L) + cs[-self.max_len:]\n        pad_t = [0]*(self.max_len - L) + tar[-self.max_len:]\n        return (\n            torch.LongTensor(pad_q),\n            torch.FloatTensor(pad_c),\n            torch.FloatTensor(pad_t)\n        )\n\nhold_loader = DataLoader(HoldDataset(holdout), batch_size=128, shuffle=False)\n\n# 4) 动态恢复和加载模型\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# LSTM\nlstm_ckpt = torch.load('best_lstm.pth')\nvocab_lstm = lstm_ckpt['emb.weight'].shape[0] - 1\nmodel_lstm = LSTMAttnModel(vocab_lstm).to(device)\nmodel_lstm.load_state_dict(lstm_ckpt)\nmodel_lstm.eval()\n\n# DKT\ndkt_ckpt = torch.load('best_dkt.pth')\nvocab_dkt = dkt_ckpt['emb.weight'].shape[0] - 1\nmodel_dkt = DKTModel(vocab_dkt).to(device)\nmodel_dkt.load_state_dict(dkt_ckpt)\nmodel_dkt.eval()\n\n# Gating\ngating_ckpt = torch.load('best_gating.pth')\ngating = GatingNet(in_dim=4).to(device)\ngating.load_state_dict(gating_ckpt)\ngating.eval()\n\n# 5) 时序特征\nrates   = np.array([np.mean(cs) for _, cs, _ in holdout], dtype=np.float32)\nlengths = np.array([len(cs)    for _, cs, _ in holdout], dtype=np.float32)\n\n# 6) 推断并收集\nall_preds = []\nidx = 0\nwith torch.no_grad():\n    for q, c, _ in hold_loader:\n        q, c = q.to(device), c.to(device)\n        pA = model_lstm(q, c)\n        pB = model_dkt (q, c)\n        b  = pA.size(0)\n        r  = torch.tensor(rates[idx:idx+b],   dtype=torch.float32, device=device).unsqueeze(1)\n        l  = torch.tensor(lengths[idx:idx+b], dtype=torch.float32, device=device).unsqueeze(1)\n        meta_in = torch.cat([pA.unsqueeze(1), pB.unsqueeze(1), r, l], dim=1)\n        w       = gating(meta_in)\n        pred    = w[:,0]*pA + w[:,1]*pB\n        all_preds.append(pred.cpu().numpy())\n        idx += b\n\ny_true = [tar[-1] for _,_,tar in holdout]\ny_pred = np.concatenate(all_preds)\n\n# 7) 评估\nprint(\"Hold-out AUC:     \", roc_auc_score(y_true, y_pred))\nprint(\"Hold-out LogLoss:\", log_loss(y_true, y_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T06:38:36.931770Z","iopub.execute_input":"2025-06-19T06:38:36.932116Z","iopub.status.idle":"2025-06-19T06:39:57.066837Z","shell.execute_reply.started":"2025-06-19T06:38:36.932093Z","shell.execute_reply":"2025-06-19T06:39:57.066183Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# E. 在 SkillBuilder 2012–2013 数据集上微调 LSTM+Attention 和 DKT（跳过大小不匹配的 embedding 层导入）\n\nimport os\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\n\n# 1) 加载 SkillBuilder 2012–2013 数据\ncsv_path = '/kaggle/input/2012data/2012-2013-data-with-predictions-4-final.csv'\ndf = pd.read_csv(csv_path)\n\n# 2) 构造新的 qid2idx & 序列，留下 90% 训练，10% 验证\nqids2 = df['problem_id'].unique()\nqid2idx2 = {q:i+1 for i,q in enumerate(qids2)}\nnum_q2   = len(qid2idx2)\n\nseqs2 = {}\nfor r in df.itertuples():\n    seqs2.setdefault(r.user_id, []).append((qid2idx2[r.problem_id], int(r.correct)))\n\ndata2 = []\nfor logs in seqs2.values():\n    if len(logs) < 2: continue\n    qs, cs = zip(*logs)\n    data2.append((list(qs[:-1]), list(cs[:-1]), list(cs[1:])))\n\ntrain2, val2 = train_test_split(data2, test_size=0.1, random_state=42, shuffle=True)\n\n# 3) Dataset & DataLoader\nclass SeqDataset2(Dataset):\n    def __init__(self, data, max_len=100):\n        self.data, self.max_len = data, max_len\n    def __len__(self): return len(self.data)\n    def __getitem__(self, i):\n        qs, cs, tar = self.data[i]\n        L = len(qs)\n        pad_q = [0]*(self.max_len-L) + qs[-self.max_len:]\n        pad_c = [0]*(self.max_len-L) + cs[-self.max_len:]\n        pad_t = [0]*(self.max_len-L) + tar[-self.max_len:]\n        return (\n            torch.LongTensor(pad_q),\n            torch.FloatTensor(pad_c),\n            torch.FloatTensor(pad_t)\n        )\n\nbatch_size = 128\ntrain_loader2 = DataLoader(SeqDataset2(train2), batch_size=batch_size, shuffle=True)\nval_loader2   = DataLoader(SeqDataset2(val2),   batch_size=batch_size)\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ncriterion = nn.BCELoss()\n\n# 4) 构建 EarlyStopping\nclass EarlyStopping:\n    def __init__(self, patience=5, delta=1e-4, path='checkpoint.pth'):\n        self.patience, self.delta, self.path = patience, delta, path\n        self.best, self.count = None, 0\n        self.stop = False\n    def __call__(self, loss, model):\n        if self.best is None or loss < self.best - self.delta:\n            self.best, self.count = loss, 0\n            torch.save(model.state_dict(), self.path)\n        else:\n            self.count += 1\n            if self.count >= self.patience:\n                self.stop = True\n\n# 5) 微调 LSTM+Attention\n#    5.1 实例化模型\nmodel_lstm_ft = LSTMAttnModel(num_q2).to(device)\n#    5.2 加载预训练权重，忽略大小不匹配的 embedding 层\npre = torch.load('best_lstm.pth')\nmdict = model_lstm_ft.state_dict()\n# 只读取尺寸匹配的键\npre_filt = {k:v for k,v in pre.items() if k in mdict and v.size()==mdict[k].size()}\nmdict.update(pre_filt)\nmodel_lstm_ft.load_state_dict(mdict)\n\nopt_lstm = torch.optim.Adam(model_lstm_ft.parameters(), lr=1e-4)\nsched_lstm = torch.optim.lr_scheduler.ReduceLROnPlateau(opt_lstm, mode='min', factor=0.5, patience=2, verbose=True)\nstopper_lstm = EarlyStopping(patience=5, path='ft_lstm.pth')\n\nfor epoch in range(1, 31):\n    # 训练\n    model_lstm_ft.train()\n    tr_loss = 0\n    for q,c,t in train_loader2:\n        q,c,t = q.to(device), c.to(device), t.to(device)\n        p = model_lstm_ft(q,c)\n        loss = criterion(p, t[:, -1])\n        opt_lstm.zero_grad(); loss.backward(); opt_lstm.step()\n        tr_loss += loss.item()\n    # 验证\n    model_lstm_ft.eval()\n    val_loss = 0\n    with torch.no_grad():\n        for q,c,t in val_loader2:\n            q,c,t = q.to(device), c.to(device), t.to(device)\n            val_loss += criterion(model_lstm_ft(q,c), t[:, -1]).item()\n    tr_loss /= len(train_loader2)\n    val_loss /= len(val_loader2)\n    sched_lstm.step(val_loss)\n    stopper_lstm(val_loss, model_lstm_ft)\n    print(f\"[LSTM ft] Epoch {epoch}: train={tr_loss:.4f}, val={val_loss:.4f}\")\n    if stopper_lstm.stop:\n        print(\"Early stopping LSTM fine-tune\")\n        break\n\nmodel_lstm_ft.load_state_dict(torch.load('ft_lstm.pth'))\n\n\n# 6) 微调 DKT\nmodel_dkt_ft = DKTModel(num_q2).to(device)\npre2 = torch.load('best_dkt.pth')\nmdict2 = model_dkt_ft.state_dict()\npre2_filt = {k:v for k,v in pre2.items() if k in mdict2 and v.size()==mdict2[k].size()}\nmdict2.update(pre2_filt)\nmodel_dkt_ft.load_state_dict(mdict2)\n\nopt_dkt = torch.optim.Adam(model_dkt_ft.parameters(), lr=1e-4)\nsched_dkt = torch.optim.lr_scheduler.ReduceLROnPlateau(opt_dkt, mode='min', factor=0.5, patience=2, verbose=True)\nstopper_dkt = EarlyStopping(patience=5, path='ft_dkt.pth')\n\nfor epoch in range(1, 31):\n    model_dkt_ft.train()\n    trl = 0\n    for q,c,t in train_loader2:\n        q,c,t = q.to(device), c.to(device), t.to(device)\n        loss = criterion(model_dkt_ft(q,c), t[:, -1])\n        opt_dkt.zero_grad(); loss.backward(); opt_dkt.step()\n        trl += loss.item()\n    model_dkt_ft.eval()\n    vll = 0\n    with torch.no_grad():\n        for q,c,t in val_loader2:\n            q,c,t = q.to(device), c.to(device), t.to(device)\n            vll += criterion(model_dkt_ft(q,c), t[:, -1]).item()\n    trl /= len(train_loader2); vll /= len(val_loader2)\n    sched_dkt.step(vll)\n    stopper_dkt(vll, model_dkt_ft)\n    print(f\"[DKT ft] Epoch {epoch}: train={trl:.4f}, val={vll:.4f}\")\n    if stopper_dkt.stop:\n        print(\"Early stopping DKT fine-tune\")\n        break\n\nmodel_dkt_ft.load_state_dict(torch.load('ft_dkt.pth'))\n\n\n# 7) 在验证集上评估微调效果\nmodel_lstm_ft.eval(); model_dkt_ft.eval()\nall_lstm, all_dkt, ys = [], [], []\n\nwith torch.no_grad():\n    for q,c,t in val_loader2:\n        q,c,t = q.to(device), c.to(device), t.to(device)\n        all_lstm.append(model_lstm_ft(q,c).cpu().numpy())\n        all_dkt.append (model_dkt_ft(q,c).cpu().numpy())\n        ys.append(t[:, -1].cpu().numpy())\n\npred_lstm = np.concatenate(all_lstm)\npred_dkt  = np.concatenate(all_dkt)\ny_true    = np.concatenate(ys)\n\nprint(\"Fine-tuned LSTM AUC:\", roc_auc_score(y_true, pred_lstm))\nprint(\"Fine-tuned DKT  AUC:\", roc_auc_score(y_true, pred_dkt))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 8) 融合 & 评估—权重平均 与 Logistic Regression Stacking\n\nimport numpy as np\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score, log_loss\n\n# --- 8.1) 收集 Fine-tuned 模型在验证集上的预测与真实标签 ---\npreds_lstm = []\npreds_dkt  = []\ny_true     = []\nwith torch.no_grad():\n    for q, c, t in val_loader2:\n        q, c = q.to(device), c.to(device)\n        preds_lstm.append(model_lstm_ft(q, c).cpu().numpy())\n        preds_dkt .append(model_dkt_ft (q, c).cpu().numpy())\n        y_true    .append(t[:, -1].cpu().numpy())\npreds_lstm = np.concatenate(preds_lstm)\npreds_dkt  = np.concatenate(preds_dkt)\ny_true     = np.concatenate(y_true)\n\n# --- 8.2) 方法 1: AUC 加权平均 (以各自 AUC 为权重) ---\nauc_lstm = roc_auc_score(y_true, preds_lstm)\nauc_dkt  = roc_auc_score(y_true, preds_dkt)\nw1 = auc_lstm / (auc_lstm + auc_dkt)\nw2 = auc_dkt  / (auc_lstm + auc_dkt)\nweighted_pred = w1 * preds_lstm + w2 * preds_dkt\nprint(f\"Weighted avg AUC:     {roc_auc_score(y_true, weighted_pred):.4f}\")\nprint(f\"Weighted avg LogLoss: {log_loss(y_true, weighted_pred):.4f}\")\n\n# --- 8.3) 方法 2: Logistic Regression Stacking ---\n# 构造特征矩阵 [pred_lstm, pred_dkt]\nX = np.vstack([preds_lstm, preds_dkt]).T\nmeta = LogisticRegression()\nmeta.fit(X, y_true)\nstack_pred = meta.predict_proba(X)[:, 1]\nprint(f\"Stacking (LR) AUC:     {roc_auc_score(y_true, stack_pred):.4f}\")\nprint(f\"Stacking (LR) LogLoss: {log_loss(y_true, stack_pred):.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}