{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":8540,"databundleVersionId":862041,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Importslib & global config\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Cấu hình hiển thị\npd.set_option(\"display.max_columns\", 50)\nsns.set(style=\"whitegrid\")\nplt.rcParams[\"figure.figsize\"] = (8, 4)\n\nDATA_DIR = Path(\"/kaggle/input/talkingdata-adtracking-fraud-detection\")\n\nprint(\"Files in DATA_DIR:\")\nfor p in DATA_DIR.iterdir():\n    print(\"  -\", p.name)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:07.772696Z","iopub.execute_input":"2025-12-24T14:28:07.773049Z","iopub.status.idle":"2025-12-24T14:28:08.696385Z","shell.execute_reply.started":"2025-12-24T14:28:07.773014Z","shell.execute_reply":"2025-12-24T14:28:08.695325Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Đọc train_sample + thống kê nhanh","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T15:38:55.791059Z","iopub.execute_input":"2025-12-24T15:38:55.791380Z","iopub.status.idle":"2025-12-24T15:38:55.794908Z","shell.execute_reply.started":"2025-12-24T15:38:55.791355Z","shell.execute_reply":"2025-12-24T15:38:55.794205Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_path = DATA_DIR / \"train_sample.csv\"\ndf = pd.read_csv(sample_path)\n\nprint(\"Shape:\", df.shape)\nprint(\"\\n5 dòng đầu:\")\ndisplay(df.head())\n\nprint(\"\\nKiểu dữ liệu:\")\nprint(df.dtypes)\n\nprint(\"\\nTỉ lệ thiếu (missing rate):\")\nprint(df.isna().mean())\n\n# Thống kê mô tả cho cột số\nprint(\"\\nDescribe numeric columns:\")\ndisplay(df.describe())\n\n# Hàm tính dung lượng bộ nhớ\ndef memory_in_mb(df_):\n    return df_.memory_usage(deep=True).sum() / 1024**2\n\nprint(f\"\\nMemory usage: {memory_in_mb(df):.2f} MB\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:08.697633Z","iopub.execute_input":"2025-12-24T14:28:08.697922Z","iopub.status.idle":"2025-12-24T14:28:08.897954Z","shell.execute_reply.started":"2025-12-24T14:28:08.697896Z","shell.execute_reply":"2025-12-24T14:28:08.897134Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature engineering cơ bản cho EDA","metadata":{}},{"cell_type":"code","source":"df[\"click_time\"] = pd.to_datetime(df[\"click_time\"])\ndf[\"hour\"] = df[\"click_time\"].dt.hour.astype(\"int8\")\ndf[\"day\"]  = df[\"click_time\"].dt.day.astype(\"int8\")\n\n# Tạm thời giữ attributed_time để quan sát, lát nữa có thể drop khi train\nprint(df[[\"click_time\", \"hour\", \"day\", \"attributed_time\"]].head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:08.898819Z","iopub.execute_input":"2025-12-24T14:28:08.899036Z","iopub.status.idle":"2025-12-24T14:28:08.943714Z","shell.execute_reply.started":"2025-12-24T14:28:08.899016Z","shell.execute_reply":"2025-12-24T14:28:08.943006Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Thống kê target","metadata":{}},{"cell_type":"code","source":"target_col = \"is_attributed\"\n\nprint(\"Đếm số lượng mỗi lớp:\")\nprint(df[target_col].value_counts())\n\nprint(\"\\nTỉ lệ mỗi lớp:\")\nprint(df[target_col].value_counts(normalize=True))\n\nfig, ax = plt.subplots(1, 2, figsize=(10, 4))\n\n# Count\ndf[target_col].value_counts().plot(\n    kind=\"bar\", ax=ax[0], title=\"Số lượng mỗi lớp (sample)\"\n)\nax[0].set_xticklabels([\"not fraud (0)\", \"fraud (1)\"], rotation=0)\n\n# Tỉ lệ %\n(df[target_col].value_counts(normalize=True) * 100).plot(\n    kind=\"bar\", ax=ax[1], title=\"Tỉ lệ mỗi lớp (%)\"\n)\nax[1].set_xticklabels([\"not fraud (0)\", \"fraud (1)\"], rotation=0)\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:08.945020Z","iopub.execute_input":"2025-12-24T14:28:08.945318Z","iopub.status.idle":"2025-12-24T14:28:09.268884Z","shell.execute_reply.started":"2025-12-24T14:28:08.945291Z","shell.execute_reply":"2025-12-24T14:28:09.268086Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# EDA theo thời gian: hour, day","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(12, 4))\n\n# Phân bố số click theo giờ (không phân lớp)\nsns.countplot(data=df, x=\"hour\", ax=axes[0])\naxes[0].set_title(\"Số click theo giờ (sample)\")\n\n# Phân bố số click theo giờ, tách theo label\nsns.countplot(data=df, x=\"hour\", hue=\"is_attributed\", ax=axes[1])\naxes[1].set_title(\"Số click theo giờ & label (sample)\")\naxes[1].legend(title=\"is_attributed\")\n\nplt.tight_layout()\nplt.show()\n\n# Tương tự cho ngày\nfig, axes = plt.subplots(1, 2, figsize=(12, 4))\n\nsns.countplot(data=df, x=\"day\", ax=axes[0])\naxes[0].set_title(\"Số click theo ngày (sample)\")\n\nsns.countplot(data=df, x=\"day\", hue=\"is_attributed\", ax=axes[1])\naxes[1].set_title(\"Số click theo ngày & label (sample)\")\naxes[1].legend(title=\"is_attributed\")\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:09.269786Z","iopub.execute_input":"2025-12-24T14:28:09.270014Z","iopub.status.idle":"2025-12-24T14:28:10.540618Z","shell.execute_reply.started":"2025-12-24T14:28:09.269995Z","shell.execute_reply":"2025-12-24T14:28:10.539851Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Top IP, app, channel theo số lượng click","metadata":{}},{"cell_type":"code","source":"# Top IP\ntop_ip = (\n    df[\"ip\"]\n    .value_counts()\n    .head(10)\n    .rename_axis(\"ip\")\n    .reset_index(name=\"count\")\n)\nprint(\"Top 10 ip theo số click:\")\ndisplay(top_ip)\n\nplt.figure(figsize=(10, 4))\nsns.barplot(data=top_ip, x=\"ip\", y=\"count\")\nplt.title(\"Top 10 IP theo số click (sample)\")\nplt.xticks(rotation=45, ha=\"right\")\nplt.tight_layout()\nplt.show()\n\n# Top app\ntop_app = (\n    df[\"app\"]\n    .value_counts()\n    .head(10)\n    .rename_axis(\"app\")\n    .reset_index(name=\"count\")\n)\nprint(\"Top 10 app theo số click:\")\ndisplay(top_app)\n\nplt.figure(figsize=(8, 4))\nsns.barplot(data=top_app, x=\"app\", y=\"count\")\nplt.title(\"Top 10 App theo số click (sample)\")\nplt.xticks(rotation=45)\nplt.tight_layout()\nplt.show()\n\n# Top channel\ntop_channel = (\n    df[\"channel\"]\n    .value_counts()\n    .head(10)\n    .rename_axis(\"channel\")\n    .reset_index(name=\"count\")\n)\nprint(\"Top 10 channel theo số click:\")\ndisplay(top_channel)\n\nplt.figure(figsize=(8, 4))\nsns.barplot(data=top_channel, x=\"channel\", y=\"count\")\nplt.title(\"Top 10 Channel theo số click (sample)\")\nplt.xticks(rotation=45)\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:10.541878Z","iopub.execute_input":"2025-12-24T14:28:10.542142Z","iopub.status.idle":"2025-12-24T14:28:11.328242Z","shell.execute_reply.started":"2025-12-24T14:28:10.542121Z","shell.execute_reply":"2025-12-24T14:28:11.327428Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import warnings\n\n# Tắt riêng FutureWarning liên quan tới use_inf_as_na (do seaborn gây ra)\nwarnings.filterwarnings(\n    \"ignore\",\n    category=FutureWarning,\n    message=\".*use_inf_as_na.*\"\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:11.329228Z","iopub.execute_input":"2025-12-24T14:28:11.329633Z","iopub.status.idle":"2025-12-24T14:28:11.334328Z","shell.execute_reply.started":"2025-12-24T14:28:11.329602Z","shell.execute_reply":"2025-12-24T14:28:11.333496Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Tỉ lệ is_attributed theo giờ & theo app (top)","metadata":{}},{"cell_type":"code","source":"# Tỉ lệ fraud theo hour\nhour_fraud = (\n    df.groupby(\"hour\")[target_col]\n    .mean()\n    .reset_index(name=\"fraud_rate\")\n)\n\nplt.figure(figsize=(8, 4))\nsns.lineplot(data=hour_fraud, x=\"hour\", y=\"fraud_rate\", marker=\"o\")\nplt.title(\"Tỉ lệ fraud theo giờ (sample)\")\nplt.ylabel(\"fraud_rate\")\nplt.tight_layout()\nplt.show()\n\n# Tỉ lệ fraud theo app (chỉ lấy top 10 app nhiều click nhất)\ntop10_app_ids = top_app[\"app\"].tolist()\ndf_app_top = df[df[\"app\"].isin(top10_app_ids)]\n\napp_fraud = (\n    df_app_top.groupby(\"app\")[target_col]\n    .mean()\n    .reset_index(name=\"fraud_rate\")\n    .sort_values(\"fraud_rate\", ascending=False)\n)\n\nplt.figure(figsize=(8, 4))\nsns.barplot(data=app_fraud, x=\"app\", y=\"fraud_rate\")\nplt.title(\"Tỉ lệ fraud theo app (top 10 app)\")\nplt.xticks(rotation=45)\nplt.tight_layout()\nplt.show()\n\nprint(\"Bảng tỉ lệ fraud theo app (top 10):\")\ndisplay(app_fraud)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:11.335314Z","iopub.execute_input":"2025-12-24T14:28:11.335666Z","iopub.status.idle":"2025-12-24T14:28:11.874795Z","shell.execute_reply.started":"2025-12-24T14:28:11.335636Z","shell.execute_reply":"2025-12-24T14:28:11.873893Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#  clean  để train (sample)","metadata":{}},{"cell_type":"code","source":"# Bỏ cả click_time (datetime) và attributed_time\ndf_model = df.drop(columns=[\"click_time\", \"attributed_time\"], errors=\"ignore\")\n\ntarget_col = \"is_attributed\"\nfeature_cols = [c for c in df_model.columns if c != target_col]\n\nX = df_model[feature_cols]\ny = df_model[target_col]\n\nprint(\"Feature columns:\", feature_cols)\nprint(df_model.dtypes)\nprint(\"X shape:\", X.shape, \"y shape:\", y.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:11.875787Z","iopub.execute_input":"2025-12-24T14:28:11.876079Z","iopub.status.idle":"2025-12-24T14:28:11.890598Z","shell.execute_reply.started":"2025-12-24T14:28:11.876060Z","shell.execute_reply":"2025-12-24T14:28:11.889865Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Cell 8 — Import lib và chia train/val cho sample","metadata":{}},{"cell_type":"code","source":"import time\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score, roc_curve\n\ntarget_col = \"is_attributed\"\n\n\nfeature_cols = [c for c in df_model.columns if c != target_col]\n\nX = df_model[feature_cols]\ny = df_model[target_col]\n\nX_train, X_val, y_train, y_val = train_test_split(\n    X, y,\n    test_size=0.2,\n    random_state=42,\n    stratify=y  # giữ tỉ lệ fraud tương tự\n)\n\nprint(\"Train shape:\", X_train.shape, \"Val shape:\", X_val.shape)\nprint(\"Positives train:\", y_train.sum(), \"Val:\", y_val.sum())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:11.893668Z","iopub.execute_input":"2025-12-24T14:28:11.893926Z","iopub.status.idle":"2025-12-24T14:28:13.434300Z","shell.execute_reply.started":"2025-12-24T14:28:11.893901Z","shell.execute_reply":"2025-12-24T14:28:13.433441Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Baseline: Logistic Regression trên tập train_sample","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import roc_auc_score\nimport time\n\n# Pipeline: chuẩn hóa feature + Logistic Regression\nlogit_clf = Pipeline([\n    (\"scaler\", StandardScaler()),  # chuẩn hóa các feature dạng số\n    (\"logreg\", LogisticRegression(\n        max_iter=1000,\n        class_weight=\"balanced\",   # xử lý mất cân bằng lớp\n        n_jobs=-1,\n        random_state=42\n    ))\n])\n\nstart = time.time()\nlogit_clf.fit(X_train, y_train)\ntrain_time_logit = time.time() - start\n\n# Dự đoán xác suất trên tập validation\ny_valid_proba_logit = logit_clf.predict_proba(X_val)[:, 1]\nauc_logit = roc_auc_score(y_val, y_valid_proba_logit)\n\nprint(f\"Thời gian train Logistic Regression (sample 100k): {train_time_logit:.2f} giây\")\nprint(f\"Valid AUC (Logistic Regression, sample 100k): {auc_logit:.6f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:13.435059Z","iopub.execute_input":"2025-12-24T14:28:13.435576Z","iopub.status.idle":"2025-12-24T14:28:14.517548Z","shell.execute_reply.started":"2025-12-24T14:28:13.435551Z","shell.execute_reply":"2025-12-24T14:28:14.516632Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import roc_curve\nimport matplotlib.pyplot as plt\n\nfpr_logit, tpr_logit, _ = roc_curve(y_val, y_valid_proba_logit)\n\nplt.figure(figsize=(6, 6))\nplt.plot(fpr_logit, tpr_logit, label=f\"Logistic Regression (AUC = {auc_logit:.4f})\")\nplt.plot([0, 1], [0, 1], linestyle=\"--\")\nplt.xlabel(\"False Positive Rate\")\nplt.ylabel(\"True Positive Rate\")\nplt.title(\"Đường cong ROC - Baseline Logistic Regression (sample 100k)\")\nplt.legend()\nplt.grid(True)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:14.518440Z","iopub.execute_input":"2025-12-24T14:28:14.518669Z","iopub.status.idle":"2025-12-24T14:28:14.745515Z","shell.execute_reply.started":"2025-12-24T14:28:14.518650Z","shell.execute_reply":"2025-12-24T14:28:14.744659Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train LightGBM trên sample","metadata":{}},{"cell_type":"code","source":"# LightGBM Dataset\ntrain_set = lgb.Dataset(X_train, label=y_train)\nval_set   = lgb.Dataset(X_val, label=y_val)\n\n# Tính tạm scale_pos_weight cho cân bằng lớp\nn_neg = (y_train == 0).sum()\nn_pos = (y_train == 1).sum()\nscale_pos_weight = n_neg / max(1, n_pos)\nprint(\"scale_pos_weight ≈\", scale_pos_weight)\n\nparams_sample = {\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"learning_rate\": 0.1,\n    \"num_leaves\": 64,\n    \"max_depth\": -1,\n    \"feature_fraction\": 0.8,\n    \"bagging_fraction\": 0.8,\n    \"bagging_freq\": 5,\n    \"min_data_in_leaf\": 100,\n    \"num_threads\": 4,          \n    \"scale_pos_weight\": scale_pos_weight,  # xử lý imbalance\n    \"verbose\": -1,\n}\n\n\nevals_result_sample = {}\n\ncallbacks_sample = [\n    lgb.early_stopping(stopping_rounds=50, verbose=True),\n    lgb.log_evaluation(period=20),\n    lgb.record_evaluation(evals_result_sample),  \n]\n\nstart = time.time()\nmodel = lgb.train(\n    params_sample,\n    train_set,\n    num_boost_round=1000,\n    valid_sets=[train_set, val_set],\n    valid_names=[\"train\", \"valid\"],\n    callbacks=callbacks_sample,\n)\n\ntrain_time_sample = time.time() - start\n\nprint(\"Train time (sample):\", train_time_sample, \"sec\")\nprint(\"Best iteration:\", model.best_iteration)\n\ny_val_pred = model.predict(X_val, num_iteration=model.best_iteration)\nauc = roc_auc_score(y_val, y_val_pred)\nprint(\"Valid AUC (sample):\", auc)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:14.746292Z","iopub.execute_input":"2025-12-24T14:28:14.746527Z","iopub.status.idle":"2025-12-24T14:28:15.197178Z","shell.execute_reply.started":"2025-12-24T14:28:14.746510Z","shell.execute_reply":"2025-12-24T14:28:15.196332Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# So sánh Logistic Regression và LightGBM trên sample 100k\n","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\n# 1) Accuracy cho Logistic Regression (baseline)\n#    (đã có y_valid_proba_logit và y_val từ cell Logistic ở trên)\n\ny_val_pred_logit = (y_valid_proba_logit >= 0.5).astype(\"int\")\nacc_logit = accuracy_score(y_val, y_val_pred_logit)\n\nprint(f\"Accuracy (Logistic Regression, sample 100k): {acc_logit:.6f}\")\n\n# 2) Accuracy cho LightGBM (sample 100k)\n#    (đã có y_val_pred và y_val từ cell LightGBM sample ở trên)\n\ny_val_pred_lgb = (y_val_pred >= 0.5).astype(\"int\")\nacc_lgb = accuracy_score(y_val, y_val_pred_lgb)\n\nprint(f\"Accuracy (LightGBM, sample 100k): {acc_lgb:.6f}\")\n\n# 3) Bảng so sánh kết quả\nresults_df = pd.DataFrame({\n    \"Model\": [\"Logistic Regression\", \"LightGBM (sample 100k)\"],\n    \"AUC\": [auc_logit, auc],\n    \"Accuracy\": [acc_logit, acc_lgb],\n    \"Train_time_sec\": [train_time_logit, train_time_sample],\n})\n\nresults_df = results_df.set_index(\"Model\")\n\nprint(\"\\nBảng so sánh Logistic Regression vs LightGBM (sample 100k):\")\ndisplay(results_df)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:15.197906Z","iopub.execute_input":"2025-12-24T14:28:15.198283Z","iopub.status.idle":"2025-12-24T14:28:15.219620Z","shell.execute_reply.started":"2025-12-24T14:28:15.198246Z","shell.execute_reply":"2025-12-24T14:28:15.218664Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#đồ thị so sánh Logistic vs LightGBM (sample 100k)\n\nmodels = results_df.index.tolist()\nauc_scores = results_df[\"AUC\"].values\nacc_scores = results_df[\"Accuracy\"].values\ntime_scores = results_df[\"Train_time_sec\"].values\n\n# Biểu đồ 1: So sánh AUC & Accuracy \nx = np.arange(len(models))  # vị trí cột\nwidth = 0.35                # độ rộng mỗi cột\n\nfig, ax = plt.subplots(figsize=(8, 5))\nax.bar(x - width/2, auc_scores, width, label=\"AUC\", color=\"red\")\nax.bar(x + width/2, acc_scores, width, label=\"Accuracy\", color=\"green\")\n\nax.set_ylabel(\"Giá trị\")\nax.set_title(\"So sánh AUC và Accuracy\\n(Logistic Regression vs LightGBM, sample 100k)\")\nax.set_xticks(x)\nax.set_xticklabels(models, rotation=15, ha=\"right\")\nax.set_ylim(0.0, 1.0)\nax.legend()\nax.grid(axis=\"y\", linestyle=\"--\", alpha=0.5)\n\nplt.tight_layout()\nplt.show()\n\n# Biểu đồ 2: So sánh thời gian train \nfig, ax = plt.subplots(figsize=(6, 4))\nax.bar(models, time_scores, color=\"purple\")  \nax.set_ylabel(\"Thời gian train (giây)\")\nax.set_title(\"So sánh thời gian huấn luyện\\n(Logistic Regression vs LightGBM, sample 100k)\")\nax.grid(axis=\"y\", linestyle=\"--\", alpha=0.5)\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:15.220328Z","iopub.execute_input":"2025-12-24T14:28:15.220648Z","iopub.status.idle":"2025-12-24T14:28:15.550590Z","shell.execute_reply.started":"2025-12-24T14:28:15.220620Z","shell.execute_reply":"2025-12-24T14:28:15.549865Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 10 — Learning curve cho sample\n\ntrain_auc_sample = evals_result_sample[\"train\"][\"auc\"]\nvalid_auc_sample = evals_result_sample[\"valid\"][\"auc\"]\n\nplt.figure(figsize=(8, 5))\nplt.plot(train_auc_sample, label=\"train AUC\")\nplt.plot(valid_auc_sample, label=\"valid AUC\")\nplt.xlabel(\"Iteration\")\nplt.ylabel(\"AUC\")\nplt.title(\"Learning curve — sample (100k dòng)\")\nplt.legend()\nplt.grid(True)\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:15.551399Z","iopub.execute_input":"2025-12-24T14:28:15.551607Z","iopub.status.idle":"2025-12-24T14:28:15.824093Z","shell.execute_reply.started":"2025-12-24T14:28:15.551592Z","shell.execute_reply":"2025-12-24T14:28:15.823287Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Feature importance (gain) cho sample\n\ndef plot_importance(model, feature_names, title):\n    importance = model.feature_importance(importance_type=\"gain\")\n    imp_df = pd.DataFrame({\n        \"feature\": feature_names,\n        \"gain\": importance,\n    }).sort_values(\"gain\", ascending=False)\n\n    plt.figure(figsize=(8, 5))\n    sns.barplot(\n        data=imp_df.head(15),\n        x=\"gain\", y=\"feature\", orient=\"h\"\n    )\n    plt.title(title)\n    plt.tight_layout()\n    plt.show()\n    return imp_df\n\nimp_sample = plot_importance(\n    model,\n    feature_cols,\n    \"Top 15 feature — sample (100k dòng)\"\n)\ndisplay(imp_sample.head(15))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:15.824902Z","iopub.execute_input":"2025-12-24T14:28:15.825111Z","iopub.status.idle":"2025-12-24T14:28:16.097573Z","shell.execute_reply.started":"2025-12-24T14:28:15.825094Z","shell.execute_reply":"2025-12-24T14:28:16.096636Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ROC curve cho sample","metadata":{}},{"cell_type":"code","source":"# ROC curve cho sample\n\nfpr_s, tpr_s, _ = roc_curve(y_val, y_val_pred)\n\nplt.figure(figsize=(7, 7))\nplt.plot(fpr_s, tpr_s, label=f\"Sample (AUC = {auc:.4f})\")\nplt.plot([0, 1], [0, 1], \"k--\", label=\"Random\")\nplt.xlabel(\"False Positive Rate\")\nplt.ylabel(\"True Positive Rate\")\nplt.title(\"ROC curve — sample (100k dòng)\")\nplt.legend()\nplt.grid(True)\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:16.098381Z","iopub.execute_input":"2025-12-24T14:28:16.098617Z","iopub.status.idle":"2025-12-24T14:28:16.367893Z","shell.execute_reply.started":"2025-12-24T14:28:16.098599Z","shell.execute_reply":"2025-12-24T14:28:16.367182Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load subset lớn từ train.csv","metadata":{}},{"cell_type":"code","source":"train_path = DATA_DIR / \"train.csv\"\n\ndtypes_big = {\n    \"ip\": \"uint32\",\n    \"app\": \"uint16\",\n    \"device\": \"uint16\",\n    \"os\": \"uint16\",\n    \"channel\": \"uint16\",\n    \"is_attributed\": \"uint8\",\n}\n\nusecols_big = [\"ip\", \"app\", \"device\", \"os\", \"channel\", \"click_time\", \"is_attributed\"]\n\nN_ROWS = 10_000_000   \n\nstart = time.time()\ndf_big = pd.read_csv(\n    train_path,\n    nrows=N_ROWS,\n    usecols=usecols_big,\n    dtype=dtypes_big,\n    parse_dates=[\"click_time\"],\n)\nload_time = time.time() - start\n\nprint(\"df_big shape:\", df_big.shape)\nprint(df_big.head())\n\nprint(\"\\nTỉ lệ thiếu:\")\nprint(df_big.isna().mean())\n\nprint(f\"\\nMemory usage df_big: {memory_in_mb(df_big):.2f} MB\")\nprint(f\"Thời gian load df_big: {load_time:.2f} sec\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:16.369741Z","iopub.execute_input":"2025-12-24T14:28:16.369969Z","iopub.status.idle":"2025-12-24T14:28:54.137790Z","shell.execute_reply.started":"2025-12-24T14:28:16.369942Z","shell.execute_reply":"2025-12-24T14:28:54.136841Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Feature engineering + split cho big subset","metadata":{}},{"cell_type":"code","source":"df_big[\"hour\"] = df_big[\"click_time\"].dt.hour.astype(\"int8\")\ndf_big[\"day\"]  = df_big[\"click_time\"].dt.day.astype(\"int8\")\ndf_big = df_big.drop(columns=[\"click_time\"])\n\ntarget_col_big = \"is_attributed\"\nfeature_cols_big = [c for c in df_big.columns if c != target_col_big]\n\nXb = df_big[feature_cols_big]\nyb = df_big[target_col_big]\n\nXb_train, Xb_val, yb_train, yb_val = train_test_split(\n    Xb, yb,\n    test_size=0.1,\n    random_state=42,\n    stratify=yb,\n)\n\nprint(\"Big Train shape:\", Xb_train.shape, \"Big Val shape:\", Xb_val.shape)\nprint(\"Positives big train:\", yb_train.sum(), \"Val:\", yb_val.sum())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:54.138661Z","iopub.execute_input":"2025-12-24T14:28:54.138938Z","iopub.status.idle":"2025-12-24T14:28:59.295042Z","shell.execute_reply.started":"2025-12-24T14:28:54.138914Z","shell.execute_reply":"2025-12-24T14:28:59.294211Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train LightGBM trên big subset (KHÔNG dùng evals_result=)","metadata":{}},{"cell_type":"code","source":"train_set_big = lgb.Dataset(Xb_train, label=yb_train)\nval_set_big   = lgb.Dataset(Xb_val, label=yb_val)\n\nn_neg_big = (yb_train == 0).sum()\nn_pos_big = (yb_train == 1).sum()\nscale_pos_weight_big = n_neg_big / max(1, n_pos_big)\nprint(\"scale_pos_weight_big ≈\", scale_pos_weight_big)\n\nparams_big = {\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"learning_rate\": 0.01,      #nhỏ hơn\n    \"num_leaves\": 64,\n    \"max_depth\": -1,\n    \"feature_fraction\": 0.8,\n    \"bagging_fraction\": 0.8,\n    \"bagging_freq\": 5,\n    \"min_data_in_leaf\": 500,    #tăng thêm regularization\n    \"num_threads\": 4,\n    \"scale_pos_weight\": scale_pos_weight_big,\n    \"verbose\": -1,\n}\n\n# dùng callback record_evaluation thay cho evals_result=\nevals_result_big = {}\n\ncallbacks_big = [\n    lgb.early_stopping(stopping_rounds=100, verbose=True),\n    lgb.log_evaluation(period=100),\n    lgb.record_evaluation(evals_result_big),  # ghi log vào dict\n]\n\nstart = time.time()\nmodel_big = lgb.train(\n    params_big,\n    train_set_big,\n    num_boost_round=2000,\n    valid_sets=[train_set_big, val_set_big],\n    valid_names=[\"train\", \"valid\"],\n    callbacks=callbacks_big, \n)\n\ntrain_time_big = time.time() - start\n\nprint(\"Train time (big subset):\", train_time_big, \"sec\")\nprint(\"Best iteration (big):\", model_big.best_iteration)\n\ny_val_pred_big = model_big.predict(Xb_val, num_iteration=model_big.best_iteration)\nauc_big = roc_auc_score(yb_val, y_val_pred_big)\nprint(\"Valid AUC (big subset):\", auc_big)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:28:59.296039Z","iopub.execute_input":"2025-12-24T14:28:59.296310Z","iopub.status.idle":"2025-12-24T14:54:29.787453Z","shell.execute_reply.started":"2025-12-24T14:28:59.296284Z","shell.execute_reply":"2025-12-24T14:54:29.786499Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Tính thêm Accuracy cho LightGBM big\n\nfrom sklearn.metrics import accuracy_score\n\n# y_val_pred_big là xác suất dự đoán của model_big trên Xb_val (Cell 15)\ny_val_pred_label_big = (y_val_pred_big >= 0.5).astype(\"int\")\nacc_lgb_big = accuracy_score(yb_val, y_val_pred_label_big)\n\n\nprint(f\"Valid ACC  (LightGBM, big subset): {acc_lgb_big:.6f}\")\nprint(f\"Thời gian train LightGBM (big subset): {train_time_big:.2f} giây\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:54:29.788239Z","iopub.execute_input":"2025-12-24T14:54:29.788473Z","iopub.status.idle":"2025-12-24T14:54:29.928710Z","shell.execute_reply.started":"2025-12-24T14:54:29.788456Z","shell.execute_reply":"2025-12-24T14:54:29.927900Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Logistic Regression trên big subset (df_big) để so sánh với LightGBM big","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import roc_auc_score, accuracy_score\nimport time\n\n# Xb_train, Xb_val, yb_train, yb_val đã được tạo ở Cell 14\n\nlogit_big_clf = Pipeline([\n    (\"scaler\", StandardScaler()),  # có thể bỏ nếu lo RAM, nhưng để công bằng với sample 100k\n    (\"logreg\", LogisticRegression(\n        solver=\"saga\",            # solver phù hợp cho dữ liệu lớn + nhiều mẫu\n        max_iter=100,              # có thể tăng nếu thấy chưa hội tụ\n        class_weight=\"balanced\",  # xử lý mất cân bằng lớp\n        n_jobs=-1,                # tận dụng tất cả CPU cores\n        random_state=42,\n        verbose=1\n    ))\n])\n\nstart = time.time()\nlogit_big_clf.fit(Xb_train, yb_train)\ntrain_time_logit_big = time.time() - start\n\n# Xác suất và nhãn dự đoán trên tập validation lớn\ny_val_proba_logit_big = logit_big_clf.predict_proba(Xb_val)[:, 1]\nauc_logit_big = roc_auc_score(yb_val, y_val_proba_logit_big)\n\ny_val_pred_logit_big = (y_val_proba_logit_big >= 0.5).astype(\"int\")\nacc_logit_big = accuracy_score(yb_val, y_val_pred_logit_big)\n\nprint(f\"Thời gian train Logistic Regression (big subset): {train_time_logit_big:.2f} giây\")\nprint(f\"Valid AUC  (Logistic Regression, big subset): {auc_logit_big:.6f}\")\nprint(f\"Valid ACC  (Logistic Regression, big subset): {acc_logit_big:.6f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T14:54:29.929611Z","iopub.execute_input":"2025-12-24T14:54:29.930094Z","iopub.status.idle":"2025-12-24T15:02:25.124174Z","shell.execute_reply.started":"2025-12-24T14:54:29.930065Z","shell.execute_reply":"2025-12-24T15:02:25.123231Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Learning curve + ROC + feature importance cho big subset","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import roc_curve\n\n\n# Learning curve (big subset)\ntrain_auc_big = np.array(evals_result_big[\"train\"][\"auc\"])\nvalid_auc_big = np.array(evals_result_big[\"valid\"][\"auc\"])\n\niters = np.arange(1, len(train_auc_big) + 1)\n\nfig, ax = plt.subplots(figsize=(8, 5))\n\n# Đường train va valid AUC\nax.plot(\n    iters,\n    train_auc_big,\n    label=\"Train AUC (big)\",\n    linewidth=2,\n    color=\"red\"\n)\nax.plot(\n    iters,\n    valid_auc_big,\n    label=\"Valid AUC (big)\",\n    linewidth=2,\n    linestyle=\"--\",\n    color=\"green\"\n)\n\n# Đánh dấu best_iteration\nbest_iter = model_big.best_iteration\nax.axvline(best_iter, linestyle=\":\", linewidth=1.5, color=\"black\")\nax.text(\n    best_iter,\n    valid_auc_big[best_iter - 1] + 0.0003,\n    f\"best_iter = {best_iter}\",\n    ha=\"center\",\n    va=\"bottom\",\n    fontsize=9,\n)\n\n# Label & title\nax.set_xlabel(\"Iteration\")\nax.set_ylabel(\"AUC\")\nax.set_title(\"Learning curve — LightGBM big subset (10M dòng)\")\n\n# Giới hạn trục Y cho dễ nhìn (có thể chỉnh lại nếu cần)\nax.set_ylim(0.94, 0.981)\n\nax.grid(True, linestyle=\"--\", alpha=0.5)\nax.legend()\nplt.tight_layout()\nplt.show()\n\n# =======================\n# Feature importance (big subset)\n# =======================\n\nimp_big = plot_importance(\n    model_big,\n    feature_cols_big,\n    \"Top 15 feature — big subset\"\n)\ndisplay(imp_big.head(15))\n\n\n# ROC curve (so sánh sample vs big)\n\n\nfpr_b, tpr_b, _ = roc_curve(yb_val, y_val_pred_big)\n\nplt.figure(figsize=(7, 7))\nplt.plot(\n    fpr_s,\n    tpr_s,\n    label=f\"Sample (AUC = {auc:.4f})\",\n    color=\"red\"\n)\nplt.plot(\n    fpr_b,\n    tpr_b,\n    label=f\"Big subset (AUC = {auc_big:.4f})\",\n    color=\"green\"\n)\nplt.plot([0, 1], [0, 1], \"k--\", label=\"Random\")\nplt.xlabel(\"False Positive Rate\")\nplt.ylabel(\"True Positive Rate\")\nplt.title(\"ROC curve — sample vs big subset\")\nplt.legend()\nplt.grid(True)\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T15:35:00.844519Z","iopub.execute_input":"2025-12-24T15:35:00.844799Z","iopub.status.idle":"2025-12-24T15:35:02.160455Z","shell.execute_reply.started":"2025-12-24T15:35:00.844781Z","shell.execute_reply":"2025-12-24T15:35:02.159634Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Bảng so sánh Logistic Regression vs LightGBM trên big subset","metadata":{}},{"cell_type":"code","source":"results_big_df = pd.DataFrame({\n    \"Model\": [\n        \"Logistic Regression (big)\",\n        \"LightGBM (big subset)\"\n    ],\n    \"AUC\": [\n        auc_logit_big,\n        auc_big\n    ],\n    \"Accuracy\": [\n        acc_logit_big,\n        acc_lgb_big\n    ],\n    \"Train_time_sec\": [\n        train_time_logit_big,\n        train_time_big\n    ]\n})\n\nresults_big_df = results_big_df.set_index(\"Model\")\n\nprint(\"So sánh Logistic Regression vs LightGBM trên big subset (df_big):\")\ndisplay(results_big_df)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T15:02:26.419679Z","iopub.execute_input":"2025-12-24T15:02:26.419910Z","iopub.status.idle":"2025-12-24T15:02:26.429995Z","shell.execute_reply.started":"2025-12-24T15:02:26.419892Z","shell.execute_reply":"2025-12-24T15:02:26.429267Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#  Đồ thị so sánh Logistic Regression vs LightGBM trên big subset","metadata":{}},{"cell_type":"code","source":"models = results_big_df.index.tolist()\nauc_scores = results_big_df[\"AUC\"].values\nacc_scores = results_big_df[\"Accuracy\"].values\ntime_scores = results_big_df[\"Train_time_sec\"].values\n\n# --- Biểu đồ 1: So sánh AUC & Accuracy ---\nx = np.arange(len(models))\nwidth = 0.35\n\nfig, ax = plt.subplots(figsize=(8, 5))\nax.bar(x - width/2, auc_scores, width, label=\"AUC\", color=\"red\")\nax.bar(x + width/2, acc_scores, width, label=\"Accuracy\", color=\"green\")\n\nax.set_ylabel(\"Giá trị\")\nax.set_title(\"So sánh AUC và Accuracy\\n(Logistic Regression vs LightGBM, big subset)\")\nax.set_xticks(x)\nax.set_xticklabels(models, rotation=15, ha=\"right\")\nax.set_ylim(0.0, 1.0)\nax.legend()\nax.grid(axis=\"y\", linestyle=\"--\", alpha=0.5)\n\nplt.tight_layout()\nplt.show()\n\n# --- Biểu đồ 2: So sánh thời gian train ---\nfig, ax = plt.subplots(figsize=(6, 4))\nax.bar(models, time_scores,color =\"blue\")\nax.set_ylabel(\"Thời gian train (giây)\")\nax.set_title(\"So sánh thời gian huấn luyện\\n(Logistic Regression vs LightGBM, big subset)\")\nax.grid(axis=\"y\", linestyle=\"--\", alpha=0.5)\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T15:26:07.879573Z","iopub.execute_input":"2025-12-24T15:26:07.879906Z","iopub.status.idle":"2025-12-24T15:26:08.243976Z","shell.execute_reply.started":"2025-12-24T15:26:07.879881Z","shell.execute_reply":"2025-12-24T15:26:08.243218Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#  So sánh num_threads=1 vs num_threads=4 (mini test song song) thử nhỏ với 2triệu dòng ","metadata":{}},{"cell_type":"code","source":"import time\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\n\n# Lấy 2 triệu dòng đầu từ df_big (đã có sẵn 10M ở trên)\nN_SMALL = 2_000_000\ndf_small = df_big.iloc[:N_SMALL].copy()\n\ntarget_col_small = \"is_attributed\"\nfeature_cols_small = [c for c in df_small.columns if c != target_col_small]\n\nXs = df_small[feature_cols_small]\nys = df_small[target_col_small]\n\nXs_train, Xs_val, ys_train, ys_val = train_test_split(\n    Xs, ys, test_size=0.1, random_state=42, stratify=ys\n)\n\ntrain_set_s = lgb.Dataset(Xs_train, label=ys_train)\nval_set_s   = lgb.Dataset(Xs_val, label=ys_val)\n\nn_neg_s = (ys_train == 0).sum()\nn_pos_s = (ys_train == 1).sum()\nscale_pos_weight_s = n_neg_s / max(1, n_pos_s)\n\nbase_params = {\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"learning_rate\": 0.05,\n    \"num_leaves\": 64,\n    \"max_depth\": -1,\n    \"feature_fraction\": 0.8,\n    \"bagging_fraction\": 0.8,\n    \"bagging_freq\": 5,\n    \"min_data_in_leaf\": 200,\n    \"scale_pos_weight\": scale_pos_weight_s,\n    \"verbose\": -1,\n}\n\ndef run_with_threads(num_threads):\n    params = base_params.copy()\n    params[\"num_threads\"] = num_threads\n\n    evals_result = {}\n    callbacks = [\n        lgb.early_stopping(stopping_rounds=50, verbose=False),\n        lgb.record_evaluation(evals_result),\n    ]\n\n    start = time.time()\n    model_tmp = lgb.train(\n        params,\n        train_set_s,\n        num_boost_round=500,\n        valid_sets=[val_set_s],\n        valid_names=[\"valid\"],\n        callbacks=callbacks,\n    )\n    t = time.time() - start\n    y_pred = model_tmp.predict(Xs_val, num_iteration=model_tmp.best_iteration)\n    auc_tmp = roc_auc_score(ys_val, y_pred)\n    return t, auc_tmp, model_tmp.best_iteration\n\ntime_1, auc_1, it_1 = run_with_threads(1)\ntime_4, auc_4, it_4 = run_with_threads(4)\n\nprint(\"num_threads=1 -> time = {:.2f}s, AUC = {:.4f}, best_iter = {}\".format(time_1, auc_1, it_1))\nprint(\"num_threads=4 -> time = {:.2f}s, AUC = {:.4f}, best_iter = {}\".format(time_4, auc_4, it_4))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T15:02:26.773735Z","iopub.execute_input":"2025-12-24T15:02:26.773923Z","iopub.status.idle":"2025-12-24T15:02:45.370534Z","shell.execute_reply.started":"2025-12-24T15:02:26.773908Z","shell.execute_reply":"2025-12-24T15:02:45.369904Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# test.csv và tạo feature hour/day","metadata":{}},{"cell_type":"code","source":"test_path = DATA_DIR / \"test.csv\"\n\ndtypes_test = {\n    \"click_id\": \"uint32\",\n    \"ip\":       \"uint32\",\n    \"app\":      \"uint16\",\n    \"device\":   \"uint16\",\n    \"os\":       \"uint16\",\n    \"channel\":  \"uint16\",\n}\n\nusecols_test = [\"click_id\", \"ip\", \"app\", \"device\", \"os\", \"channel\", \"click_time\"]\n\nprint(\"Đang load test.csv ...\")\ntest_df = pd.read_csv(\n    test_path,\n    usecols=usecols_test,\n    dtype=dtypes_test,\n    parse_dates=[\"click_time\"],\n)\n\nprint(\"test_df shape:\", test_df.shape)\ndisplay(test_df.head())\n\n# Feature engineering giống y train\ntest_df[\"hour\"] = test_df[\"click_time\"].dt.hour.astype(\"int8\")\ntest_df[\"day\"]  = test_df[\"click_time\"].dt.day.astype(\"int8\")\n\n# Bỏ cột thời gian thô\ntest_df = test_df.drop(columns=[\"click_time\"])\n\nprint(\"\\nSau khi thêm feature:\")\ndisplay(test_df.head())\nprint(test_df.dtypes)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T15:02:45.373592Z","iopub.execute_input":"2025-12-24T15:02:45.373991Z","iopub.status.idle":"2025-12-24T15:03:53.897001Z","shell.execute_reply.started":"2025-12-24T15:02:45.373972Z","shell.execute_reply":"2025-12-24T15:03:53.896140Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# X_test và predict với cả LightGBM & Logistic Regression","metadata":{}},{"cell_type":"code","source":"print(\"Feature dùng để train (big):\", feature_cols_big)\n\n# Đảm bảo test_df có đủ các cột \nmissing_cols = set(feature_cols_big) - set(test_df.columns)\nprint(\"Missing cols in test:\", missing_cols)\n\nif len(missing_cols) > 0:\n    raise ValueError(f\"Thiếu cột trong test_df: {missing_cols}\")\n\nX_test = test_df[feature_cols_big]\n\nprint(\"X_test shape:\", X_test.shape)\ndisplay(X_test.head())\n\n#Dự đoán xác suất is_attributed=1 bằng LightGBM \ntest_pred_lgb = model_big.predict(\n    X_test,\n    num_iteration=model_big.best_iteration\n)\n\nprint(\"\\nVí dụ 10 giá trị dự đoán đầu (LightGBM):\")\nprint(test_pred_lgb[:10])\n\n#Dự đoán xác suất is_attributed=1 bằng Logistic Regression \ntest_pred_logit = logit_big_clf.predict_proba(X_test)[:, 1]\n\nprint(\"\\nVí dụ 10 giá trị dự đoán đầu (Logistic Regression):\")\nprint(test_pred_logit[:10])\n\n# So sánh phân bố xác suất của 2 mô hình \nplt.figure(figsize=(8, 4))\nplt.hist(test_pred_lgb, bins=50, alpha=0.5, label=\"LightGBM\")\nplt.hist(test_pred_logit, bins=50, alpha=0.5, label=\"Logistic\")\nplt.title(\"Phân bố xác suất dự đoán trên test.csv\\n(so sánh LightGBM vs Logistic)\")\nplt.xlabel(\"P(is_attributed = 1)\")\nplt.ylabel(\"Count\")\nplt.legend()\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T15:03:53.897833Z","iopub.execute_input":"2025-12-24T15:03:53.898043Z","iopub.status.idle":"2025-12-24T15:14:07.757345Z","shell.execute_reply.started":"2025-12-24T15:03:53.898025Z","shell.execute_reply":"2025-12-24T15:14:07.756688Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Tạo 2 file submission: LightGBM và Logistic Regression\n","metadata":{}},{"cell_type":"code","source":"sub_path = DATA_DIR / \"sample_submission.csv\"\nsub_base = pd.read_csv(sub_path)\n\nprint(\"sample_submission head:\")\ndisplay(sub_base.head())\n\n#LightGBM \nsub_lgb = sub_base.copy()\nsub_lgb[\"is_attributed\"] = test_pred_lgb\n\nout_path_lgb = Path(\"submission_lightgbm_10m.csv\")\nsub_lgb.to_csv(out_path_lgb, index=False)\nprint(\"Đã lưu file submission LightGBM:\", out_path_lgb)\n\n#Logistic Regression\nsub_logit = sub_base.copy()\nsub_logit[\"is_attributed\"] = test_pred_logit\n\nout_path_logit = Path(\"submission_logit_10m.csv\")\nsub_logit.to_csv(out_path_logit, index=False)\nprint(\"Đã lưu file submission Logistic:\", out_path_logit)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T15:14:07.758126Z","iopub.execute_input":"2025-12-24T15:14:07.758499Z","iopub.status.idle":"2025-12-24T15:15:22.701970Z","shell.execute_reply.started":"2025-12-24T15:14:07.758474Z","shell.execute_reply":"2025-12-24T15:15:22.700955Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Gộp dự đoán của 2 model vào test_df để trực quan hóa","metadata":{}},{"cell_type":"code","source":"test_vis_df = test_df.copy()\ntest_vis_df[\"pred_lgb\"]   = test_pred_lgb\ntest_vis_df[\"pred_logit\"] = test_pred_logit\n\ndisplay(test_vis_df.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T15:15:22.702866Z","iopub.execute_input":"2025-12-24T15:15:22.703088Z","iopub.status.idle":"2025-12-24T15:15:22.961173Z","shell.execute_reply.started":"2025-12-24T15:15:22.703068Z","shell.execute_reply":"2025-12-24T15:15:22.960321Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Scatter plot giữa 2 xác suất dự đoán (Logistic vs LightGBM) thử với 100k thôi nặng","metadata":{}},{"cell_type":"code","source":"\n\n\n#mẫu 100k \nN_SAMPLE = 100_000\nif len(test_vis_df) > N_SAMPLE:\n    vis_sample = test_vis_df.sample(N_SAMPLE, random_state=42)\nelse:\n    vis_sample = test_vis_df\n\nplt.figure(figsize=(6, 6))\nsns.scatterplot(\n    data=vis_sample,\n    x=\"pred_logit\",\n    y=\"pred_lgb\",\n    alpha=0.3,\n    s=10\n)\nplt.xlabel(\"P(is_attributed = 1) - Logistic\")\nplt.ylabel(\"P(is_attributed = 1) - LightGBM\")\nplt.title(\"So sánh xác suất dự đoán của 2 mô hình\\n(trên mẫu từ test.csv)\")\nplt.grid(True, linestyle=\"--\", alpha=0.5)\nplt.tight_layout()\nplt.show()\n\n# In hệ số tương quan\ncorr = vis_sample[[\"pred_logit\", \"pred_lgb\"]].corr().iloc[0,1]\nprint(f\"Hệ số tương quan Pearson giữa 2 model: {corr:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-24T15:15:24.311744Z","iopub.execute_input":"2025-12-24T15:15:24.311961Z","iopub.status.idle":"2025-12-24T15:15:25.606948Z","shell.execute_reply.started":"2025-12-24T15:15:24.311938Z","shell.execute_reply":"2025-12-24T15:15:25.606324Z"}},"outputs":[],"execution_count":null}]}