{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import polars as pl\nfrom pathlib import Path\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nimport numpy as np\nimport pandas as pd\nfrom collections import defaultdict\nfrom catboost import CatBoostClassifier, Pool\nimport lightgbm as lgb\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport sklearn.neighbors, sklearn.metrics, sklearn.preprocessing\nfrom xgboost import XGBClassifier\nimport polars.selectors as cs\nfrom sklearn.metrics import cohen_kappa_score, ConfusionMatrixDisplay\nimport numpy as np\nfrom colorama import Fore, Style\nimport random\n\nimport polars as pl\nimport numpy as np\nfrom sklearn.preprocessing import StandardScaler\nfrom tqdm import tqdm\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport gc\nimport itertools\nimport pickle\nimport re\nimport time\nimport os\nimport logging \nfrom scipy.stats.mstats import winsorize\n\nN_THREADS = 3\nN_BAGS = 1\nN_FOLDS = 5\nN_SEEDS = 1\nSEED = np.random.randint(10000)\nGPU = 0\n\nos.environ['PYTHONHASHSEED'] = str(SEED)\nrandom.seed(SEED)\nnp.random.seed(SEED)\nos.environ['POLARS_MAX_THREADS'] = str(N_THREADS)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:21:06.107520Z","iopub.execute_input":"2024-12-19T01:21:06.108092Z","iopub.status.idle":"2024-12-19T01:21:06.122964Z","shell.execute_reply.started":"2024-12-19T01:21:06.108039Z","shell.execute_reply":"2024-12-19T01:21:06.121590Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:21:06.125405Z","iopub.execute_input":"2024-12-19T01:21:06.125882Z","iopub.status.idle":"2024-12-19T01:21:06.147434Z","shell.execute_reply.started":"2024-12-19T01:21:06.125831Z","shell.execute_reply":"2024-12-19T01:21:06.146180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MyLogger:\n    \"\"\"\n    This class helps to suppress logs in lightgbm and Optuna\n    Source - https://github.com/microsoft/LightGBM/issues/6014\n    \"\"\"\n\n    def init(self, logging_lbl: str):\n        self.logger = logging.getLogger(logging_lbl)\n        self.logger.setLevel(logging.ERROR)\n\n    def info(self, message):\n        pass\n\n    def warning(self, message):\n        pass\n\n    def error(self, message):\n        self.logger.error(message)\n\nl = MyLogger()\nl.init(logging_lbl = \"lightgbm_custom\")\nlgb.register_logger(l)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:21:06.148793Z","iopub.execute_input":"2024-12-19T01:21:06.149168Z","iopub.status.idle":"2024-12-19T01:21:06.160144Z","shell.execute_reply.started":"2024-12-19T01:21:06.149135Z","shell.execute_reply":"2024-12-19T01:21:06.158971Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"season_dtype = pl.Enum(['Spring', 'Summer', 'Fall', 'Winter','Missing'])\ntarget_labels = ['None', 'Mild', 'Moderate', 'Severe']\n\ntrain = (\n    pl.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\n    .with_columns(pl.col('^.*Season$').cast(season_dtype))\n    .drop('^PCIAT.*$','^PAQ_*$')\n)\n\ntest = (\n    pl.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n    .with_columns(pl.col('^.*Season$').cast(season_dtype))\n    .drop('^PCIAT.*$','^PAQ_*$')\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:21:06.161752Z","iopub.execute_input":"2024-12-19T01:21:06.162181Z","iopub.status.idle":"2024-12-19T01:21:06.191119Z","shell.execute_reply.started":"2024-12-19T01:21:06.162144Z","shell.execute_reply":"2024-12-19T01:21:06.190068Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing = train.select([(pl.col(c).is_null().mean()).alias(c) for c in train.columns])\n\nmissing = (\n    missing.transpose(include_header=True)  \n    .filter(pl.col(\"column_0\") < 0.5)       \n    .with_columns((pl.col(\"column_0\") * 100).round(4).alias('%')) \n    .sort('%', descending=True)         \n)\n\nprint(f\"{missing.height} columns have less than 50% missing.\")","metadata":{"execution":{"iopub.status.busy":"2024-12-19T01:21:06.193335Z","iopub.execute_input":"2024-12-19T01:21:06.193682Z","iopub.status.idle":"2024-12-19T01:21:06.203914Z","shell.execute_reply.started":"2024-12-19T01:21:06.193649Z","shell.execute_reply":"2024-12-19T01:21:06.202739Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols = [i for i in train.columns if i in missing['column'].to_numpy()]\ndftr = train[cols]\ndfte = test[cols[:-1]]\n\n# DON'T WANT TO USE SII.. yet anyways.\ndftr = dftr.filter(~pl.col('sii').is_null())","metadata":{"execution":{"iopub.status.busy":"2024-12-19T01:21:06.205560Z","iopub.execute_input":"2024-12-19T01:21:06.205967Z","iopub.status.idle":"2024-12-19T01:21:06.217849Z","shell.execute_reply.started":"2024-12-19T01:21:06.205928Z","shell.execute_reply":"2024-12-19T01:21:06.216663Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def quadratic_weighted_kappa(preds, data):\n    y_true = data.get_label()\n    y_pred = preds.clip(y_min, y_max).round()\n    qwk = cohen_kappa_score(y_true, y_pred, weights=\"quadratic\")\n    return 'QWK', qwk, True\n\n\ndef qwk_obj(preds, dtrain):\n    labels = dtrain.get_label()\n    preds = preds.clip(y_min, y_max)\n    f = 1/2 * np.sum((preds - labels)**2)\n    g = 1/2 * np.sum((preds - a)**2 + b)\n    df = preds - labels\n    dg = preds - a\n    grad = (df/g - f*dg/g**2)*len(labels)\n    hess = np.ones(len(labels))\n    return grad, hess","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:21:06.219309Z","iopub.execute_input":"2024-12-19T01:21:06.219759Z","iopub.status.idle":"2024-12-19T01:21:06.231463Z","shell.execute_reply.started":"2024-12-19T01:21:06.219714Z","shell.execute_reply":"2024-12-19T01:21:06.230340Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def feature_eng(dftr):\n    median_df = (\n        dftr.group_by(\"Basic_Demos-Age\")\n        .agg(\n            (pl.col(\"Physical-Height\").median()).alias(\"Feat_Height\"),\n            (pl.col(\"Physical-Weight\").median()).alias(\"Feat_Weight\"),\n        )\n    )\n\n    dftr = dftr.join(median_df, on=\"Basic_Demos-Age\", how=\"left\").with_columns(\n        pl.when(pl.col(\"Physical-Height\").is_null())\n        .then(pl.col(\"Feat_Height\"))\n        .otherwise(pl.col(\"Physical-Height\"))\n        .alias(\"Physical-Height\"),\n        \n        pl.when(pl.col(\"Physical-Weight\").is_null())\n        .then(pl.col(\"Feat_Weight\"))\n        .otherwise(pl.col(\"Physical-Weight\"))\n        .alias(\"Physical-Weight\"),\n    ).drop([\"Feat_Height\", \"Feat_Weight\"])\n\n    dftr = dftr.with_columns([\n        pl.when(pl.col('Basic_Demos-Age') < 10)\n        .then(0)\n        .when((pl.col('Basic_Demos-Age') >= 10) & (pl.col('Basic_Demos-Age') < 14))\n        .then(1)\n        .otherwise(2)\n        .alias('age_group')\n    ])\n\n    return dftr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:21:06.233375Z","iopub.execute_input":"2024-12-19T01:21:06.233716Z","iopub.status.idle":"2024-12-19T01:21:06.247149Z","shell.execute_reply.started":"2024-12-19T01:21:06.233684Z","shell.execute_reply":"2024-12-19T01:21:06.246048Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\nunique_ids = dftr.to_pandas().drop_duplicates('id')[['id', 'sii']]\nIDS = sorted(list(unique_ids['id']))\n\nFOLDS = []\nstrat_kf = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)\n\nfor _ in range(N_BAGS):\n    ids_with_folds = []\n    for fold_idx, (train_idx, val_idx) in enumerate(strat_kf.split(unique_ids['id'], unique_ids['sii'])):\n        val_ids = unique_ids['id'].iloc[val_idx].tolist()\n        ids_with_folds.append(val_ids)\n    FOLDS.append(ids_with_folds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:21:06.405185Z","iopub.execute_input":"2024-12-19T01:21:06.406071Z","iopub.status.idle":"2024-12-19T01:21:06.434142Z","shell.execute_reply.started":"2024-12-19T01:21:06.406025Z","shell.execute_reply":"2024-12-19T01:21:06.432818Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cross_validate_model(\n    bst,\n    model_train_func,\n    clip_range=(0, 3),\n    init_score=2.0,\n    SEED=1\n\n):\n    global dftr, dfte\n    \n    feature_importance = np.zeros(X_te.shape[1])\n    oof_raw_preds = np.zeros(len(y), dtype=float)\n    tr_preds = np.zeros(len(X), dtype=float)\n    te_preds = np.zeros(len(X_te), dtype=float)\n    n_splits = N_BAGS * N_FOLDS\n    diff = 0\n        \n    for bag_idx, bag in enumerate(FOLDS):\n        for fold_idx in range(N_FOLDS):\n            fold_num = bag_idx * N_FOLDS + fold_idx\n\n            valid_ids = bag[fold_idx]\n            train_ids = [i for fold in bag if fold != valid_ids for i in fold]\n\n            idx_tr = dftr.to_pandas().index[dftr.to_pandas()['id'].isin(train_ids)].to_numpy()\n            idx_va = dftr.to_pandas().index[dftr.to_pandas()['id'].isin(valid_ids)].to_numpy()\n\n            X_tr, X_va = X.iloc[idx_tr], X.iloc[idx_va]\n            y_tr, y_va = y[idx_tr], y[idx_va]\n\n            # ------------- Feature Eng\n            X_tr, X_va = (feature_eng(pl.from_pandas(X_tr)).to_pandas(),\n                          feature_eng(pl.from_pandas(X_va)).to_pandas())\n\n            model = model_train_func(X_tr, y_tr, X_va, y_va, init_score, SEED)\n\n            # ------------- training \n            tr_pred = np.clip(model.predict(X_tr) + init_score, *clip_range)\n            tr_preds[idx_tr] += tr_pred\n\n            # ------------- validation \n            y_pred = np.clip(model.predict(X_va) + init_score, *clip_range)\n            oof_raw_preds[idx_va] = y_pred\n\n            # ------------- test \n            te_pred = np.clip(model.predict(X_te) + init_score, *clip_range)\n            te_preds += te_pred\n\n            if hasattr(model, \"feature_importance\"):\n                feature_importance += model.feature_importance()\n\n            trn_eval = cohen_kappa_score(y_tr, np.round(tr_pred).astype(int), weights='quadratic')\n            val_eval = cohen_kappa_score(y_va, np.round(y_pred).astype(int), weights='quadratic')\n\n            diff+= abs(trn_eval - val_eval)\n            \n            # print(f\"# Processing Fold {fold_num + 1}/{n_splits}...\")\n            # print(f\"TRAIN: {trn_eval:.3f}, EVAL: {val_eval:.3f}, DIFF: {abs(trn_eval - val_eval):.3f}\")\n            # print()\n\n    tr_preds /= n_splits\n    te_preds /= n_splits\n    oof_score = cohen_kappa_score(y, np.round(oof_raw_preds).astype(int), weights='quadratic')\n    tr_score = cohen_kappa_score(y, np.round(tr_preds).astype(int), weights='quadratic')\n\n    # print('~' * 50)\n    # print(f\"# TRAIN: {tr_score:.3f}\")\n    # print(f\"# OOF:   {oof_score:.3f}\")\n    # print(f\"# DIFF:  {(diff/n_splits):.3f}\")\n    # print('~' * 50)\n\n    return {\n        \"oof\": oof_raw_preds,\n        \"trn\": tr_preds,\n        \"test\": te_preds,\n        \"oof_score\": oof_score,\n        \"train_score\": tr_score,\n        \"feature_importance\": feature_importance,\n    }\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:21:06.439077Z","iopub.execute_input":"2024-12-19T01:21:06.439471Z","iopub.status.idle":"2024-12-19T01:21:06.453954Z","shell.execute_reply.started":"2024-12-19T01:21:06.439434Z","shell.execute_reply":"2024-12-19T01:21:06.452510Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_lgb_model(X_tr, y_tr, X_va, y_va, init_score, SEED):\n    lgb_train = lgb.Dataset(X_tr, label=y_tr.to_numpy(),\n                            init_score=[init_score]*len(X_tr))\n    \n    lgb_val = lgb.Dataset(X_va, label=y_va.to_numpy(),\n                          init_score=[init_score]*len(X_va))\n\n    params = dict(\n        objective=qwk_obj,\n        metric=\"None\",\n        feature_fraction=0.5,\n        colsample_bytree=0.2,\n        max_bin=124,\n        max_depth=6,\n        num_leaves=2**4,\n        min_data_in_leaf=30,\n        path_smooth=4,\n        verbosity=-1,\n        random_state=SEED,\n        extra_seed=SEED,\n        feature_fraction_seed=SEED,\n        extra_trees=True\n    )\n\n    model = lgb.train(\n        params,\n        lgb_train,\n        valid_sets=[lgb_val],\n        num_boost_round=10000,\n        feval=quadratic_weighted_kappa,\n        callbacks=[\n            lgb.early_stopping(50, verbose=-1),\n            lgb.log_evaluation(0)\n        ]\n    )\n    return model\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:21:06.455533Z","iopub.execute_input":"2024-12-19T01:21:06.456085Z","iopub.status.idle":"2024-12-19T01:21:06.475807Z","shell.execute_reply.started":"2024-12-19T01:21:06.456015Z","shell.execute_reply":"2024-12-19T01:21:06.474634Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bst = ['PreInt_EduHx-computerinternet_hoursday',\n       'SDS-SDS_Total_T',\n       'SDS-SDS_Total_Raw',\n       'Basic_Demos-Sex',\n       'Basic_Demos-Age',\n       'Physical-Weight',\n       'Physical-Height',\n       'FGC-FGC_CU',\n       'FGC-FGC_PU',\n       'FGC-FGC_SRL_Zone']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:21:06.477382Z","iopub.execute_input":"2024-12-19T01:21:06.477842Z","iopub.status.idle":"2024-12-19T01:21:06.492002Z","shell.execute_reply.started":"2024-12-19T01:21:06.477797Z","shell.execute_reply":"2024-12-19T01:21:06.490746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target='sii'\n\ny = dftr.get_column(target)\nX = dftr.drop(['id', target]).to_pandas()[bst]\n\ndfte = feature_eng(dfte)\nX_te = dfte.drop('id').to_pandas()[bst + ['age_group']]\n\na, b = y.mean(), y.var(ddof=0)\ny_min, y_max = y.min(), y.max()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:21:06.495244Z","iopub.execute_input":"2024-12-19T01:21:06.495746Z","iopub.status.idle":"2024-12-19T01:21:06.524446Z","shell.execute_reply.started":"2024-12-19T01:21:06.495696Z","shell.execute_reply":"2024-12-19T01:21:06.523171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport random\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom collections import Counter\nfrom colorama import Fore, Style\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\n\nk_ave = 0\nn = 50\nseed_performance = []\noof_predictions = []\ntr_predictions = []\nte_predictions = []\nseen = set()\n\ndef calculate_mode(predictions_list):\n    return np.array([Counter(col).most_common(1)[0][0] for col in zip(*predictions_list)])\n\nfor seed_idx in tqdm(range(n), total=n):\n    g2g = False\n    while not g2g:\n        SEED = np.random.randint(1e8)\n        if SEED not in seen:\n            seen.add(SEED)\n            g2g = True\n            \n    os.environ['PYTHONHASHSEED'] = str(SEED)\n    random.seed(SEED)\n    np.random.seed(SEED)\n    os.environ['POLARS_MAX_THREADS'] = str(3)\n\n    unique_ids = dftr.to_pandas().drop_duplicates('id')[['id', 'sii']]\n    strat_kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)\n    FOLDS = [\n        [unique_ids['id'].iloc[val_idx].tolist() for _, val_idx in strat_kf.split(unique_ids['id'], unique_ids['sii'])]\n    ]\n\n    lgb_results = cross_validate_model(bst, train_lgb_model, SEED=SEED)\n    \n    oof_preds = np.round(lgb_results['oof']).astype(int)\n    tr_preds = np.round(lgb_results['trn']).astype(int)\n    te_preds = np.round(lgb_results['test']).astype(int)\n   \n    oof_predictions.append(oof_preds)\n    tr_predictions.append(tr_preds)\n    te_predictions.append(te_preds)\n    \n    oof_score = cohen_kappa_score(y.to_numpy(), oof_preds, weights='quadratic')\n    tr_score = cohen_kappa_score(y.to_numpy(), tr_preds, weights='quadratic')\n    seed_performance.append({'SEED': SEED, 'Train Score': tr_score, 'OOF Score': oof_score})\n\n    # print(f\"{Fore.YELLOW}{Style.BRIGHT}# SEED {SEED} - TRAIN: {tr_score:.3f}{Style.RESET_ALL}\")\n    # print(f\"{Fore.GREEN}{Style.BRIGHT}# SEED {SEED} - OOF: {oof_score:.3f}{Style.RESET_ALL}\")\n    # print(f\"{Fore.RED}{Style.BRIGHT}# SEED {SEED} - DIFF: {tr_score - oof_score:.3f}{Style.RESET_ALL}\")\n    # print()\n\n    k_ave += oof_score\n\naverage_oof_score = k_ave / n\nprint(f\"Average OOF Score: {average_oof_score:.3f}\")\n\nfinal_oof_predictions = calculate_mode(oof_predictions)\nfinal_tr_predictions = calculate_mode(tr_predictions)\nfinal_te_predictions = calculate_mode(te_predictions)\n\nensemble_oof_score = cohen_kappa_score(y.to_numpy(), final_oof_predictions, weights='quadratic')\nensemble_tr_score = cohen_kappa_score(y.to_numpy(), final_tr_predictions, weights='quadratic')\n\nprint(f\"{Fore.BLUE}{Style.BRIGHT}Final Ensemble TRAIN Score: {ensemble_tr_score:.3f}{Style.RESET_ALL}\")\nprint(f\"{Fore.BLUE}{Style.BRIGHT}Final Ensemble OOF Score: {ensemble_oof_score:.3f}{Style.RESET_ALL}\")\n\nseed_perf_df = pd.DataFrame(seed_performance)\nseed_perf_df['Difference'] = seed_perf_df['Train Score'] - seed_perf_df['OOF Score']\nmean_diff = seed_perf_df['Difference'].mean()\nseed_perf_df = seed_perf_df.sort_values('SEED')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:21:06.526025Z","iopub.execute_input":"2024-12-19T01:21:06.526412Z","iopub.status.idle":"2024-12-19T01:22:25.586065Z","shell.execute_reply.started":"2024-12-19T01:21:06.526375Z","shell.execute_reply":"2024-12-19T01:22:25.585063Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate mean and standard deviation\ntrain_score_mean = seed_perf_df['Train Score'].mean()\ntrain_score_std = seed_perf_df['Train Score'].std()\n\noof_score_mean = seed_perf_df['OOF Score'].mean()\noof_score_std = seed_perf_df['OOF Score'].std()\n\ndifference_mean = seed_perf_df['Difference'].mean()\ndifference_std = seed_perf_df['Difference'].std()\n\n# Print summary statistics\nprint(f\"Train Score: Mean = {train_score_mean:.3f}, Std = {train_score_std:.3f}\")\nprint(f\"OOF Score: Mean = {oof_score_mean:.3f}, Std = {oof_score_std:.3f}\")\nprint(f\"Difference (Train - OOF): Mean = {difference_mean:.3f}, Std = {difference_std:.3f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:22:25.587850Z","iopub.execute_input":"2024-12-19T01:22:25.588253Z","iopub.status.idle":"2024-12-19T01:22:25.597316Z","shell.execute_reply.started":"2024-12-19T01:22:25.588207Z","shell.execute_reply":"2024-12-19T01:22:25.596021Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(18, 10))\nplt.style.use('seaborn-whitegrid')\n\n# Top subplot\nplt.subplot(2, 1, 1)\nplt.plot(seed_perf_df['SEED'], seed_perf_df['Train Score'], \n            label='Train Score', color='blue', alpha=0.8, marker='o', markersize=6)\n\nplt.plot(seed_perf_df['SEED'], seed_perf_df['OOF Score'], \n            label='OOF Score', color='red', alpha=0.8, marker='s', markersize=6)\n\nplt.axhline(average_oof_score, linestyle='--', color='green', linewidth=2, label='Average OOF Score')\n\nplt.title(\"Seed Performance Across Runs\", fontsize=14, fontweight='bold')\nplt.xlabel(\"Seed Index\")\nplt.ylabel(\"Quadratic Weighted Kappa Score\")\nplt.legend(loc='best')\nplt.grid(visible=True, which='major', linestyle='--', linewidth=0.5, alpha=0.7)\n\n# Bottom subplot\nplt.subplot(2, 1, 2)\nplt.plot(seed_perf_df['SEED'], seed_perf_df['Difference'], \n            label='Difference (Train - OOF)', color='purple', alpha=0.8, marker='^', markersize=6)\n\nplt.axhline(mean_diff, linestyle='--', color='orange', linewidth=2, label=f'Mean Difference = {mean_diff:.3f}')\nplt.title(\"Difference Between Train and OOF Scores Across Seeds\", fontsize=14, fontweight='bold')\nplt.xlabel(\"Seed Index\")\nplt.ylabel(\"Difference (Train - OOF)\")\nplt.ylim(0.0, 0.1) \nplt.legend(loc='best')\nplt.grid(visible=True, which='major', linestyle='--', linewidth=0.5, alpha=0.7)\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:22:25.598661Z","iopub.execute_input":"2024-12-19T01:22:25.599044Z","iopub.status.idle":"2024-12-19T01:22:26.302963Z","shell.execute_reply.started":"2024-12-19T01:22:25.599004Z","shell.execute_reply":"2024-12-19T01:22:26.301891Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_test = test.to_pandas()\nsub_test['sii'] = np.round(final_te_predictions).astype(int)\nsub_test['sii'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-12-19T01:22:26.304567Z","iopub.execute_input":"2024-12-19T01:22:26.304916Z","iopub.status.idle":"2024-12-19T01:22:26.321982Z","shell.execute_reply.started":"2024-12-19T01:22:26.304883Z","shell.execute_reply":"2024-12-19T01:22:26.320931Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_test[['id','sii']].to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T01:22:26.323407Z","iopub.execute_input":"2024-12-19T01:22:26.323784Z","iopub.status.idle":"2024-12-19T01:22:26.331099Z","shell.execute_reply.started":"2024-12-19T01:22:26.323749Z","shell.execute_reply":"2024-12-19T01:22:26.330018Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_test[['id','sii']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:22:26.332443Z","iopub.execute_input":"2024-12-19T01:22:26.332817Z","iopub.status.idle":"2024-12-19T01:22:26.348687Z","shell.execute_reply.started":"2024-12-19T01:22:26.332786Z","shell.execute_reply":"2024-12-19T01:22:26.347352Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# TRAIN: tr_score=0.493 \n# OOF: oof_score=0.478 \n# LB = 0.455\n# OOF/LB = 0.466\n\n# TRAIN: tr_score=0.490 \n# OOF: oof_score=0.480 \n# LB = 0.449\n# OOF/LB = 0.464\n\n# TRAIN: tr_score=0.494 \n# OOF: oof_score=0.481 \n# LB = 0.462\n# OOF/LB = 0.0.472\n\n# TRAIN: tr_score=0.515 \n# OOF: oof_score=0.488 \n# LB = 0.463\n# OOF/LB = 0.476\n\n# TRAIN: tr_score=0.504 \n# OOF: oof_score=0.494 \n# LB = 0.457\n# OOF/LB = 0.476","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:22:26.350172Z","iopub.execute_input":"2024-12-19T01:22:26.350635Z","iopub.status.idle":"2024-12-19T01:22:26.360652Z","shell.execute_reply.started":"2024-12-19T01:22:26.350550Z","shell.execute_reply":"2024-12-19T01:22:26.359533Z"}},"outputs":[],"execution_count":null}]}