{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.18","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-24T19:37:27.485442Z","iopub.execute_input":"2025-07-24T19:37:27.485784Z","iopub.status.idle":"2025-07-24T19:37:27.954815Z","shell.execute_reply.started":"2025-07-24T19:37:27.485760Z","shell.execute_reply":"2025-07-24T19:37:27.950265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CFG:\n    train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    test_path = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    sample_sub_path = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n\n    target = \"label\"\n    n_folds = 5\n    seed = 42\n\n    run_optuna = True\n    n_optuna_trials = 250","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T19:37:27.955503Z","iopub.execute_input":"2025-07-24T19:37:27.955753Z","iopub.status.idle":"2025-07-24T19:37:27.966790Z","shell.execute_reply.started":"2025-07-24T19:37:27.955732Z","shell.execute_reply":"2025-07-24T19:37:27.960944Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_parquet(CFG.train_path).reset_index()\nsub  = pd.read_csv(CFG.sample_sub_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T19:37:27.969807Z","iopub.execute_input":"2025-07-24T19:37:27.970038Z","iopub.status.idle":"2025-07-24T19:37:51.037494Z","shell.execute_reply.started":"2025-07-24T19:37:27.970016Z","shell.execute_reply":"2025-07-24T19:37:51.032853Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test  = pd.read_parquet(CFG.test_path) .reset_index()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T19:37:51.038350Z","iopub.execute_input":"2025-07-24T19:37:51.038556Z","iopub.status.idle":"2025-07-24T19:38:09.626239Z","shell.execute_reply.started":"2025-07-24T19:37:51.038535Z","shell.execute_reply":"2025-07-24T19:38:09.620912Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def optimize_memory(df, verbose=True):\n    \"\"\"\n    Optimize memory usage by downcasting numeric types where possible.\n    \"\"\"\n    import numpy as np\n\n    if verbose:\n        start_mem = df.memory_usage(deep=True).sum() / 1024**2\n        print(f'Memory usage before optimization: {start_mem:.2f} MB')\n\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if pd.api.types.is_numeric_dtype(col_type):\n            c_min = df[col].min()\n            c_max = df[col].max()\n\n            if pd.api.types.is_integer_dtype(col_type):\n                if c_min >= np.iinfo(np.int8).min and c_max <= np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min >= np.iinfo(np.int16).min and c_max <= np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min >= np.iinfo(np.int32).min and c_max <= np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                else:\n                    df[col] = df[col].astype(np.int64)\n\n            elif pd.api.types.is_float_dtype(col_type):\n                if c_min >= np.finfo(np.float32).min and c_max <= np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n\n    if verbose:\n        end_mem = df.memory_usage(deep=True).sum() / 1024**2\n        print(f'Memory usage after optimization: {end_mem:.2f} MB')\n        print(f'Decreased by {100 * (start_mem - end_mem) / start_mem:.1f}%')\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T19:38:09.628132Z","iopub.execute_input":"2025-07-24T19:38:09.628390Z","iopub.status.idle":"2025-07-24T19:38:09.641525Z","shell.execute_reply.started":"2025-07-24T19:38:09.628364Z","shell.execute_reply":"2025-07-24T19:38:09.637837Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = optimize_memory(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T19:38:09.643205Z","iopub.execute_input":"2025-07-24T19:38:09.643462Z","iopub.status.idle":"2025-07-24T19:38:13.020385Z","shell.execute_reply.started":"2025-07-24T19:38:09.643440Z","shell.execute_reply":"2025-07-24T19:38:13.016784Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T15:11:27.775714Z","iopub.execute_input":"2025-07-24T15:11:27.776934Z","iopub.status.idle":"2025-07-24T15:11:27.825555Z","shell.execute_reply.started":"2025-07-24T15:11:27.776906Z","shell.execute_reply":"2025-07-24T15:11:27.820228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T15:11:27.827909Z","iopub.execute_input":"2025-07-24T15:11:27.828148Z","iopub.status.idle":"2025-07-24T15:11:27.841957Z","shell.execute_reply.started":"2025-07-24T15:11:27.828126Z","shell.execute_reply":"2025-07-24T15:11:27.836633Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = df.iloc[int(len(df) * 0.6):].reset_index(drop=True) ## only the last 40% rows","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T15:11:27.844154Z","iopub.execute_input":"2025-07-24T15:11:27.844396Z","iopub.status.idle":"2025-07-24T15:11:28.881303Z","shell.execute_reply.started":"2025-07-24T15:11:27.844373Z","shell.execute_reply":"2025-07-24T15:11:28.876062Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nimport optuna\nfrom sklearn.model_selection import KFold\nfrom sklearn.ensemble import RandomForestRegressor\nfrom scipy.stats import pearsonr\nimport pandas as pd\nimport numpy as np\n\ndf = df.drop(columns='index')\ndf = df.dropna()\n\ntarget = \"label\" \nfeatures = [col for col in df.columns if col != target]\nX = df[features]\ny = df[target]\n\n# Low Variance Filter \nfrom sklearn.feature_selection import VarianceThreshold\n\nvar_thresh = VarianceThreshold(threshold=0.01)\nX_reduced = var_thresh.fit_transform(X)\nreduced_features = X.columns[var_thresh.get_support()].tolist()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T15:12:19.088422Z","iopub.execute_input":"2025-07-24T15:12:19.088777Z","iopub.status.idle":"2025-07-24T15:12:23.637810Z","shell.execute_reply.started":"2025-07-24T15:12:19.088732Z","shell.execute_reply":"2025-07-24T15:12:23.632129Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_reduced.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T15:12:23.639318Z","iopub.execute_input":"2025-07-24T15:12:23.639666Z","iopub.status.idle":"2025-07-24T15:12:23.651660Z","shell.execute_reply.started":"2025-07-24T15:12:23.639642Z","shell.execute_reply":"2025-07-24T15:12:23.647526Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n#  Remove Highly Correlated Features \ndef remove_high_correlation(df, threshold=0.98):\n    corr_matrix = df.corr().abs()\n    upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool))\n    drop_cols = [column for column in upper.columns if any(upper[column] > threshold)]\n    return df.drop(columns=drop_cols), [c for c in df.columns if c not in drop_cols]\n\n\nX_filtered, filtered_features = remove_high_correlation(X[reduced_features])\nX = X_filtered\nfeatures = filtered_features  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T15:12:23.653864Z","iopub.execute_input":"2025-07-24T15:12:23.654496Z","iopub.status.idle":"2025-07-24T15:17:29.850074Z","shell.execute_reply.started":"2025-07-24T15:12:23.654470Z","shell.execute_reply":"2025-07-24T15:17:29.844170Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T15:17:29.850996Z","iopub.execute_input":"2025-07-24T15:17:29.851255Z","iopub.status.idle":"2025-07-24T15:17:29.865456Z","shell.execute_reply.started":"2025-07-24T15:17:29.851227Z","shell.execute_reply":"2025-07-24T15:17:29.861497Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.feature_selection import SelectKBest, f_regression\n\nX = X_filtered\ny = df[\"label\"]\n\n# Applying SelectKBest with ANOVA F-value\nselector = SelectKBest(score_func=f_regression, k='all')\nselector.fit(X, y)\n\n# Displaying scores for each feature\nfeature_scores = pd.DataFrame({'Feature': X.columns, 'Score': selector.scores_})\nprint(feature_scores.sort_values(by='Score', ascending=False))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T15:17:29.866274Z","iopub.execute_input":"2025-07-24T15:17:29.866502Z","iopub.status.idle":"2025-07-24T15:17:30.576157Z","shell.execute_reply.started":"2025-07-24T15:17:29.866475Z","shell.execute_reply":"2025-07-24T15:17:30.568153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"threshold = 1.0  \nselected_features = feature_scores[feature_scores['Score'] > threshold]['Feature'].tolist()\nX = X[selected_features]\nX.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T15:17:30.579743Z","iopub.execute_input":"2025-07-24T15:17:30.580156Z","iopub.status.idle":"2025-07-24T15:17:30.749615Z","shell.execute_reply.started":"2025-07-24T15:17:30.580105Z","shell.execute_reply":"2025-07-24T15:17:30.746169Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features = X.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T15:17:30.751370Z","iopub.execute_input":"2025-07-24T15:17:30.751621Z","iopub.status.idle":"2025-07-24T15:17:30.763444Z","shell.execute_reply.started":"2025-07-24T15:17:30.751596Z","shell.execute_reply":"2025-07-24T15:17:30.757742Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import time\nfrom lightgbm import LGBMRegressor\n\n# === Config ===\nSEED = 42\nkf = KFold(n_splits=5, shuffle=True, random_state=SEED)\nsplits = list(kf.split(X))\nN_TRIALS = 60\nMIN_FEATURES = 15\nPENALTY = 1e-4\n\n# === Loss Function ===\ndef negative_pearson(y_true, y_pred):\n    corr, _ = pearsonr(y_true, y_pred)\n    return -corr\n\n# === Optuna Objective ===\nclass FeatureSelectionOptuna:\n    def __init__(self, model, features, X, y, splits, loss_fn, min_features, penalty):\n        self.model = model\n        self.features = features\n        self.X = X\n        self.y = y\n        self.splits = splits\n        self.loss_fn = loss_fn\n        self.min_features = min_features\n        self.penalty = penalty\n\n    def __call__(self, trial):\n        # Use a single binary vector instead of categorical sampling\n        mask = [trial.suggest_int(f\"f_{i}\", 0, 1) for i in range(len(self.features))]\n        selected = [f for f, m in zip(self.features, mask) if m]\n\n        if len(selected) < self.min_features:\n            return float(\"inf\")\n\n        total_loss = 0\n        for train_idx, val_idx in self.splits:\n            X_train = self.X.iloc[train_idx][selected]\n            y_train = self.y.iloc[train_idx]\n            X_val = self.X.iloc[val_idx][selected]\n            y_val = self.y.iloc[val_idx]\n\n            self.model.fit(X_train, y_train)\n            preds = self.model.predict(X_val)\n            total_loss += self.loss_fn(y_val, preds)\n\n        avg_loss = total_loss / len(self.splits)\n        return avg_loss + self.penalty * len(selected)\n\n# === Define Study ===\nmodel = LGBMRegressor(n_jobs=-1, random_state=SEED)\n\nstudy = optuna.create_study(direction=\"minimize\", sampler=optuna.samplers.TPESampler(seed=SEED))\nstart = time.time()\n\nstudy.optimize(\n    FeatureSelectionOptuna(\n        model=model,\n        features=features,\n        X=X,\n        y=y,\n        splits=splits,\n        loss_fn=negative_pearson,\n        min_features=MIN_FEATURES,\n        penalty=PENALTY,\n    ),\n    n_trials=N_TRIALS,\n    show_progress_bar=True,\n)\n\nprint(f\"\\n Total time: {time.time() - start:.2f} seconds\")\nbest_features = [f for i, f in enumerate(features) if study.best_params[f\"f_{i}\"]]\nprint(\" Best features selected:\", best_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T15:18:01.054953Z","iopub.execute_input":"2025-07-24T15:18:01.055342Z","iopub.status.idle":"2025-07-24T15:48:09.206438Z","shell.execute_reply.started":"2025-07-24T15:18:01.055307Z","shell.execute_reply":"2025-07-24T15:48:09.200013Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_features = ['bid_qty', 'X2', 'X3', 'X9', 'X12', 'X18', 'X19', 'X21', 'X22', 'X23', 'X24', 'X25', 'X30', 'X34', 'X39', 'X40', 'X45',\n                 'X51', 'X58', 'X60', 'X61', 'X62', 'X66', 'X67', 'X68', 'X69', 'X71', 'X73', 'X75', 'X77', 'X84', 'X86', 'X90', 'X92', \n                 'X93', 'X95', 'X99', 'X100', 'X103', 'X106', 'X108', 'X112', 'X117', 'X121', 'X123', 'X124', 'X127', 'X129', 'X132',\n                 'X133', 'X135', 'X137', 'X138', 'X139', 'X144', 'X150', 'X151', 'X159', 'X163', 'X165', 'X167', 'X171', 'X173', 'X175',\n                 'X177', 'X180', 'X187', 'X189', 'X197', 'X199', 'X201', 'X202', 'X203', 'X205', 'X206', 'X207', 'X209', 'X212', 'X214',\n                 'X216', 'X219', 'X220', 'X223', 'X227', 'X230', 'X232', 'X235', 'X239', 'X250', 'X251', 'X253', 'X266', 'X267', 'X268',\n                 'X271', 'X273', 'X274', 'X276', 'X278', 'X281', 'X287', 'X298', 'X304', 'X306', 'X310', 'X311', 'X317', 'X319', 'X321',\n                 'X325', 'X327', 'X333', 'X334', 'X335', 'X338', 'X339', 'X340', 'X341', 'X342', 'X343', 'X346', 'X347', 'X353', 'X355',\n                 'X358', 'X360', 'X361', 'X364', 'X368', 'X370', 'X376', 'X378', 'X382', 'X386', 'X389', 'X391', 'X395', 'X398', 'X400',\n                 'X402', 'X406', 'X407', 'X408', 'X410', 'X415', 'X416', 'X419', 'X430', 'X431', 'X432', 'X436', 'X437', 'X443', 'X445',\n                 'X446', 'X448', 'X454', 'X459', 'X460', 'X463', 'X468', 'X469', 'X471', 'X473', 'X474', 'X476', 'X477', 'X478', 'X480',\n                 'X482', 'X485', 'X487', 'X495', 'X496', 'X501', 'X503', 'X504', 'X506', 'X507', 'X510', 'X511', 'X514', 'X519', 'X520',\n                 'X526', 'X529', 'X532', 'X533', 'X535', 'X537', 'X540', 'X544', 'X545', 'X547', 'X549', 'X552', 'X560', 'X562', 'X564',\n                 'X568', 'X569', 'X576', 'X577', 'X580', 'X581', 'X582', 'X584', 'X588', 'X589', 'X596', 'X601', 'X602', 'X603', 'X605',\n                 'X606', 'X608', 'X610', 'X612', 'X614', 'X620', 'X626', 'X629', 'X630', 'X633', 'X640', 'X642', 'X646', 'X648', 'X657',\n                 'X659', 'X666', 'X669', 'X670', 'X674', 'X677', 'X679', 'X684', 'X688', 'X692', 'X693', 'X694', 'X696', 'X697', 'X701',\n                 'X709', 'X710', 'X713', 'X714', 'X716', 'X720','X725', 'X728', 'X733', 'X734', 'X738', 'X742', 'X744', 'X746', 'X750',\n                 'X752', 'X753', 'X755', 'X756', 'X759', 'X760', 'X762', 'X768', 'X770', 'X772', 'X778']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T19:38:22.581881Z","iopub.execute_input":"2025-07-24T19:38:22.582175Z","iopub.status.idle":"2025-07-24T19:38:22.597256Z","shell.execute_reply.started":"2025-07-24T19:38:22.582151Z","shell.execute_reply":"2025-07-24T19:38:22.593238Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[best_features].shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T19:38:23.550345Z","iopub.execute_input":"2025-07-24T19:38:23.550801Z","iopub.status.idle":"2025-07-24T19:38:24.398615Z","shell.execute_reply.started":"2025-07-24T19:38:23.550757Z","shell.execute_reply":"2025-07-24T19:38:24.393328Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ensemble ridge","metadata":{}},{"cell_type":"code","source":"df = df.iloc[int(len(df) * 0.4):].reset_index(drop=True) ## only the last 40% rows","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T19:38:28.828263Z","iopub.execute_input":"2025-07-24T19:38:28.828539Z","iopub.status.idle":"2025-07-24T19:38:30.362066Z","shell.execute_reply.started":"2025-07-24T19:38:28.828515Z","shell.execute_reply":"2025-07-24T19:38:30.357474Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import KFold\nfrom sklearn.linear_model import RidgeCV\nfrom sklearn.metrics import make_scorer\nfrom sklearn.ensemble import RandomForestRegressor\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom scipy.stats import pearsonr\nfrom lightgbm import LGBMRegressor\n\n# ------------------ Config ------------------ #\nEARLY_PERCENTAGE = 0.4\n\nclass Config:\n    FEATURES = best_features\n    LABEL_COLUMN = \"label\"\n    N_FOLDS = 5\n    RANDOM_STATE = 42\n\n# ------------------ Load and Prepare Data ------------------ #\n# Ensure no 'label' in features\nfeatures = [f for f in Config.FEATURES if f != Config.LABEL_COLUMN]\n\n# Defensive copies to avoid SettingWithCopyWarning\ntrain_df = df[features + [Config.LABEL_COLUMN]].copy()\ntest_df = test[features].copy()\nsubmission_df = sub.copy()\n\n# Optional: optimize memory\ntrain_df[features] = train_df[features].astype(np.float32)\ntrain_df[Config.LABEL_COLUMN] = train_df[Config.LABEL_COLUMN].astype(np.float32)\ntest_df[features] = test_df[features].astype(np.float32)\n\nX_train = train_df[features]\ny_train = train_df[Config.LABEL_COLUMN]\n\n# ------------------ Base Models ------------------ #\nMODELS = [\n    (\"xgb\", XGBRegressor(\n        tree_method='hist',\n        device='gpu',\n        n_jobs=-1,\n        learning_rate=0.015,\n        max_depth=6,\n        subsample=0.7,\n        colsample_bytree=0.5,\n        reg_alpha=10,\n        reg_lambda=80,\n        n_estimators=400,\n        verbosity=0,\n        random_state=Config.RANDOM_STATE\n    )),\n    (\"lgb\", LGBMRegressor(\n        boosting_type='gbdt',\n        objective='regression',\n        learning_rate=0.01,\n        n_estimators=400,\n        max_depth=6,\n        subsample=0.7,\n        colsample_bytree=0.5,\n        reg_alpha=5,\n        reg_lambda=80,\n        random_state=Config.RANDOM_STATE\n    )),\n    (\"rf\", RandomForestRegressor(\n        n_estimators=300,\n        max_depth=10,\n        n_jobs=-1,\n        random_state=Config.RANDOM_STATE\n    ))\n]\n\n# ------------------ Ridge Stacker ------------------ #\ndef negative_pearson(y_true, y_pred):\n    return -pearsonr(y_true, y_pred)[0]\n\nscorer = make_scorer(negative_pearson, greater_is_better=False)\n\nkf = KFold(n_splits=Config.N_FOLDS, shuffle=True, random_state=Config.RANDOM_STATE)\n\nmeta_train = np.zeros((len(X_train), len(MODELS)))\nmeta_test = np.zeros((len(test_df), len(MODELS)))\n\nfor i, (name, model) in enumerate(MODELS):\n    print(f\"\\nTraining base model: {name}\")\n    fold_preds = np.zeros(len(X_train))\n    fold_test = np.zeros((len(test_df), Config.N_FOLDS))\n\n    for fold, (train_idx, val_idx) in enumerate(kf.split(X_train)):\n        print(f\"  Fold {fold+1}/{Config.N_FOLDS}\")\n        model.fit(X_train.iloc[train_idx], y_train.iloc[train_idx])\n        fold_preds[val_idx] = model.predict(X_train.iloc[val_idx])\n        fold_test[:, fold] = model.predict(test_df)\n\n    meta_train[:, i] = fold_preds\n    meta_test[:, i] = fold_test.mean(axis=1)\n\n# ------------------ RidgeCV Stacking ------------------ #\nprint(\"\\nTraining RidgeCV stacker\")\nridge = RidgeCV(alphas=np.logspace(-4, 2, 50), cv=Config.N_FOLDS, scoring=scorer)\nridge.fit(meta_train, y_train)\n\nfinal_preds = ridge.predict(meta_test)\nsubmission_df[\"prediction\"] = final_preds\nsubmission_df.to_csv(\"stacked_submission.csv\", index=False)\n\nprint(\"\\nSaved: stacked_submission.csv\")\nprint(\"Ridge Coefficients:\", ridge.coef_)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T17:53:00.498886Z","iopub.execute_input":"2025-07-24T17:53:00.499253Z","iopub.status.idle":"2025-07-24T18:40:03.347906Z","shell.execute_reply.started":"2025-07-24T17:53:00.499225Z","shell.execute_reply":"2025-07-24T18:40:03.341139Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# NN ","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, TensorDataset, random_split\nimport pandas as pd\n\nclass Autoencoder(nn.Module):\n    def __init__(self, input_dim, bottleneck_dim=128):\n        super().__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, 512),\n            nn.ReLU(),\n            nn.LayerNorm(512),\n            nn.Linear(512, 256),\n            nn.ReLU(),\n            nn.Linear(256, bottleneck_dim)\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(bottleneck_dim, 256),\n            nn.ReLU(),\n            nn.Linear(256, 512),\n            nn.ReLU(),\n            nn.Linear(512, input_dim)\n        )\n\n    def forward(self, x):\n        z = self.encoder(x)\n        return self.decoder(z)\n\n    def encode(self, x):\n        return self.encoder(x)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T19:38:34.485335Z","iopub.execute_input":"2025-07-24T19:38:34.485633Z","iopub.status.idle":"2025-07-24T19:39:01.261667Z","shell.execute_reply.started":"2025-07-24T19:38:34.485607Z","shell.execute_reply":"2025-07-24T19:39:01.255681Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler\nimport numpy as np\nimport pandas as pd\n\ndef smart_scaling(df: pd.DataFrame):\n    \"\"\"Scale each column with an optimal scaler based on distribution heuristics.\"\"\"\n    scaled_data = {}\n    scalers = {}\n\n    for col in df.columns:\n        values = df[col].values.astype(np.float32)\n        skewness = pd.Series(values).skew()\n\n        if abs(skewness) > 1.5:\n            scaler = RobustScaler()\n        elif np.min(values) >= 0 and np.max(values) <= 1e3:\n            scaler = MinMaxScaler()\n        else:\n            scaler = StandardScaler()\n\n        reshaped = values.reshape(-1, 1)\n        scaled = scaler.fit_transform(reshaped).ravel()\n        scaled_data[col] = scaled\n        scalers[col] = scaler  # Optional: if you want to transform test data later\n\n    return pd.DataFrame(scaled_data, columns=df.columns, dtype=np.float32), scalers\n    \nX_top, _ = smart_scaling(df[best_features])  # Only keep the scaled DataFrame\n\ny = df['label']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T19:39:01.264093Z","iopub.execute_input":"2025-07-24T19:39:01.264503Z","iopub.status.idle":"2025-07-24T19:39:04.192530Z","shell.execute_reply.started":"2025-07-24T19:39:01.264474Z","shell.execute_reply":"2025-07-24T19:39:04.186326Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_autoencoder(X_top, bottleneck_dim=128, epochs=200, batch_size=2048, patience=20):\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    X_tensor = torch.tensor(X_top.values, dtype=torch.float32)\n    dataset = TensorDataset(X_tensor)\n    loader = DataLoader(dataset, batch_size=batch_size, shuffle=True, pin_memory=True)\n\n    model = Autoencoder(input_dim=X_top.shape[1], bottleneck_dim=bottleneck_dim).to(device)\n    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=1e-4)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', patience=5, factor=0.5, verbose=True)\n    criterion = nn.MSELoss()\n\n    best_loss = float('inf')\n    wait = 0\n\n    model.train()\n    for epoch in range(epochs):\n        total_loss = 0\n        for (x_batch,) in loader:\n            x_batch = x_batch.to(device)\n            optimizer.zero_grad()\n            recon = model(x_batch)\n            loss = criterion(recon, x_batch)\n            loss.backward()\n            optimizer.step()\n            total_loss += loss.item() * x_batch.size(0)\n\n        epoch_loss = total_loss / len(loader.dataset)\n        scheduler.step(epoch_loss)\n\n        print(f\"Epoch {epoch+1:03d} | Loss: {epoch_loss:.6f}\")\n\n        # Early stopping\n        if epoch_loss < best_loss:\n            best_loss = epoch_loss\n            wait = 0\n            torch.save(model.state_dict(), \"best_autoencoder.pt\")\n        else:\n            wait += 1\n            if wait >= patience:\n                print(\"Early stopping triggered.\")\n                break\n\n    model.load_state_dict(torch.load(\"best_autoencoder.pt\"))\n    return model.eval(), device","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T19:40:44.070215Z","iopub.execute_input":"2025-07-24T19:40:44.070746Z","iopub.status.idle":"2025-07-24T19:40:44.088303Z","shell.execute_reply.started":"2025-07-24T19:40:44.070710Z","shell.execute_reply":"2025-07-24T19:40:44.082163Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train and extract compressed features\nautoencoder, device = train_autoencoder(X_top)\n\nwith torch.no_grad():\n    compressed_features = autoencoder.encode(\n        torch.tensor(X_top.values, dtype=torch.float32).to(device)\n    ).cpu().numpy()\n\ncompressed_df = pd.DataFrame(compressed_features, columns=[f'ae_{i}' for i in range(compressed_features.shape[1])])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T19:40:45.881583Z","iopub.execute_input":"2025-07-24T19:40:45.881951Z","iopub.status.idle":"2025-07-24T20:03:10.896909Z","shell.execute_reply.started":"2025-07-24T19:40:45.881921Z","shell.execute_reply":"2025-07-24T20:03:10.891281Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"compressed_features = autoencoder.encode(\n    torch.tensor(X_top.values, dtype=torch.float32).to(device)\n).detach().cpu().numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T20:03:10.898871Z","iopub.execute_input":"2025-07-24T20:03:10.899549Z","iopub.status.idle":"2025-07-24T20:03:11.506629Z","shell.execute_reply.started":"2025-07-24T20:03:10.899521Z","shell.execute_reply":"2025-07-24T20:03:11.502674Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, TensorDataset\nimport numpy as np\n# ====================== Model ====================== #\nclass CryptoNetV3(nn.Module):\n    def __init__(self, input_dim):\n        super().__init__()\n\n        self.norm_input = nn.LayerNorm(input_dim)\n\n        self.block1 = nn.Sequential(\n            nn.Linear(input_dim, 512),\n            nn.GELU(),\n            nn.LayerNorm(512),\n            nn.Dropout(0.2)\n        )\n        self.block2 = nn.Sequential(\n            nn.Linear(512, 256),\n            nn.GELU(),\n            nn.LayerNorm(256),\n            nn.Dropout(0.3)\n        )\n        self.block3 = nn.Sequential(\n            nn.Linear(256, 128),\n            nn.GELU(),\n            nn.LayerNorm(128),\n            nn.Dropout(0.3)\n        )\n        self.block4 = nn.Sequential(\n            nn.Linear(128, 64),\n            nn.GELU(),\n            nn.LayerNorm(64),\n            nn.Dropout(0.2)\n        )\n        self.output_layer = nn.Linear(64, 1)\n\n        self._init_weights()\n\n    def forward(self, x):\n        x = self.norm_input(x)\n        x = self.block1(x)\n        x = self.block2(x)\n        x = self.block3(x)\n        x = self.block4(x)\n        return self.output_layer(x)\n\n    def _init_weights(self):\n        for m in self.modules():\n            if isinstance(m, nn.Linear):\n                nn.init.kaiming_normal_(m.weight, nonlinearity='relu')\n                nn.init.constant_(m.bias, 0)\n\n# ====================== Pearson Correlation ====================== #\ndef pearson_corr(y_true, y_pred):\n    vx = y_true - y_true.mean()\n    vy = y_pred - y_pred.mean()\n    corr = (vx * vy).sum() / (torch.sqrt((vx ** 2).sum()) * torch.sqrt((vy ** 2).sum()))\n    return corr.item() if not torch.isnan(corr) else 0.0\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T20:03:11.510651Z","iopub.execute_input":"2025-07-24T20:03:11.510922Z","iopub.status.idle":"2025-07-24T20:03:11.528535Z","shell.execute_reply.started":"2025-07-24T20:03:11.510875Z","shell.execute_reply":"2025-07-24T20:03:11.522692Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch.utils.data import random_split, DataLoader, TensorDataset\n\n# Make sure compressed_features is a numpy array\nX_tensor = torch.tensor(compressed_features, dtype=torch.float32)\n\n# If y_recent is a pandas Series, use `.values`; if it's already a numpy array, just use it directly\nif isinstance(y, pd.Series):\n    y_array = y.values\nelse:\n    y_array = y\n\ny_tensor = torch.tensor(y_array.reshape(-1, 1), dtype=torch.float32)\n\n# Create dataset\ndataset = TensorDataset(X_tensor, y_tensor)\n\n# Split sizes\ntrain_size = int(0.8 * len(dataset))\nval_size = len(dataset) - train_size\n\n# Split dataset\ntrain_set, val_set = random_split(dataset, [train_size, val_size])\n\n# DataLoaders\ntrain_loader = DataLoader(train_set, batch_size=1024, shuffle=True)\nval_loader = DataLoader(val_set, batch_size=1024)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T20:03:11.530105Z","iopub.execute_input":"2025-07-24T20:03:11.530481Z","iopub.status.idle":"2025-07-24T20:03:11.573189Z","shell.execute_reply.started":"2025-07-24T20:03:11.530457Z","shell.execute_reply":"2025-07-24T20:03:11.567989Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, TensorDataset\nimport numpy as np\nimport random\n\n# For reproducibility\ndef set_seed(seed=42):\n    torch.manual_seed(seed)\n    np.random.seed(seed)\n    random.seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n\nset_seed(42)\n\n# ====================== Improved CryptoNet ====================== #\nclass CryptoNetV4(nn.Module):\n    def __init__(self, inp):\n        super().__init__()\n        self.ln0 = nn.LayerNorm(inp)\n        def block(in_f, out_f, p):\n            return nn.Sequential(\n                nn.Linear(in_f, out_f),\n                nn.SiLU(),\n                nn.Dropout(p),\n                nn.LayerNorm(out_f),\n            )\n        self.b1 = block(inp, 512, 0.2)\n        self.b2 = block(512, 256, 0.3)\n        self.b3 = block(256, 128, 0.3)\n        self.b4 = block(128, 64, 0.2)\n        self.out = nn.Linear(64, 1)\n        self._init()\n    def _init(self):\n        for m in self.modules():\n            if isinstance(m, nn.Linear):\n                nn.init.kaiming_normal_(m.weight, nonlinearity='relu')\n                nn.init.zeros_(m.bias)\n    def forward(self, x):\n        x = self.ln0(x)\n        for b in (self.b1, self.b2, self.b3, self.b4):\n            x = b(x)\n        return self.out(x).squeeze(-1)\n\n\n# ====================== Pearson Correlation ====================== #\ndef pearson_corr(y_true, y_pred):\n    vx = y_true - y_true.mean()\n    vy = y_pred - y_pred.mean()\n    corr = (vx * vy).sum() / (torch.sqrt((vx ** 2).sum()) * torch.sqrt((vy ** 2).sum()))\n    return corr.item() if not torch.isnan(corr) else 0.0\n\n# ====================== Training Loop ====================== #\ndef train_model(X_train, y_train, X_val, y_val, input_dim, batch_size=1024, lr=5e-4, epochs=200, patience=20):\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n    train_loader = DataLoader(TensorDataset(X_train, y_train), batch_size=batch_size, shuffle=True, pin_memory=True)\n    val_loader = DataLoader(TensorDataset(X_val, y_val), batch_size=batch_size, pin_memory=True)\n\n    model = CryptoNetV3(input_dim=input_dim).to(device)\n    optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-4)\n    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs)\n    criterion = nn.MSELoss()\n\n    best_corr = -np.inf\n    wait = 0\n\n    for epoch in range(1, epochs + 1):\n        model.train()\n        total_loss = 0.0\n        for xb, yb in train_loader:\n            xb, yb = xb.to(device), yb.to(device)\n            optimizer.zero_grad()\n            preds = model(xb)\n            loss = criterion(preds, yb)\n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n            optimizer.step()\n            total_loss += loss.item()\n\n        scheduler.step()\n\n        # Validation\n        model.eval()\n        val_preds, val_targets = [], []\n        with torch.no_grad():\n            for xb, yb in val_loader:\n                xb = xb.to(device)\n                preds = model(xb).cpu()\n                val_preds.append(preds)\n                val_targets.append(yb)\n\n        val_preds = torch.cat(val_preds)\n        val_targets = torch.cat(val_targets)\n        val_corr = pearson_corr(val_targets, val_preds)\n\n        print(f\"Epoch {epoch:03d} | Loss: {total_loss:.4f} | Val Corr: {val_corr:.5f}\")\n\n        if val_corr > best_corr:\n            best_corr = val_corr\n            wait = 0\n            torch.save(model.state_dict(), \"best_model.pt\")\n        else:\n            wait += 1\n            if wait >= patience:\n                print(f\"Early stopping at epoch {epoch}\")\n                break\n\n    model.load_state_dict(torch.load(\"best_model.pt\"))\n    return model\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T20:15:46.433806Z","iopub.execute_input":"2025-07-24T20:15:46.434123Z","iopub.status.idle":"2025-07-24T20:15:46.462744Z","shell.execute_reply.started":"2025-07-24T20:15:46.434098Z","shell.execute_reply":"2025-07-24T20:15:46.455945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_tensor = torch.tensor(compressed_df.values, dtype=torch.float32)\ny_tensor = torch.tensor(df[\"label\"].values, dtype=torch.float32)\n\n# Split to train/val\nval_split = 0.2\nsplit_idx = int(len(X_tensor) * (1 - val_split))\nX_train, X_val = X_tensor[:split_idx], X_tensor[split_idx:]\ny_train, y_val = y_tensor[:split_idx], y_tensor[split_idx:]\n\nmodel = train_model(\n    X_train, y_train, X_val, y_val,\n    input_dim=X_tensor.shape[1],\n    lr=3e-4,\n    epochs=300,\n    batch_size=1024,\n    patience=10\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T20:15:47.944345Z","iopub.execute_input":"2025-07-24T20:15:47.944662Z","iopub.status.idle":"2025-07-24T20:17:49.440190Z","shell.execute_reply.started":"2025-07-24T20:15:47.944635Z","shell.execute_reply":"2025-07-24T20:17:49.434541Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## SUBMISSION\n# 1. Encode the test features using the trained autoencoder\nX_test  = test[best_features]\nautoencoder.eval()\nwith torch.no_grad():\n    test_tensor = torch.tensor(X_test.values, dtype=torch.float32).to(device)\n    compressed_test = autoencoder.encode(test_tensor).cpu().numpy()\n\n# 2. Create test DataLoader\ntest_dataset = TensorDataset(torch.tensor(compressed_test, dtype=torch.float32))\ntest_loader = DataLoader(test_dataset, batch_size=2048, shuffle=False)\n\n# 3. Predict using CryptoNetV2\nmodel.eval()\ntest_preds = []\n\nwith torch.no_grad():\n    for (X_batch,) in test_loader:\n        X_batch = X_batch.to(device)\n        preds = model(X_batch).squeeze().cpu().numpy()\n        test_preds.extend(preds)\n\n# 4. Create submission\nsubmission = sub\nsubmission[\"prediction\"] = test_preds\nsubmission.to_csv(\"submission_autoencodernet.csv\", index=False)\n\nprint(\"✅ Submission saved as: submission_autoencodernet.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T20:09:35.258385Z","iopub.execute_input":"2025-07-24T20:09:35.258701Z","iopub.status.idle":"2025-07-24T20:09:41.683028Z","shell.execute_reply.started":"2025-07-24T20:09:35.258676Z","shell.execute_reply":"2025-07-24T20:09:41.677464Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}