{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import KFold\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom sklearn.linear_model import (\n    HuberRegressor, RANSACRegressor, TheilSenRegressor,\n    Lasso, ElasticNet, Ridge\n)\nfrom sklearn.cross_decomposition import PLSRegression\nfrom sklearn.preprocessing import StandardScaler, RobustScaler\nfrom sklearn.ensemble import RandomForestRegressor\nfrom scipy.stats import pearsonr\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# ===== Feature Engineering =====\ndef feature_engineering(df):\n    \"\"\"Original features plus new robust features\"\"\"\n    # Original features\n    df['volume_weighted_sell'] = df['sell_qty'] * df['volume']\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-8)\n    df['selling_pressure'] = df['sell_qty'] / (df['volume'] + 1e-8)\n    df['effective_spread_proxy'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['volume'] + 1e-8)\n    \n    # New robust features\n    df['log_volume'] = np.log1p(df['volume'])\n    df['bid_ask_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['bid_qty'] + df['ask_qty'] + 1e-8)\n    df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + 1e-8)\n    df['liquidity_ratio'] = (df['bid_qty'] + df['ask_qty']) / (df['volume'] + 1e-8)\n    \n    # Handle infinities and NaN\n    df = df.replace([np.inf, -np.inf], np.nan)\n    \n    # For each column, replace NaN with median for robustness\n    for col in df.columns:\n        if df[col].isna().any():\n            median_val = df[col].median()\n            df[col] = df[col].fillna(median_val if not pd.isna(median_val) else 0)\n    \n    return df\n\n# ===== Configuration =====\nclass Config:\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    \n    # Original features plus additional market features\n    FEATURES = [\n        \"X863\", \"X856\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X855\", \"X174\", \"X302\", \"X178\", \"X168\", \"X612\",\n        \"buy_qty\", \"sell_qty\", \"volume\", \"X888\", \"X421\", \"X333\",\n        \"bid_qty\", \"ask_qty\"\n    ]\n    \n    LABEL_COLUMN = \"label\"\n    N_FOLDS = 3\n    RANDOM_STATE = 42\n\n# ===== Model Parameters =====\n# Original XGBoost parameters\nXGB_PARAMS = {\n    \"tree_method\": \"hist\",\n    \"device\": \"gpu\",\n    \"colsample_bylevel\": 0.4778,\n    \"colsample_bynode\": 0.3628,\n    \"colsample_bytree\": 0.7107,\n    \"gamma\": 1.7095,\n    \"learning_rate\": 0.02213,\n    \"max_depth\": 20,\n    \"max_leaves\": 12,\n    \"min_child_weight\": 16,\n    \"n_estimators\": 1667,\n    \"subsample\": 0.06567,\n    \"reg_alpha\": 39.3524,\n    \"reg_lambda\": 75.4484,\n    \"verbosity\": 0,\n    \"random_state\": Config.RANDOM_STATE,\n    \"n_jobs\": -1\n}\n\n# LightGBM parameters (simpler model for noisy data)\nLGBM_PARAMS = {\n    \"n_estimators\": 500,\n    \"learning_rate\": 0.03,\n    \"num_leaves\": 31,\n    \"min_child_samples\": 50,\n    \"subsample\": 0.8,\n    \"colsample_bytree\": 0.8,\n    \"reg_alpha\": 10,\n    \"reg_lambda\": 10,\n    \"random_state\": Config.RANDOM_STATE,\n    \"device\": \"gpu\",\n    \"verbosity\": -1,\n    \"n_jobs\": -1\n}\n\n# Define all learners\nLEARNERS = [\n    {\"name\": \"xgb_baseline\", \"Estimator\": XGBRegressor, \"params\": XGB_PARAMS, \"need_scale\": False},\n    {\"name\": \"lgbm\", \"Estimator\": LGBMRegressor, \"params\": LGBM_PARAMS, \"need_scale\": False},\n    {\"name\": \"huber\", \"Estimator\": HuberRegressor, \"params\": {\"epsilon\": 1.5, \"alpha\": 0.01, \"max_iter\": 500}, \"need_scale\": True},\n    {\"name\": \"ransac\", \"Estimator\": RANSACRegressor, \"params\": {\"min_samples\": 0.7, \"max_trials\": 100, \"random_state\": Config.RANDOM_STATE}, \"need_scale\": True},\n    {\"name\": \"theilsen\", \"Estimator\": TheilSenRegressor, \"params\": {\"max_subpopulation\": 10000, \"random_state\": Config.RANDOM_STATE}, \"need_scale\": True},\n    {\"name\": \"lasso\", \"Estimator\": Lasso, \"params\": {\"alpha\": 0.001, \"max_iter\": 1000}, \"need_scale\": True},\n    {\"name\": \"elasticnet\", \"Estimator\": ElasticNet, \"params\": {\"alpha\": 0.001, \"l1_ratio\": 0.5, \"max_iter\": 1000}, \"need_scale\": True},\n    {\"name\": \"pls\", \"Estimator\": PLSRegression, \"params\": {\"n_components\": 50}, \"need_scale\": True},\n]\n\n# ===== Data Loading =====\ndef create_time_decay_weights(n: int, decay: float = 0.9) -> np.ndarray:\n    \"\"\"Create time decay weights for more recent data importance\"\"\"\n    positions = np.arange(n)\n    normalized = positions / (n - 1)\n    weights = decay ** (1.0 - normalized)\n    return weights * n / weights.sum()\n\ndef load_data():\n    \"\"\"Load and preprocess data\"\"\"\n    train_df = pd.read_parquet(Config.TRAIN_PATH, columns=Config.FEATURES + [Config.LABEL_COLUMN])\n    test_df = pd.read_parquet(Config.TEST_PATH, columns=Config.FEATURES)\n    submission_df = pd.read_csv(Config.SUBMISSION_PATH)\n    \n    # Apply feature engineering\n    train_df = feature_engineering(train_df)\n    test_df = feature_engineering(test_df)\n    \n    # Update features list with engineered features\n    engineered_features = [\n        \"volume_weighted_sell\", \"buy_sell_ratio\", \"selling_pressure\", \n        \"effective_spread_proxy\", \"log_volume\", \"bid_ask_imbalance\",\n        \"order_flow_imbalance\", \"liquidity_ratio\"\n    ]\n    Config.FEATURES = list(set(Config.FEATURES + engineered_features))\n    \n    print(f\"Loaded data - Train: {train_df.shape}, Test: {test_df.shape}, Submission: {submission_df.shape}\")\n    print(f\"Total features: {len(Config.FEATURES)}\")\n    \n    return train_df.reset_index(drop=True), test_df.reset_index(drop=True), submission_df\n\n# ===== Model Training =====\ndef get_model_slices(n_samples: int):\n    \"\"\"Define different data slices for training\"\"\"\n    return [\n        {\"name\": \"full_data\", \"cutoff\": 0},\n        {\"name\": \"last_75pct\", \"cutoff\": int(0.25 * n_samples)},\n        {\"name\": \"last_50pct\", \"cutoff\": int(0.50 * n_samples)},\n    ]\n\ndef train_single_model(X_train, y_train, X_valid, y_valid, X_test, learner, sample_weights=None):\n    \"\"\"Train a single model with appropriate scaling if needed\"\"\"\n    if learner[\"need_scale\"]:\n        scaler = RobustScaler()  # More robust to outliers than StandardScaler\n        X_train_scaled = scaler.fit_transform(X_train)\n        X_valid_scaled = scaler.transform(X_valid)\n        X_test_scaled = scaler.transform(X_test)\n    else:\n        X_train_scaled = X_train\n        X_valid_scaled = X_valid\n        X_test_scaled = X_test\n    \n    model = learner[\"Estimator\"](**learner[\"params\"])\n    \n    # Handle different model training approaches\n    if learner[\"name\"] in [\"xgb_baseline\", \"lgbm\"]:\n        if learner[\"name\"] == \"xgb_baseline\":\n            model.fit(X_train_scaled, y_train, sample_weight=sample_weights, \n                     eval_set=[(X_valid_scaled, y_valid)], verbose=False)\n        else:  # LightGBM\n            model.fit(X_train_scaled, y_train, sample_weight=sample_weights,\n                     eval_set=[(X_valid_scaled, y_valid)], callbacks=[])\n    elif learner[\"name\"] in [\"huber\", \"lasso\", \"elasticnet\"]:\n        model.fit(X_train_scaled, y_train, sample_weight=sample_weights)\n    else:\n        # RANSAC, TheilSen, PLS don't support sample weights\n        model.fit(X_train_scaled, y_train)\n    \n    valid_pred = model.predict(X_valid_scaled)\n    test_pred = model.predict(X_test_scaled)\n    \n    return valid_pred, test_pred\n\ndef train_and_evaluate(train_df, test_df):\n    \"\"\"Train all models with cross-validation\"\"\"\n    n_samples = len(train_df)\n    model_slices = get_model_slices(n_samples)\n    \n    # Initialize prediction dictionaries\n    oof_preds = {\n        learner[\"name\"]: {s[\"name\"]: np.zeros(n_samples) for s in model_slices}\n        for learner in LEARNERS\n    }\n    test_preds = {\n        learner[\"name\"]: {s[\"name\"]: np.zeros(len(test_df)) for s in model_slices}\n        for learner in LEARNERS\n    }\n    \n    full_weights = create_time_decay_weights(n_samples)\n    kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n    \n    for fold, (train_idx, valid_idx) in enumerate(kf.split(train_df), start=1):\n        print(f\"\\n--- Fold {fold}/{Config.N_FOLDS} ---\")\n        X_valid = train_df.iloc[valid_idx][Config.FEATURES]\n        y_valid = train_df.iloc[valid_idx][Config.LABEL_COLUMN]\n        X_test = test_df[Config.FEATURES]\n        \n        for s in model_slices:\n            cutoff = s[\"cutoff\"]\n            slice_name = s[\"name\"]\n            subset = train_df.iloc[cutoff:].reset_index(drop=True)\n            rel_idx = train_idx[train_idx >= cutoff] - cutoff\n            \n            if len(rel_idx) == 0:\n                continue\n                \n            X_train = subset.iloc[rel_idx][Config.FEATURES]\n            y_train = subset.iloc[rel_idx][Config.LABEL_COLUMN]\n            sw = create_time_decay_weights(len(subset))[rel_idx] if cutoff > 0 else full_weights[train_idx]\n            \n            print(f\"  Training slice: {slice_name}, samples: {len(X_train)}\")\n            \n            for learner in LEARNERS:\n                try:\n                    valid_pred, test_pred = train_single_model(\n                        X_train, y_train, X_valid, y_valid, X_test, learner, sw\n                    )\n                    \n                    # Store OOF predictions\n                    mask = valid_idx >= cutoff\n                    if mask.any():\n                        idxs = valid_idx[mask]\n                        X_valid_subset = train_df.iloc[idxs][Config.FEATURES]\n                        if learner[\"need_scale\"]:\n                            scaler = RobustScaler()\n                            scaler.fit(X_train)\n                            valid_pred_subset = learner[\"Estimator\"](**learner[\"params\"]).fit(\n                                scaler.transform(X_train), y_train\n                            ).predict(scaler.transform(X_valid_subset))\n                            oof_preds[learner[\"name\"]][slice_name][idxs] = valid_pred_subset\n                        else:\n                            oof_preds[learner[\"name\"]][slice_name][idxs] = valid_pred[mask]\n                    \n                    if cutoff > 0 and (~mask).any():\n                        oof_preds[learner[\"name\"]][slice_name][valid_idx[~mask]] = \\\n                            oof_preds[learner[\"name\"]][\"full_data\"][valid_idx[~mask]]\n                    \n                    test_preds[learner[\"name\"]][slice_name] += test_pred\n                    \n                except Exception as e:\n                    print(f\"    Error training {learner['name']}: {str(e)}\")\n                    continue\n    \n    # Normalize test predictions\n    for learner_name in test_preds:\n        for slice_name in test_preds[learner_name]:\n            test_preds[learner_name][slice_name] /= Config.N_FOLDS\n    \n    return oof_preds, test_preds, model_slices\n\n# ===== Ensemble and Submission =====\ndef create_submissions(train_df, oof_preds, test_preds, submission_df):\n    \"\"\"Create multiple submission files for different strategies\"\"\"\n    all_submissions = {}\n    \n    # 1. Original baseline (XGBoost only)\n    if \"xgb_baseline\" in oof_preds:\n        xgb_oof = np.mean(list(oof_preds[\"xgb_baseline\"].values()), axis=0)\n        xgb_test = np.mean(list(test_preds[\"xgb_baseline\"].values()), axis=0)\n        xgb_score = pearsonr(train_df[Config.LABEL_COLUMN], xgb_oof)[0]\n        print(f\"\\nXGBoost Baseline Score: {xgb_score:.4f}\")\n        \n        submission_xgb = submission_df.copy()\n        submission_xgb[\"prediction\"] = xgb_test\n        submission_xgb.to_csv(\"submission_xgb_baseline.csv\", index=False)\n        all_submissions[\"xgb_baseline\"] = xgb_score\n    \n    # 2. Robust methods ensemble\n    robust_methods = [\"huber\", \"ransac\", \"theilsen\"]\n    robust_oof_list = []\n    robust_test_list = []\n    \n    for method in robust_methods:\n        if method in oof_preds:\n            method_oof = np.mean(list(oof_preds[method].values()), axis=0)\n            method_test = np.mean(list(test_preds[method].values()), axis=0)\n            method_score = pearsonr(train_df[Config.LABEL_COLUMN], method_oof)[0]\n            print(f\"{method.upper()} Score: {method_score:.4f}\")\n            \n            if not np.isnan(method_score):\n                robust_oof_list.append(method_oof)\n                robust_test_list.append(method_test)\n    \n    if robust_oof_list:\n        robust_oof = np.mean(robust_oof_list, axis=0)\n        robust_test = np.mean(robust_test_list, axis=0)\n        robust_score = pearsonr(train_df[Config.LABEL_COLUMN], robust_oof)[0]\n        print(f\"\\nRobust Ensemble Score: {robust_score:.4f}\")\n        \n        submission_robust = submission_df.copy()\n        submission_robust[\"prediction\"] = robust_test\n        submission_robust.to_csv(\"submission_robust_ensemble.csv\", index=False)\n        all_submissions[\"robust_ensemble\"] = robust_score\n    \n    # 3. Regularized methods ensemble\n    regularized_methods = [\"lasso\", \"elasticnet\"]\n    reg_oof_list = []\n    reg_test_list = []\n    \n    for method in regularized_methods:\n        if method in oof_preds:\n            method_oof = np.mean(list(oof_preds[method].values()), axis=0)\n            method_test = np.mean(list(test_preds[method].values()), axis=0)\n            method_score = pearsonr(train_df[Config.LABEL_COLUMN], method_oof)[0]\n            print(f\"{method.upper()} Score: {method_score:.4f}\")\n            \n            if not np.isnan(method_score):\n                reg_oof_list.append(method_oof)\n                reg_test_list.append(method_test)\n    \n    if reg_oof_list:\n        reg_oof = np.mean(reg_oof_list, axis=0)\n        reg_test = np.mean(reg_test_list, axis=0)\n        reg_score = pearsonr(train_df[Config.LABEL_COLUMN], reg_oof)[0]\n        print(f\"\\nRegularized Ensemble Score: {reg_score:.4f}\")\n        \n        submission_reg = submission_df.copy()\n        submission_reg[\"prediction\"] = reg_test\n        submission_reg.to_csv(\"submission_regularized_ensemble.csv\", index=False)\n        all_submissions[\"regularized_ensemble\"] = reg_score\n    \n    # 4. Tree-based ensemble (XGB + LightGBM)\n    tree_methods = [\"xgb_baseline\", \"lgbm\"]\n    tree_oof_list = []\n    tree_test_list = []\n    \n    for method in tree_methods:\n        if method in oof_preds:\n            method_oof = np.mean(list(oof_preds[method].values()), axis=0)\n            method_test = np.mean(list(test_preds[method].values()), axis=0)\n            tree_oof_list.append(method_oof)\n            tree_test_list.append(method_test)\n    \n    if tree_oof_list:\n        tree_oof = np.mean(tree_oof_list, axis=0)\n        tree_test = np.mean(tree_test_list, axis=0)\n        tree_score = pearsonr(train_df[Config.LABEL_COLUMN], tree_oof)[0]\n        print(f\"\\nTree Ensemble Score: {tree_score:.4f}\")\n        \n        submission_tree = submission_df.copy()\n        submission_tree[\"prediction\"] = tree_test\n        submission_tree.to_csv(\"submission_tree_ensemble.csv\", index=False)\n        all_submissions[\"tree_ensemble\"] = tree_score\n    \n    # 5. Full ensemble (weighted by performance)\n    all_oof_scores = {}\n    all_oof_preds = {}\n    all_test_preds = {}\n    \n    for learner_name in oof_preds:\n        learner_oof = np.mean(list(oof_preds[learner_name].values()), axis=0)\n        learner_test = np.mean(list(test_preds[learner_name].values()), axis=0)\n        score = pearsonr(train_df[Config.LABEL_COLUMN], learner_oof)[0]\n        \n        if not np.isnan(score) and score > 0:  # Only include positive correlations\n            all_oof_scores[learner_name] = score\n            all_oof_preds[learner_name] = learner_oof\n            all_test_preds[learner_name] = learner_test\n    \n    # Weighted ensemble\n    if all_oof_scores:\n        total_score = sum(all_oof_scores.values())\n        weights = {k: v/total_score for k, v in all_oof_scores.items()}\n        \n        weighted_oof = sum(weights[k] * all_oof_preds[k] for k in weights)\n        weighted_test = sum(weights[k] * all_test_preds[k] for k in weights)\n        weighted_score = pearsonr(train_df[Config.LABEL_COLUMN], weighted_oof)[0]\n        \n        print(f\"\\nWeighted Full Ensemble Score: {weighted_score:.4f}\")\n        print(\"Weights:\", {k: f\"{v:.3f}\" for k, v in weights.items()})\n        \n        submission_weighted = submission_df.copy()\n        submission_weighted[\"prediction\"] = weighted_test\n        submission_weighted.to_csv(\"submission_weighted_ensemble.csv\", index=False)\n        all_submissions[\"weighted_ensemble\"] = weighted_score\n    \n    # 6. Simple average of all valid models\n    simple_oof = np.mean(list(all_oof_preds.values()), axis=0)\n    simple_test = np.mean(list(all_test_preds.values()), axis=0)\n    simple_score = pearsonr(train_df[Config.LABEL_COLUMN], simple_oof)[0]\n    \n    print(f\"\\nSimple Full Ensemble Score: {simple_score:.4f}\")\n    \n    submission_simple = submission_df.copy()\n    submission_simple[\"prediction\"] = simple_test\n    submission_simple.to_csv(\"submission_simple_ensemble.csv\", index=False)\n    all_submissions[\"simple_ensemble\"] = simple_score\n    \n    # Print summary\n    print(\"\\n\" + \"=\"*50)\n    print(\"SUBMISSION SUMMARY:\")\n    print(\"=\"*50)\n    for name, score in sorted(all_submissions.items(), key=lambda x: x[1], reverse=True):\n        print(f\"{name:25s}: {score:.4f}\")\n    \n    return all_submissions\n\n# ===== Main Execution =====\nif __name__ == \"__main__\":\n    print(\"Loading data...\")\n    train_df, test_df, submission_df = load_data()\n    \n    print(\"\\nTraining models...\")\n    oof_preds, test_preds, model_slices = train_and_evaluate(train_df, test_df)\n    \n    print(\"\\nCreating submissions...\")\n    submission_scores = create_submissions(train_df, oof_preds, test_preds, submission_df)\n    \n    print(\"\\nAll submissions created successfully!\")\n    print(\"Files created:\")\n    print(\"- submission_xgb_baseline.csv (original baseline)\")\n    print(\"- submission_robust_ensemble.csv (Huber + RANSAC + TheilSen)\")\n    print(\"- submission_regularized_ensemble.csv (Lasso + ElasticNet)\")\n    print(\"- submission_tree_ensemble.csv (XGBoost + LightGBM)\")\n    print(\"- submission_weighted_ensemble.csv (weighted by performance)\")\n    print(\"- submission_simple_ensemble.csv (simple average)\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}