{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Single Variation Runner - Set EARLY_PERCENTAGE to desired value\n\n# ========== DETERMINISTIC SETUP ==========\nimport os\nimport random\nimport numpy as np\n\n# Set environment variables BEFORE importing other libraries\nos.environ['PYTHONHASHSEED'] = '42'\nos.environ['OMP_NUM_THREADS'] = '1'\nos.environ['MKL_NUM_THREADS'] = '1'\nos.environ['OPENBLAS_NUM_THREADS'] = '1'\nos.environ['VECLIB_MAXIMUM_THREADS'] = '1'\nos.environ['NUMEXPR_NUM_THREADS'] = '1'\n\n# Set all random seeds\nSEED = 42\nrandom.seed(SEED)\nnp.random.seed(SEED)\n\n# ========== CONFIGURATION - CHANGE THIS VALUE ==========\nEARLY_PERCENTAGE = 0.35  # Change this to 0.20, 0.25, 0.30, 0.35, 0.40, or 0.45\n# ======================================================\n\n# Imports\nimport sys\nimport pandas as pd\nfrom sklearn.model_selection import KFold\nfrom xgboost import XGBRegressor\nfrom scipy.stats import pearsonr\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Feature Engineering\ndef feature_engineering(df):\n    # Create a copy to avoid modifying the original\n    df = df.copy()\n    \n    # Use explicit order of operations\n    df['volume_weighted_sell'] = df['sell_qty'].values * df['volume'].values\n    df['buy_sell_ratio'] = df['buy_qty'].values / (df['sell_qty'].values + 1e-8)\n    df['selling_pressure'] = df['sell_qty'].values / (df['volume'].values + 1e-8)\n    df['effective_spread_proxy'] = np.abs(df['buy_qty'].values - df['sell_qty'].values) / (df['volume'].values + 1e-8)\n    \n    # Replace inf values\n    df = df.replace([np.inf, -np.inf], np.nan)\n    df = df.fillna(0)\n    \n    return df \n\n# Configuration\nclass Config:\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    \n    FEATURES = [\n        \"X863\", \"X856\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X855\", \"X174\", \"X302\", \"X178\", \"X168\", \"X612\",\n        \"buy_qty\", \"sell_qty\", \"volume\", \"X888\", \"X421\", \"X333\", \"X292\",\n    ]\n    \n    LABEL_COLUMN = \"label\"\n    N_FOLDS = 3\n    RANDOM_STATE = SEED\n\n# XGBoost parameters for deterministic behavior\nXGB_PARAMS = {\n    # Core parameters\n    'tree_method': 'exact',  # Changed from 'hist' for determinism\n    'device': 'cpu',\n    'n_jobs': 1,  # Single thread\n    \n    # Model parameters\n    'colsample_bytree': 0.499384, \n    'colsample_bynode': 0.748391,\n    'gamma': 7.34723, \n    'learning_rate': 0.4129738, \n    'max_depth': 7, \n    'max_leaves': 40, \n    'n_estimators': 500,\n    'reg_alpha': 27.791606770656145, \n    'reg_lambda': 84.90603428439086,\n    'subsample': 0.06567,\n    \n    # Deterministic parameters\n    'random_state': Config.RANDOM_STATE,\n    'seed': Config.RANDOM_STATE,\n    'verbosity': 0,\n}\n\nLEARNERS = [\n    {\"name\": \"xgb\", \"Estimator\": XGBRegressor, \"params\": XGB_PARAMS},\n]\n\n# Loading Data\ndef create_time_decay_weights(n: int, decay: float = 0.9, reverse: bool = False) -> np.ndarray:\n    \"\"\"Create time decay weights with deterministic computation.\"\"\"\n    positions = np.arange(n, dtype=np.float64)\n    \n    if reverse:\n        normalized = 1.0 - (positions / (n - 1))\n    else:\n        normalized = positions / (n - 1)\n    \n    # Use float64 for consistent precision\n    weights = np.power(decay, (1.0 - normalized), dtype=np.float64)\n    \n    # Normalize weights\n    weight_sum = np.sum(weights, dtype=np.float64)\n    weights = weights * n / weight_sum\n    \n    return weights.astype(np.float32)  # Convert back to float32 for memory efficiency\n\ndef load_data():\n    train_df = pd.read_parquet(Config.TRAIN_PATH, columns=Config.FEATURES + [Config.LABEL_COLUMN])\n    test_df = pd.read_parquet(Config.TEST_PATH, columns=Config.FEATURES)\n    submission_df = pd.read_csv(Config.SUBMISSION_PATH)\n    \n    train_df = feature_engineering(train_df)\n    test_df = feature_engineering(test_df)\n    \n    print(f\"Loaded data - Train: {train_df.shape}, Test: {test_df.shape}, Submission: {submission_df.shape}\")\n    return train_df.reset_index(drop=True), test_df.reset_index(drop=True), submission_df\n\nConfig.FEATURES += [\"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"]\nConfig.FEATURES = list(set(Config.FEATURES))  # remove duplicates\n\n# Training and Evaluation\ndef get_model_slices(n_samples: int):\n    return [\n        {\"name\": \"full_data\", \"type\": \"full\", \"cutoff\": 0},\n        {\"name\": \"last_75pct\", \"type\": \"recent\", \"cutoff\": int(0.25 * n_samples)},\n        {\"name\": \"last_50pct\", \"type\": \"recent\", \"cutoff\": int(0.50 * n_samples)},\n        {\"name\": f\"first_{int(EARLY_PERCENTAGE*100)}pct\", \"type\": \"early\", \"cutoff\": int(EARLY_PERCENTAGE * n_samples)},\n    ]\n\ndef train_and_evaluate(train_df, test_df):\n    n_samples = len(train_df)\n    model_slices = get_model_slices(n_samples)\n    \n    oof_preds = {\n        learner[\"name\"]: {s[\"name\"]: np.zeros(n_samples) for s in model_slices}\n        for learner in LEARNERS\n    }\n    test_preds = {\n        learner[\"name\"]: {s[\"name\"]: np.zeros(len(test_df)) for s in model_slices}\n        for learner in LEARNERS\n    }\n    \n    full_weights = create_time_decay_weights(n_samples)\n    kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n    \n    for fold, (train_idx, valid_idx) in enumerate(kf.split(train_df), start=1):\n        print(f\"\\n--- Fold {fold}/{Config.N_FOLDS} ---\")\n        X_valid = train_df.iloc[valid_idx][Config.FEATURES]\n        y_valid = train_df.iloc[valid_idx][Config.LABEL_COLUMN]\n        \n        for s in model_slices:\n            cutoff = s[\"cutoff\"]\n            slice_name = s[\"name\"]\n            slice_type = s[\"type\"]\n            \n            if slice_type == \"full\":\n                # Use all data\n                subset = train_df.reset_index(drop=True)\n                rel_idx = train_idx\n                sw = full_weights[train_idx]\n                \n            elif slice_type == \"recent\":\n                # Use data from cutoff to end (recent data)\n                subset = train_df.iloc[cutoff:].reset_index(drop=True)\n                rel_idx = train_idx[train_idx >= cutoff] - cutoff\n                if cutoff > 0:\n                    sw = create_time_decay_weights(len(subset))[rel_idx]\n                else:\n                    sw = full_weights[train_idx]\n                    \n            elif slice_type == \"early\":\n                # Use data from start to cutoff (early data)\n                subset = train_df.iloc[:cutoff].reset_index(drop=True)\n                rel_idx = train_idx[train_idx < cutoff]\n                if len(rel_idx) > 0:\n                    # For early data, we might want to give more weight to later samples within the subset\n                    sw = create_time_decay_weights(len(subset))[rel_idx]\n                else:\n                    sw = np.array([])\n            \n            # Skip if no training data available for this slice\n            if len(rel_idx) == 0:\n                print(f\"  Skipping slice: {slice_name} (no training data in fold)\")\n                continue\n            \n            X_train = subset.iloc[rel_idx][Config.FEATURES]\n            y_train = subset.iloc[rel_idx][Config.LABEL_COLUMN]\n            \n            X_train_np = X_train.values\n            y_train_np = y_train.values\n            X_valid_np = X_valid.values\n            y_valid_np = y_valid.values\n            \n            print(f\"  Training slice: {slice_name}, samples: {len(X_train)}\")\n            \n            for learner in LEARNERS:\n                model = learner[\"Estimator\"](**learner[\"params\"])\n                model.fit(X_train_np, y_train_np, sample_weight=sw, \n                          eval_set=[(X_valid_np, y_valid_np)], verbose=False)\n                \n                # Handle predictions based on slice type\n                if slice_type == \"early\":\n                    # For early slice, only predict on validation samples that were in the training range\n                    mask = valid_idx < cutoff\n                    if mask.any():\n                        idxs = valid_idx[mask]\n                        oof_preds[learner[\"name\"]][slice_name][idxs] = model.predict(train_df.iloc[idxs][Config.FEATURES])\n                    # For validation samples outside the early training range, use full_data predictions\n                    if (~mask).any():\n                        oof_preds[learner[\"name\"]][slice_name][valid_idx[~mask]] = oof_preds[learner[\"name\"]][\"full_data\"][valid_idx[~mask]]\n                else:\n                    # For recent slices and full data\n                    mask = valid_idx >= cutoff if slice_type == \"recent\" else np.ones(len(valid_idx), dtype=bool)\n                    if mask.any():\n                        idxs = valid_idx[mask]\n                        oof_preds[learner[\"name\"]][slice_name][idxs] = model.predict(train_df.iloc[idxs][Config.FEATURES])\n                    if slice_type == \"recent\" and cutoff > 0 and (~mask).any():\n                        oof_preds[learner[\"name\"]][slice_name][valid_idx[~mask]] = oof_preds[learner[\"name\"]][\"full_data\"][valid_idx[~mask]]\n                \n                # Test predictions (always use the model regardless of slice type)\n                test_preds[learner[\"name\"]][slice_name] += model.predict(test_df[Config.FEATURES])\n    \n    # Normalize test predictions\n    for learner_name in test_preds:\n        for slice_name in test_preds[learner_name]:\n            test_preds[learner_name][slice_name] /= Config.N_FOLDS\n    \n    return oof_preds, test_preds, model_slices\n\n# Submission\ndef ensemble_and_submit(train_df, oof_preds, test_preds, submission_df):\n    learner_ensembles = {}\n    \n    print(\"\\nIndividual Slice Scores:\")\n    for learner_name in oof_preds:\n        scores = {}\n        for s in oof_preds[learner_name]:\n            # Calculate score only on samples where the model made actual predictions\n            # (not filled from other models)\n            score = pearsonr(train_df[Config.LABEL_COLUMN], oof_preds[learner_name][s])[0]\n            scores[s] = score\n            print(f\"  {learner_name} - {s}: {score:.4f}\")\n        \n        total_score = sum(scores.values())\n        \n        # Simple average ensemble\n        oof_simple = np.mean(list(oof_preds[learner_name].values()), axis=0)\n        test_simple = np.mean(list(test_preds[learner_name].values()), axis=0)\n        score_simple = pearsonr(train_df[Config.LABEL_COLUMN], oof_simple)[0]\n        \n        # Weighted ensemble based on OOF scores\n        oof_weighted = sum(scores[s] / total_score * oof_preds[learner_name][s] for s in scores)\n        test_weighted = sum(scores[s] / total_score * test_preds[learner_name][s] for s in scores)\n        score_weighted = pearsonr(train_df[Config.LABEL_COLUMN], oof_weighted)[0]\n        \n        print(f\"\\n{learner_name.upper()} Simple Ensemble Pearson:   {score_simple:.4f}\")\n        print(f\"{learner_name.upper()} Weighted Ensemble Pearson: {score_weighted:.4f}\")\n        \n        # Store the better performing ensemble\n        if score_weighted > score_simple:\n            learner_ensembles[learner_name] = {\n                \"oof\": oof_weighted,\n                \"test\": test_weighted,\n                \"type\": \"weighted\"\n            }\n        else:\n            learner_ensembles[learner_name] = {\n                \"oof\": oof_simple,\n                \"test\": test_simple,\n                \"type\": \"simple\"\n            }\n    \n    # Final ensemble across all learners\n    final_oof = np.mean([le[\"oof\"] for le in learner_ensembles.values()], axis=0)\n    final_test = np.mean([le[\"test\"] for le in learner_ensembles.values()], axis=0)\n    final_score = pearsonr(train_df[Config.LABEL_COLUMN], final_oof)[0]\n    \n    print(f\"\\nFINAL ensemble across learners Pearson: {final_score:.4f}\")\n    print(f\"Ensemble types used: {[le['type'] for le in learner_ensembles.values()]}\")\n    \n    # Save with percentage in filename\n    filename = f\"submission_early_{int(EARLY_PERCENTAGE*100)}pct.csv\"\n    submission_df[\"prediction\"] = final_test\n    submission_df.to_csv(filename, index=False)\n    print(f\"\\nSaved: {filename}\")\n\n# Main\nif __name__ == \"__main__\":\n    print(f\"\\nRunning with EARLY_PERCENTAGE = {EARLY_PERCENTAGE} ({int(EARLY_PERCENTAGE*100)}%)\")\n    train_df, test_df, submission_df = load_data()\n    oof_preds, test_preds, model_slices = train_and_evaluate(train_df, test_df)\n    ensemble_and_submit(train_df, oof_preds, test_preds, submission_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}