{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#!/usr/bin/env python3\n\"\"\"\nDRW Crypto Advanced Non-Linear Feature Engineering\n=================================================\n\nThis implementation uses Random Forest and Gradient Boosting based transformations\nto capture complex non-linear relationships without the memory constraints of PCA.\nMaintains important feature preservation while applying advanced transformations\nto the remaining features.\n\"\"\"\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nimport os\nimport time\nfrom datetime import datetime\nwarnings.filterwarnings('ignore')\n\n# Core ML libraries\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, QuantileTransformer\nfrom sklearn.feature_selection import SelectKBest, f_regression, mutual_info_regression, VarianceThreshold\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, ExtraTreesRegressor\nfrom sklearn.linear_model import ElasticNet, Ridge, Lasso\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.preprocessing import PolynomialFeatures\nfrom sklearn.feature_extraction import FeatureHasher\nfrom scipy.stats import pearsonr\nimport joblib\n\n# Deep learning (optional)\ntry:\n    import tensorflow as tf\n    from tensorflow import keras\n    from tensorflow.keras import layers, Model, regularizers\n    tf.random.set_seed(42)\n    TENSORFLOW_AVAILABLE = True\nexcept ImportError:\n    TENSORFLOW_AVAILABLE = False\n    print(\"TensorFlow not available - deep learning methods will be skipped\")\n\n# Set random seed\nnp.random.seed(42)\n\n# Define important features - original plus high-importance augmented features\nIMPORTANT_FEATURES = [\n    # Original important features\n    \"X863\", \"X856\", \"X344\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n    \"X415\", \"X345\", \"X137\", \"X855\", \"X174\", \"X302\", \"X178\", \"X532\", \"X168\", \"X612\",\n    \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\", \"X888\", \"X421\", \"X333\",\n    # High-importance augmented features\n    \"X862_minus_X852\", \"X598_minus_X862\", \"regime_indicator_137_302\", \"X852_minus_X345\",\n    \"rbf_862_852\", \"multidim_distance_kernel\", \"X168_minus_X612\", \"X532_plus_X888\",\n    \"complex_interaction_862_852_345\", \"volume_weighted_technical\"\n]\n\n\nclass FeatureAugmenter:\n    \"\"\"Creates augmented features that have proven to be highly predictive\"\"\"\n    \n    def __init__(self):\n        self.feature_stats = {}\n        \n    def fit(self, df):\n        \"\"\"Store statistics needed for transformation\"\"\"\n        for col in ['X862', 'X852', 'X345', 'X532', 'X888', 'X137', 'X302', \n                   'X178', 'X168', 'X612', 'X598', 'volume']:\n            if col in df.columns:\n                col_data = df[col].replace([np.inf, -np.inf], np.nan).dropna()\n                if len(col_data) > 0:\n                    self.feature_stats[f'{col}_mean'] = col_data.mean()\n                    self.feature_stats[f'{col}_std'] = col_data.std()\n                    self.feature_stats[f'{col}_p25'] = col_data.quantile(0.25)\n                    self.feature_stats[f'{col}_p75'] = col_data.quantile(0.75)\n                else:\n                    self.feature_stats[f'{col}_mean'] = 0\n                    self.feature_stats[f'{col}_std'] = 1\n                    self.feature_stats[f'{col}_p25'] = -1\n                    self.feature_stats[f'{col}_p75'] = 1\n        return self\n    \n    def transform(self, df):\n        \"\"\"Create augmented features with robust handling\"\"\"\n        augmented_features = []\n        feature_names = []\n        \n        # Complex interactions between top features\n        if all(f in df.columns for f in ['X862', 'X852', 'X345']):\n            feat1 = np.nan_to_num(df['X862'].values, nan=0.0, posinf=1e10, neginf=-1e10)\n            feat2 = np.nan_to_num(df['X852'].values, nan=0.0, posinf=1e10, neginf=-1e10)\n            feat3 = np.nan_to_num(df['X345'].values, nan=0.0, posinf=1e10, neginf=-1e10)\n            \n            interaction = np.tanh(np.clip(feat1, -10, 10)) * \\\n                         np.exp(-np.clip(np.abs(feat2) / 2, 0, 10)) * \\\n                         np.sign(feat3)\n            augmented_features.append(interaction)\n            feature_names.append('complex_interaction_862_852_345')\n            \n            ratio_poly = (feat1 ** 2) / (feat2 ** 2 + 1)\n            ratio_poly = np.clip(ratio_poly, -1e6, 1e6)\n            augmented_features.append(ratio_poly)\n            feature_names.append('poly_ratio_862_852')\n            \n            augmented_features.append(feat1 - feat2)\n            feature_names.append('X862_minus_X852')\n            \n            augmented_features.append(feat2 - feat3)\n            feature_names.append('X852_minus_X345')\n            \n            rbf_sigma = max(self.feature_stats.get('X862_std', 1), 0.1)\n            diff = np.clip(feat1 - feat2, -100, 100)\n            rbf_862_852 = np.exp(-((diff) ** 2) / (2 * rbf_sigma ** 2))\n            augmented_features.append(rbf_862_852)\n            feature_names.append('rbf_862_852')\n        \n        # Market microstructure features\n        if all(f in df.columns for f in ['bid_qty', 'ask_qty', 'volume', 'buy_qty', 'sell_qty']):\n            bid = np.nan_to_num(df['bid_qty'].values, nan=0.0, posinf=1e10, neginf=0)\n            ask = np.nan_to_num(df['ask_qty'].values, nan=0.0, posinf=1e10, neginf=0)\n            vol = np.nan_to_num(df['volume'].values, nan=0.0, posinf=1e10, neginf=0)\n            buy = np.nan_to_num(df['buy_qty'].values, nan=0.0, posinf=1e10, neginf=0)\n            sell = np.nan_to_num(df['sell_qty'].values, nan=0.0, posinf=1e10, neginf=0)\n            \n            order_imbalance = np.clip((bid - ask) / (bid + ask + 1), -1, 1)\n            flow_imbalance = np.clip((buy - sell) / (buy + sell + 1), -1, 1)\n            kyle_lambda = flow_imbalance * np.sqrt(np.abs(order_imbalance)) / (np.log1p(vol) + 1)\n            kyle_lambda = np.clip(kyle_lambda, -10, 10)\n            augmented_features.append(kyle_lambda)\n            feature_names.append('kyle_lambda_complex')\n            \n            total_pressure = bid + ask\n            vol_mean = max(self.feature_stats.get('volume_mean', 1), 1)\n            vol_adj_pressure = np.log1p(total_pressure) * np.exp(-np.clip(vol / vol_mean, 0, 10))\n            augmented_features.append(vol_adj_pressure)\n            feature_names.append('vol_adjusted_pressure')\n            \n            buy_intensity = np.clip(buy / (vol + 1), 0, 1)\n            sell_intensity = np.clip(sell / (vol + 1), 0, 1)\n            intensity_diff = buy_intensity - sell_intensity\n            intensity_asymmetry = np.sign(intensity_diff) * np.log1p(np.abs(intensity_diff))\n            augmented_features.append(intensity_asymmetry)\n            feature_names.append('trade_intensity_asymmetry')\n        \n        # Cross-domain interactions\n        if all(f in df.columns for f in ['X532', 'X888', 'volume']):\n            x532 = np.nan_to_num(df['X532'].values, nan=0.0, posinf=1e10, neginf=-1e10)\n            x888 = np.nan_to_num(df['X888'].values, nan=0.0, posinf=1e10, neginf=-1e10)\n            vol = np.nan_to_num(df['volume'].values, nan=0.0, posinf=1e10, neginf=0)\n            \n            technical = np.clip(x532 * x888, -1e10, 1e10)\n            vol_std = max(self.feature_stats.get('volume_std', 1), 0.1)\n            vol_weighted_tech = technical * np.log1p(vol) / vol_std\n            vol_weighted_tech = np.clip(vol_weighted_tech, -1e6, 1e6)\n            augmented_features.append(vol_weighted_tech)\n            feature_names.append('volume_weighted_technical')\n            \n            augmented_features.append(x532 + x888)\n            feature_names.append('X532_plus_X888')\n        \n        # Regime-based features\n        if 'X137' in df.columns and 'X302' in df.columns:\n            feat1 = np.nan_to_num(df['X137'].values, nan=0.0, posinf=1e10, neginf=-1e10)\n            feat2 = np.nan_to_num(df['X302'].values, nan=0.0, posinf=1e10, neginf=-1e10)\n            \n            p75_1 = self.feature_stats.get('X137_p75', np.percentile(feat1[np.isfinite(feat1)], 75) if np.any(np.isfinite(feat1)) else 1)\n            p75_2 = self.feature_stats.get('X302_p75', np.percentile(feat2[np.isfinite(feat2)], 75) if np.any(np.isfinite(feat2)) else 1)\n            p25_1 = self.feature_stats.get('X137_p25', np.percentile(feat1[np.isfinite(feat1)], 25) if np.any(np.isfinite(feat1)) else -1)\n            p25_2 = self.feature_stats.get('X302_p25', np.percentile(feat2[np.isfinite(feat2)], 25) if np.any(np.isfinite(feat2)) else -1)\n            \n            regime_indicator = np.where(\n                (feat1 > p75_1) & (feat2 > p75_2), 1,\n                np.where((feat1 < p25_1) & (feat2 < p25_2), -1, 0)\n            ).astype(np.float32)\n            augmented_features.append(regime_indicator)\n            feature_names.append('regime_indicator_137_302')\n        \n        # Distance-based features\n        if all(f in df.columns for f in ['X178', 'X168', 'X612']):\n            x178 = np.nan_to_num(df['X178'].values, nan=0.0, posinf=1e10, neginf=-1e10)\n            x168 = np.nan_to_num(df['X168'].values, nan=0.0, posinf=1e10, neginf=-1e10)\n            x612 = np.nan_to_num(df['X612'].values, nan=0.0, posinf=1e10, neginf=-1e10)\n            \n            feat1 = (x178 - self.feature_stats.get('X178_mean', 0)) / max(self.feature_stats.get('X178_std', 1), 0.1)\n            feat2 = (x168 - self.feature_stats.get('X168_mean', 0)) / max(self.feature_stats.get('X168_std', 1), 0.1)\n            feat3 = (x612 - self.feature_stats.get('X612_mean', 0)) / max(self.feature_stats.get('X612_std', 1), 0.1)\n            \n            feat1 = np.clip(feat1, -10, 10)\n            feat2 = np.clip(feat2, -10, 10)\n            feat3 = np.clip(feat3, -10, 10)\n            \n            distance = np.sqrt(feat1**2 + feat2**2 + feat3**2)\n            distance_kernel = np.exp(-np.clip(distance**2 / 2, 0, 50))\n            augmented_features.append(distance_kernel)\n            feature_names.append('multidim_distance_kernel')\n            \n            augmented_features.append(x168 - x612)\n            feature_names.append('X168_minus_X612')\n        \n        # Additional top feature interactions\n        if 'X598' in df.columns and 'X862' in df.columns:\n            f598 = np.nan_to_num(df['X598'].values, nan=0.0, posinf=1e10, neginf=-1e10)\n            f862 = np.nan_to_num(df['X862'].values, nan=0.0, posinf=1e10, neginf=-1e10)\n            \n            augmented_features.append(f598 - f862)\n            feature_names.append('X598_minus_X862')\n        \n        if augmented_features:\n            for i, feat in enumerate(augmented_features):\n                augmented_features[i] = np.nan_to_num(feat, nan=0.0, posinf=1e6, neginf=-1e6)\n            \n            augmented_df = pd.DataFrame(\n                np.column_stack(augmented_features),\n                columns=feature_names,\n                index=df.index\n            )\n            return pd.concat([df, augmented_df], axis=1)\n        else:\n            return df\n    \n    def fit_transform(self, df):\n        self.fit(df)\n        return self.transform(df)\n\n\nclass RandomForestFeatureTransformer:\n    \"\"\"Uses Random Forest to create non-linear feature transformations\"\"\"\n    \n    def __init__(self, n_estimators=100, max_leaf_nodes=32, n_jobs=-1):\n        self.n_estimators = n_estimators\n        self.max_leaf_nodes = max_leaf_nodes\n        self.n_jobs = n_jobs\n        self.forest = None\n        self.leaf_encoders = {}\n        self.n_features_out = None\n        \n    def fit(self, X, y):\n        \"\"\"Fit the random forest and prepare for feature extraction\"\"\"\n        print(f\"Training Random Forest with {self.n_estimators} trees...\")\n        \n        self.forest = RandomForestRegressor(\n            n_estimators=self.n_estimators,\n            max_leaf_nodes=self.max_leaf_nodes,\n            max_features='sqrt',\n            min_samples_leaf=50,\n            random_state=42,\n            n_jobs=self.n_jobs\n        )\n        \n        self.forest.fit(X, y)\n        \n        # Calculate output feature dimensions\n        self.n_features_out = 0\n        for i in range(self.n_estimators):\n            tree = self.forest.estimators_[i]\n            n_leaves = tree.tree_.node_count\n            self.n_features_out += n_leaves\n        \n        print(f\"Random Forest trained. Will generate {self.n_features_out} features\")\n        \n        return self\n    \n    def transform(self, X):\n        \"\"\"Transform features using tree leaf indices\"\"\"\n        # Get leaf indices for each tree\n        leaf_indices = self.forest.apply(X)\n        \n        # Create feature matrix using hashing trick to reduce dimensionality\n        hasher = FeatureHasher(n_features=min(200, self.n_features_out), \n                              input_type='pair')\n        \n        # Create feature pairs for hashing\n        feature_dicts = []\n        for sample_idx in range(X.shape[0]):\n            pairs = [(f'tree_{tree_idx}', int(leaf_indices[sample_idx, tree_idx]))\n                    for tree_idx in range(self.n_estimators)]\n            feature_dicts.append(pairs)\n        \n        # Apply hashing\n        hashed_features = hasher.transform(feature_dicts).toarray()\n        \n        return hashed_features\n    \n    def fit_transform(self, X, y):\n        self.fit(X, y)\n        return self.transform(X)\n\n\nclass GradientBoostingFeatureEngineering:\n    \"\"\"Creates features by capturing patterns at different scales using gradient boosting\"\"\"\n    \n    def __init__(self, n_rounds=3, trees_per_round=50):\n        self.n_rounds = n_rounds\n        self.trees_per_round = trees_per_round\n        self.models = []\n        self.feature_importances = []\n        \n    def fit(self, X, y):\n        \"\"\"Fit gradient boosting models to capture residual patterns\"\"\"\n        print(f\"Training {self.n_rounds} rounds of gradient boosting...\")\n        \n        residuals = y.copy()\n        \n        for round_idx in range(self.n_rounds):\n            print(f\"  Round {round_idx + 1}/{self.n_rounds}...\")\n            \n            gb = GradientBoostingRegressor(\n                n_estimators=self.trees_per_round,\n                max_depth=4,\n                learning_rate=0.1,\n                subsample=0.8,\n                random_state=42 + round_idx\n            )\n            \n            gb.fit(X, residuals)\n            self.models.append(gb)\n            \n            # Update residuals\n            predictions = gb.predict(X)\n            residuals = residuals - predictions\n            \n            # Store feature importances\n            self.feature_importances.append(gb.feature_importances_)\n        \n        return self\n    \n    def transform(self, X):\n        \"\"\"Transform features using the trained models\"\"\"\n        all_features = []\n        \n        # Get predictions from each round\n        for round_idx, model in enumerate(self.models):\n            # Predictions as features\n            predictions = model.predict(X).reshape(-1, 1)\n            all_features.append(predictions)\n            \n            # Extract leaf indices and create hashed features\n            leaf_features = []\n            for tree_idx in range(len(model.estimators_)):\n                tree = model.estimators_[tree_idx, 0]\n                leaf_indices = tree.apply(X)\n                leaf_features.append(leaf_indices)\n            \n            # Hash tree features\n            hasher = FeatureHasher(n_features=30, input_type='pair')\n            hashed = hasher.transform(\n                [[(f'r{round_idx}_t{tree_idx}', int(leaf_features[tree_idx][sample_idx]))\n                  for tree_idx in range(len(leaf_features))]\n                 for sample_idx in range(X.shape[0])]\n            ).toarray()\n            \n            all_features.append(hashed)\n        \n        return np.hstack(all_features)\n    \n    def fit_transform(self, X, y):\n        self.fit(X, y)\n        return self.transform(X)\n\n\nclass PolynomialInteractionExtractor:\n    \"\"\"Extracts polynomial interactions from top features\"\"\"\n    \n    def __init__(self, n_top_features=20, degree=2):\n        self.n_top_features = n_top_features\n        self.degree = degree\n        self.selector = None\n        self.poly = None\n        self.selected_indices = None\n        \n    def fit(self, X, y):\n        \"\"\"Select top features and fit polynomial transformer\"\"\"\n        # Select top features using mutual information\n        self.selector = SelectKBest(mutual_info_regression, k=min(self.n_top_features, X.shape[1]))\n        X_selected = self.selector.fit_transform(X, y)\n        self.selected_indices = self.selector.get_support(indices=True)\n        \n        # Create polynomial features\n        self.poly = PolynomialFeatures(degree=self.degree, interaction_only=True, include_bias=False)\n        self.poly.fit(X_selected)\n        \n        return self\n    \n    def transform(self, X):\n        \"\"\"Transform using fitted components\"\"\"\n        X_selected = self.selector.transform(X)\n        X_poly = self.poly.transform(X_selected)\n        \n        # Remove the original features (keep only interactions)\n        X_interactions = X_poly[:, self.n_top_features:]\n        \n        return X_interactions\n    \n    def fit_transform(self, X, y):\n        self.fit(X, y)\n        return self.transform(X)\n\n\nclass DeepAutoencoderReducer:\n    \"\"\"Neural network based dimensionality reduction (if TensorFlow available)\"\"\"\n    \n    def __init__(self, encoding_dim=50, hidden_layers=[256, 128]):\n        self.encoding_dim = encoding_dim\n        self.hidden_layers = hidden_layers\n        self.encoder = None\n        self.autoencoder = None\n        self.input_dim = None\n        \n    def build_model(self, input_dim):\n        \"\"\"Build the autoencoder architecture\"\"\"\n        if not TENSORFLOW_AVAILABLE:\n            raise ImportError(\"TensorFlow not available\")\n        \n        self.input_dim = input_dim\n        \n        # Encoder\n        encoder_input = layers.Input(shape=(input_dim,))\n        x = encoder_input\n        \n        # Add noise for denoising autoencoder\n        x = layers.GaussianNoise(0.1)(x)\n        \n        for units in self.hidden_layers:\n            x = layers.Dense(units)(x)\n            x = layers.BatchNormalization()(x)\n            x = layers.LeakyReLU(alpha=0.1)(x)\n            x = layers.Dropout(0.2)(x)\n        \n        # Bottleneck\n        encoded = layers.Dense(self.encoding_dim, activation='linear',\n                              kernel_regularizer=regularizers.l1(1e-5),\n                              name='encoded')(x)\n        \n        # Decoder\n        x = encoded\n        for units in reversed(self.hidden_layers):\n            x = layers.Dense(units)(x)\n            x = layers.BatchNormalization()(x)\n            x = layers.LeakyReLU(alpha=0.1)(x)\n        \n        decoded = layers.Dense(input_dim, activation='linear')(x)\n        \n        # Create models\n        self.autoencoder = Model(encoder_input, decoded)\n        self.encoder = Model(encoder_input, encoded)\n        \n        # Compile\n        self.autoencoder.compile(\n            optimizer=keras.optimizers.Adam(learning_rate=0.001),\n            loss='mse'\n        )\n        \n        return self\n    \n    def fit(self, X, epochs=50, batch_size=256, validation_split=0.1):\n        \"\"\"Train the autoencoder\"\"\"\n        if not TENSORFLOW_AVAILABLE:\n            print(\"Skipping deep autoencoder - TensorFlow not available\")\n            return self\n        \n        if self.encoder is None:\n            self.build_model(X.shape[1])\n        \n        # Train with early stopping\n        early_stop = keras.callbacks.EarlyStopping(\n            monitor='val_loss',\n            patience=5,\n            restore_best_weights=True\n        )\n        \n        history = self.autoencoder.fit(\n            X, X,\n            epochs=epochs,\n            batch_size=batch_size,\n            validation_split=validation_split,\n            callbacks=[early_stop],\n            verbose=0\n        )\n        \n        print(f\"Autoencoder trained. Final val_loss: {history.history['val_loss'][-1]:.4f}\")\n        \n        return self\n    \n    def transform(self, X):\n        \"\"\"Transform using the encoder\"\"\"\n        if not TENSORFLOW_AVAILABLE or self.encoder is None:\n            # Return truncated version if TensorFlow not available\n            return X[:, :self.encoding_dim]\n        \n        return self.encoder.predict(X, verbose=0)\n    \n    def fit_transform(self, X, **kwargs):\n        self.fit(X, **kwargs)\n        return self.transform(X)\n\n\nclass AdvancedNonLinearReducer:\n    \"\"\"Main class that combines all non-linear transformation methods\"\"\"\n    \n    def __init__(self, important_features, target_dims=100, use_deep_learning=False):\n        self.important_features = important_features\n        self.target_dims = target_dims\n        self.use_deep_learning = use_deep_learning and TENSORFLOW_AVAILABLE\n        \n        # Initialize components\n        self.rf_transformer = RandomForestFeatureTransformer(n_estimators=50, max_leaf_nodes=32)\n        self.gb_engineer = GradientBoostingFeatureEngineering(n_rounds=3, trees_per_round=30)\n        self.poly_extractor = PolynomialInteractionExtractor(n_top_features=15, degree=2)\n        \n        if self.use_deep_learning:\n            self.deep_reducer = DeepAutoencoderReducer(encoding_dim=30)\n        \n        self.feature_selector = None\n        self.important_scaler = None\n        self.other_scaler = None\n        self.important_indices = None\n        self.other_indices = None\n        \n    def fit(self, X, feature_names, y):\n        \"\"\"Fit all transformation components\"\"\"\n        print(\"\\nAdvanced Non-Linear Feature Engineering\")\n        print(\"=\" * 50)\n        \n        # Separate important and other features\n        self.important_indices = [i for i, name in enumerate(feature_names) \n                                 if name in self.important_features]\n        self.other_indices = [i for i in range(len(feature_names)) \n                             if i not in self.important_indices]\n        \n        print(f\"Preserving {len(self.important_indices)} important features\")\n        print(f\"Transforming {len(self.other_indices)} other features\")\n        \n        if len(self.other_indices) == 0:\n            print(\"Warning: All features marked as important. No transformation needed.\")\n            return self\n        \n        X_important = X[:, self.important_indices]\n        X_other = X[:, self.other_indices]\n        \n        # Scale features\n        self.important_scaler = RobustScaler()\n        X_important_scaled = self.important_scaler.fit_transform(X_important)\n        \n        self.other_scaler = RobustScaler()\n        X_other_scaled = self.other_scaler.fit_transform(X_other)\n        \n        # Apply transformations\n        print(\"\\n1. Random Forest transformation...\")\n        rf_features = self.rf_transformer.fit_transform(X_other_scaled, y)\n        \n        print(\"\\n2. Gradient Boosting feature engineering...\")\n        gb_features = self.gb_engineer.fit_transform(X_other_scaled, y)\n        \n        print(\"\\n3. Polynomial interaction extraction...\")\n        poly_features = self.poly_extractor.fit_transform(X_other_scaled, y)\n        \n        # Combine all engineered features\n        all_engineered = [rf_features, gb_features, poly_features]\n        \n        if self.use_deep_learning:\n            print(\"\\n4. Deep autoencoder transformation...\")\n            deep_features = self.deep_reducer.fit_transform(X_other_scaled, epochs=30)\n            all_engineered.append(deep_features)\n        \n        X_all_engineered = np.hstack(all_engineered)\n        \n        print(f\"\\nTotal engineered features: {X_all_engineered.shape[1]}\")\n        \n        # Select best engineered features to reach target dimensions\n        n_select = min(self.target_dims, X_all_engineered.shape[1])\n        print(f\"Selecting top {n_select} engineered features...\")\n        \n        self.feature_selector = SelectKBest(f_regression, k=n_select)\n        self.feature_selector.fit(X_all_engineered, y)\n        \n        return self\n    \n    def transform(self, X):\n        \"\"\"Transform new data using fitted components\"\"\"\n        if len(self.other_indices) == 0:\n            # All features are important, just scale\n            return self.important_scaler.transform(X)\n        \n        # Separate features\n        X_important = X[:, self.important_indices]\n        X_other = X[:, self.other_indices]\n        \n        # Scale\n        X_important_scaled = self.important_scaler.transform(X_important)\n        X_other_scaled = self.other_scaler.transform(X_other)\n        \n        # Apply transformations\n        rf_features = self.rf_transformer.transform(X_other_scaled)\n        gb_features = self.gb_engineer.transform(X_other_scaled)\n        poly_features = self.poly_extractor.transform(X_other_scaled)\n        \n        all_engineered = [rf_features, gb_features, poly_features]\n        \n        if self.use_deep_learning and hasattr(self, 'deep_reducer'):\n            deep_features = self.deep_reducer.transform(X_other_scaled)\n            all_engineered.append(deep_features)\n        \n        X_all_engineered = np.hstack(all_engineered)\n        \n        # Select features\n        X_selected = self.feature_selector.transform(X_all_engineered)\n        \n        # Combine with important features\n        X_final = np.hstack([X_important_scaled, X_selected])\n        \n        return X_final\n    \n    def fit_transform(self, X, feature_names, y):\n        self.fit(X, feature_names, y)\n        return self.transform(X)\n    \n    def get_feature_info(self):\n        \"\"\"Get information about the transformation\"\"\"\n        n_engineered = self.feature_selector.k if self.feature_selector else 0\n        \n        info = {\n            'n_important': len(self.important_indices),\n            'n_other': len(self.other_indices),\n            'n_engineered': n_engineered,\n            'total_output': len(self.important_indices) + n_engineered,\n            'rf_features': 200,  # Hashed features\n            'gb_features': self.gb_engineer.n_rounds * (1 + 30),  # predictions + hashed\n            'poly_features': self.poly_extractor.poly.n_output_features_ - self.poly_extractor.n_top_features if hasattr(self.poly_extractor, 'poly') and self.poly_extractor.poly else 0\n        }\n        \n        if self.use_deep_learning:\n            info['deep_features'] = self.deep_reducer.encoding_dim\n        \n        return info\n\n\nclass AdvancedCryptoPreprocessor:\n    \"\"\"Enhanced preprocessor for crypto regression task\"\"\"\n    \n    def __init__(self):\n        self.scaler = None\n        self.variance_selector = None\n        self.outlier_percentiles = {}\n        self.feature_augmenter = FeatureAugmenter()\n        self.nan_fill_values = {}\n    \n    def fit_transform(self, X, feature_names, y=None):\n        \"\"\"Fit and transform for regression task\"\"\"\n        \n        # Convert to DataFrame for feature augmentation\n        df = pd.DataFrame(X, columns=feature_names)\n        \n        # Apply feature augmentation\n        df_augmented = self.feature_augmenter.fit_transform(df)\n        \n        # Extract augmented arrays and names\n        feature_names_augmented = list(df_augmented.columns)\n        X_augmented = df_augmented.values\n        \n        print(f\"Created {len(feature_names_augmented) - len(feature_names)} augmented features\")\n        \n        # Store median values for each feature\n        for i, feat_name in enumerate(feature_names_augmented):\n            finite_vals = X_augmented[:, i][np.isfinite(X_augmented[:, i])]\n            if len(finite_vals) > 0:\n                self.nan_fill_values[i] = np.median(finite_vals)\n            else:\n                self.nan_fill_values[i] = 0.0\n        \n        # Handle infinite and NaN values\n        for i in range(X_augmented.shape[1]):\n            col = X_augmented[:, i]\n            \n            finite_vals = col[np.isfinite(col)]\n            if len(finite_vals) > 0:\n                max_val = np.percentile(finite_vals, 99.9)\n                min_val = np.percentile(finite_vals, 0.1)\n                col = np.where(np.isposinf(col), max_val * 10, col)\n                col = np.where(np.isneginf(col), min_val * 10, col)\n            else:\n                col = np.where(np.isposinf(col), 1e6, col)\n                col = np.where(np.isneginf(col), -1e6, col)\n            \n            col = np.where(np.isnan(col), self.nan_fill_values[i], col)\n            X_augmented[:, i] = col\n        \n        # Remove zero variance features\n        self.variance_selector = VarianceThreshold(threshold=1e-8)\n        X_var_filtered = self.variance_selector.fit_transform(X_augmented)\n        feature_mask = self.variance_selector.get_support()\n        feature_names_filtered = [f for f, m in zip(feature_names_augmented, feature_mask) if m]\n        \n        print(f\"Removed {len(feature_names_augmented) - len(feature_names_filtered)} zero-variance features\")\n        \n        # Robust scaling with outlier clipping\n        self.outlier_percentiles = {}\n        X_clipped = X_var_filtered.copy()\n        \n        for i in range(X_clipped.shape[1]):\n            col = X_clipped[:, i]\n            p01 = np.percentile(col, 1)\n            p99 = np.percentile(col, 99)\n            self.outlier_percentiles[i] = (p01, p99)\n            X_clipped[:, i] = np.clip(col, p01, p99)\n        \n        # Scale\n        self.scaler = RobustScaler()\n        X_scaled = self.scaler.fit_transform(X_clipped)\n        \n        # Final check\n        X_scaled = np.nan_to_num(X_scaled, nan=0.0, posinf=5.0, neginf=-5.0)\n        \n        return X_scaled, feature_names_filtered\n    \n    def transform(self, X, feature_names):\n        \"\"\"Transform new data\"\"\"\n        df = pd.DataFrame(X, columns=feature_names)\n        df_augmented = self.feature_augmenter.transform(df)\n        X_augmented = df_augmented.values\n        \n        # Handle infinite and NaN values\n        for i in range(X_augmented.shape[1]):\n            col = X_augmented[:, i]\n            \n            if i in self.outlier_percentiles:\n                p01, p99 = self.outlier_percentiles[i]\n                col = np.where(np.isposinf(col), p99 * 10, col)\n                col = np.where(np.isneginf(col), p01 * 10, col)\n            else:\n                col = np.where(np.isposinf(col), 1e6, col)\n                col = np.where(np.isneginf(col), -1e6, col)\n            \n            fill_value = self.nan_fill_values.get(i, 0.0)\n            col = np.where(np.isnan(col), fill_value, col)\n            X_augmented[:, i] = col\n        \n        X_var_filtered = self.variance_selector.transform(X_augmented)\n        \n        X_clipped = X_var_filtered.copy()\n        for i, (p01, p99) in self.outlier_percentiles.items():\n            if i < X_clipped.shape[1]:\n                X_clipped[:, i] = np.clip(X_clipped[:, i], p01, p99)\n        \n        X_scaled = self.scaler.transform(X_clipped)\n        X_scaled = np.nan_to_num(X_scaled, nan=0.0, posinf=5.0, neginf=-5.0)\n        \n        return X_scaled\n\n\ndef evaluate_model(X_train, X_test, y_train, y_test, model_name, model):\n    \"\"\"Evaluate a single model and return metrics\"\"\"\n    try:\n        model.fit(X_train, y_train)\n        y_pred = model.predict(X_test)\n        \n        if np.any(~np.isfinite(y_pred)):\n            y_pred = np.nan_to_num(y_pred, nan=np.mean(y_train))\n        \n        mse = mean_squared_error(y_test, y_pred)\n        r2 = r2_score(y_test, y_pred)\n        pearson_corr, _ = pearsonr(y_test, y_pred)\n        \n        return {\n            'model': model_name,\n            'mse': mse,\n            'r2': r2,\n            'pearson': pearson_corr\n        }\n    except Exception as e:\n        print(f\"Model {model_name} failed: {e}\")\n        return {\n            'model': model_name,\n            'mse': np.inf,\n            'r2': -np.inf,\n            'pearson': -1\n        }\n\n\ndef main():\n    \"\"\"Main execution function\"\"\"\n    \n    print(\"DRW Crypto Competition - Advanced Non-Linear Feature Engineering\")\n    print(\"=\" * 80)\n    print(f\"Started at: {datetime.now()}\")\n    \n    # Determine file path\n    if os.path.exists('/kaggle/input/drw-crypto-market-prediction/train.parquet'):\n        file_path = '/kaggle/input/drw-crypto-market-prediction/train.parquet'\n    elif os.path.exists('train.parquet'):\n        file_path = 'train.parquet'\n    else:\n        print(\"\\nERROR: train.parquet not found!\")\n        print(\"Please ensure the data file is in the current directory or Kaggle input.\")\n        return None\n    \n    # Load data\n    print(f\"\\nLoading data from {file_path}...\")\n    df = pd.read_parquet(file_path)\n    print(f\"Loaded {len(df)} total samples\")\n    \n    # Sample for faster experimentation (use all data for final submission)\n    n_samples = min(35000, len(df))\n    if len(df) > n_samples:\n        df_subset = df.sample(n=n_samples, random_state=42)\n    else:\n        df_subset = df.copy()\n    \n    print(f\"Using {len(df_subset)} samples for analysis\")\n    \n    # Get feature columns\n    feature_cols = [col for col in df_subset.columns if col not in ['timestamp', 'label']]\n    X = df_subset[feature_cols].values\n    y = df_subset['label'].values\n    \n    print(f\"\\nOriginal dataset shape: {X.shape}\")\n    \n    # Preprocess\n    print(\"\\nPreprocessing data and creating augmented features...\")\n    preprocessor = AdvancedCryptoPreprocessor()\n    X_processed, feature_cols_filtered = preprocessor.fit_transform(X, feature_cols, y)\n    \n    print(f\"After augmentation and preprocessing: {X_processed.shape}\")\n    \n    # Split data\n    X_train, X_test, y_train, y_test = train_test_split(\n        X_processed, y, test_size=0.3, random_state=42, shuffle=True\n    )\n    \n    print(f\"Train shape: {X_train.shape}, Test shape: {X_test.shape}\")\n    \n    # Test different approaches\n    print(\"\\n\" + \"=\" * 60)\n    print(\"TESTING NON-LINEAR TRANSFORMATION APPROACHES\")\n    print(\"=\" * 60)\n    \n    results = []\n    \n    # 1. Baseline: No reduction\n    print(\"\\n1. Baseline (no reduction)...\")\n    start_time = time.time()\n    \n    models = {\n        'RandomForest': RandomForestRegressor(n_estimators=100, max_depth=10, random_state=42, n_jobs=-1),\n        'GradientBoosting': GradientBoostingRegressor(n_estimators=100, max_depth=5, random_state=42),\n        'ElasticNet': ElasticNet(alpha=0.01, random_state=42, max_iter=2000),\n        'Ridge': Ridge(alpha=1.0, random_state=42)\n    }\n    \n    baseline_results = []\n    for model_name, model in models.items():\n        result = evaluate_model(X_train, X_test, y_train, y_test, model_name, model)\n        baseline_results.append(result)\n    \n    best_baseline = max(baseline_results, key=lambda x: x['pearson'])\n    print(f\"Best baseline: {best_baseline['model']} - Pearson: {best_baseline['pearson']:.4f}\")\n    print(f\"Time: {time.time() - start_time:.2f}s\")\n    \n    results.append({\n        'approach': 'Baseline',\n        'dimensions': X_train.shape[1],\n        'best_pearson': best_baseline['pearson'],\n        'best_model': best_baseline['model'],\n        'time': time.time() - start_time\n    })\n    \n    # 2. Advanced Non-Linear Reduction\n    for target_dims in [50, 100, 150]:\n        print(f\"\\n2. Advanced Non-Linear Reduction (target_dims={target_dims})...\")\n        start_time = time.time()\n        \n        reducer = AdvancedNonLinearReducer(\n            important_features=IMPORTANT_FEATURES,\n            target_dims=target_dims,\n            use_deep_learning=TENSORFLOW_AVAILABLE\n        )\n        \n        X_train_reduced = reducer.fit_transform(X_train, feature_cols_filtered, y_train)\n        X_test_reduced = reducer.transform(X_test)\n        \n        # Get info\n        info = reducer.get_feature_info()\n        print(f\"Output dimensions: {X_train_reduced.shape[1]}\")\n        print(f\"  Important features: {info['n_important']}\")\n        print(f\"  Engineered features: {info['n_engineered']}\")\n        \n        # Evaluate\n        approach_results = []\n        for model_name, model in models.items():\n            result = evaluate_model(X_train_reduced, X_test_reduced, y_train, y_test, model_name, model)\n            approach_results.append(result)\n        \n        best_result = max(approach_results, key=lambda x: x['pearson'])\n        print(f\"Best model: {best_result['model']} - Pearson: {best_result['pearson']:.4f}\")\n        print(f\"Time: {time.time() - start_time:.2f}s\")\n        \n        results.append({\n            'approach': f'NonLinear-{target_dims}',\n            'dimensions': X_train_reduced.shape[1],\n            'best_pearson': best_result['pearson'],\n            'best_model': best_result['model'],\n            'time': time.time() - start_time\n        })\n    \n    # Summary\n    print(\"\\n\" + \"=\" * 80)\n    print(\"SUMMARY RESULTS\")\n    print(\"=\" * 80)\n    \n    results_df = pd.DataFrame(results)\n    results_df = results_df.sort_values('best_pearson', ascending=False)\n    \n    print(\"\\nPerformance Summary:\")\n    print(results_df.to_string(index=False))\n    \n    # Best approach\n    best_approach = results_df.iloc[0]\n    \n    print(f\"\\n✅ BEST APPROACH: {best_approach['approach']}\")\n    print(f\"   Pearson Correlation: {best_approach['best_pearson']:.4f}\")\n    print(f\"   Dimensions: {best_approach['dimensions']}\")\n    print(f\"   Best Model: {best_approach['best_model']}\")\n    \n    # Save results\n    results_df.to_csv('nonlinear_reduction_results.csv', index=False)\n    print(\"\\nResults saved to 'nonlinear_reduction_results.csv'\")\n    \n    # Save the best transformer for production use\n    if 'NonLinear' in best_approach['approach']:\n        print(\"\\nSaving best transformer for production use...\")\n        \n        # Retrain on full training data\n        best_dims = int(best_approach['approach'].split('-')[1])\n        final_reducer = AdvancedNonLinearReducer(\n            important_features=IMPORTANT_FEATURES,\n            target_dims=best_dims,\n            use_deep_learning=TENSORFLOW_AVAILABLE\n        )\n        \n        # Use all available data for final training\n        X_full = np.vstack([X_train, X_test])\n        y_full = np.hstack([y_train, y_test])\n        \n        final_reducer.fit(X_full, feature_cols_filtered, y_full)\n        \n        # Save components\n        joblib.dump(preprocessor, 'preprocessor.pkl')\n        joblib.dump(final_reducer, 'nonlinear_reducer.pkl')\n        joblib.dump(feature_cols, 'original_feature_names.pkl')\n        \n        print(\"Saved: preprocessor.pkl, nonlinear_reducer.pkl, original_feature_names.pkl\")\n    \n    print(f\"\\nCompleted at: {datetime.now()}\")\n    \n    return results_df\n\n\nif __name__ == \"__main__\":\n    results = main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}