{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#!/usr/bin/env python\n# coding: utf-8\n\n\"\"\"\nDRW Crypto Market Prediction Competition\nComplete Solution with Advanced Feature Engineering\n\"\"\"\n\n# Install required packages\nimport subprocess\nimport sys\n\ndef install_packages():\n    \"\"\"Install required packages if not already installed\"\"\"\n    packages = ['pandas', 'numpy', 'lightgbm', 'scikit-learn', 'scipy']\n    for package in packages:\n        subprocess.check_call([sys.executable, '-m', 'pip', 'install', package, '-q'])\n\n# Run installation\ninstall_packages()\n\n# Import libraries\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nfrom scipy.stats import pearsonr\nfrom sklearn.preprocessing import PolynomialFeatures\nfrom sklearn.kernel_approximation import RBFSampler, Nystroem\nimport gc\nimport warnings\nimport os\nfrom datetime import datetime\n\nwarnings.filterwarnings('ignore')\n\n# Configuration\nclass Config:\n    \"\"\"Configuration settings for the model\"\"\"\n    # Paths - Updated for Kaggle environment\n    train_path = '/kaggle/input/drw-crypto-market-prediction/train.parquet'\n    test_path = '/kaggle/input/drw-crypto-market-prediction/test.parquet'\n    sample_submission_path = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n    submission_path = 'submission.csv'\n    \n    # Model parameters\n    seed = 42\n    n_folds = 5\n    \n    # Memory-efficient LightGBM parameters\n    lgb_params = {\n        'objective': 'regression',\n        'metric': 'rmse',\n        'boosting_type': 'gbdt',\n        'num_leaves': 31,\n        'learning_rate': 0.05,\n        'feature_fraction': 0.8,\n        'bagging_fraction': 0.8,\n        'bagging_freq': 5,\n        'verbosity': -1,\n        'seed': seed,\n        'n_jobs': -1,\n        'min_child_samples': 20,\n        'reg_alpha': 0.1,\n        'reg_lambda': 0.1,\n        # Memory optimization parameters\n        'max_bin': 63,\n        'min_data_in_bin': 5,\n        'feature_pre_filter': False,\n        'force_col_wise': True,\n        'histogram_pool_size': -1,\n    }\n    \n    # Training parameters\n    early_stopping_rounds = 100\n    n_estimators = 1000\n    verbose_eval = 100\n    \n    # Feature engineering\n    use_recent_months_only = True\n    recent_months = 3\n    \n    # Feature selection parameters\n    use_feature_selection = True\n    feature_selection_threshold = 0.7\n    min_features_to_keep = 100\n    \n    # Feature engineering parameters\n    feature_engineering_config = {\n        'max_new_features_per_type': 5,\n        'feature_generation_log': [],\n        'round_1': {\n            'polynomial': {\n                'enabled': True,\n                'degree': 2,\n                'interaction_only': True,\n                'include_bias': False,\n                'max_features': 10,\n                'feature_selection': 'importance',\n                'max_new_features': 5\n            },\n            'kernel': {\n                'enabled': True,\n                'kernels': ['rbf'],\n                'n_components': 5,\n                'gamma': 0.1,\n                'subset_size': 0.3,\n                'max_new_features': 5\n            },\n            'statistical': {\n                'enabled': True,\n                'transforms': ['log1p', 'sqrt'],\n                'threshold': 0.01,\n                'max_new_features': 5\n            },\n            'custom': {\n                'enabled': True,\n                'ratios': True,\n                'differences': False,\n                'products': True,\n                'max_new_features': 5\n            }\n        },\n        'round_2': {\n            'polynomial': {\n                'enabled': True,\n                'degree': 2,\n                'interaction_only': True,\n                'include_bias': False,\n                'max_features': 8,\n                'feature_selection': 'importance',\n                'max_new_features': 5\n            },\n            'kernel': {\n                'enabled': False,\n                'kernels': ['rbf'],\n                'n_components': 5,\n                'gamma': 0.05,\n                'subset_size': 0.2,\n                'max_new_features': 5\n            },\n            'statistical': {\n                'enabled': False,\n                'transforms': ['log1p'],\n                'threshold': 0.01,\n                'max_new_features': 5\n            },\n            'custom': {\n                'enabled': True,\n                'ratios': True,\n                'differences': False,\n                'products': True,\n                'max_new_features': 5\n            }\n        }\n    }\n\ndef reduce_mem_usage(dataframe, dataset_name):\n    \"\"\"Reduce memory usage by optimizing data types\"\"\"\n    print(f'Reducing memory usage for: {dataset_name}')\n    initial_mem_usage = dataframe.memory_usage().sum() / 1024**2\n\n    for col in dataframe.columns:\n        col_type = dataframe[col].dtype\n        \n        if col_type != 'object' and col != 'timestamp':\n            c_min = dataframe[col].min()\n            c_max = dataframe[col].max()\n            \n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    dataframe[col] = dataframe[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    dataframe[col] = dataframe[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    dataframe[col] = dataframe[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    dataframe[col] = dataframe[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    dataframe[col] = dataframe[col].astype(np.float32)\n                else:\n                    dataframe[col] = dataframe[col].astype(np.float64)\n\n    final_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    print(f'--- Memory usage before: {initial_mem_usage:.2f} MB')\n    print(f'--- Memory usage after: {final_mem_usage:.2f} MB')\n    print(f'--- Decreased memory usage by {100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage:.1f}%\\n')\n\n    return dataframe\n\ndef log_feature_generation(round_num, transform_type, feature_names, original_count):\n    \"\"\"Log information about generated features\"\"\"\n    log_entry = {\n        'round': round_num,\n        'transform_type': transform_type,\n        'features_generated': feature_names,\n        'num_generated': len(feature_names),\n        'original_candidates': original_count,\n        'timestamp': pd.Timestamp.now()\n    }\n    Config.feature_engineering_config['feature_generation_log'].append(log_entry)\n    \n    print(f\"Generated {len(feature_names)} {transform_type} features from {original_count} candidates\")\n\ndef apply_statistical_transforms(X, config, round_num):\n    \"\"\"Apply statistical transformations to features with controlled generation\"\"\"\n    if not config['enabled']:\n        return X\n    \n    X_transformed = X.copy()\n    potential_features = []\n    \n    for col in X.columns:\n        if X[col].std() > config['threshold']:\n            for transform in config['transforms']:\n                if transform == 'log1p':\n                    if X[col].min() >= 0:\n                        new_col = np.log1p(X[col])\n                        potential_features.append((f'{col}_log1p', new_col))\n                elif transform == 'sqrt':\n                    if X[col].min() >= 0:\n                        new_col = np.sqrt(X[col])\n                        potential_features.append((f'{col}_sqrt', new_col))\n    \n    original_count = len(potential_features)\n    if len(potential_features) > config['max_new_features']:\n        np.random.seed(42 + round_num)\n        indices = np.random.choice(len(potential_features), config['max_new_features'], replace=False)\n        selected_features = [potential_features[i] for i in indices]\n    else:\n        selected_features = potential_features\n    \n    feature_names = []\n    for name, values in selected_features:\n        X_transformed[name] = values.astype(np.float32)\n        feature_names.append(name)\n    \n    log_feature_generation(round_num, 'statistical', feature_names, original_count)\n    \n    return X_transformed\n\ndef apply_custom_features(X, config, round_num):\n    \"\"\"Apply custom feature engineering with controlled generation\"\"\"\n    if not config['enabled']:\n        return X\n    \n    X_transformed = X.copy()\n    potential_features = []\n    \n    feature_cols = X.columns.tolist()\n    n_features = min(10, len(feature_cols))\n    np.random.seed(42 + round_num)\n    selected_features = np.random.choice(feature_cols, n_features, replace=False)\n    \n    for i, col1 in enumerate(selected_features):\n        for col2 in selected_features[i+1:]:\n            if config['ratios']:\n                denominator = X[col2] + 1e-8\n                ratio = X[col1] / denominator\n                if ratio.std() > 0.01:\n                    potential_features.append((f'{col1}_ratio_{col2}', ratio, 'ratio'))\n            \n            if config['products']:\n                prod = X[col1] * X[col2]\n                if prod.std() > 0.01:\n                    potential_features.append((f'{col1}_prod_{col2}', prod, 'product'))\n    \n    original_count = len(potential_features)\n    if len(potential_features) > config['max_new_features']:\n        np.random.seed(43 + round_num)\n        indices = np.random.choice(len(potential_features), config['max_new_features'], replace=False)\n        selected_features = [potential_features[i] for i in indices]\n    else:\n        selected_features = potential_features\n    \n    feature_names = []\n    feature_types = {'ratio': 0, 'difference': 0, 'product': 0}\n    for name, values, feat_type in selected_features:\n        X_transformed[name] = values.astype(np.float32)\n        feature_names.append(name)\n        feature_types[feat_type] += 1\n    \n    log_entry_detail = f\"Generated: {feature_types['ratio']} ratios, {feature_types['product']} products\"\n    print(f\"  {log_entry_detail}\")\n    log_feature_generation(round_num, 'custom', feature_names, original_count)\n    \n    return X_transformed\n\ndef apply_polynomial_features(X, config, feature_importance, round_num):\n    \"\"\"Apply polynomial feature transformations with controlled generation\"\"\"\n    if not config['enabled']:\n        return X\n    \n    if config['feature_selection'] == 'importance' and feature_importance is not None:\n        top_features = feature_importance.head(config['max_features'])['feature'].tolist()\n        top_features = [f for f in top_features if f in X.columns]\n    else:\n        np.random.seed(44 + round_num)\n        top_features = np.random.choice(X.columns, \n                                      min(config['max_features'], len(X.columns)), \n                                      replace=False).tolist()\n    \n    print(f\"Creating polynomial features from {len(top_features)} selected features\")\n    \n    poly = PolynomialFeatures(\n        degree=config['degree'],\n        interaction_only=config['interaction_only'],\n        include_bias=config['include_bias']\n    )\n    \n    X_poly = poly.fit_transform(X[top_features])\n    feature_names = poly.get_feature_names_out(top_features)\n    \n    poly_df = pd.DataFrame(X_poly, columns=feature_names, index=X.index)\n    poly_df = poly_df.drop(columns=top_features, errors='ignore')\n    \n    potential_features = [(col, poly_df[col]) for col in poly_df.columns]\n    original_count = len(potential_features)\n    \n    if len(potential_features) > config['max_new_features']:\n        np.random.seed(45 + round_num)\n        indices = np.random.choice(len(potential_features), config['max_new_features'], replace=False)\n        selected_features = [potential_features[i] for i in indices]\n    else:\n        selected_features = potential_features\n    \n    X_transformed = X.copy()\n    feature_names = []\n    for name, values in selected_features:\n        X_transformed[name] = values.astype(np.float32)\n        feature_names.append(name)\n    \n    log_feature_generation(round_num, 'polynomial', feature_names, original_count)\n    \n    del poly_df, X_poly\n    gc.collect()\n    \n    return X_transformed\n\ndef apply_kernel_features(X, config, round_num):\n    \"\"\"Apply kernel transformations with controlled generation\"\"\"\n    if not config['enabled']:\n        return X\n    \n    X_transformed = X.copy()\n    \n    n_samples = int(len(X) * config['subset_size'])\n    sample_indices = np.random.choice(len(X), n_samples, replace=False)\n    \n    for kernel_idx, kernel_type in enumerate(config['kernels']):\n        print(f\"Applying {kernel_type} kernel transformation...\")\n        \n        max_components_per_kernel = config['max_new_features'] // len(config['kernels'])\n        n_components = min(config['n_components'], max_components_per_kernel)\n        \n        feature_names = []\n        \n        if kernel_type == 'rbf':\n            rbf_sampler = RBFSampler(\n                gamma=config['gamma'],\n                n_components=n_components,\n                random_state=42 + round_num + kernel_idx\n            )\n            \n            rbf_sampler.fit(X.iloc[sample_indices])\n            X_kernel = rbf_sampler.transform(X)\n            \n            for i in range(X_kernel.shape[1]):\n                feat_name = f'rbf_component_{i}'\n                X_transformed[feat_name] = X_kernel[:, i].astype(np.float32)\n                feature_names.append(feat_name)\n        \n        log_feature_generation(round_num, f'{kernel_type}_kernel', feature_names, n_components)\n        \n        del X_kernel\n        gc.collect()\n    \n    return X_transformed\n\ndef apply_feature_engineering_round(X, config_round, round_num, feature_importance=None):\n    \"\"\"Apply a complete round of feature engineering with controlled generation\"\"\"\n    print(f\"\\n=== Feature Engineering Round {round_num} ===\")\n    initial_features = X.shape[1]\n    \n    X_transformed = X\n    \n    if 'statistical' in config_round:\n        X_transformed = apply_statistical_transforms(X_transformed, config_round['statistical'], round_num)\n        gc.collect()\n    \n    if 'custom' in config_round:\n        X_transformed = apply_custom_features(X_transformed, config_round['custom'], round_num)\n        gc.collect()\n    \n    if 'polynomial' in config_round:\n        X_transformed = apply_polynomial_features(X_transformed, config_round['polynomial'], feature_importance, round_num)\n        gc.collect()\n    \n    if 'kernel' in config_round:\n        X_transformed = apply_kernel_features(X_transformed, config_round['kernel'], round_num)\n        gc.collect()\n    \n    print(f\"Features increased from {initial_features} to {X_transformed.shape[1]}\")\n    print(f\"Net new features added: {X_transformed.shape[1] - initial_features}\")\n    \n    variance_threshold = 1e-8\n    low_variance_cols = X_transformed.columns[X_transformed.var() < variance_threshold]\n    if len(low_variance_cols) > 0:\n        X_transformed = X_transformed.drop(columns=low_variance_cols)\n        print(f\"Removed {len(low_variance_cols)} low variance features\")\n    \n    return X_transformed\n\ndef remove_highly_correlated_features(X, threshold=0.99):\n    \"\"\"Remove features with correlation above threshold\"\"\"\n    print(f\"\\nRemoving features with correlation > {threshold}\")\n    \n    n_features = X.shape[1]\n    chunk_size = 100\n    to_remove = set()\n    \n    for i in range(0, n_features, chunk_size):\n        for j in range(i, n_features, chunk_size):\n            chunk_i = X.iloc[:, i:min(i+chunk_size, n_features)]\n            chunk_j = X.iloc[:, j:min(j+chunk_size, n_features)]\n            \n            corr_chunk = chunk_i.corrwith(chunk_j)\n            \n            for idx, corr_val in corr_chunk.items():\n                if abs(corr_val) > threshold and chunk_i.columns[0] != idx:\n                    if chunk_i.columns[0] < idx:\n                        to_remove.add(idx)\n                    else:\n                        to_remove.add(chunk_i.columns[0])\n    \n    X_reduced = X.drop(columns=list(to_remove))\n    print(f\"Removed {len(to_remove)} highly correlated features\")\n    \n    return X_reduced\n\ndef save_feature_generation_report(log_data, filename='feature_generation_report.txt'):\n    \"\"\"Save a detailed report of all feature generation steps\"\"\"\n    with open(filename, 'w') as f:\n        f.write(\"Feature Generation Report\\n\")\n        f.write(\"=\" * 80 + \"\\n\\n\")\n        \n        for entry in log_data:\n            f.write(f\"Round {entry['round']} - {entry['transform_type']}\\n\")\n            f.write(f\"Timestamp: {entry['timestamp']}\\n\")\n            f.write(f\"Generated {entry['num_generated']} features from {entry['original_candidates']} candidates\\n\")\n            f.write(f\"Selected ratio: {entry['num_generated']/max(entry['original_candidates'], 1):.2%}\\n\")\n            f.write(\"Generated features:\\n\")\n            for i, feat in enumerate(entry['features_generated']):\n                f.write(f\"  {i+1}. {feat}\\n\")\n            f.write(\"\\n\" + \"-\" * 80 + \"\\n\\n\")\n        \n        f.write(\"Summary Statistics\\n\")\n        f.write(\"=\" * 80 + \"\\n\")\n        total_generated = sum(entry['num_generated'] for entry in log_data)\n        total_candidates = sum(entry['original_candidates'] for entry in log_data)\n        f.write(f\"Total features generated: {total_generated}\\n\")\n        f.write(f\"Total candidate features considered: {total_candidates}\\n\")\n        f.write(f\"Overall selection ratio: {total_generated/max(total_candidates, 1):.2%}\\n\")\n    \n    print(f\"\\nFeature generation report saved to {filename}\")\n\ndef load_data():\n    \"\"\"Load train and test data from parquet files with memory optimization\"\"\"\n    print(\"Loading data...\")\n    \n    train_df = pd.read_parquet(Config.train_path)\n    test_df = pd.read_parquet(Config.test_path)\n    \n    print(f\"Initial train shape: {train_df.shape}\")\n    print(f\"Initial test shape: {test_df.shape}\")\n    \n    train_df = reduce_mem_usage(train_df, 'train')\n    test_df = reduce_mem_usage(test_df, 'test')\n    \n    gc.collect()\n    \n    return train_df, test_df\n\ndef preprocess_data(train_df, test_df):\n    \"\"\"Preprocess the data with memory efficiency in mind\"\"\"\n    print(\"\\nPreprocessing data...\")\n    \n    if Config.use_recent_months_only and 'timestamp' in train_df.columns:\n        train_df['timestamp'] = pd.to_datetime(train_df['timestamp'])\n        \n        cutoff_date = train_df['timestamp'].max() - pd.DateOffset(months=Config.recent_months)\n        train_df = train_df[train_df['timestamp'] >= cutoff_date].reset_index(drop=True)\n        print(f\"Using data from {cutoff_date} onwards. New train shape: {train_df.shape}\")\n        \n        gc.collect()\n    \n    if 'timestamp' in train_df.columns:\n        train_df = train_df.drop('timestamp', axis=1)\n    if 'timestamp' in test_df.columns:\n        test_df = test_df.drop('timestamp', axis=1)\n    \n    feature_cols = [col for col in train_df.columns if col != 'label']\n    \n    X_train = train_df[feature_cols].copy()\n    y_train = train_df['label'].copy()\n    X_test = test_df[feature_cols].copy()\n    \n    del train_df\n    gc.collect()\n    \n    if X_train.isnull().sum().sum() > 0:\n        print(\"Handling missing values...\")\n        X_train = X_train.fillna(0)\n        X_test = X_test.fillna(0)\n    \n    print(\"Adding engineered features...\")\n    \n    X_train['bid_ask_imbalance'] = (X_train['bid_qty'] - X_train['ask_qty']) / (X_train['bid_qty'] + X_train['ask_qty'] + 1e-8)\n    X_train['buy_sell_imbalance'] = (X_train['buy_qty'] - X_train['sell_qty']) / (X_train['buy_qty'] + X_train['sell_qty'] + 1e-8)\n    \n    X_test['bid_ask_imbalance'] = (X_test['bid_qty'] - X_test['ask_qty']) / (X_test['bid_qty'] + X_test['ask_qty'] + 1e-8)\n    X_test['buy_sell_imbalance'] = (X_test['buy_qty'] - X_test['sell_qty']) / (X_test['buy_qty'] + X_test['sell_qty'] + 1e-8)\n    \n    for col in ['bid_ask_imbalance', 'buy_sell_imbalance']:\n        X_train[col] = X_train[col].astype(np.float32)\n        X_test[col] = X_test[col].astype(np.float32)\n    \n    print(f\"Final train features: {X_train.shape[1]}\")\n    \n    gc.collect()\n    \n    return X_train, y_train, X_test\n\ndef pearson_correlation(y_true, y_pred):\n    \"\"\"Calculate Pearson correlation coefficient\"\"\"\n    return pearsonr(y_true, y_pred)[0]\n\ndef select_features_by_importance(X_train, y_train, X_test, threshold=0.7, min_features=100):\n    \"\"\"Perform feature selection based on LightGBM feature importance\"\"\"\n    print(\"\\nPerforming feature selection...\")\n    \n    params = {\n        'objective': 'regression',\n        'metric': 'rmse',\n        'boosting_type': 'gbdt',\n        'num_leaves': 31,\n        'learning_rate': 0.1,\n        'feature_fraction': 0.9,\n        'verbosity': -1,\n        'seed': 42,\n        'n_jobs': -1,\n        'force_col_wise': True,\n        'max_bin': 63,\n    }\n    \n    sample_size = min(50000, len(X_train))\n    sample_idx = np.random.choice(len(X_train), sample_size, replace=False)\n    X_sample = X_train.iloc[sample_idx]\n    y_sample = y_train.iloc[sample_idx]\n    \n    train_set = lgb.Dataset(X_sample, y_sample, free_raw_data=True)\n    \n    model = lgb.train(\n        params,\n        train_set,\n        num_boost_round=100,\n        callbacks=[lgb.log_evaluation(0)]\n    )\n    \n    importance_df = pd.DataFrame({\n        'feature': X_train.columns,\n        'importance': model.feature_importance(importance_type='gain')\n    }).sort_values('importance', ascending=False)\n    \n    n_features = max(int(len(X_train.columns) * threshold), min_features)\n    n_features = min(n_features, len(X_train.columns))\n    \n    selected_features = importance_df.head(n_features)['feature'].tolist()\n    \n    print(f\"Selected {len(selected_features)} features out of {len(X_train.columns)}\")\n    print(f\"Top 10 features: {selected_features[:10]}\")\n    \n    del model, train_set, X_sample, y_sample\n    gc.collect()\n    \n    return X_train[selected_features], X_test[selected_features], selected_features\n\ndef train_lightgbm_single_fold(X_train, y_train, X_test, fold_data):\n    \"\"\"Train a single fold of LightGBM with memory optimization\"\"\"\n    fold, train_idx, valid_idx = fold_data\n    print(f\"\\nFold {fold + 1}/{Config.n_folds}\")\n    \n    X_tr, X_val = X_train.iloc[train_idx], X_train.iloc[valid_idx]\n    y_tr, y_val = y_train.iloc[train_idx], y_train.iloc[valid_idx]\n    \n    train_set = lgb.Dataset(X_tr, y_tr, free_raw_data=True)\n    valid_set = lgb.Dataset(X_val, y_val, free_raw_data=True, reference=train_set)\n    \n    model = lgb.train(\n        Config.lgb_params,\n        train_set,\n        num_boost_round=Config.n_estimators,\n        valid_sets=[valid_set],\n        valid_names=['valid'],\n        callbacks=[\n            lgb.early_stopping(Config.early_stopping_rounds),\n            lgb.log_evaluation(Config.verbose_eval)\n        ]\n    )\n    \n    val_predictions = model.predict(X_val, num_iteration=model.best_iteration)\n    test_predictions = model.predict(X_test, num_iteration=model.best_iteration)\n    \n    fold_score = pearson_correlation(y_val, val_predictions)\n    print(f\"Fold {fold + 1} Pearson correlation: {fold_score:.6f}\")\n    \n    importance = model.feature_importance(importance_type='gain')\n    \n    del train_set, valid_set, X_tr, X_val, y_tr, y_val\n    gc.collect()\n    \n    return val_predictions, test_predictions, valid_idx, importance, fold_score\n\ndef train_lightgbm_cv(X_train, y_train, X_test):\n    \"\"\"Train LightGBM with cross-validation using memory-efficient approach\"\"\"\n    print(\"\\nTraining LightGBM with cross-validation...\")\n    \n    oof_predictions = np.zeros(len(X_train), dtype=np.float32)\n    test_predictions = np.zeros(len(X_test), dtype=np.float32)\n    \n    feature_importance = pd.DataFrame()\n    feature_importance['feature'] = X_train.columns\n    \n    kf = KFold(n_splits=Config.n_folds, shuffle=True, random_state=Config.seed)\n    \n    scores = []\n    \n    for fold, (train_idx, valid_idx) in enumerate(kf.split(X_train)):\n        fold_data = (fold, train_idx, valid_idx)\n        val_preds, test_preds, valid_idx, importance, fold_score = train_lightgbm_single_fold(\n            X_train, y_train, X_test, fold_data\n        )\n        \n        oof_predictions[valid_idx] = val_preds\n        test_predictions += test_preds / Config.n_folds\n        \n        feature_importance[f'fold_{fold + 1}'] = importance\n        scores.append(fold_score)\n        \n        gc.collect()\n    \n    oof_score = pearson_correlation(y_train, oof_predictions)\n    print(f\"\\nOverall OOF Pearson correlation: {oof_score:.6f}\")\n    print(f\"Mean CV score: {np.mean(scores):.6f} (+/- {np.std(scores):.6f})\")\n    \n    feature_importance['importance'] = feature_importance[[f'fold_{i+1}' for i in range(Config.n_folds)]].mean(axis=1)\n    feature_importance = feature_importance[['feature', 'importance']].sort_values('importance', ascending=False)\n    \n    print(\"\\nTop 20 most important features:\")\n    print(feature_importance.head(20))\n    \n    return test_predictions, oof_predictions, feature_importance\n\ndef create_submission(test_predictions, test_df):\n    \"\"\"Create submission file\"\"\"\n    print(\"\\nCreating submission file...\")\n    \n    # Load sample submission to get the correct format\n    sample_sub = pd.read_csv(Config.sample_submission_path)\n    \n    # Check what columns are available in sample submission\n    print(f\"Sample submission columns: {sample_sub.columns.tolist()}\")\n    print(f\"Sample submission shape: {sample_sub.shape}\")\n    \n    # Create submission dataframe based on available columns\n    if 'id' in sample_sub.columns:\n        submission = pd.DataFrame({\n            'id': sample_sub['id'],\n            'prediction': test_predictions\n        })\n    elif 'ID' in sample_sub.columns:\n        submission = pd.DataFrame({\n            'ID': sample_sub['ID'],\n            'prediction': test_predictions\n        })\n    elif len(sample_sub.columns) == 1:\n        # If sample submission only has label column, use index as ID\n        submission = pd.DataFrame({\n            'prediction': test_predictions\n        })\n        submission.index = sample_sub.index\n    else:\n        # Use the first column as ID column (whatever it's named)\n        id_column = sample_sub.columns[0]\n        submission = pd.DataFrame({\n            id_column: sample_sub[id_column],\n            'prediction': test_predictions\n        })\n    \n    # Ensure we have predictions for all test samples\n    assert len(submission) == len(sample_sub), f\"Submission length mismatch: {len(submission)} vs {len(sample_sub)}\"\n    \n    # Save submission with the same index structure as sample\n    submission.to_csv(Config.submission_path, index=('id' not in sample_sub.columns and 'ID' not in sample_sub.columns))\n    print(f\"Submission saved to {Config.submission_path}\")\n    \n    # Display statistics\n    print(f\"\\nSubmission statistics:\")\n    print(f\"Mean: {test_predictions.mean():.6f}\")\n    print(f\"Std: {test_predictions.std():.6f}\")\n    print(f\"Min: {test_predictions.min():.6f}\")\n    print(f\"Max: {test_predictions.max():.6f}\")\n    \n    return submission\n\ndef main():\n    \"\"\"Main execution function with two-stage feature engineering\"\"\"\n    print(\"DRW Crypto Price Movement Prediction - Advanced Feature Engineering\")\n    print(\"=\" * 65)\n    print(f\"Execution started at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\")\n    \n    # Set random seed for reproducibility\n    np.random.seed(Config.seed)\n    \n    # Load data with memory optimization\n    train_df, test_df = load_data()\n    \n    # Store test_df reference for submission\n    test_df_copy = test_df.copy()\n    \n    # Initial preprocessing\n    X_train, y_train, X_test = preprocess_data(train_df, test_df)\n    \n    # Clean up\n    del train_df, test_df\n    gc.collect()\n    \n    print(\"\\n=== Stage 1: Initial Feature Reduction ===\")\n    \n    # Remove zero-variance and duplicate columns\n    initial_features = X_train.shape[1]\n    X_train = X_train.loc[:, X_train.nunique() > 1]\n    X_test = X_test[X_train.columns]\n    \n    # Remove duplicate columns\n    duplicate_cols = []\n    for i in range(len(X_train.columns)):\n        for j in range(i + 1, len(X_train.columns)):\n            if X_train.iloc[:, i].equals(X_train.iloc[:, j]):\n                duplicate_cols.append(X_train.columns[j])\n    \n    X_train = X_train.drop(columns=duplicate_cols)\n    X_test = X_test.drop(columns=duplicate_cols)\n    print(f\"Removed {initial_features - X_train.shape[1]} zero-variance/duplicate features\")\n    \n    # Remove highly correlated features\n    X_train = remove_highly_correlated_features(X_train, threshold=0.99)\n    X_test = X_test[X_train.columns]\n    \n    gc.collect()\n    \n    # First LightGBM feature selection\n    if Config.use_feature_selection:\n        print(\"\\n=== First LightGBM Feature Selection ===\")\n        X_train, X_test, selected_features = select_features_by_importance(\n            X_train, y_train, X_test, \n            threshold=0.5,\n            min_features=200\n        )\n        gc.collect()\n    \n    # First round of feature engineering\n    if 'round_1' in Config.feature_engineering_config:\n        X_train = apply_feature_engineering_round(\n            X_train, \n            Config.feature_engineering_config['round_1'], \n            round_num=1\n        )\n        X_test = apply_feature_engineering_round(\n            X_test, \n            Config.feature_engineering_config['round_1'], \n            round_num=1\n        )\n        \n        # Ensure same columns in train and test\n        common_cols = X_train.columns.intersection(X_test.columns)\n        X_train = X_train[common_cols]\n        X_test = X_test[common_cols]\n        \n        gc.collect()\n    \n    # Second LightGBM feature selection\n    print(\"\\n=== Second LightGBM Feature Selection ===\")\n    X_train, X_test, selected_features = select_features_by_importance(\n        X_train, y_train, X_test, \n        threshold=Config.feature_selection_threshold,\n        min_features=Config.min_features_to_keep\n    )\n    \n    # Get feature importance for second round\n    temp_importance = pd.DataFrame({\n        'feature': selected_features[:30],\n        'importance': range(30, 0, -1)\n    })\n    \n    gc.collect()\n    \n    # Second round of feature engineering\n    if 'round_2' in Config.feature_engineering_config:\n        X_train = apply_feature_engineering_round(\n            X_train, \n            Config.feature_engineering_config['round_2'], \n            round_num=2,\n            feature_importance=temp_importance\n        )\n        X_test = apply_feature_engineering_round(\n            X_test, \n            Config.feature_engineering_config['round_2'], \n            round_num=2,\n            feature_importance=temp_importance\n        )\n        \n        # Ensure same columns\n        common_cols = X_train.columns.intersection(X_test.columns)\n        X_train = X_train[common_cols]\n        X_test = X_test[common_cols]\n        \n        gc.collect()\n    \n    # Final feature selection\n    print(\"\\n=== Final Feature Selection ===\")\n    X_train, X_test, selected_features = select_features_by_importance(\n        X_train, y_train, X_test, \n        threshold=0.8,\n        min_features=100\n    )\n    \n    print(f\"\\nFinal feature count: {X_train.shape[1]}\")\n    gc.collect()\n    \n    # Train final model with cross-validation\n    print(\"\\n=== Training Final Model ===\")\n    cv_predictions, oof_predictions, feature_importance = train_lightgbm_cv(X_train, y_train, X_test)\n    \n    # Use CV predictions\n    test_predictions = cv_predictions\n    \n    # Create submission\n    submission = create_submission(test_predictions, test_df_copy)\n    \n    # Final cleanup\n    del X_train, y_train, X_test, test_df_copy\n    gc.collect()\n    \n    # Save feature generation report\n    if len(Config.feature_engineering_config['feature_generation_log']) > 0:\n        save_feature_generation_report(\n            Config.feature_engineering_config['feature_generation_log'],\n            'feature_generation_report.txt'\n        )\n    \n    print(\"\\nTraining complete!\")\n    print(f\"Final model used {len(selected_features)} features after two rounds of engineering\")\n    print(f\"Execution completed at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\")\n    \n    return submission, feature_importance\n\nif __name__ == \"__main__\":\n    submission, feature_importance = main()\n    print(\"\\nSubmission file created successfully!\")\n    print(\"Please submit the 'submission.csv' file to the competition.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}