{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-28T20:07:42.039419Z","iopub.execute_input":"2025-05-28T20:07:42.039676Z","iopub.status.idle":"2025-05-28T20:07:42.403946Z","shell.execute_reply.started":"2025-05-28T20:07:42.03963Z","shell.execute_reply":"2025-05-28T20:07:42.403334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Install required packages\n!pip install pandas numpy lightgbm scikit-learn scipy\n\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom scipy.stats import pearsonr\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Configuration\nclass Config:\n    # Paths - Updated for Kaggle environment\n    train_path = '/kaggle/input/drw-crypto-market-prediction/train.parquet'\n    test_path = '/kaggle/input/drw-crypto-market-prediction/test.parquet'\n    sample_submission_path = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n    submission_path = 'submission.csv'\n    \n    # Model parameters\n    seed = 42\n    n_folds = 5\n    test_size = 0.2\n    \n    # Memory-efficient LightGBM parameters\n    lgb_params = {\n        'objective': 'regression',\n        'metric': 'rmse',\n        'boosting_type': 'gbdt',\n        'num_leaves': 31,\n        'learning_rate': 0.05,\n        'feature_fraction': 0.8,\n        'bagging_fraction': 0.8,\n        'bagging_freq': 5,\n        'verbosity': -1,\n        'seed': seed,\n        'n_jobs': -1,\n        'min_child_samples': 20,\n        'reg_alpha': 0.1,\n        'reg_lambda': 0.1,\n        # Memory optimization parameters\n        'max_bin': 63,  # Reduced from default 255\n        'min_data_in_bin': 5,\n        'feature_pre_filter': False,\n        'force_col_wise': True,  # Force column-wise histogram building for better memory efficiency\n        'histogram_pool_size': -1,  # Use all available memory\n    }\n    \n    # Training parameters\n    early_stopping_rounds = 100\n    n_estimators = 1000\n    verbose_eval = 100\n    \n    # Feature engineering\n    use_recent_months_only = True  # As suggested in the tips\n    recent_months = 3  # Reduced from 6 to save memory\n\ndef reduce_mem_usage(dataframe, dataset_name):\n    \"\"\"Reduce memory usage by optimizing data types\"\"\"\n    print(f'Reducing memory usage for: {dataset_name}')\n    initial_mem_usage = dataframe.memory_usage().sum() / 1024**2\n\n    for col in dataframe.columns:\n        col_type = dataframe[col].dtype\n        \n        if col_type != 'object' and col != 'timestamp':  # Don't convert timestamp or object columns\n            c_min = dataframe[col].min()\n            c_max = dataframe[col].max()\n            \n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    dataframe[col] = dataframe[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    dataframe[col] = dataframe[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    dataframe[col] = dataframe[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    dataframe[col] = dataframe[col].astype(np.int64)\n            else:\n                # Use float32 as minimum to maintain precision for financial data\n                if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    dataframe[col] = dataframe[col].astype(np.float32)\n                else:\n                    dataframe[col] = dataframe[col].astype(np.float64)\n\n    final_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    print(f'--- Memory usage before: {initial_mem_usage:.2f} MB')\n    print(f'--- Memory usage after: {final_mem_usage:.2f} MB')\n    print(f'--- Decreased memory usage by {100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage:.1f}%\\n')\n\n    return dataframe\n\ndef load_data():\n    \"\"\"Load train and test data from parquet files with memory optimization\"\"\"\n    print(\"Loading data...\")\n    \n    # Load data\n    train_df = pd.read_parquet(Config.train_path)\n    test_df = pd.read_parquet(Config.test_path)\n    \n    print(f\"Initial train shape: {train_df.shape}\")\n    print(f\"Initial test shape: {test_df.shape}\")\n    \n    # Reduce memory usage\n    train_df = reduce_mem_usage(train_df, 'train')\n    test_df = reduce_mem_usage(test_df, 'test')\n    \n    # Force garbage collection\n    gc.collect()\n    \n    return train_df, test_df\n\ndef preprocess_data(train_df, test_df):\n    \"\"\"Preprocess the data with memory efficiency in mind\"\"\"\n    print(\"\\nPreprocessing data...\")\n    \n    # If using recent months only\n    if Config.use_recent_months_only and 'timestamp' in train_df.columns:\n        # Convert timestamp to datetime if it's not already\n        train_df['timestamp'] = pd.to_datetime(train_df['timestamp'])\n        \n        # Get the last N months\n        cutoff_date = train_df['timestamp'].max() - pd.DateOffset(months=Config.recent_months)\n        train_df = train_df[train_df['timestamp'] >= cutoff_date].reset_index(drop=True)\n        print(f\"Using data from {cutoff_date} onwards. New train shape: {train_df.shape}\")\n        \n        # Force garbage collection after filtering\n        gc.collect()\n    \n    # Drop timestamp column as it's not useful for prediction\n    if 'timestamp' in train_df.columns:\n        train_df = train_df.drop('timestamp', axis=1)\n    if 'timestamp' in test_df.columns:\n        test_df = test_df.drop('timestamp', axis=1)\n    \n    # Separate features and target\n    feature_cols = [col for col in train_df.columns if col != 'label']\n    \n    X_train = train_df[feature_cols].copy()\n    y_train = train_df['label'].copy()\n    X_test = test_df[feature_cols].copy()\n    \n    # Delete original dataframes to free memory\n    del train_df\n    gc.collect()\n    \n    # Handle missing values if any\n    if X_train.isnull().sum().sum() > 0:\n        print(\"Handling missing values...\")\n        X_train = X_train.fillna(0)\n        X_test = X_test.fillna(0)\n    \n    # Feature engineering (simplified to save memory)\n    print(\"Adding engineered features...\")\n    \n    # Create features in-place to save memory\n    X_train['bid_ask_imbalance'] = (X_train['bid_qty'] - X_train['ask_qty']) / (X_train['bid_qty'] + X_train['ask_qty'] + 1e-8)\n    X_train['buy_sell_imbalance'] = (X_train['buy_qty'] - X_train['sell_qty']) / (X_train['buy_qty'] + X_train['sell_qty'] + 1e-8)\n    \n    X_test['bid_ask_imbalance'] = (X_test['bid_qty'] - X_test['ask_qty']) / (X_test['bid_qty'] + X_test['ask_qty'] + 1e-8)\n    X_test['buy_sell_imbalance'] = (X_test['buy_qty'] - X_test['sell_qty']) / (X_test['buy_qty'] + X_test['sell_qty'] + 1e-8)\n    \n    # Convert engineered features to float32\n    for col in ['bid_ask_imbalance', 'buy_sell_imbalance']:\n        X_train[col] = X_train[col].astype(np.float32)\n        X_test[col] = X_test[col].astype(np.float32)\n    \n    print(f\"Final train features: {X_train.shape[1]}\")\n    \n    # Force garbage collection\n    gc.collect()\n    \n    return X_train, y_train, X_test\n\ndef pearson_correlation(y_true, y_pred):\n    \"\"\"Calculate Pearson correlation coefficient\"\"\"\n    return pearsonr(y_true, y_pred)[0]\n\ndef train_lightgbm_single_fold(X_train, y_train, X_test, fold_data):\n    \"\"\"Train a single fold of LightGBM with memory optimization\"\"\"\n    fold, train_idx, valid_idx = fold_data\n    print(f\"\\nFold {fold + 1}/{Config.n_folds}\")\n    \n    # Split data\n    X_tr, X_val = X_train.iloc[train_idx], X_train.iloc[valid_idx]\n    y_tr, y_val = y_train.iloc[train_idx], y_train.iloc[valid_idx]\n    \n    # Create LightGBM datasets with free_raw_data to save memory\n    train_set = lgb.Dataset(X_tr, y_tr, free_raw_data=True)\n    valid_set = lgb.Dataset(X_val, y_val, free_raw_data=True, reference=train_set)\n    \n    # Train model\n    model = lgb.train(\n        Config.lgb_params,\n        train_set,\n        num_boost_round=Config.n_estimators,\n        valid_sets=[valid_set],\n        valid_names=['valid'],\n        callbacks=[\n            lgb.early_stopping(Config.early_stopping_rounds),\n            lgb.log_evaluation(Config.verbose_eval)\n        ]\n    )\n    \n    # Predictions\n    val_predictions = model.predict(X_val, num_iteration=model.best_iteration)\n    test_predictions = model.predict(X_test, num_iteration=model.best_iteration)\n    \n    # Calculate fold score\n    fold_score = pearson_correlation(y_val, val_predictions)\n    print(f\"Fold {fold + 1} Pearson correlation: {fold_score:.6f}\")\n    \n    # Get feature importance\n    importance = model.feature_importance(importance_type='gain')\n    \n    # Clean up\n    del train_set, valid_set, X_tr, X_val, y_tr, y_val\n    gc.collect()\n    \n    return val_predictions, test_predictions, valid_idx, importance, fold_score\n\ndef train_lightgbm_cv(X_train, y_train, X_test):\n    \"\"\"Train LightGBM with cross-validation using memory-efficient approach\"\"\"\n    print(\"\\nTraining LightGBM with cross-validation...\")\n    \n    # Store out-of-fold predictions\n    oof_predictions = np.zeros(len(X_train), dtype=np.float32)\n    test_predictions = np.zeros(len(X_test), dtype=np.float32)\n    \n    # Store feature importance\n    feature_importance = pd.DataFrame()\n    feature_importance['feature'] = X_train.columns\n    \n    # K-Fold cross-validation\n    kf = KFold(n_splits=Config.n_folds, shuffle=True, random_state=Config.seed)\n    \n    scores = []\n    \n    # Process one fold at a time to save memory\n    for fold, (train_idx, valid_idx) in enumerate(kf.split(X_train)):\n        fold_data = (fold, train_idx, valid_idx)\n        val_preds, test_preds, valid_idx, importance, fold_score = train_lightgbm_single_fold(\n            X_train, y_train, X_test, fold_data\n        )\n        \n        # Store predictions\n        oof_predictions[valid_idx] = val_preds\n        test_predictions += test_preds / Config.n_folds\n        \n        # Store feature importance\n        feature_importance[f'fold_{fold + 1}'] = importance\n        scores.append(fold_score)\n        \n        # Force garbage collection after each fold\n        gc.collect()\n    \n    # Calculate overall OOF score\n    oof_score = pearson_correlation(y_train, oof_predictions)\n    print(f\"\\nOverall OOF Pearson correlation: {oof_score:.6f}\")\n    print(f\"Mean CV score: {np.mean(scores):.6f} (+/- {np.std(scores):.6f})\")\n    \n    # Calculate mean feature importance\n    feature_importance['importance'] = feature_importance[[f'fold_{i+1}' for i in range(Config.n_folds)]].mean(axis=1)\n    feature_importance = feature_importance[['feature', 'importance']].sort_values('importance', ascending=False)\n    \n    print(\"\\nTop 20 most important features:\")\n    print(feature_importance.head(20))\n    \n    return test_predictions, oof_predictions, feature_importance\n\ndef create_submission(test_predictions, test_df):\n    \"\"\"Create submission file\"\"\"\n    print(\"\\nCreating submission file...\")\n    \n    # Load sample submission to get the correct format\n    try:\n        sample_sub = pd.read_csv(Config.sample_submission_path)\n        submission = sample_sub.copy()\n        submission['label'] = test_predictions\n    except:\n        # If sample submission not available, create from scratch\n        submission = pd.DataFrame({\n            'id': range(len(test_predictions)),\n            'label': test_predictions\n        })\n    \n    # Save submission\n    submission.to_csv(Config.submission_path, index=False)\n    print(f\"Submission saved to {Config.submission_path}\")\n    \n    # Display statistics\n    print(f\"\\nSubmission statistics:\")\n    print(f\"Mean: {submission['label'].mean():.6f}\")\n    print(f\"Std: {submission['label'].std():.6f}\")\n    print(f\"Min: {submission['label'].min():.6f}\")\n    print(f\"Max: {submission['label'].max():.6f}\")\n    \n    return submission\n\ndef main():\n    \"\"\"Main execution function with memory management\"\"\"\n    print(\"DRW Crypto Price Movement Prediction - Memory Efficient Version\")\n    print(\"=\" * 60)\n    \n    # Load data with memory optimization\n    train_df, test_df = load_data()\n    \n    # Store test_df reference for submission\n    test_df_copy = test_df.copy()\n    \n    # Preprocess data\n    X_train, y_train, X_test = preprocess_data(train_df, test_df)\n    \n    # Clean up\n    del train_df, test_df\n    gc.collect()\n    \n    # Train with cross-validation\n    cv_predictions, oof_predictions, feature_importance = train_lightgbm_cv(X_train, y_train, X_test)\n    \n    # Use CV predictions\n    test_predictions = cv_predictions\n    \n    # Create submission\n    submission = create_submission(test_predictions, test_df_copy)\n    \n    # Final cleanup\n    del X_train, y_train, X_test, test_df_copy\n    gc.collect()\n    \n    print(\"\\nTraining complete!\")\n    \n    return submission, feature_importance\n\nif __name__ == \"__main__\":\n    submission, feature_importance = main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T20:44:46.516605Z","iopub.execute_input":"2025-05-28T20:44:46.51729Z","iopub.status.idle":"2025-05-28T20:45:29.481637Z","shell.execute_reply.started":"2025-05-28T20:44:46.517262Z","shell.execute_reply":"2025-05-28T20:45:29.477685Z"}},"outputs":[],"execution_count":null}]}