{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#FEATURE IMPORTANCE ANALYSIS     \n# Analyses a sample of 4,500,000 rows from the 'train.parquet' and identifies top 10 features grouped by symbol_ids.\n# Example of Symbol_id = 17. Adjust as necessary\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport polars as pl\nimport numpy as np\nimport joblib\nimport tensorflow as tf\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, BatchNormalization, Dropout, Input\nimport matplotlib.pyplot as plt\nimport random\n\n# Step 1: Define Data Directory and Required Columns\ndata_dir = '/kaggle/input/jane-street-real-time-market-data-forecasting'\nfeatures = [f'feature_{i:02d}' for i in range(79)]\n\n# Focus on responder_0 and responder_3\nresponders = [f'responder_{i}' for i in range(9)]\nrequired_columns = features + responders + ['weight', 'symbol_id', 'date_id', 'time_id']\n\n# Step 2: Load and Sample Data\ntrain = pl.read_parquet(os.path.join(data_dir, 'train.parquet'), columns=required_columns)\ntrain_sample = train.sample(n=4_500_000)\n\n#Top Important Features for Each Group\n\nimport pandas as pd\nimport numpy as np\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\n\n# Exclude rows corresponding to symbol_id values 0 to 12 - ADJUST \nexcluded_symbol_ids = list(range(0, 17)) + list(range(18,39)) # Exclude all but idxs 16, 17\ntrain_sample = train_sample.filter(~pl.col(\"symbol_id\").is_in(list(excluded_symbol_ids)))\n\n# Convert Polars to Pandas for processing\ndata_pandas = train_sample.to_pandas()\n\n# Grouping data by 'symbol_id'\ngrouped = data_pandas.groupby('symbol_id')\n\n# Prepare to collect feature importance results\nfeature_importances = []\n\n# Define features and target responder\nfeatures = [col for col in data_pandas.columns if col.startswith('feature_')]\ntarget = 'responder_6'\n\n# Perform feature importance analysis for each 'symbol_id'\nfor symbol_id, group_data in grouped:\n    print(f\"Processing symbol_id: {symbol_id}\")\n    \n    # Drop rows with NaN values in target or features\n    group_data = group_data.dropna(subset=features + [target])\n    \n    # Define features (X) and target (y)\n    X = group_data[features]\n    y = group_data[target]\n    \n    # Skip if not enough data\n    if len(X) < 100:\n        print(f\"Not enough data for symbol_id {symbol_id}. Skipping.\")\n        continue\n    \n    # Split data into train and test\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n    \n    # Train a Random Forest model\n    model = RandomForestRegressor(n_estimators=100, random_state=42)\n    model.fit(X_train, y_train)\n    \n    # Get feature importances\n    importances = model.feature_importances_\n    importance_df = pd.DataFrame({\n        'feature': features,\n        'importance': importances\n    }).sort_values(by='importance', ascending=False)\n    \n    # Save top 10 important features for the group\n    top_features = importance_df.head(10)\n    top_features['symbol_id'] = symbol_id\n    top_features.to_csv(f\"top_features_symbol_{symbol_id}.csv\", index=False)\n    print(f\"Top features for symbol_id {symbol_id} saved to 'top_features_symbol_{symbol_id}.csv'.\")\n    \n    # Collect for overall analysis\n    feature_importances.append(top_features)\n\n# Combine all top features across symbol_id groups\ncombined_top_features = pd.concat(feature_importances, axis=0)\n\n# Save combined top features to a CSV file\ncombined_top_features.to_csv('combined_top_features_by_symbol_id.csv', index=False)\nprint(\"Combined top features saved to 'combined_top_features_by_symbol_id.csv'.\")\n\n# Plot example: top overall important features\noverall_importance = combined_top_features.groupby('feature')['importance'].mean().sort_values(ascending=False)\n\nplt.figure(figsize=(12, 8))\noverall_importance.head(20).plot(kind='bar', color='skyblue')\nplt.title(\"Top 20 Most Important Features (Aggregated Across symbol_id)\")\nplt.xlabel(\"Features\")\nplt.ylabel(\"Average Importance\")\nplt.xticks(rotation=45)\nplt.tight_layout()\nplt.savefig(\"overall_top_features.jpeg\", format=\"jpeg\", dpi=300)\nplt.show()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null}]}