{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaL4","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"},{"sourceId":91496,"databundleVersionId":11802066,"sourceType":"competition"},{"sourceId":11955382,"sourceType":"datasetVersion","datasetId":7516050}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport xgboost as xgb\nfrom sklearn.model_selection import train_test_split\nfrom scipy.stats import pearsonr\nfrom sklearn.utils.random import check_random_state\n\n# === Data Type Optimization ===\ndef optimize_dataframe(df):\n    for col in df.columns:\n        col_type = df[col].dtype\n        if pd.api.types.is_numeric_dtype(col_type):\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if pd.api.types.is_integer_dtype(col_type):\n                if c_min >= np.iinfo(np.int8).min and c_max <= np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min >= np.iinfo(np.int16).min and c_max <= np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min >= np.iinfo(np.int32).min and c_max <= np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n            else:\n                if c_min >= np.finfo(np.float16).min and c_max <= np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min >= np.finfo(np.float32).min and c_max <= np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n    return df\n\n# === Load datasets ===\ntrain = pd.read_parquet(\"/kaggle/input/drw-crypto-market-prediction/train.parquet\")\ntest = pd.read_parquet(\"/kaggle/input/drw-crypto-market-prediction/test.parquet\")\nsubmission = pd.read_csv(\"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\")\n\n# === Extract periodic time features BEFORE dropping timestamp (from index)\ntimestamp = pd.to_datetime(train.index)\ndayth = timestamp.dayofyear\nminute = timestamp.hour * 60 + timestamp.minute\n\n# Encode sin/cos for daily and yearly periodicity\ntrain[\"day_sin\"] = np.sin(2 * np.pi * dayth / 365)\ntrain[\"day_cos\"] = np.cos(2 * np.pi * dayth / 365)\ntrain[\"minute_sin\"] = np.sin(2 * np.pi * minute / 1440)\ntrain[\"minute_cos\"] = np.cos(2 * np.pi * minute / 1440)\n\n# Store target for timestamp prediction\ny_time = train[[\"day_sin\", \"day_cos\", \"minute_sin\", \"minute_cos\"]].copy()\n\n# Drop timestamp only from test (train uses it as index)\ntest.drop(columns=[\"timestamp\"], inplace=True, errors=\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-28T08:53:21.057241Z","iopub.execute_input":"2025-05-28T08:53:21.057859Z","iopub.status.idle":"2025-05-28T08:54:03.825327Z","shell.execute_reply.started":"2025-05-28T08:53:21.057836Z","shell.execute_reply":"2025-05-28T08:54:03.824713Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras import layers, models\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.metrics.pairwise import cosine_similarity\nimport tensorflow as tf\nimport pandas as pd\nimport numpy as np\n\n# === Ensure top_features is defined\ntop_features = pd.read_csv(\"/kaggle/input/shapfeature/shap_selected_features.csv\")[\"feature\"].tolist()\n\n# === Inputs and targets for time prediction model\nX_time = train[top_features]\ny_time_target = y_time[[\"day_sin\", \"day_cos\", \"minute_sin\", \"minute_cos\"]]\n\n# === Split into train/val\nX_tr, X_val, y_tr, y_val = train_test_split(X_time, y_time_target, test_size=0.2, random_state=42)\n\n# === Confirm GPU usage\nprint(\"🚀 Available devices:\", tf.config.list_physical_devices())\nprint(\"🧠 Using GPU:\" if tf.config.list_physical_devices('GPU') else \"⚠️ GPU not found, running on CPU\")\n\n# === Define the model\ndef build_time_predictor(input_dim):\n    model = models.Sequential([\n        layers.Dense(256, activation='relu', input_shape=(input_dim,)),\n        layers.Dense(128, activation='relu'),\n        layers.Dense(4)  # Predict: day_sin, day_cos, minute_sin, minute_cos\n    ])\n    model.compile(optimizer='adam', loss='mse')\n    return model\n\nmodel_time = build_time_predictor(X_tr.shape[1])\n\n# === Train the model\nmodel_time.fit(X_tr, y_tr, validation_data=(X_val, y_val), epochs=20, batch_size=1024, verbose=1)\n\n# === Evaluate performance\ny_val_pred = model_time.predict(X_val)\nmae = mean_absolute_error(y_val, y_val_pred)\ncos_sim = cosine_similarity(y_val, y_val_pred).diagonal().mean()\n\nprint(f\"\\n📊 Time Feature Model Evaluation\")\nprint(f\"🔹 MAE: {mae:.5f}\")\nprint(f\"🔹 Avg. Cosine Similarity: {cos_sim:.5f}\")\n\n# === Predict time features for test set\ntest_time_pred = model_time.predict(test[top_features])\n\n# === Append predicted features to test set\ntest[\"day_sin\"] = test_time_pred[:, 0]\ntest[\"day_cos\"] = test_time_pred[:, 1]\ntest[\"minute_sin\"] = test_time_pred[:, 2]\ntest[\"minute_cos\"] = test_time_pred[:, 3]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T08:54:13.305506Z","iopub.execute_input":"2025-05-28T08:54:13.306035Z","iopub.status.idle":"2025-05-28T08:55:50.923633Z","shell.execute_reply.started":"2025-05-28T08:54:13.306011Z","shell.execute_reply":"2025-05-28T08:55:50.922949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install gplearn\n\nimport xgboost as xgb\nfrom scipy.stats import pearsonr\nfrom gplearn.genetic import SymbolicTransformer\n\n# === Add time columns to feature set\nall_features = top_features + [\"day_sin\", \"day_cos\", \"minute_sin\", \"minute_cos\"]\n\n# === Redefine target (if lost)\ny = train[\"label\"]\n\n# === Optimize types again\nX = optimize_dataframe(train[all_features])\ntest = optimize_dataframe(test[all_features])\n\n# === Clip to prevent symbolic overflow\nX_clipped = X.clip(lower=-1e3, upper=1e3)\ntest_clipped = test.clip(lower=-1e3, upper=1e3)\n\n# === gplearn symbolic transformation\nfunction_set = ['add', 'sub', 'mul', 'sqrt', 'abs', 'neg']\ngp = SymbolicTransformer(\n    generations=10,\n    population_size=1000,\n    hall_of_fame=100,\n    n_components=10,\n    function_set=function_set,\n    parsimony_coefficient=0.01,\n    max_samples=0.9,\n    verbose=1,\n    random_state=42,\n    n_jobs=-1\n)\n\ngp.fit(X_clipped, y)\ngp_features_train = gp.transform(X_clipped)\ngp_features_test = gp.transform(test_clipped)\n\n# === Final feature matrices\nX_full = np.hstack([X, gp_features_train])\ntest_full = np.hstack([test, gp_features_test])\n\n# === Train-validation split\nX_train, X_val, y_train, y_val = train_test_split(X_full, y, test_size=0.2, random_state=42)\n\n# === XGBoost training\nmodel = xgb.XGBRegressor(\n    tree_method=\"hist\",\n    device=\"cuda\",\n    n_estimators=500,\n    max_depth=8,\n    learning_rate=0.05,\n    subsample=0.6,\n    colsample_bytree=0.8,\n    reg_alpha=0.1,\n    reg_lambda=1.0,\n    random_state=42\n)\n\nmodel.fit(X_train, y_train,\n          eval_set=[(X_val, y_val)],\n          early_stopping_rounds=20,\n          verbose=True)\n\n# === Evaluate\ny_pred_val = model.predict(X_val)\npearson_corr, _ = pearsonr(y_val, y_pred_val)\n\nprint(f\"\\n📊 Final GP+Time+SHAP Model Evaluation\")\nprint(f\"📈 Validation Pearson Correlation: {pearson_corr:.5f}\")\n\n# === Predict and Save submission\ny_test_pred = model.predict(test_full)\nsubmission[\"prediction\"] = y_test_pred\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"📦 Saved: submission.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T09:00:36.460382Z","iopub.execute_input":"2025-05-28T09:00:36.461141Z","iopub.status.idle":"2025-05-28T09:01:57.932576Z","shell.execute_reply.started":"2025-05-28T09:00:36.461114Z","shell.execute_reply":"2025-05-28T09:01:57.931925Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"📋 Final XGBoost Feature Columns (train):\", X.shape[1])\nprint(X.columns.tolist() if isinstance(X, pd.DataFrame) else \"➡️ Not a DataFrame — likely NumPy array after hstack\")\n\nprint(\"\\n📋 Final XGBoost Feature Columns (test):\", test.shape[1])\nprint(test.columns.tolist() if isinstance(test, pd.DataFrame) else \"➡️ Not a DataFrame — likely NumPy array after hstack\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T09:03:42.486285Z","iopub.execute_input":"2025-05-28T09:03:42.486543Z","iopub.status.idle":"2025-05-28T09:03:42.491061Z","shell.execute_reply.started":"2025-05-28T09:03:42.486524Z","shell.execute_reply":"2025-05-28T09:03:42.490363Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Show first 5 rows of test data before symbolic features were added\nprint(\"📊 Sample of test data (with SHAP + predicted time features):\")\nprint(test.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T09:08:47.701682Z","iopub.execute_input":"2025-05-28T09:08:47.702368Z","iopub.status.idle":"2025-05-28T09:08:47.719923Z","shell.execute_reply.started":"2025-05-28T09:08:47.702346Z","shell.execute_reply":"2025-05-28T09:08:47.719378Z"}},"outputs":[],"execution_count":null}]}