{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"},{"sourceId":248874877,"sourceType":"kernelVersion"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pathlib import Path\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Deep learning imports\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers, models, callbacks, regularizers\nfrom tensorflow.keras.layers import Dense, Dropout, GaussianNoise, BatchNormalization, LayerNormalization, Add\nfrom sklearn.preprocessing import StandardScaler, RobustScaler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error, r2_score\n\n# GPU Configuration\nprint(\"=\"*80)\nprint(\"GPU CONFIGURATION\")\nprint(\"=\"*80)\n\ngpus = tf.config.list_physical_devices('GPU')\nif gpus:\n    print(f\"✓ Found {len(gpus)} GPU(s)\")\n    try:\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n        print(\"✓ GPU memory growth enabled\")\n    except:\n        pass\n    \n    try:\n        from tensorflow.keras import mixed_precision\n        policy = mixed_precision.Policy('mixed_float16')\n        mixed_precision.set_global_policy(policy)\n        print(\"✓ Mixed precision enabled\")\n    except:\n        pass\nelse:\n    print(\"⚠️ No GPU found. Running on CPU.\")\n\n# Set random seeds\nnp.random.seed(42)\ntf.random.set_seed(42)\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"ADVANCED NEURAL NETWORKS FOR CRYPTO PREDICTION\")\nprint(\"=\"*80)\n\n# Load data\nprint(\"\\nLoading datasets...\")\ntrain_df = pd.read_parquet('/kaggle/input/step-3-drw-remove-columns-with-little-variance/train_reduced_variance.parquet')\ntest_df = pd.read_parquet('/kaggle/input/step-3-drw-remove-columns-with-little-variance/test_reduced_variance.parquet')\n\nprint(f\"Train shape: {train_df.shape}\")\nprint(f\"Test shape: {test_df.shape}\")\n\n# Prepare features\nmetadata_cols = [col for col in train_df.columns if col.startswith('_')]\nfeature_cols = [col for col in train_df.columns if col not in ['label'] + metadata_cols]\n\nX_train_full = train_df[feature_cols].values.astype(np.float32)\ny_train_full = train_df['label'].values.astype(np.float32)\nX_test = test_df[feature_cols].values.astype(np.float32)\n\n# Handle missing values\nX_train_full = np.nan_to_num(X_train_full, 0)\nX_test = np.nan_to_num(X_test, 0)\n\n# Split data\nX_train, X_val, y_train, y_val = train_test_split(X_train_full, y_train_full, test_size=0.2, random_state=42)\n\nprint(f\"\\nTrain samples: {len(X_train):,}\")\nprint(f\"Val samples: {len(X_val):,}\")\n\n# Scale data\nprint(\"\\nScaling data...\")\nscaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(X_train).astype(np.float32)\nX_val_scaled = scaler.transform(X_val).astype(np.float32)\nX_test_scaled = scaler.fit_transform(X_train_full).astype(np.float32)\nX_test_final = scaler.transform(X_test).astype(np.float32)\n\n# Define architectures\ndef create_residual_network(input_dim, name='ResNet'):\n    \"\"\"Residual Network\"\"\"\n    inputs = layers.Input(shape=(input_dim,), name=f'{name}_input')\n    \n    # Initial layer\n    x = Dense(512, activation='relu')(inputs)\n    x = BatchNormalization()(x)\n    x = Dropout(0.3)(x)\n    \n    # Residual blocks\n    for units in [512, 256, 256, 128]:\n        shortcut = x\n        x = Dense(units, activation='relu')(x)\n        x = BatchNormalization()(x)\n        x = Dropout(0.3)(x)\n        x = Dense(units)(x)\n        x = BatchNormalization()(x)\n        \n        # Adjust shortcut if needed\n        if shortcut.shape[-1] != units:\n            shortcut = Dense(units)(shortcut)\n        \n        x = Add()([shortcut, x])\n        x = layers.Activation('relu')(x)\n    \n    outputs = Dense(1, name=f'{name}_output')(x)\n    return models.Model(inputs, outputs, name=name)\n\ndef create_deep_network(input_dim, name='DeepNet'):\n    \"\"\"Deep Dense Network\"\"\"\n    inputs = layers.Input(shape=(input_dim,), name=f'{name}_input')\n    \n    x = GaussianNoise(0.1)(inputs)\n    \n    # Deep layers\n    for units in [1024, 512, 512, 256, 256, 128, 64]:\n        x = Dense(units, activation='elu')(x)\n        x = BatchNormalization()(x)\n        x = Dropout(0.4)(x)\n    \n    outputs = Dense(1, name=f'{name}_output')(x)\n    return models.Model(inputs, outputs, name=name)\n\ndef create_wide_network(input_dim, name='WideNet'):\n    \"\"\"Wide Network\"\"\"\n    inputs = layers.Input(shape=(input_dim,), name=f'{name}_input')\n    \n    x = inputs\n    \n    # Wide layers\n    for units in [2048, 1024, 512, 256]:\n        x = Dense(units, activation='relu', kernel_regularizer=regularizers.l2(0.0001))(x)\n        x = BatchNormalization()(x)\n        x = Dropout(0.5)(x)\n    \n    outputs = Dense(1, name=f'{name}_output')(x)\n    return models.Model(inputs, outputs, name=name)\n\ndef create_regularized_network(input_dim, name='RegNet'):\n    \"\"\"Heavily Regularized Network\"\"\"\n    inputs = layers.Input(shape=(input_dim,), name=f'{name}_input')\n    \n    x = GaussianNoise(0.2)(inputs)\n    \n    for i, units in enumerate([512, 256, 128, 64]):\n        x = Dense(units, activation='relu', kernel_regularizer=regularizers.l2(0.001))(x)\n        x = LayerNormalization()(x)\n        x = Dropout(0.5 - i * 0.1)(x)  # Decreasing dropout\n    \n    outputs = Dense(1, name=f'{name}_output')(x)\n    return models.Model(inputs, outputs, name=name)\n\n# Training setup\nprint(\"\\n\" + \"=\"*80)\nprint(\"TRAINING MODELS\")\nprint(\"=\"*80)\n\n# Model configurations\nmodel_configs = [\n    {'name': 'ResNet', 'fn': create_residual_network, 'optimizer': 'adam', 'epochs': 50},\n    {'name': 'DeepNet', 'fn': create_deep_network, 'optimizer': 'adam', 'epochs': 50},\n    {'name': 'WideNet', 'fn': create_wide_network, 'optimizer': 'adam', 'epochs': 40},\n    {'name': 'RegNet', 'fn': create_regularized_network, 'optimizer': 'adam', 'epochs': 60},\n]\n\n# Train models\nresults = {}\nall_predictions = []\n\nfor config in model_configs:\n    print(f\"\\n{'='*60}\")\n    print(f\"Training {config['name']}\")\n    print(f\"{'='*60}\")\n    \n    # Clear session\n    tf.keras.backend.clear_session()\n    gc.collect()\n    \n    # Create model\n    model = config['fn'](X_train_scaled.shape[1], name=config['name'])\n    \n    # Compile\n    optimizer = keras.optimizers.Adam(learning_rate=0.001)\n    if gpus and 'mixed_float16' in str(mixed_precision.global_policy().name):\n        try:\n            optimizer = mixed_precision.LossScaleOptimizer(optimizer)\n        except:\n            pass\n    \n    model.compile(\n        optimizer=optimizer,\n        loss='mse',\n        metrics=['mae']\n    )\n    \n    # Callbacks\n    early_stopping = callbacks.EarlyStopping(\n        monitor='val_loss',\n        patience=15,\n        restore_best_weights=True,\n        verbose=0\n    )\n    \n    reduce_lr = callbacks.ReduceLROnPlateau(\n        monitor='val_loss',\n        factor=0.5,\n        patience=5,\n        min_lr=0.00001,\n        verbose=0\n    )\n    \n    # Train\n    history = model.fit(\n        X_train_scaled, y_train,\n        validation_data=(X_val_scaled, y_val),\n        epochs=config['epochs'],\n        batch_size=512,\n        callbacks=[early_stopping, reduce_lr],\n        verbose=1\n    )\n    \n    # Evaluate\n    val_pred = model.predict(X_val_scaled, verbose=0).flatten()\n    val_rmse = np.sqrt(mean_squared_error(y_val, val_pred))\n    val_r2 = r2_score(y_val, val_pred)\n    \n    print(f\"\\nValidation RMSE: {val_rmse:.6f}\")\n    print(f\"Validation R²: {val_r2:.6f}\")\n    \n    # Test predictions\n    test_pred = model.predict(X_test_final, verbose=0).flatten()\n    \n    # Store results\n    results[config['name']] = {\n        'model': model,\n        'val_rmse': val_rmse,\n        'val_r2': val_r2,\n        'test_pred': test_pred,\n        'history': history.history\n    }\n    all_predictions.append(test_pred)\n\n# Create ensemble\nprint(\"\\n\" + \"=\"*80)\nprint(\"CREATING ENSEMBLE\")\nprint(\"=\"*80)\n\n# Sort by performance\nsorted_models = sorted(results.items(), key=lambda x: x[1]['val_rmse'])\n\nprint(\"\\nModel Performance:\")\nfor name, result in sorted_models:\n    print(f\"{name}: RMSE={result['val_rmse']:.6f}, R²={result['val_r2']:.6f}\")\n\n# Weighted ensemble\nweights = []\npredictions = []\nfor name, result in sorted_models:\n    weight = 1.0 / result['val_rmse']\n    weights.append(weight)\n    predictions.append(result['test_pred'])\n\nweights = np.array(weights)\nweights = weights / weights.sum()\n\nensemble_pred = np.average(predictions, axis=0, weights=weights)\n\n# Create submission\nprint(\"\\n\" + \"=\"*80)\nprint(\"CREATING SUBMISSION\")\nprint(\"=\"*80)\n\nsubmission = pd.DataFrame({\n    'ID': range(1, len(ensemble_pred) + 1),\n    'prediction': ensemble_pred\n})\n\nprint(f\"\\nPredictions: {len(submission)}\")\nprint(f\"Range: [{submission['prediction'].min():.4f}, {submission['prediction'].max():.4f}]\")\nprint(f\"Mean: {submission['prediction'].mean():.4f}\")\nprint(f\"Std: {submission['prediction'].std():.4f}\")\n\n# Save\noutput_dir = Path('/kaggle/working')\nsubmission.to_csv(output_dir / 'submission.csv', index=False)\nprint(f\"\\n✓ Saved to: submission.csv\")\n\n# Visualizations\nfig, axes = plt.subplots(2, 2, figsize=(12, 10))\n\n# 1. Model comparison\nax = axes[0, 0]\nnames = list(results.keys())\nrmses = [results[n]['val_rmse'] for n in names]\nbars = ax.bar(names, rmses, color=['green' if r == min(rmses) else 'blue' for r in rmses])\nax.set_ylabel('Validation RMSE')\nax.set_title('Model Performance')\nfor bar, rmse in zip(bars, rmses):\n    ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.001,\n            f'{rmse:.4f}', ha='center', va='bottom', fontsize=9)\n\n# 2. Training history (best model)\nax = axes[0, 1]\nbest_name = sorted_models[0][0]\nhistory = results[best_name]['history']\nax.plot(history['loss'], label='Train')\nax.plot(history['val_loss'], label='Val')\nax.set_xlabel('Epoch')\nax.set_ylabel('Loss')\nax.set_title(f'Training History - {best_name}')\nax.legend()\nax.set_yscale('log')\n\n# 3. Prediction distribution\nax = axes[1, 0]\nax.hist(ensemble_pred, bins=50, color='green', alpha=0.7, edgecolor='black')\nax.set_xlabel('Prediction')\nax.set_ylabel('Count')\nax.set_title('Ensemble Prediction Distribution')\n\n# 4. Model weights\nax = axes[1, 1]\nax.bar(names, weights, color='orange')\nax.set_ylabel('Ensemble Weight')\nax.set_title('Model Weights in Ensemble')\nfor i, (name, weight) in enumerate(zip(names, weights)):\n    ax.text(i, weight + 0.01, f'{weight:.3f}', ha='center', va='bottom')\n\nplt.tight_layout()\nplt.savefig(output_dir / 'model_analysis.png', dpi=150, bbox_inches='tight')\nplt.show()\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"COMPLETE!\")\nprint(\"=\"*80)\nprint(f\"\\n✓ Trained {len(results)} models\")\nprint(f\"✓ Best model: {sorted_models[0][0]} (RMSE: {sorted_models[0][1]['val_rmse']:.6f})\")\nprint(f\"✓ Ensemble weights: {dict(zip(names, weights.round(3)))}\")\nprint(\"\\n✅ Done!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}