{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ============================================================\n# SECTION 0: IMPORTS AND CONFIGURATION\n# ============================================================\nimport os\nimport sys\nimport gc\nimport time\nimport glob\nimport warnings\nimport random\nfrom pathlib import Path\nfrom collections import defaultdict\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib\nmatplotlib.use('Agg')\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, TensorDataset, Dataset\n\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.preprocessing import RobustScaler, StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import IsolationForest, RandomForestClassifier\nfrom sklearn.metrics import (\n classification_report, confusion_matrix, precision_recall_curve,\n roc_curve, roc_auc_score, average_precision_score, fbeta_score,\n f1_score, precision_score, recall_score, accuracy_score\n)\nfrom sklearn.calibration import CalibratedClassifierCV\n\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom catboost import CatBoostClassifier\n\nfrom imblearn.over_sampling import SMOTE\nfrom imblearn.under_sampling import RandomUnderSampler\n\nimport joblib\n\nwarnings.filterwarnings('ignore')\n\n# ---- Reproducibility ----\nSEED = 42\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed_all(SEED)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\n\n# ---- GPU Detection ----\nif torch.cuda.is_available():\n DEVICE = torch.device('cuda')\n print(f\"[GPU] CUDA available: {torch.cuda.get_device_name(0)}\")\n print(f\"[GPU] Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB\")\nelse:\n DEVICE = torch.device('cpu')\n print(\"[CPU] No CUDA GPU detected — using CPU\")\n\n# ---- Paths ----\nDATA_ROOT = None\nfor p in [\n '/kaggle/input/competitions/smartphone-decimeter-2023/sdc2023/train',\n '/kaggle/input/smartphone-decimeter-2023/sdc2023/train',\n '/kaggle/input/smartphone-decimeter-2023/train'\n]:\n if os.path.isdir(p):\n DATA_ROOT = p\n break\nif DATA_ROOT is None:\n raise FileNotFoundError(\"Train folder not found.\")\n\nSAVE_DIR = '/kaggle/working/models/'\nPLOT_DIR = '/kaggle/working/plots/'\nos.makedirs(SAVE_DIR, exist_ok=True)\nos.makedirs(PLOT_DIR, exist_ok=True)\n\nprint(f\"Data root: {DATA_ROOT}\")\n\n# ---- Timing utility ----\nNOTEBOOK_START = time.time()\n\ndef elapsed():\n s = int(time.time() - NOTEBOOK_START)\n return f\"[{s//3600:02d}h{(s%3600)//60:02d}m{s%60:02d}s]\"\n\ndef log(msg):\n print(f\"{elapsed()} {msg}\", flush=True)\n\n# ---- Heartbeat ----\nimport threading\n\ndef heartbeat(interval=60):\n while True:\n time.sleep(interval)\n s = int(time.time() - NOTEBOOK_START)\n print(f\" ♥ still running — {s//3600:02d}h{(s%3600)//60:02d}m{s%60:02d}s\", flush=True)\n\nhb = threading.Thread(target=heartbeat, daemon=True)\nhb.start()\n\n# ============================================================\n# COLUMN CONFIGURATION\n# ============================================================\nKEEP_COLS = [\n 'MultipathIndicator',\n 'Cn0DbHz', 'SvElevationDegrees', 'PseudorangeRateMetersPerSecond',\n 'TroposphericDelayMeters', 'ReceivedSvTimeUncertaintyNanos',\n 'BiasUncertaintyNanos', 'DriftUncertaintyNanosPerSecond',\n 'AccumulatedDeltaRangeMeters', 'IsrbMeters', 'RawPseudorangeMeters',\n 'PseudorangeRateUncertaintyMetersPerSecond',\n 'AccumulatedDeltaRangeUncertaintyMeters',\n 'AccumulatedDeltaRangeState', 'CarrierFrequencyHz',\n 'TimeNanos', 'Svid', 'ConstellationType',\n]\n\n# Base features (31)\nBASE_FEATURES = [\n 'Cn0DbHz', 'SvElevationDegrees', 'PseudorangeRateMetersPerSecond',\n 'TroposphericDelayMeters', 'ReceivedSvTimeUncertaintyNanos',\n 'BiasUncertaintyNanos', 'DriftUncertaintyNanosPerSecond',\n 'AccumulatedDeltaRangeMeters',\n 'PseudorangeRateUncertaintyMetersPerSecond',\n 'AccumulatedDeltaRangeUncertaintyMeters',\n 'elevation_sin', 'log_time_unc', 'log_bias_unc', 'prr_abs',\n 'log_prr_unc', 'log_adr_unc',\n 'adr_cycle_slip', 'adr_quality_score',\n 'wavelength_cm',\n 'low_elev_flag', 'low_cno_flag', 'high_unc_flag',\n 'cno_unc_ratio', 'prr_unc_product', 'prr_bias_product',\n 'tropo_bias_product', 'cno_bias_ratio', 'elev_bias_product',\n 'code_carrier_div',\n 'drift_unc_missing', 'sv_data_missing',\n]\n\n# Temporal features (added in Section 3B)\nTEMPORAL_FEATURES = [\n # Rolling statistics per satellite\n 'cno_roll3_mean', 'cno_roll3_std',\n 'prr_roll3_mean', 'prr_roll3_std',\n 'time_unc_roll3_mean',\n 'ccd_roll3_mean', 'ccd_roll3_std',\n # Lag features\n 'cno_lag1', 'cno_lag2',\n 'prr_lag1', 'prr_lag2',\n 'ccd_lag1', 'ccd_lag2',\n # Rate of change\n 'cno_diff1', 'prr_diff1', 'ccd_diff1',\n # Satellite-level aggregation\n 'sv_mean_cno', 'sv_std_cno',\n 'sv_mean_ccd', 'sv_std_ccd',\n # Epoch-level features\n 'epoch_sv_count', 'epoch_mean_cno', 'epoch_mean_elev',\n 'epoch_multipath_risk',\n]\n\nFEATURE_COLS = BASE_FEATURES + TEMPORAL_FEATURES\n\nprint(f\"Base features: {len(BASE_FEATURES)}\")\nprint(f\"Temporal features: {len(TEMPORAL_FEATURES)}\")\nprint(f\"Total features: {len(FEATURE_COLS)}\")\nprint(\"=\" * 60)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 1: DATA LOADING\n# ============================================================\nlog(\"=\" * 60)\nlog(\"SECTION 1: DATA LOADING — started\")\nlog(\"=\" * 60)\n\nt0 = time.time()\n\ncsv_files = glob.glob(os.path.join(DATA_ROOT, '*', '*', 'device_gnss.csv'))\nlog(f\"Found {len(csv_files)} device_gnss.csv files\")\n\nframes = []\nfor fpath in tqdm(csv_files, desc=\"Loading CSVs\"):\n try:\n header_df = pd.read_csv(fpath, nrows=0)\n available_cols = [c for c in KEEP_COLS if c in header_df.columns]\n non_float_cols = {'MultipathIndicator', 'AccumulatedDeltaRangeState',\n 'Svid', 'ConstellationType'}\n dtype_map = {c: 'float32' for c in available_cols if c not in non_float_cols}\n chunk = pd.read_csv(fpath, usecols=available_cols, dtype=dtype_map)\n for c in KEEP_COLS:\n if c not in chunk.columns:\n chunk[c] = np.nan\n parts = Path(fpath).parts\n chunk['_drive_id'] = parts[-3]\n chunk['_phone_name'] = parts[-2]\n frames.append(chunk)\n except Exception as e:\n print(f\" [WARN] Skipping {fpath}: {e}\")\n\ndf = pd.concat(frames, ignore_index=True)\ndel frames\ngc.collect()\n\nfor col in df.select_dtypes(include=['float64']).columns:\n df[col] = df[col].astype('float32')\n\nlog(f\"Total rows loaded: {len(df):,}\")\nlog(f\"MultipathIndicator NaN count: {df['MultipathIndicator'].isna().sum():,} \"\n f\"({df['MultipathIndicator'].isna().mean():.2%})\")\nlog(f\"MultipathIndicator distribution (excluding NaN): {df['MultipathIndicator'].value_counts().to_dict()}\")\nlog(f\"SECTION 1 complete — took {time.time()-t0:.1f}s\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 2: DATA CLEANING\n# ============================================================\nlog(\"=\" * 60)\nlog(\"SECTION 2: DATA CLEANING — started\")\nlog(\"=\" * 60)\n\nt0 = time.time()\n\n# Sentinel values\nisrb_mask = df['IsrbMeters'].abs() > 1_000_000\nlog(f\"IsrbMeters sentinels replaced: {isrb_mask.sum():,}\")\ndf.loc[isrb_mask, 'IsrbMeters'] = np.nan\n\npseudo_mask = (df['RawPseudorangeMeters'] < 15_000_000) | \\\n (df['RawPseudorangeMeters'] > 35_000_000)\nlog(f\"RawPseudorangeMeters sentinels replaced: {pseudo_mask.sum():,}\")\ndf.loc[pseudo_mask, 'RawPseudorangeMeters'] = np.nan\n\n# Missing flags\ndf['drift_unc_missing'] = df['DriftUncertaintyNanosPerSecond'].isna().astype('int8')\ndf['sv_data_missing'] = df['SvElevationDegrees'].isna().astype('int8')\nlog(f\"drift_unc_missing rate: {df['drift_unc_missing'].mean():.4f}\")\nlog(f\"sv_data_missing rate: {df['sv_data_missing'].mean():.4f}\")\n\nlog(f\"SECTION 2 complete — took {time.time()-t0:.1f}s\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 3A: BASE FEATURE ENGINEERING\n# ============================================================\nlog(\"=\" * 60)\nlog(\"SECTION 3A: BASE FEATURE ENGINEERING — started\")\nlog(\"=\" * 60)\n\nt0 = time.time()\n\nSPEED_OF_LIGHT = 299_792_458\n\n# Transforms\ndf['elevation_sin'] = np.sin(np.radians(df['SvElevationDegrees'])).astype('float32')\ndf['log_time_unc'] = np.log1p(df['ReceivedSvTimeUncertaintyNanos']).astype('float32')\ndf['log_bias_unc'] = np.log1p(df['BiasUncertaintyNanos']).astype('float32')\ndf['prr_abs'] = df['PseudorangeRateMetersPerSecond'].abs().astype('float32')\ndf['log_prr_unc'] = np.log1p(df['PseudorangeRateUncertaintyMetersPerSecond']).astype('float32')\ndf['log_adr_unc'] = np.log1p(df['AccumulatedDeltaRangeUncertaintyMeters']).astype('float32')\n\n# ADR State bits\nadr_state = df['AccumulatedDeltaRangeState'].fillna(0).values.astype(np.int64)\ndf['adr_cycle_slip'] = ((adr_state >> 2) & 1).astype(np.int8)\ndf['adr_quality_score'] = (((adr_state & 1) - ((adr_state >> 1) & 1) -\n ((adr_state >> 2) & 1))).astype(np.int8)\ndel adr_state\n\n# Carrier frequency\ndf['wavelength_cm'] = np.where(\n df['CarrierFrequencyHz'].notna() & (df['CarrierFrequencyHz'] > 0),\n (SPEED_OF_LIGHT / df['CarrierFrequencyHz']) * 100,\n np.nan\n).astype('float32')\n\n# Flags\ndf['low_elev_flag'] = (df['SvElevationDegrees'].fillna(999) < 15).astype('int8')\ndf['low_cno_flag'] = (df['Cn0DbHz'].fillna(999) < 22).astype('int8')\ndf['high_unc_flag'] = (df['ReceivedSvTimeUncertaintyNanos'].fillna(0) > 100).astype('int8')\n\n# Interactions\ndf['cno_unc_ratio'] = (df['Cn0DbHz'] / (df['log_time_unc'] + 1e-6)).astype('float32')\ndf['prr_unc_product'] = (df['prr_abs'] * df['log_time_unc']).astype('float32')\ndf['prr_bias_product'] = (df['prr_abs'] * df['log_bias_unc']).astype('float32')\ndf['tropo_bias_product'] = (df['TroposphericDelayMeters'] * df['log_bias_unc']).astype('float32')\ndf['cno_bias_ratio'] = (df['Cn0DbHz'] / (df['log_bias_unc'] + 1e-6)).astype('float32')\ndf['elev_bias_product'] = (df['elevation_sin'] * df['log_bias_unc']).astype('float32')\n\n# Gold standard: Code-Carrier Divergence\ndf['code_carrier_div'] = (df['AccumulatedDeltaRangeMeters'] -\n df['RawPseudorangeMeters']).abs().astype('float32')\n\n# Replace inf\nfor col in BASE_FEATURES:\n if col in df.columns and df[col].dtype in ['float32', 'float64']:\n mask = np.isinf(df[col].values)\n if mask.any():\n log(f\" Replacing {mask.sum()} inf values in {col}\")\n df.loc[mask, col] = np.nan\n\nlog(f\"SECTION 3A complete — took {time.time()-t0:.1f}s\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 3B: TEMPORAL FEATURE ENGINEERING\n# ============================================================\nlog(\"=\" * 60)\nlog(\"SECTION 3B: TEMPORAL FEATURE ENGINEERING — started\")\nlog(\"=\" * 60)\n\nt0 = time.time()\n\n# Sort by drive, phone, satellite, time — CRITICAL for temporal features\nlog(\"Sorting data for temporal computation...\")\ndf['_sort_key'] = (df['_drive_id'].astype(str) + '_' +\n df['_phone_name'].astype(str) + '_' +\n df['Svid'].fillna(-1).astype(str) + '_' +\n df['ConstellationType'].fillna(-1).astype(str))\n\ndf = df.sort_values(['_sort_key',\n 'TimeNanos' if 'TimeNanos' in df.columns else '_sort_key'],\n ignore_index=True)\n\nlog(\"Computing per-satellite temporal features...\")\n\n# Group key for per-satellite time series\nsat_group_key = '_sort_key'\n\n# ---- Rolling statistics (window=3) per satellite ----\nfor col, new_name in [\n ('Cn0DbHz', 'cno_roll3_mean'),\n ('Cn0DbHz', 'cno_roll3_std'),\n ('PseudorangeRateMetersPerSecond', 'prr_roll3_mean'),\n ('PseudorangeRateMetersPerSecond', 'prr_roll3_std'),\n ('ReceivedSvTimeUncertaintyNanos', 'time_unc_roll3_mean'),\n ('code_carrier_div', 'ccd_roll3_mean'),\n ('code_carrier_div', 'ccd_roll3_std'),\n]:\n stat = 'std' if 'std' in new_name else 'mean'\n log(f\" Computing {new_name}...\")\n df[new_name] = (df.groupby(sat_group_key)[col]\n .transform(lambda x: x.rolling(3, min_periods=1).agg(stat))\n .astype('float32'))\n\n# ---- Lag features ----\nlog(\"Computing lag features...\")\nfor col, lag, new_name in [\n ('Cn0DbHz', 1, 'cno_lag1'),\n ('Cn0DbHz', 2, 'cno_lag2'),\n ('PseudorangeRateMetersPerSecond', 1, 'prr_lag1'),\n ('PseudorangeRateMetersPerSecond', 2, 'prr_lag2'),\n ('code_carrier_div', 1, 'ccd_lag1'),\n ('code_carrier_div', 2, 'ccd_lag2'),\n]:\n df[new_name] = (df.groupby(sat_group_key)[col]\n .transform(lambda x: x.shift(lag))\n .astype('float32'))\n\n# ---- Rate of change features ----\nlog(\"Computing diff features...\")\nfor col, new_name in [\n ('Cn0DbHz', 'cno_diff1'),\n ('PseudorangeRateMetersPerSecond', 'prr_diff1'),\n ('code_carrier_div', 'ccd_diff1'),\n]:\n df[new_name] = (df.groupby(sat_group_key)[col]\n .transform(lambda x: x.diff(1))\n .astype('float32'))\n\n# ---- Satellite-level aggregation (across all time for this satellite) ----\nlog(\"Computing satellite-level statistics...\")\nsat_stats = df.groupby(sat_group_key).agg(\n sv_mean_cno=('Cn0DbHz', 'mean'),\n sv_std_cno=('Cn0DbHz', 'std'),\n sv_mean_ccd=('code_carrier_div', 'mean'),\n sv_std_ccd=('code_carrier_div', 'std'),\n).astype('float32')\n\ndf = df.join(sat_stats, on=sat_group_key)\ndel sat_stats\n\n# ---- Epoch-level features (all satellites at same time point) ----\nlog(\"Computing epoch-level statistics...\")\n\n# Epoch key: drive + phone + time\nif 'TimeNanos' in df.columns:\n df['_epoch_key'] = (df['_drive_id'].astype(str) + '_' +\n df['_phone_name'].astype(str) + '_' +\n df['TimeNanos'].fillna(-1).astype(str))\nelse:\n # fallback: use row index groups\n df['_epoch_key'] = (df['_drive_id'].astype(str) + '_' +\n df['_phone_name'].astype(str))\n\nepoch_stats = df.groupby('_epoch_key').agg(\n epoch_sv_count=('Cn0DbHz', 'count'),\n epoch_mean_cno=('Cn0DbHz', 'mean'),\n epoch_mean_elev=('SvElevationDegrees', 'mean'),\n epoch_multipath_risk=('low_elev_flag', 'mean'),\n).astype('float32')\n\ndf = df.join(epoch_stats, on='_epoch_key')\ndel epoch_stats\n\n# Replace inf in temporal features\nfor col in TEMPORAL_FEATURES:\n if col in df.columns and df[col].dtype in ['float32', 'float64']:\n mask = np.isinf(df[col].values)\n if mask.any():\n df.loc[mask, col] = np.nan\n\nlog(f\"DataFrame shape after temporal features: {df.shape}\")\n\n# Drop helper columns\ndrop_cols = ['_sort_key', '_epoch_key', '_drive_id', '_phone_name',\n 'IsrbMeters', 'RawPseudorangeMeters', 'AccumulatedDeltaRangeState',\n 'CarrierFrequencyHz', 'TimeNanos', 'Svid', 'ConstellationType']\ndf.drop(columns=[c for c in drop_cols if c in df.columns], inplace=True)\ngc.collect()\n\nlog(f\"SECTION 3B complete — took {time.time()-t0:.1f}s\")\nlog(f\"Final feature count: {len(FEATURE_COLS)}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 4: TRAIN / VAL / TEST SPLIT (FIXED)\n# ============================================================\nlog(\"=\" * 60)\nlog(\"SECTION 4: TRAIN/VAL/TEST SPLIT — started\")\nlog(\"=\" * 60)\n\nt0 = time.time()\n\n# ---- Step 1: Handle NaN in target by DROPPING ----\nnan_target_count = df['MultipathIndicator'].isna().sum()\nif nan_target_count > 0:\n log(f\"WARNING: Found {nan_target_count:,} rows with NaN MultipathIndicator \"\n f\"({nan_target_count/len(df):.2%})\")\n log(f\"Dropping these rows - cannot train on unknown targets\")\n df = df[df['MultipathIndicator'].notna()].copy()\n log(f\"Rows remaining: {len(df):,}\")\n\n# Convert target to integer (safe now - no NaN)\ndf['MultipathIndicator'] = df['MultipathIndicator'].astype(int)\n\n# Verify target is clean\nassert df['MultipathIndicator'].isna().sum() == 0, \"Target still has NaN after cleaning!\"\nassert df['MultipathIndicator'].isin([0, 1]).all(), \"Target has values other than 0/1!\"\n\n# ---- Step 2: Filter available features ----\nFEATURE_COLS = [c for c in FEATURE_COLS if c in df.columns]\nlog(f\"Available features: {len(FEATURE_COLS)} / {len(BASE_FEATURES) + len(TEMPORAL_FEATURES)} expected\")\n\n# Warn about missing features\nmissing_features = set(BASE_FEATURES + TEMPORAL_FEATURES) - set(FEATURE_COLS)\nif missing_features:\n log(f\"WARNING: {len(missing_features)} features missing: {sorted(missing_features)}\")\n\n# ---- Step 3: Separate features and target ----\nX = df[FEATURE_COLS].copy()\ny = df['MultipathIndicator'].copy()\n\n# Log final target distribution\nclass_counts = y.value_counts()\nlog(f\"Final target distribution:\")\nlog(f\" Class 0 (Normal): {class_counts.get(0, 0):,} ({class_counts.get(0, 0)/len(y):.1%})\")\nlog(f\" Class 1 (Multipath): {class_counts.get(1, 0):,} ({class_counts.get(1, 0)/len(y):.1%})\")\nif class_counts.get(1, 0) > 0:\n log(f\" Imbalance ratio: {class_counts.get(0, 0)/class_counts.get(1, 0):.1f}:1\")\n\ndel df\ngc.collect()\n\n# ---- Step 4: Train/Val/Test Split ----\nlog(\"Splitting 80/10/10 stratified...\")\nX_trainval, X_test, y_trainval, y_test = train_test_split(\n X, y, test_size=0.1, stratify=y, random_state=SEED)\n\nX_train, X_val, y_train, y_val = train_test_split(\n X_trainval, y_trainval, test_size=0.1111, stratify=y_trainval, random_state=SEED)\n\nX_train = X_train.copy()\nX_val = X_val.copy()\nX_test = X_test.copy()\ndel X, y, X_trainval, y_trainval\ngc.collect()\n\nlog(f\"Train: {len(X_train):,} Val: {len(X_val):,} Test: {len(X_test):,}\")\nlog(f\"Multipath rates — train: {y_train.mean():.4f} \"\n f\"val: {y_val.mean():.4f} test: {y_test.mean():.4f}\")\n\n# ---- Step 5: Outlier clipping on training set only ----\nlog(\"Clipping outliers...\")\nclip_cols = ['ReceivedSvTimeUncertaintyNanos', 'BiasUncertaintyNanos', 'TroposphericDelayMeters']\nclip_cols = [c for c in clip_cols if c in FEATURE_COLS]\nclip_bounds = {}\nif clip_cols:\n for col in clip_cols:\n lo = X_train[col].quantile(0.01)\n hi = X_train[col].quantile(0.99)\n clip_bounds[col] = (lo, hi)\n for split_name, split_X in [('train', X_train), ('val', X_val), ('test', X_test)]:\n for col, (lo, hi) in clip_bounds.items():\n split_X[col] = split_X[col].clip(lo, hi)\n log(f\" Clipped {split_name}: {len(clip_cols)} columns\")\nelse:\n log(\" No clip columns found in features\")\n\n# ---- Step 6: Median imputation from training set only ----\nlog(\"Median imputation...\")\nnumeric_impute = [c for c in FEATURE_COLS\n if X_train[c].dtype in ['float32', 'float64', 'int8', 'int16', 'int32', 'int64']]\ntrain_medians = X_train[numeric_impute].median()\n\nX_train = X_train.fillna(train_medians)\nX_val = X_val.fillna(train_medians)\nX_test = X_test.fillna(train_medians)\n\n# ---- Step 7: Final inf cleanup ----\nfor split_X in [X_train, X_val, X_test]:\n for col in FEATURE_COLS:\n if split_X[col].dtype in ['float32', 'float64']:\n mask = np.isinf(split_X[col].values)\n if mask.any():\n split_X.loc[mask, col] = float(train_medians[col])\n\nlog(f\"NaN after imputation — \"\n f\"train:{X_train.isna().sum().sum()} \"\n f\"val:{X_val.isna().sum().sum()} \"\n f\"test:{X_test.isna().sum().sum()}\")\nlog(f\"SECTION 4 complete — took {time.time()-t0:.1f}s\")\nprint(\"=\" * 60)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 5: CLASS IMBALANCE — SMOTE ONLY (fast) (FIXED)\n# ============================================================\nlog(\"=\" * 60)\nlog(\"SECTION 5: CLASS IMBALANCE HANDLING — started\")\nlog(\"=\" * 60)\n\nt0 = time.time()\n\n# Step 1: Undersample to 5:1\nlog(\"Step 1/2 — RandomUnderSampler (5:1)...\")\nrus = RandomUnderSampler(sampling_strategy=0.2, random_state=SEED)\nX_under, y_under = rus.fit_resample(X_train, y_train)\nlog(f\" After undersampling: {len(X_under):,} \"\n f\"dist={pd.Series(y_under).value_counts().to_dict()}\")\n\n# Safety check: ensure enough minority samples for SMOTE\nminority_count = pd.Series(y_under).value_counts().min()\nif minority_count < 2:\n log(f\" ERROR: Only {minority_count} minority samples after undersampling!\")\n log(f\" SMOTE impossible - using undersampled data directly\")\n X_balanced, y_balanced = X_under, y_under\nelse:\n if minority_count < 5:\n log(f\" WARNING: Only {minority_count} minority samples after undersampling!\")\n log(f\" Reducing SMOTE k_neighbors from 5 to {max(1, minority_count - 1)}\")\n k_neighbors = max(1, minority_count - 1)\n else:\n k_neighbors = 5\n \n # Step 2: SMOTE only (no Tomek — orders of magnitude faster)\n log(f\"Step 2/2 — SMOTE (3:1) — k_neighbors={k_neighbors}...\")\n smote = SMOTE(sampling_strategy=0.33, random_state=SEED, k_neighbors=k_neighbors)\n X_balanced, y_balanced = smote.fit_resample(X_under, y_under)\n log(f\" After SMOTE: {len(X_balanced):,} \"\n f\"dist={pd.Series(y_balanced).value_counts().to_dict()}\")\n\ndel X_under, y_under\ngc.collect()\nlog(f\"SECTION 5 complete — took {time.time()-t0:.1f}s\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# INITIALIZE PREDICTION ARRAYS (safety fallback)\n# ============================================================\nlog(\"Initializing prediction arrays...\")\nlgb_val_probs = np.zeros(len(X_val), dtype=np.float32)\nlgb_test_probs = np.zeros(len(X_test), dtype=np.float32)\nxgb_val_prob = np.zeros(len(X_val), dtype=np.float32)\nxgb_test_prob = np.zeros(len(X_test), dtype=np.float32)\ncat_val_prob = np.zeros(len(X_val), dtype=np.float32)\ncat_test_prob = np.zeros(len(X_test), dtype=np.float32)\nmlp_val_prob = np.full(len(X_val), 0.5, dtype=np.float32)\nmlp_test_prob = np.full(len(X_test), 0.5, dtype=np.float32)\nft_val_prob = np.full(len(X_val), 0.5, dtype=np.float32)\nft_test_prob = np.full(len(X_test), 0.5, dtype=np.float32)\nvae_val_norm = np.zeros(len(X_val), dtype=np.float32)\nvae_test_norm = np.zeros(len(X_test), dtype=np.float32)\niso_val_norm = np.zeros(len(X_val), dtype=np.float32)\niso_test_norm = np.zeros(len(X_test), dtype=np.float32)\nlstm_val_prob = np.full(len(X_val), 0.5, dtype=np.float32)\nlstm_test_prob = np.full(len(X_test), 0.5, dtype=np.float32)\nlog(\"Prediction arrays initialized ✓\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 6: MODEL 1 — LightGBM (Manual BalancedBagging)\n# ============================================================\nlog(\"=\" * 60)\nlog(\"SECTION 6: MODEL 1 — LightGBM BalancedBagging — started\")\nlog(\"=\" * 60)\n\nt0 = time.time()\n\nN_LGB_BAGS = 20\nLGB_PARAMS = dict(\n n_estimators=500, max_depth=8, num_leaves=63, learning_rate=0.05,\n subsample=0.8, colsample_bytree=0.8, min_child_samples=50,\n scale_pos_weight=5, reg_alpha=0.1, reg_lambda=1.0,\n random_state=SEED, verbose=-1, n_jobs=-1,\n device='gpu' if torch.cuda.is_available() else 'cpu'\n)\n\nlog(f\"Manual BalancedBagging: {N_LGB_BAGS} estimators × LGB(500 trees)\")\nlog(f\"Training on full train set ({len(X_train):,} rows), undersampling per bag...\")\n\nlgb_estimators = []\nn_minority = (y_train == 1).sum()\nn_majority_per_bag = n_minority * 5\n\n# Safety check\nif n_minority == 0:\n log(\"ERROR: No minority class samples in training data!\")\n log(\"Cannot train LightGBM with balanced bagging\")\n lgb_val_probs = np.zeros(len(X_val))\n lgb_test_probs = np.zeros(len(X_test))\nelse:\n minority_idx = np.where(y_train.values == 1)[0]\n majority_idx = np.where(y_train.values == 0)[0]\n\n lgb_val_probs = np.zeros(len(X_val))\n lgb_test_probs = np.zeros(len(X_test))\n\n X_train_np = X_train.values\n y_train_np = y_train.values\n X_val_np = X_val.values\n X_test_np = X_test.values\n\n for i in range(N_LGB_BAGS):\n log(f\" [{i+1}/{N_LGB_BAGS}] Undersampling...\")\n rng = np.random.RandomState(SEED + i)\n maj_sample = rng.choice(majority_idx, size=n_majority_per_bag, replace=False)\n bag_idx = np.concatenate([minority_idx, maj_sample])\n rng.shuffle(bag_idx)\n X_bag = X_train_np[bag_idx]\n y_bag = y_train_np[bag_idx]\n\n log(f\" [{i+1}/{N_LGB_BAGS}] Training LGB on {len(bag_idx):,} rows...\")\n t_bag = time.time()\n est = lgb.LGBMClassifier(**LGB_PARAMS)\n est.fit(X_bag, y_bag,\n eval_set=[(X_val_np, y_val.values)],\n callbacks=[lgb.early_stopping(50, verbose=False),\n lgb.log_evaluation(-1)])\n lgb_estimators.append(est)\n lgb_val_probs += est.predict_proba(X_val_np)[:, 1]\n lgb_test_probs += est.predict_proba(X_test_np)[:, 1]\n log(f\" [{i+1}/{N_LGB_BAGS}] Done — {time.time()-t_bag:.1f}s ✓\")\n\n lgb_val_probs /= N_LGB_BAGS\n lgb_test_probs /= N_LGB_BAGS\n\nlgb_val_pred = (lgb_val_probs >= 0.5).astype(int)\npr_auc = average_precision_score(y_val, lgb_val_probs)\nroc_auc = roc_auc_score(y_val, lgb_val_probs)\nf2 = fbeta_score(y_val, lgb_val_pred, beta=2)\nlog(f\"PR-AUC: {pr_auc:.4f} ROC-AUC: {roc_auc:.4f} F2: {f2:.4f}\")\nprint(classification_report(y_val, lgb_val_pred, target_names=['Normal', 'Multipath']))\nlog(f\"SECTION 6 complete — took {time.time()-t0:.1f}s\")\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 7: MODEL 2 — XGBoost (FIXED)\n# ============================================================\nlog(\"=\" * 60)\nlog(\"SECTION 7: MODEL 2 — XGBoost — started\")\nlog(\"=\" * 60)\n\nt0 = time.time()\n\nneg_count = (y_balanced == 0).sum()\npos_count = (y_balanced == 1).sum()\nscale_pw = neg_count / max(pos_count, 1)\n\n# Convert to numpy arrays consistently (fixes feature name mismatch)\nX_balanced_np = X_balanced.values if hasattr(X_balanced, 'values') else np.array(X_balanced)\ny_balanced_np = y_balanced.values if hasattr(y_balanced, 'values') else np.array(y_balanced)\nX_val_np = X_val.values\ny_val_np = y_val.values\nX_test_np = X_test.values\n\nxgb_model = xgb.XGBClassifier(\n n_estimators=500,\n max_depth=7,\n learning_rate=0.05,\n subsample=0.8,\n colsample_bytree=0.8,\n min_child_weight=50,\n scale_pos_weight=scale_pw,\n reg_alpha=0.1,\n reg_lambda=1.0,\n eval_metric='aucpr',\n early_stopping_rounds=50,\n random_state=SEED,\n verbosity=0,\n nthread=-1,\n device='cuda' if torch.cuda.is_available() else 'cpu',\n tree_method='hist',\n)\n\nlog(f\"Training XGBoost on balanced data ({len(X_balanced_np):,} rows)...\")\nxgb_model.fit(\n X_balanced_np, y_balanced_np,\n eval_set=[(X_val_np, y_val_np)],\n verbose=False\n)\n\nxgb_val_prob = xgb_model.predict_proba(X_val_np)[:, 1]\nxgb_test_prob = xgb_model.predict_proba(X_test_np)[:, 1]\nxgb_val_pred = (xgb_val_prob >= 0.5).astype(int)\n\npr_auc = average_precision_score(y_val, xgb_val_prob)\nroc_auc = roc_auc_score(y_val, xgb_val_prob)\nf2 = fbeta_score(y_val, xgb_val_pred, beta=2)\nlog(f\"XGBoost — PR-AUC: {pr_auc:.4f} ROC-AUC: {roc_auc:.4f} F2: {f2:.4f}\")\nprint(classification_report(y_val, xgb_val_pred, target_names=['Normal', 'Multipath']))\nlog(f\"SECTION 7 complete — took {time.time()-t0:.1f}s\")\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 8: MODEL 3 — CatBoost (FIXED - No subsample)\n# ============================================================\nlog(\"=\" * 60)\nlog(\"SECTION 8: MODEL 3 — CatBoost — started\")\nlog(\"=\" * 60)\n\nt0 = time.time()\n\n# Convert to numpy arrays consistently\nX_balanced_np = X_balanced.values if hasattr(X_balanced, 'values') else np.array(X_balanced)\ny_balanced_np = y_balanced.values if hasattr(y_balanced, 'values') else np.array(y_balanced)\nX_val_np = X_val.values\ny_val_np = y_val.values\nX_test_np = X_test.values\n\n# Safe GPU detection for CatBoost\nuse_gpu = torch.cuda.is_available()\ncat_params = dict(\n iterations=500,\n depth=8,\n learning_rate=0.05,\n l2_leaf_reg=3.0,\n auto_class_weights='Balanced',\n eval_metric='PRAUC',\n early_stopping_rounds=50,\n random_seed=SEED,\n verbose=0,\n)\n\nif use_gpu:\n try:\n cat_model = CatBoostClassifier(**cat_params, task_type='GPU')\n log(\"CatBoost using GPU\")\n except Exception as e:\n log(f\"CatBoost GPU initialization failed: {e}\")\n log(\"Falling back to CPU\")\n cat_model = CatBoostClassifier(**cat_params, task_type='CPU')\nelse:\n cat_model = CatBoostClassifier(**cat_params, task_type='CPU')\n log(\"CatBoost using CPU\")\n\nlog(f\"Training CatBoost on balanced data ({len(X_balanced_np):,} rows)...\")\ncat_model.fit(\n X_balanced_np, y_balanced_np,\n eval_set=(X_val_np, y_val_np),\n use_best_model=True,\n verbose=False\n)\n\ncat_val_prob = cat_model.predict_proba(X_val_np)[:, 1]\ncat_test_prob = cat_model.predict_proba(X_test_np)[:, 1]\ncat_val_pred = (cat_val_prob >= 0.5).astype(int)\n\npr_auc = average_precision_score(y_val, cat_val_prob)\nroc_auc = roc_auc_score(y_val, cat_val_prob)\nf2 = fbeta_score(y_val, cat_val_pred, beta=2)\nlog(f\"CatBoost — PR-AUC: {pr_auc:.4f} ROC-AUC: {roc_auc:.4f} F2: {f2:.4f}\")\nprint(classification_report(y_val, cat_val_pred, target_names=['Normal', 'Multipath']))\nlog(f\"SECTION 8 complete — took {time.time()-t0:.1f}s\")\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 9: MODEL 4 — MLP Focal Loss (FIXED)\n# ============================================================\nlog(\"=\" * 60)\nlog(\"SECTION 9: MODEL 4 — MLP Focal Loss — started\")\nlog(\"=\" * 60)\n\nt0 = time.time()\n\n\n# Define classes OUTSIDE if/else so Section 10 can use them\nclass FocalLoss(nn.Module):\n def __init__(self, alpha=0.75, gamma=2.0):\n super().__init__()\n self.alpha = alpha\n self.gamma = gamma\n\n def forward(self, inputs, targets):\n bce = F.binary_cross_entropy_with_logits(inputs, targets, reduction='none')\n pt = torch.exp(-bce)\n alpha_t = self.alpha * targets + (1 - self.alpha) * (1 - targets)\n return (alpha_t * (1 - pt) ** self.gamma * bce).mean()\n\n\nclass MultipathMLP(nn.Module):\n def __init__(self, n_features):\n super().__init__()\n self.net = nn.Sequential(\n nn.Linear(n_features, 256),\n nn.BatchNorm1d(256),\n nn.GELU(),\n nn.Dropout(0.3),\n nn.Linear(256, 128),\n nn.BatchNorm1d(128),\n nn.GELU(),\n nn.Dropout(0.3),\n nn.Linear(128, 64),\n nn.BatchNorm1d(64),\n nn.GELU(),\n nn.Dropout(0.2),\n nn.Linear(64, 32),\n nn.BatchNorm1d(32),\n nn.GELU(),\n nn.Dropout(0.1),\n nn.Linear(32, 1)\n )\n for m in self.modules():\n if isinstance(m, nn.Linear):\n nn.init.kaiming_normal_(m.weight, nonlinearity='relu')\n if m.bias is not None:\n nn.init.zeros_(m.bias)\n\n def forward(self, x):\n return self.net(x)\n\n\ndef batched_inference(model, data_np, device, batch_size=4096):\n model.eval()\n probs = []\n with torch.no_grad():\n for i in range(0, len(data_np), batch_size):\n xb = torch.tensor(data_np[i:i+batch_size], dtype=torch.float32).to(device)\n probs.append(torch.sigmoid(model(xb).squeeze(1)).cpu().numpy())\n return np.concatenate(probs)\n\n\n# Safety check for balanced data\nif len(X_balanced) == 0:\n log(\"ERROR: No balanced data for MLP training!\")\n mlp_val_prob = np.full(len(X_val), 0.5, dtype=np.float32)\n mlp_test_prob = np.full(len(X_test), 0.5, dtype=np.float32)\n scaler_mlp = None\nelse:\n # Convert to numpy arrays consistently\n X_balanced_np = X_balanced.values if hasattr(X_balanced, 'values') else np.array(X_balanced)\n y_balanced_np = y_balanced.values if hasattr(y_balanced, 'values') else np.array(y_balanced)\n X_val_np = X_val.values\n y_val_np = y_val.values\n X_test_np = X_test.values\n \n scaler_mlp = RobustScaler()\n X_bal_scaled = scaler_mlp.fit_transform(X_balanced_np).astype(np.float32)\n X_val_scaled = scaler_mlp.transform(X_val_np).astype(np.float32)\n X_test_scaled = scaler_mlp.transform(X_test_np).astype(np.float32)\n\n n_features = len(FEATURE_COLS)\n mlp_model = MultipathMLP(n_features).to(DEVICE)\n focal_loss = FocalLoss(alpha=0.75, gamma=2.0)\n\n optimizer_mlp = torch.optim.AdamW(mlp_model.parameters(), lr=1e-4, weight_decay=1e-4)\n scheduler_mlp = torch.optim.lr_scheduler.OneCycleLR(\n optimizer_mlp,\n max_lr=1e-3,\n steps_per_epoch=max(1, len(X_bal_scaled) // 1024 + 1),\n epochs=50,\n pct_start=0.1\n )\n\n train_ds = TensorDataset(\n torch.tensor(X_bal_scaled, dtype=torch.float32),\n torch.tensor(y_balanced_np, dtype=torch.float32)\n )\n val_ds = TensorDataset(\n torch.tensor(X_val_scaled, dtype=torch.float32),\n torch.tensor(y_val_np, dtype=torch.float32)\n )\n train_loader = DataLoader(train_ds, batch_size=1024, shuffle=True,\n num_workers=0, pin_memory=True)\n val_loader = DataLoader(val_ds, batch_size=2048, shuffle=False,\n num_workers=0, pin_memory=True)\n\n MLP_EPOCHS = 50\n MLP_PATIENCE = 10\n best_mlp_f2 = -1\n mlp_patience = 0\n best_mlp_state = None\n\n log(f\"Training MLP ({n_features} features, \"\n f\"{sum(p.numel() for p in mlp_model.parameters()):,} params)...\")\n\n for epoch in range(MLP_EPOCHS):\n mlp_model.train()\n train_loss = 0.0\n for xb, yb in train_loader:\n xb, yb = xb.to(DEVICE), yb.to(DEVICE)\n optimizer_mlp.zero_grad()\n loss = focal_loss(mlp_model(xb).squeeze(1), yb)\n \n if torch.isnan(loss):\n continue\n \n loss.backward()\n torch.nn.utils.clip_grad_norm_(mlp_model.parameters(), max_norm=1.0)\n optimizer_mlp.step()\n scheduler_mlp.step()\n train_loss += loss.item()\n\n mlp_model.eval()\n val_preds = []\n val_loss = 0.0\n with torch.no_grad():\n for xb, yb in val_loader:\n xb, yb = xb.to(DEVICE), yb.to(DEVICE)\n logits = mlp_model(xb).squeeze(1)\n loss = focal_loss(logits, yb)\n if not torch.isnan(loss):\n val_loss += loss.item()\n val_preds.append(torch.sigmoid(logits).cpu().numpy())\n\n val_probs = np.concatenate(val_preds)\n val_probs = np.nan_to_num(val_probs, nan=0.5)\n val_f2 = fbeta_score(y_val_np, (val_probs >= 0.5).astype(int), beta=2)\n\n if val_f2 > best_mlp_f2:\n best_mlp_f2 = val_f2\n mlp_patience = 0\n best_mlp_state = {k: v.cpu().clone() for k, v in mlp_model.state_dict().items()}\n log(f\" Ep {epoch+1:02d}/{MLP_EPOCHS} | \"\n f\"train={train_loss/len(train_loader):.4f} \"\n f\"val={val_loss/len(val_loader):.4f} F2={val_f2:.4f} ★ BEST\")\n else:\n mlp_patience += 1\n if (epoch + 1) % 5 == 0:\n log(f\" Ep {epoch+1:02d}/{MLP_EPOCHS} | \"\n f\"train={train_loss/len(train_loader):.4f} \"\n f\"val={val_loss/len(val_loader):.4f} F2={val_f2:.4f} \"\n f\"(patience {mlp_patience}/{MLP_PATIENCE})\")\n if mlp_patience >= MLP_PATIENCE:\n log(f\" Early stopping at epoch {epoch+1}\")\n break\n\n if best_mlp_state is not None:\n mlp_model.load_state_dict(best_mlp_state)\n mlp_model.eval()\n log(f\"Best MLP val F2: {best_mlp_f2:.4f}\")\n\n mlp_val_prob = batched_inference(mlp_model, X_val_scaled, DEVICE)\n mlp_test_prob = batched_inference(mlp_model, X_test_scaled, DEVICE)\n \n mlp_val_prob = np.nan_to_num(mlp_val_prob, nan=0.5)\n mlp_test_prob = np.nan_to_num(mlp_test_prob, nan=0.5)\n\n del train_ds, val_ds, train_loader, val_loader, X_bal_scaled\n gc.collect()\n torch.cuda.empty_cache() if torch.cuda.is_available() else None\n\nlog(f\"SECTION 9 complete — took {time.time()-t0:.1f}s\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 10: MODEL 5 — FT-Transformer (FIXED)\n# ============================================================\nlog(\"=\" * 60)\nlog(\"SECTION 10: MODEL 5 — FT-Transformer — started\")\nlog(\"=\" * 60)\n\nt0 = time.time()\n\n\nclass FTTransformer(nn.Module):\n def __init__(self, n_features, d_token=64, n_heads=4, n_blocks=3, dropout=0.1):\n super().__init__()\n self.n_features = n_features\n self.feature_embeddings = nn.ModuleList([\n nn.Linear(1, d_token) for _ in range(n_features)\n ])\n self.cls_token = nn.Parameter(torch.zeros(1, 1, d_token))\n nn.init.trunc_normal_(self.cls_token, std=0.02)\n encoder_layer = nn.TransformerEncoderLayer(\n d_model=d_token, nhead=n_heads, dim_feedforward=d_token * 4,\n dropout=dropout, activation='gelu', batch_first=True, norm_first=True\n )\n self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=n_blocks)\n self.norm = nn.LayerNorm(d_token)\n self.head = nn.Linear(d_token, 1)\n self._init_weights()\n\n def _init_weights(self):\n for emb in self.feature_embeddings:\n nn.init.xavier_uniform_(emb.weight)\n nn.init.zeros_(emb.bias)\n nn.init.xavier_uniform_(self.head.weight)\n nn.init.zeros_(self.head.bias)\n\n def forward(self, x):\n B = x.size(0)\n tokens = torch.stack(\n [emb(x[:, i:i+1]) for i, emb in enumerate(self.feature_embeddings)], dim=1\n )\n cls = self.cls_token.expand(B, -1, -1)\n tokens = torch.cat([cls, tokens], dim=1)\n out = self.transformer(tokens)\n cls_out = self.norm(out[:, 0, :])\n return self.head(cls_out)\n\n\nif len(X_balanced) == 0:\n log(\"ERROR: No balanced data for FT-Transformer training!\")\n ft_val_prob = np.full(len(X_val), 0.5, dtype=np.float32)\n ft_test_prob = np.full(len(X_test), 0.5, dtype=np.float32)\nelse:\n X_balanced_np = X_balanced.values if hasattr(X_balanced, 'values') else np.array(X_balanced)\n y_balanced_np = y_balanced.values if hasattr(y_balanced, 'values') else np.array(y_balanced)\n X_val_np = X_val.values\n y_val_np = y_val.values\n X_test_np = X_test.values\n \n try:\n _ = scaler_mlp\n log(\"Using scaler_mlp from Section 9\")\n X_bal_ft = scaler_mlp.transform(X_balanced_np).astype(np.float32)\n X_val_ft = scaler_mlp.transform(X_val_np).astype(np.float32)\n X_test_ft = scaler_mlp.transform(X_test_np).astype(np.float32)\n except NameError:\n log(\"Creating new scaler for FT-Transformer\")\n scaler_mlp = RobustScaler()\n X_bal_ft = scaler_mlp.fit_transform(X_balanced_np).astype(np.float32)\n X_val_ft = scaler_mlp.transform(X_val_np).astype(np.float32)\n X_test_ft = scaler_mlp.transform(X_test_np).astype(np.float32)\n\n try:\n ft_focal = FocalLoss(alpha=0.75, gamma=2.0)\n except NameError:\n log(\"WARNING: FocalLoss not defined, using BCEWithLogitsLoss\")\n pos_weight = torch.tensor([(y_balanced_np == 0).sum() / max((y_balanced_np == 1).sum(), 1)], \n dtype=torch.float32).to(DEVICE)\n ft_focal = nn.BCEWithLogitsLoss(pos_weight=pos_weight)\n\n ft_model = FTTransformer(n_features=n_features, d_token=64, n_heads=4,\n n_blocks=3, dropout=0.1).to(DEVICE)\n optimizer_ft = torch.optim.AdamW(ft_model.parameters(), lr=3e-5, weight_decay=1e-4)\n scheduler_ft = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer_ft, T_max=30)\n\n ft_train_ds = TensorDataset(\n torch.tensor(X_bal_ft, dtype=torch.float32),\n torch.tensor(y_balanced_np, dtype=torch.float32)\n )\n ft_val_ds = TensorDataset(\n torch.tensor(X_val_ft, dtype=torch.float32),\n torch.tensor(y_val_np, dtype=torch.float32)\n )\n ft_train_loader = DataLoader(ft_train_ds, batch_size=512, shuffle=True,\n num_workers=0, pin_memory=True)\n ft_val_loader = DataLoader(ft_val_ds, batch_size=1024, shuffle=False,\n num_workers=0, pin_memory=True)\n\n FT_EPOCHS = 30\n FT_PATIENCE = 7\n best_ft_f2 = -1\n ft_patience = 0\n best_ft_state = None\n\n log(f\"Training FT-Transformer ({sum(p.numel() for p in ft_model.parameters()):,} params)...\")\n\n for epoch in range(FT_EPOCHS):\n ft_model.train()\n train_loss = 0.0\n nan_batches = 0\n for xb, yb in ft_train_loader:\n xb, yb = xb.to(DEVICE), yb.to(DEVICE)\n optimizer_ft.zero_grad()\n logits = ft_model(xb).squeeze(1)\n loss = ft_focal(logits, yb)\n if torch.isnan(loss):\n nan_batches += 1\n continue\n loss.backward()\n torch.nn.utils.clip_grad_norm_(ft_model.parameters(), max_norm=0.5)\n optimizer_ft.step()\n train_loss += loss.item()\n scheduler_ft.step()\n\n if nan_batches > 0:\n log(f\" Ep {epoch+1:02d} — WARNING: {nan_batches} NaN batches skipped\")\n\n ft_model.eval()\n val_preds = []\n val_loss = 0.0\n with torch.no_grad():\n for xb, yb in ft_val_loader:\n xb, yb = xb.to(DEVICE), yb.to(DEVICE)\n logits = ft_model(xb).squeeze(1)\n loss = ft_focal(logits, yb)\n if not torch.isnan(loss):\n val_loss += loss.item()\n val_preds.append(torch.sigmoid(logits).cpu().numpy())\n\n val_probs = np.concatenate(val_preds)\n val_probs = np.nan_to_num(val_probs, nan=0.5)\n val_f2 = fbeta_score(y_val_np, (val_probs >= 0.5).astype(int), beta=2)\n\n if val_f2 > best_ft_f2:\n best_ft_f2 = val_f2\n ft_patience = 0\n best_ft_state = {k: v.cpu().clone() for k, v in ft_model.state_dict().items()}\n log(f\" Ep {epoch+1:02d}/{FT_EPOCHS} | F2={val_f2:.4f} ★ BEST\")\n else:\n ft_patience += 1\n if (epoch + 1) % 5 == 0:\n log(f\" Ep {epoch+1:02d}/{FT_EPOCHS} | F2={val_f2:.4f} (patience {ft_patience}/{FT_PATIENCE})\")\n if ft_patience >= FT_PATIENCE:\n log(f\" Early stopping at epoch {epoch+1}\")\n break\n\n if best_ft_state is not None:\n ft_model.load_state_dict(best_ft_state)\n ft_model.eval()\n log(f\"Best FT-Transformer val F2: {best_ft_f2:.4f}\")\n\n try:\n ft_val_prob = batched_inference(ft_model, X_val_ft, DEVICE, batch_size=1024)\n ft_test_prob = batched_inference(ft_model, X_test_ft, DEVICE, batch_size=1024)\n except NameError:\n def batched_inference(model, data_np, device, batch_size=4096):\n model.eval()\n probs = []\n with torch.no_grad():\n for i in range(0, len(data_np), batch_size):\n xb = torch.tensor(data_np[i:i+batch_size], dtype=torch.float32).to(device)\n probs.append(torch.sigmoid(model(xb).squeeze(1)).cpu().numpy())\n return np.concatenate(probs)\n ft_val_prob = batched_inference(ft_model, X_val_ft, DEVICE, batch_size=1024)\n ft_test_prob = batched_inference(ft_model, X_test_ft, DEVICE, batch_size=1024)\n \n ft_val_prob = np.nan_to_num(ft_val_prob, nan=0.5)\n ft_test_prob = np.nan_to_num(ft_test_prob, nan=0.5)\n\n del ft_train_ds, ft_val_ds, ft_train_loader, ft_val_loader, X_bal_ft\n gc.collect()\n torch.cuda.empty_cache() if torch.cuda.is_available() else None\n\nlog(f\"SECTION 10 complete — took {time.time()-t0:.1f}s\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 11: MODEL 6 — Variational Autoencoder (VAE) (FIXED)\n# ============================================================\nlog(\"=\" * 60)\nlog(\"SECTION 11: MODEL 6 — Variational Autoencoder — started\")\nlog(\"=\" * 60)\n\nt0 = time.time()\n\n\nclass VAE(nn.Module):\n \"\"\"\n Variational Autoencoder for anomaly detection.\n Better than plain AE: learns a probability distribution\n over the latent space → better uncertainty estimates.\n Anomaly score = reconstruction loss + KL divergence.\n \"\"\"\n def __init__(self, n_features, latent_dim=8):\n super().__init__()\n # Encoder\n self.encoder = nn.Sequential(\n nn.Linear(n_features, 128),\n nn.BatchNorm1d(128),\n nn.GELU(),\n nn.Linear(128, 64),\n nn.BatchNorm1d(64),\n nn.GELU(),\n )\n self.fc_mu = nn.Linear(64, latent_dim)\n self.fc_logvar = nn.Linear(64, latent_dim)\n\n # Decoder\n self.decoder = nn.Sequential(\n nn.Linear(latent_dim, 64),\n nn.BatchNorm1d(64),\n nn.GELU(),\n nn.Linear(64, 128),\n nn.BatchNorm1d(128),\n nn.GELU(),\n nn.Linear(128, n_features)\n )\n\n for m in self.modules():\n if isinstance(m, nn.Linear):\n nn.init.kaiming_normal_(m.weight)\n nn.init.zeros_(m.bias)\n\n def encode(self, x):\n h = self.encoder(x)\n return self.fc_mu(h), self.fc_logvar(h)\n\n def reparameterize(self, mu, logvar):\n if self.training:\n std = torch.exp(0.5 * logvar)\n eps = torch.randn_like(std)\n return mu + eps * std\n return mu\n\n def decode(self, z):\n return self.decoder(z)\n\n def forward(self, x):\n mu, logvar = self.encode(x)\n z = self.reparameterize(mu, logvar)\n x_hat = self.decode(z)\n return x_hat, mu, logvar\n\n def anomaly_score(self, x):\n \"\"\"Per-sample anomaly score = reconstruction MSE + KL\"\"\"\n self.eval()\n with torch.no_grad():\n mu, logvar = self.encode(x)\n z = mu # Use mean for deterministic inference\n x_hat = self.decode(z)\n recon = ((x - x_hat) ** 2).mean(dim=1)\n kl = -0.5 * (1 + logvar - mu.pow(2) - logvar.exp()).sum(dim=1)\n return recon + 0.001 * kl\n\n\ndef vae_loss(x, x_hat, mu, logvar, beta=1.0):\n recon = F.mse_loss(x_hat, x, reduction='sum') / x.size(0)\n kl = -0.5 * torch.mean(1 + logvar - mu.pow(2) - logvar.exp())\n return recon + beta * kl, recon.item(), kl.item()\n\n\n# Train on class-0 only\nX_train_normal = X_train[y_train == 0]\nval_normal_mask = (y_val.values == 0)\n\n# Safety check\nif len(X_train_normal) < 100:\n log(f\"WARNING: Only {len(X_train_normal)} class-0 samples for VAE training!\")\n log(\"VAE needs sufficient normal samples - using fallback\")\n vae_val_score = np.zeros(len(X_val))\n vae_test_score = np.zeros(len(X_test))\n vae_val_norm = np.zeros(len(X_val))\n vae_test_norm = np.zeros(len(X_test))\n vae_p5, vae_p95 = 0, 1\nelse:\n log(f\"Class-0 samples for VAE: {len(X_train_normal):,}\")\n\n scaler_vae = StandardScaler()\n X_vae_train = scaler_vae.fit_transform(X_train_normal).astype(np.float32)\n X_vae_val = scaler_vae.transform(X_val).astype(np.float32)\n X_vae_test = scaler_vae.transform(X_test).astype(np.float32)\n\n vae_model = VAE(n_features=n_features, latent_dim=8).to(DEVICE)\n optimizer_vae = torch.optim.Adam(vae_model.parameters(), lr=1e-3)\n scheduler_vae = torch.optim.lr_scheduler.ReduceLROnPlateau(\n optimizer_vae, patience=3, factor=0.5)\n\n vae_train_ds = TensorDataset(torch.tensor(X_vae_train, dtype=torch.float32))\n vae_train_loader = DataLoader(vae_train_ds, batch_size=2048, shuffle=True,\n num_workers=0, pin_memory=True)\n\n VAE_EPOCHS = 50\n VAE_PATIENCE = 10\n best_vae_loss = float('inf')\n vae_patience = 0\n best_vae_state = None\n\n # Prepare validation normal samples\n if val_normal_mask.any():\n X_vae_val_normal = torch.tensor(X_vae_val[val_normal_mask], dtype=torch.float32)\n log(f\"Validation normal samples: {val_normal_mask.sum():,}\")\n else:\n X_vae_val_normal = torch.tensor(X_vae_val[:1], dtype=torch.float32)\n log(\"WARNING: No normal samples in validation set, using dummy\")\n\n log(f\"Training VAE ({sum(p.numel() for p in vae_model.parameters()):,} params)...\")\n\n for epoch in range(VAE_EPOCHS):\n vae_model.train()\n train_loss = 0.0\n for (xb,) in vae_train_loader:\n xb = xb.to(DEVICE)\n optimizer_vae.zero_grad()\n x_hat, mu, logvar = vae_model(xb)\n loss, _, _ = vae_loss(xb, x_hat, mu, logvar, beta=1.0)\n \n if torch.isnan(loss):\n continue\n \n loss.backward()\n torch.nn.utils.clip_grad_norm_(vae_model.parameters(), max_norm=1.0)\n optimizer_vae.step()\n train_loss += loss.item()\n\n # Val loss on normal class\n vae_model.eval()\n with torch.no_grad():\n xv = X_vae_val_normal.to(DEVICE)\n xv_hat, mu_v, lv_v = vae_model(xv)\n val_loss, recon_v, kl_v = vae_loss(xv, xv_hat, mu_v, lv_v)\n\n scheduler_vae.step(val_loss)\n\n if val_loss < best_vae_loss:\n best_vae_loss = val_loss\n vae_patience = 0\n best_vae_state = {k: v.cpu().clone() for k, v in vae_model.state_dict().items()}\n log(f\" Ep {epoch+1:02d}/{VAE_EPOCHS} | \"\n f\"train={train_loss/len(vae_train_loader):.4f} \"\n f\"val={val_loss:.4f} (recon={recon_v:.4f} kl={kl_v:.4f}) ★ BEST\")\n else:\n vae_patience += 1\n if (epoch + 1) % 5 == 0:\n log(f\" Ep {epoch+1:02d}/{VAE_EPOCHS} | \"\n f\"train={train_loss/len(vae_train_loader):.4f} \"\n f\"val={val_loss:.4f} (patience {vae_patience}/{VAE_PATIENCE})\")\n if vae_patience >= VAE_PATIENCE:\n log(f\" Early stopping at epoch {epoch+1}\")\n break\n\n if best_vae_state is not None:\n vae_model.load_state_dict(best_vae_state)\n vae_model.eval()\n\n\n def compute_vae_scores(model, data_np, device, batch_size=4096):\n scores = []\n with torch.no_grad():\n for i in range(0, len(data_np), batch_size):\n xb = torch.tensor(data_np[i:i+batch_size], dtype=torch.float32).to(device)\n s = model.anomaly_score(xb).cpu().numpy()\n scores.append(s)\n return np.concatenate(scores)\n\n\n vae_val_score = compute_vae_scores(vae_model, X_vae_val, DEVICE)\n vae_test_score = compute_vae_scores(vae_model, X_vae_test, DEVICE)\n\n # Normalize scores to [0, 1] probability-like range\n vae_p5 = np.percentile(vae_val_score[val_normal_mask], 5) if val_normal_mask.any() else 0\n vae_p95 = np.percentile(vae_val_score, 95)\n vae_val_norm = np.clip((vae_val_score - vae_p5) / (vae_p95 - vae_p5 + 1e-8), 0, 1)\n vae_test_norm = np.clip((vae_test_score - vae_p5) / (vae_p95 - vae_p5 + 1e-8), 0, 1)\n\n if val_normal_mask.any():\n log(f\"VAE anomaly scores — class0: {np.median(vae_val_score[val_normal_mask]):.4f} \"\n f\"class1: {np.median(vae_val_score[~val_normal_mask]):.4f} \"\n f\"ratio: {np.median(vae_val_score[~val_normal_mask])/max(np.median(vae_val_score[val_normal_mask]), 1e-8):.2f}x\")\n\n del vae_train_ds, vae_train_loader, X_vae_train\n gc.collect()\n torch.cuda.empty_cache() if torch.cuda.is_available() else None\n\nlog(f\"SECTION 11 complete — took {time.time()-t0:.1f}s\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 12: MODEL 7 — Isolation Forest (FIXED)\n# ============================================================\nlog(\"=\" * 60)\nlog(\"SECTION 12: MODEL 7 — Isolation Forest — started\")\nlog(\"=\" * 60)\n\nt0 = time.time()\n\n# Train on full training set (both classes) — IF learns the normal manifold\n# Use a subsample for speed\nIF_SUBSAMPLE = min(100_000, len(X_train))\n\nif IF_SUBSAMPLE < 100:\n log(f\"WARNING: Only {IF_SUBSAMPLE} samples for Isolation Forest!\")\n log(\"Isolation Forest needs more data - using fallback\")\n iso_val_norm = np.zeros(len(X_val))\n iso_test_norm = np.zeros(len(X_test))\n iso_min, iso_max = 0, 1\nelse:\n rng = np.random.RandomState(SEED)\n if_idx = rng.choice(len(X_train), IF_SUBSAMPLE, replace=False)\n\n # Calculate contamination rate safely\n contamination_rate = y_train.mean()\n if contamination_rate <= 0 or contamination_rate >= 1:\n contamination_rate = 0.1 # Default if all same class\n log(f\"WARNING: Adjusting contamination to {contamination_rate}\")\n\n log(f\"Training Isolation Forest on {IF_SUBSAMPLE:,} samples \"\n f\"(contamination={contamination_rate:.4f})...\")\n\n iso_forest = IsolationForest(\n n_estimators=200,\n max_samples=256,\n contamination=contamination_rate,\n random_state=SEED,\n n_jobs=-1,\n verbose=0\n )\n iso_forest.fit(X_train.values[if_idx])\n\n # IF score: more negative = more anomalous → negate for probability interpretation\n log(\"Computing Isolation Forest scores...\")\n iso_val_score = -iso_forest.score_samples(X_val.values)\n iso_test_score = -iso_forest.score_samples(X_test.values)\n\n # Normalize\n iso_min = np.percentile(iso_val_score, 1)\n iso_max = np.percentile(iso_val_score, 99)\n iso_val_norm = np.clip((iso_val_score - iso_min) / (iso_max - iso_min + 1e-8), 0, 1)\n iso_test_norm = np.clip((iso_test_score - iso_min) / (iso_max - iso_min + 1e-8), 0, 1)\n\n iso_pr_auc = average_precision_score(y_val, iso_val_norm)\n iso_roc_auc = roc_auc_score(y_val, iso_val_norm)\n log(f\"Isolation Forest — PR-AUC: {iso_pr_auc:.4f} ROC-AUC: {iso_roc_auc:.4f}\")\n\nlog(f\"SECTION 12 complete — took {time.time()-t0:.1f}s\")\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 13: MODEL 8 — LSTM Temporal Classifier (FIXED)\n# ============================================================\nlog(\"=\" * 60)\nlog(\"SECTION 13: MODEL 8 — LSTM Temporal — started\")\nlog(\"=\" * 60)\n\nt0 = time.time()\n\n# Build sequences from the training data\nSEQ_LEN = 10 # 10 consecutive measurements per satellite\n\n# Safety check for minimum data size\nif len(X_train) < SEQ_LEN or len(X_val) < SEQ_LEN:\n log(f\"WARNING: Data too small for LSTM (need at least {SEQ_LEN} samples)\")\n log(f\"Train: {len(X_train)}, Val: {len(X_val)}\")\n log(\"Skipping LSTM model, using default 0.5 predictions\")\n lstm_val_prob = np.full(len(X_val), 0.5, dtype=np.float32)\n lstm_test_prob = np.full(len(X_test), 0.5, dtype=np.float32)\n best_lstm_f2 = 0.0\nelse:\n scaler_lstm = RobustScaler()\n X_train_lstm_scaled = scaler_lstm.fit_transform(X_train).astype(np.float32)\n X_val_lstm_scaled = scaler_lstm.transform(X_val).astype(np.float32)\n X_test_lstm_scaled = scaler_lstm.transform(X_test).astype(np.float32)\n\n\n class LSTMClassifier(nn.Module):\n \"\"\"\n Bidirectional LSTM for sequential GNSS data.\n Takes a window of SEQ_LEN measurements and predicts\n multipath for the CENTER element.\n \"\"\"\n def __init__(self, n_features, hidden_dim=128, n_layers=2, dropout=0.3):\n super().__init__()\n self.lstm = nn.LSTM(\n input_size=n_features,\n hidden_size=hidden_dim,\n num_layers=n_layers,\n batch_first=True,\n bidirectional=True,\n dropout=dropout if n_layers > 1 else 0.0\n )\n self.attention = nn.Sequential(\n nn.Linear(hidden_dim * 2, 64),\n nn.Tanh(),\n nn.Linear(64, 1)\n )\n self.head = nn.Sequential(\n nn.Linear(hidden_dim * 2, 64),\n nn.GELU(),\n nn.Dropout(dropout),\n nn.Linear(64, 1)\n )\n\n for name, param in self.lstm.named_parameters():\n if 'weight_ih' in name:\n nn.init.xavier_uniform_(param.data)\n elif 'weight_hh' in name:\n nn.init.orthogonal_(param.data)\n elif 'bias' in name:\n nn.init.zeros_(param.data)\n\n def forward(self, x):\n # x: (B, SEQ_LEN, n_features)\n out, _ = self.lstm(x) # (B, SEQ_LEN, hidden*2)\n\n # Attention pooling\n attn_weights = torch.softmax(self.attention(out), dim=1) # (B, SEQ_LEN, 1)\n context = (attn_weights * out).sum(dim=1) # (B, hidden*2)\n\n return self.head(context)\n\n\n class SequenceDataset(Dataset):\n \"\"\"\n Creates sliding windows of length SEQ_LEN.\n Label is for the center element of the window.\n \"\"\"\n def __init__(self, X_np, y_np, seq_len=10, stride=1):\n self.X = X_np\n self.y = y_np\n self.seq_len = seq_len\n self.stride = stride\n self.half = seq_len // 2\n # Valid center indices\n self.centers = list(range(self.half,\n len(X_np) - self.half,\n stride))\n\n def __len__(self):\n return len(self.centers)\n\n def __getitem__(self, idx):\n c = self.centers[idx]\n start = c - self.half\n end = start + self.seq_len\n x_seq = self.X[start:end] # (SEQ_LEN, n_features)\n y_label = self.y[c]\n return (torch.tensor(x_seq, dtype=torch.float32),\n torch.tensor(y_label, dtype=torch.float32))\n\n\n # Use stride=3 to reduce dataset size but maintain coverage\n lstm_train_ds = SequenceDataset(X_train_lstm_scaled, y_train.values,\n seq_len=SEQ_LEN, stride=3)\n lstm_val_ds = SequenceDataset(X_val_lstm_scaled, y_val.values,\n seq_len=SEQ_LEN, stride=1)\n lstm_test_ds = SequenceDataset(X_test_lstm_scaled, y_test.values,\n seq_len=SEQ_LEN, stride=1)\n\n log(f\"LSTM dataset sizes — train: {len(lstm_train_ds):,} \"\n f\"val: {len(lstm_val_ds):,} test: {len(lstm_test_ds):,}\")\n\n # Safety check for empty datasets\n if len(lstm_train_ds) == 0 or len(lstm_val_ds) == 0:\n log(\"ERROR: LSTM datasets empty!\")\n log(\"Skipping LSTM model\")\n lstm_val_prob = np.full(len(X_val), 0.5, dtype=np.float32)\n lstm_test_prob = np.full(len(X_test), 0.5, dtype=np.float32)\n best_lstm_f2 = 0.0\n else:\n lstm_train_loader = DataLoader(lstm_train_ds, batch_size=512, shuffle=True,\n num_workers=0, pin_memory=True)\n lstm_val_loader = DataLoader(lstm_val_ds, batch_size=1024, shuffle=False,\n num_workers=0, pin_memory=True)\n lstm_test_loader = DataLoader(lstm_test_ds, batch_size=1024, shuffle=False,\n num_workers=0, pin_memory=True)\n\n lstm_model = LSTMClassifier(n_features=n_features, hidden_dim=128,\n n_layers=2, dropout=0.3).to(DEVICE)\n\n # Class weights for BCE loss\n n_pos = (y_train == 1).sum()\n n_neg = (y_train == 0).sum()\n pos_weight = torch.tensor([n_neg / max(n_pos, 1)], dtype=torch.float32).to(DEVICE)\n lstm_loss_fn = nn.BCEWithLogitsLoss(pos_weight=pos_weight)\n\n optimizer_lstm = torch.optim.AdamW(lstm_model.parameters(), lr=1e-3, weight_decay=1e-4)\n scheduler_lstm = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer_lstm, T_max=30)\n\n LSTM_EPOCHS = 30\n LSTM_PATIENCE = 7\n best_lstm_f2 = -1\n lstm_patience = 0\n best_lstm_state = None\n\n log(f\"Training LSTM ({sum(p.numel() for p in lstm_model.parameters()):,} params)...\")\n\n for epoch in range(LSTM_EPOCHS):\n lstm_model.train()\n train_loss = 0.0\n for xb, yb in lstm_train_loader:\n xb, yb = xb.to(DEVICE), yb.to(DEVICE)\n optimizer_lstm.zero_grad()\n logits = lstm_model(xb).squeeze(1)\n loss = lstm_loss_fn(logits, yb)\n \n if torch.isnan(loss):\n continue\n \n loss.backward()\n torch.nn.utils.clip_grad_norm_(lstm_model.parameters(), max_norm=1.0)\n optimizer_lstm.step()\n train_loss += loss.item()\n scheduler_lstm.step()\n\n lstm_model.eval()\n val_preds = []\n with torch.no_grad():\n for xb, yb in lstm_val_loader:\n xb = xb.to(DEVICE)\n logits = lstm_model(xb).squeeze(1)\n val_preds.append(torch.sigmoid(logits).cpu().numpy())\n\n val_probs = np.concatenate(val_preds)\n \n # FIXED: Use actual center indices for validation labels\n val_centers_y = y_val.values[lstm_val_ds.centers]\n \n # Verify length match\n if len(val_probs) != len(val_centers_y):\n log(f\" WARNING: Length mismatch - probs:{len(val_probs)} vs centers:{len(val_centers_y)}\")\n min_len = min(len(val_probs), len(val_centers_y))\n val_probs = val_probs[:min_len]\n val_centers_y = val_centers_y[:min_len]\n \n val_f2 = fbeta_score(val_centers_y,\n (val_probs >= 0.5).astype(int), beta=2)\n\n if val_f2 > best_lstm_f2:\n best_lstm_f2 = val_f2\n lstm_patience = 0\n best_lstm_state = {k: v.cpu().clone() for k, v in lstm_model.state_dict().items()}\n log(f\" Ep {epoch+1:02d}/{LSTM_EPOCHS} | F2={val_f2:.4f} ★ BEST\")\n else:\n lstm_patience += 1\n if (epoch + 1) % 5 == 0:\n log(f\" Ep {epoch+1:02d}/{LSTM_EPOCHS} | F2={val_f2:.4f} \"\n f\"(patience {lstm_patience}/{LSTM_PATIENCE})\")\n if lstm_patience >= LSTM_PATIENCE:\n log(f\" Early stopping at epoch {epoch+1}\")\n break\n\n if best_lstm_state is not None:\n lstm_model.load_state_dict(best_lstm_state)\n lstm_model.eval()\n log(f\"Best LSTM val F2: {best_lstm_f2:.4f}\")\n\n # Inference — LSTM only covers center elements, pad edges with 0.5\n def lstm_full_inference(model, loader, n_total, centers, device):\n probs_center = []\n model.eval()\n with torch.no_grad():\n for xb, _ in loader:\n xb = xb.to(device)\n logits = model(xb).squeeze(1)\n probs_center.append(torch.sigmoid(logits).cpu().numpy())\n probs_center = np.concatenate(probs_center)\n full_probs = np.full(n_total, 0.5, dtype=np.float32)\n \n # FIXED: Verify lengths before assignment\n if len(probs_center) == len(centers):\n for i, c in enumerate(centers):\n full_probs[c] = probs_center[i]\n else:\n log(f\" WARNING: LSTM inference length mismatch\")\n min_len = min(len(probs_center), len(centers))\n for i in range(min_len):\n full_probs[centers[i]] = probs_center[i]\n \n return full_probs\n\n lstm_val_prob = lstm_full_inference(\n lstm_model, lstm_val_loader, len(y_val), lstm_val_ds.centers, DEVICE)\n lstm_test_prob = lstm_full_inference(\n lstm_model, lstm_test_loader, len(y_test), lstm_test_ds.centers, DEVICE)\n\n del lstm_train_ds, lstm_val_ds, lstm_test_ds\n del lstm_train_loader, lstm_val_loader, lstm_test_loader\n gc.collect()\n torch.cuda.empty_cache() if torch.cuda.is_available() else None\n\nlog(f\"SECTION 13 complete — took {time.time()-t0:.1f}s\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 14: OOF META-STACKING (FIXED - NaN cleanup)\n# ============================================================\nlog(\"=\" * 60)\nlog(\"SECTION 14: OOF META-STACKING — started\")\nlog(\"=\" * 60)\n\nt0 = time.time()\n\n# ---- Clean NaN from all prediction arrays FIRST ----\nlog(\"Cleaning NaN from prediction arrays...\")\nlgb_val_probs = np.nan_to_num(lgb_val_probs, nan=0.0)\nlgb_test_probs = np.nan_to_num(lgb_test_probs, nan=0.0)\nxgb_val_prob = np.nan_to_num(xgb_val_prob, nan=0.0)\nxgb_test_prob = np.nan_to_num(xgb_test_prob, nan=0.0)\ncat_val_prob = np.nan_to_num(cat_val_prob, nan=0.0)\ncat_test_prob = np.nan_to_num(cat_test_prob, nan=0.0)\nmlp_val_prob = np.nan_to_num(mlp_val_prob, nan=0.5)\nmlp_test_prob = np.nan_to_num(mlp_test_prob, nan=0.5)\nft_val_prob = np.nan_to_num(ft_val_prob, nan=0.5)\nft_test_prob = np.nan_to_num(ft_test_prob, nan=0.5)\nvae_val_norm = np.nan_to_num(vae_val_norm, nan=0.0)\nvae_test_norm = np.nan_to_num(vae_test_norm, nan=0.0)\niso_val_norm = np.nan_to_num(iso_val_norm, nan=0.0)\niso_test_norm = np.nan_to_num(iso_test_norm, nan=0.0)\nlstm_val_prob = np.nan_to_num(lstm_val_prob, nan=0.5)\nlstm_test_prob = np.nan_to_num(lstm_test_prob, nan=0.5)\nlog(\"NaN cleanup complete ✓\")\n\n# ---- Collect all val/test predictions ----\nmeta_val = np.column_stack([\n lgb_val_probs.ravel(),\n xgb_val_prob.ravel(),\n cat_val_prob.ravel(),\n mlp_val_prob.ravel(),\n ft_val_prob.ravel(),\n vae_val_norm.ravel(),\n iso_val_norm.ravel(),\n lstm_val_prob.ravel(),\n])\n\nmeta_test = np.column_stack([\n lgb_test_probs.ravel(),\n xgb_test_prob.ravel(),\n cat_test_prob.ravel(),\n mlp_test_prob.ravel(),\n ft_test_prob.ravel(),\n vae_test_norm.ravel(),\n iso_test_norm.ravel(),\n lstm_test_prob.ravel(),\n])\n\n# Final safety: replace any remaining NaN/inf\nmeta_val = np.nan_to_num(meta_val, nan=0.0, posinf=1.0, neginf=0.0)\nmeta_test = np.nan_to_num(meta_test, nan=0.0, posinf=1.0, neginf=0.0)\n\nmodel_names = ['LightGBM', 'XGBoost', 'CatBoost', 'MLP',\n 'FT-Transformer', 'VAE', 'IsoForest', 'LSTM']\n\nlog(\"Individual model VALIDATION PR-AUCs:\")\nfor name, probs in zip(model_names, meta_val.T):\n try:\n auc = average_precision_score(y_val, probs)\n log(f\" {name:<15s}: PR-AUC={auc:.4f}\")\n except Exception as e:\n log(f\" {name:<15s}: PR-AUC=ERROR ({e})\")\n\n# ---- Meta-learner: XGBoost ----\nlog(\"Training XGBoost meta-learner on VALIDATION predictions...\")\nneg_c = (y_val == 0).sum()\npos_c = (y_val == 1).sum()\n\nmeta_xgb = xgb.XGBClassifier(\n n_estimators=200,\n max_depth=4,\n learning_rate=0.05,\n scale_pos_weight=neg_c / max(pos_c, 1),\n eval_metric='aucpr',\n random_state=SEED,\n verbosity=0,\n nthread=-1,\n device='cuda' if torch.cuda.is_available() else 'cpu',\n tree_method='hist',\n)\nmeta_xgb.fit(meta_val, y_val.values)\n\nmeta_val_prob_xgb = meta_xgb.predict_proba(meta_val)[:, 1]\nmeta_xgb_val_pr = average_precision_score(y_val, meta_val_prob_xgb)\nlog(f\"Meta-learner (XGBoost) VAL PR-AUC: {meta_xgb_val_pr:.4f}\")\n\n# ---- Meta-learner: Logistic Regression ----\nmeta_lr = LogisticRegression(class_weight='balanced', C=1.0,\n max_iter=1000, random_state=SEED)\nmeta_lr.fit(meta_val, y_val.values)\nmeta_val_prob_lr = meta_lr.predict_proba(meta_val)[:, 1]\nmeta_lr_val_pr = average_precision_score(y_val, meta_val_prob_lr)\nlog(f\"Meta-learner (LogReg) VAL PR-AUC: {meta_lr_val_pr:.4f}\")\n\n# Select best meta-learner using VALIDATION set only\nif meta_xgb_val_pr >= meta_lr_val_pr:\n log(f\"Using XGBoost meta-learner (VAL PR-AUC: {meta_xgb_val_pr:.4f})\")\n final_test_prob = meta_xgb.predict_proba(meta_test)[:, 1]\n final_val_prob = meta_val_prob_xgb\nelse:\n log(f\"Using LogReg meta-learner (VAL PR-AUC: {meta_lr_val_pr:.4f})\")\n final_test_prob = meta_lr.predict_proba(meta_test)[:, 1]\n final_val_prob = meta_val_prob_lr\n\n# Clean final predictions\nfinal_test_prob = np.nan_to_num(final_test_prob, nan=0.0)\nfinal_val_prob = np.nan_to_num(final_val_prob, nan=0.0)\n\nfinal_test_pr = average_precision_score(y_test, final_test_prob)\nlog(f\"Selected meta-learner TEST PR-AUC: {final_test_pr:.4f} (informational only)\")\n\nlog(f\"SECTION 14 complete — took {time.time()-t0:.1f}s\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 15: THRESHOLD OPTIMIZATION (FIXED)\n# ============================================================\nlog(\"=\" * 60)\nlog(\"SECTION 15: THRESHOLD OPTIMIZATION — started\")\nlog(\"=\" * 60)\n\nt0 = time.time()\n\n# Optimize on VAL set (not test — avoid leakage)\nif len(final_val_prob) == 0 or np.allclose(final_val_prob, final_val_prob[0]):\n log(\"WARNING: All predictions identical, using default threshold 0.5\")\n optimal_threshold = 0.5\n best_f2 = 0.0\nelse:\n precisions, recalls, thresholds = precision_recall_curve(y_val, final_val_prob)\n \n if len(thresholds) == 0:\n log(\"WARNING: No valid thresholds found, using default 0.5\")\n optimal_threshold = 0.5\n best_f2 = 0.0\n else:\n f2_scores = []\n for p, r in zip(precisions[:-1], recalls[:-1]):\n denom = 4 * p + r\n f2 = (5 * p * r) / (denom + 1e-8) if denom > 0 else 0.0\n f2_scores.append(f2)\n \n optimal_idx = np.argmax(f2_scores)\n optimal_threshold = thresholds[optimal_idx]\n best_f2 = f2_scores[optimal_idx]\n \n log(f\"Optimal threshold (from val): {optimal_threshold:.4f}\")\n log(f\"F2 at optimal threshold: {best_f2:.4f}\")\n log(f\"Precision at optimal: {precisions[optimal_idx]:.4f}\")\n log(f\"Recall at optimal: {recalls[optimal_idx]:.4f}\")\n\nlog(f\"SECTION 15 complete — took {time.time()-t0:.1f}s\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 16: COMPREHENSIVE EVALUATION (FIXED)\n# ============================================================\nlog(\"=\" * 60)\nlog(\"SECTION 16: COMPREHENSIVE EVALUATION — started\")\nlog(\"=\" * 60)\n\nt0 = time.time()\n\n\ndef evaluate_model(y_true, y_prob, model_name, threshold=0.5):\n y_pred = (y_prob >= threshold).astype(int)\n metrics = {\n 'Model': model_name,\n 'Threshold': threshold,\n 'Accuracy': accuracy_score(y_true, y_pred),\n 'Precision': precision_score(y_true, y_pred, zero_division=0),\n 'Recall': recall_score(y_true, y_pred, zero_division=0),\n 'F1': f1_score(y_true, y_pred, zero_division=0),\n 'F2': fbeta_score(y_true, y_pred, beta=2, zero_division=0),\n }\n try:\n metrics['ROC_AUC'] = roc_auc_score(y_true, y_prob)\n metrics['PR_AUC'] = average_precision_score(y_true, y_prob)\n except Exception:\n metrics['ROC_AUC'] = 0.0\n metrics['PR_AUC'] = 0.0\n return metrics, y_pred\n\n\n# Evaluate all models on TEST set\nmodels_eval = {\n 'LightGBM': lgb_test_probs.ravel(),\n 'XGBoost': xgb_test_prob.ravel(),\n 'CatBoost': cat_test_prob.ravel(),\n 'MLP-Focal': mlp_test_prob.ravel(),\n 'FT-Transformer': ft_test_prob.ravel(),\n 'VAE': vae_test_norm.ravel(),\n 'IsoForest': iso_test_norm.ravel(),\n 'LSTM': lstm_test_prob.ravel(),\n 'Ensemble': final_test_prob.ravel(),\n}\n\nall_results = []\nall_preds = {}\n\nfor name, probs in models_eval.items():\n thr = optimal_threshold if name == 'Ensemble' else 0.5\n # Verify probabilities are valid\n if np.all(probs == probs[0]):\n log(f\" WARNING: {name} predictions are constant ({probs[0]:.4f})\")\n m, preds = evaluate_model(y_test, probs, name, threshold=thr)\n all_results.append(m)\n all_preds[name] = preds\n print(f\"\\n{'='*50}\")\n print(f\" {name} (threshold={thr:.4f})\")\n print('='*50)\n print(classification_report(y_test, preds,\n target_names=['Normal', 'Multipath'],\n zero_division=0))\n\nresults_df = pd.DataFrame(all_results)\nprint(\"\\n\" + \"=\" * 80)\nprint(\"MODEL COMPARISON SUMMARY (TEST SET)\")\nprint(\"=\" * 80)\nprint(results_df.to_string(index=False, float_format='%.4f'))\n\nresults_df.to_csv('/kaggle/working/evaluation_results.csv', index=False)\n\n# ---- PLOTS ----\n\n# 1. PR Curves\ntry:\n fig, ax = plt.subplots(figsize=(10, 7))\n colors = plt.cm.tab10(np.linspace(0, 1, len(models_eval)))\n for (name, probs), color in zip(models_eval.items(), colors):\n try:\n if np.all(probs == probs[0]):\n log(f\" Skipping PR curve for {name} (constant predictions)\")\n continue\n pr, rec, _ = precision_recall_curve(y_test, probs)\n ap = average_precision_score(y_test, probs)\n lw = 2.5 if name == 'Ensemble' else 1.5\n ax.plot(rec, pr, label=f'{name} (AP={ap:.3f})', lw=lw, color=color)\n except Exception as e:\n log(f\" PR curve failed for {name}: {e}\")\n baseline = y_test.mean()\n ax.axhline(baseline, color='gray', linestyle='--', alpha=0.5, label=f'Baseline (={baseline:.3f})')\n ax.set_xlabel('Recall', fontsize=12)\n ax.set_ylabel('Precision', fontsize=12)\n ax.set_title('Precision-Recall Curves — All Models (Test Set)', fontsize=14)\n ax.legend(loc='upper right', fontsize=9)\n ax.grid(True, alpha=0.3)\n plt.tight_layout()\n plt.savefig(os.path.join(PLOT_DIR, 'pr_curves_all.png'), dpi=150)\n plt.close()\n log(\"PR curves saved ✓\")\nexcept Exception as e:\n log(f\"PR curves plot failed: {e}\")\n\n# 2. ROC Curves\ntry:\n fig, ax = plt.subplots(figsize=(8, 7))\n for (name, probs), color in zip(models_eval.items(), colors):\n try:\n if np.all(probs == probs[0]):\n continue\n fpr, tpr, _ = roc_curve(y_test, probs)\n auc_val = roc_auc_score(y_test, probs)\n lw = 2.5 if name == 'Ensemble' else 1.5\n ax.plot(fpr, tpr, label=f'{name} (AUC={auc_val:.3f})', lw=lw, color=color)\n except Exception:\n pass\n ax.plot([0, 1], [0, 1], 'k--', alpha=0.3)\n ax.set_xlabel('FPR', fontsize=12)\n ax.set_ylabel('TPR', fontsize=12)\n ax.set_title('ROC Curves — All Models (Test Set)', fontsize=14)\n ax.legend(loc='lower right', fontsize=9)\n ax.grid(True, alpha=0.3)\n plt.tight_layout()\n plt.savefig(os.path.join(PLOT_DIR, 'roc_curves_all.png'), dpi=150)\n plt.close()\n log(\"ROC curves saved ✓\")\nexcept Exception as e:\n log(f\"ROC curves plot failed: {e}\")\n\n# 3. Confusion matrices\ntry:\n n_models = len(all_preds)\n ncols = 3\n nrows = (n_models + ncols - 1) // ncols\n fig, axes = plt.subplots(nrows, ncols, figsize=(6*ncols, 5*nrows))\n axes = axes.ravel() if n_models > 1 else [axes]\n for ax, (name, preds) in zip(axes, all_preds.items()):\n cm = confusion_matrix(y_test, preds)\n sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', ax=ax,\n xticklabels=['Normal', 'Multipath'],\n yticklabels=['Normal', 'Multipath'])\n pr = precision_score(y_test, preds, zero_division=0)\n rc = recall_score(y_test, preds, zero_division=0)\n ax.set_title(f'{name}\\nPrec={pr:.3f} Rec={rc:.3f}', fontsize=11)\n ax.set_ylabel('True')\n ax.set_xlabel('Predicted')\n for ax in axes[n_models:]:\n ax.set_visible(False)\n plt.tight_layout()\n plt.savefig(os.path.join(PLOT_DIR, 'confusion_matrices_all.png'), dpi=150)\n plt.close()\n log(\"Confusion matrices saved ✓\")\nexcept Exception as e:\n log(f\"Confusion matrices plot failed: {e}\")\n\n# 4. F2 vs threshold for ensemble\ntry:\n fig, ax = plt.subplots(figsize=(8, 5))\n ax.plot(thresholds, f2_scores, color='darkorange', lw=2)\n ax.axvline(optimal_threshold, color='red', linestyle='--',\n label=f'Optimal={optimal_threshold:.3f} F2={best_f2:.4f}')\n ax.set_xlabel('Threshold', fontsize=12)\n ax.set_ylabel('F2 Score', fontsize=12)\n ax.set_title('Ensemble: F2 Score vs Decision Threshold', fontsize=14)\n ax.legend(fontsize=11)\n ax.grid(True, alpha=0.3)\n plt.tight_layout()\n plt.savefig(os.path.join(PLOT_DIR, 'f2_vs_threshold.png'), dpi=150)\n plt.close()\n log(\"F2 threshold plot saved ✓\")\nexcept Exception as e:\n log(f\"F2 threshold plot failed: {e}\")\n\n# 5. Feature importance (LightGBM aggregate)\ntry:\n if len(lgb_estimators) > 0:\n importances = np.zeros(len(FEATURE_COLS))\n for est in lgb_estimators:\n importances += est.feature_importances_\n importances /= len(lgb_estimators)\n feat_imp = pd.DataFrame({'feature': FEATURE_COLS, 'importance': importances})\n feat_imp = feat_imp.sort_values('importance', ascending=True).tail(25)\n fig, ax = plt.subplots(figsize=(8, 10))\n bars = ax.barh(feat_imp['feature'], feat_imp['importance'], color='steelblue')\n ax.set_xlabel('Mean Importance')\n ax.set_title('Top 25 LightGBM Feature Importances')\n plt.tight_layout()\n plt.savefig(os.path.join(PLOT_DIR, 'lgb_feature_importance.png'), dpi=150)\n plt.close()\n log(\"Feature importance plot saved ✓\")\n log(\"Top 10 features:\")\n for _, row in feat_imp.tail(10).iterrows():\n log(f\" {row['feature']:<30s}: {row['importance']:.1f}\")\nexcept Exception as e:\n log(f\"Feature importance plot failed: {e}\")\n\n# 6. Model comparison bar chart\ntry:\n fig, axes = plt.subplots(1, 3, figsize=(18, 6))\n for ax, metric in zip(axes, ['F2', 'PR_AUC', 'ROC_AUC']):\n vals = results_df.set_index('Model')[metric].sort_values()\n colors_bar = ['crimson' if idx == 'Ensemble' else 'steelblue' for idx in vals.index]\n vals.plot(kind='barh', ax=ax, color=colors_bar, alpha=0.85)\n ax.set_title(metric, fontsize=13)\n ax.set_xlim(0, 1)\n ax.grid(True, alpha=0.3, axis='x')\n for i, v in enumerate(vals):\n ax.text(v + 0.005, i, f'{v:.3f}', va='center', fontsize=9)\n plt.suptitle('Model Comparison — Test Set', fontsize=15, y=1.02)\n plt.tight_layout()\n plt.savefig(os.path.join(PLOT_DIR, 'model_comparison.png'), dpi=150, bbox_inches='tight')\n plt.close()\n log(\"Model comparison plot saved ✓\")\nexcept Exception as e:\n log(f\"Model comparison plot failed: {e}\")\n\n# 7. Meta-learner feature importance\ntry:\n meta_imp = meta_xgb.feature_importances_\n fig, ax = plt.subplots(figsize=(8, 5))\n ax.barh(model_names, meta_imp, color='teal', alpha=0.8)\n ax.set_xlabel('Importance')\n ax.set_title('Meta-Learner Model Weights (XGBoost importance)')\n plt.tight_layout()\n plt.savefig(os.path.join(PLOT_DIR, 'meta_learner_importance.png'), dpi=150)\n plt.close()\n log(\"Meta-learner importance plot saved ✓\")\nexcept Exception as e:\n log(f\"Meta-learner importance plot failed: {e}\")\n\nlog(f\"SECTION 16 complete — took {time.time()-t0:.1f}s\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 17: SAVING ALL ARTIFACTS (FIXED)\n# ============================================================\nlog(\"=\" * 60)\nlog(\"SECTION 17: SAVING ALL ARTIFACTS — started\")\nlog(\"=\" * 60)\n\nt0 = time.time()\n\n# LightGBM estimators\ntry:\n for i, est in enumerate(lgb_estimators):\n est.booster_.save_model(os.path.join(SAVE_DIR, f'lgb_bag_{i:02d}.txt'))\n log(f\" Saved {len(lgb_estimators)} LGB estimators\")\nexcept Exception as e:\n log(f\" LGB save failed: {e}\")\n\n# XGBoost\ntry:\n xgb_model.save_model(os.path.join(SAVE_DIR, 'xgb_model.json'))\n log(\" Saved XGBoost model\")\nexcept Exception as e:\n log(f\" XGBoost save failed: {e}\")\n\n# CatBoost\ntry:\n cat_model.save_model(os.path.join(SAVE_DIR, 'catboost_model.cbm'))\n log(\" Saved CatBoost model\")\nexcept Exception as e:\n log(f\" CatBoost save failed: {e}\")\n\n# PyTorch models\ntry:\n if 'mlp_model' in dir():\n torch.save(mlp_model.state_dict(), os.path.join(SAVE_DIR, 'mlp_model.pt'))\n log(\" Saved MLP model\")\nexcept Exception as e:\n log(f\" MLP save failed: {e}\")\n\ntry:\n if 'ft_model' in dir():\n torch.save(ft_model.state_dict(), os.path.join(SAVE_DIR, 'ft_transformer.pt'))\n log(\" Saved FT-Transformer model\")\nexcept Exception as e:\n log(f\" FT-Transformer save failed: {e}\")\n\ntry:\n if 'vae_model' in dir():\n torch.save(vae_model.state_dict(), os.path.join(SAVE_DIR, 'vae_model.pt'))\n log(\" Saved VAE model\")\nexcept Exception as e:\n log(f\" VAE save failed: {e}\")\n\ntry:\n if 'lstm_model' in dir():\n torch.save(lstm_model.state_dict(), os.path.join(SAVE_DIR, 'lstm_model.pt'))\n log(\" Saved LSTM model\")\nexcept Exception as e:\n log(f\" LSTM save failed: {e}\")\n\n# Isolation Forest\ntry:\n if 'iso_forest' in dir():\n joblib.dump(iso_forest, os.path.join(SAVE_DIR, 'iso_forest.pkl'))\n log(\" Saved Isolation Forest\")\nexcept Exception as e:\n log(f\" Isolation Forest save failed: {e}\")\n\n# Meta-learner\ntry:\n joblib.dump(meta_xgb, os.path.join(SAVE_DIR, 'meta_xgb.pkl'))\n log(\" Saved meta-learner (XGBoost)\")\nexcept Exception as e:\n log(f\" Meta-learner save failed: {e}\")\n\ntry:\n joblib.dump(meta_lr, os.path.join(SAVE_DIR, 'meta_lr.pkl'))\n log(\" Saved meta-learner (LogReg)\")\nexcept Exception as e:\n log(f\" LogReg save failed: {e}\")\n\n# Scalers\ntry:\n if 'scaler_mlp' in dir():\n joblib.dump(scaler_mlp, os.path.join(SAVE_DIR, 'scaler_mlp.pkl'))\n log(\" Saved scaler_mlp\")\nexcept Exception as e:\n log(f\" Scaler_mlp save failed: {e}\")\n\ntry:\n if 'scaler_vae' in dir():\n joblib.dump(scaler_vae, os.path.join(SAVE_DIR, 'scaler_vae.pkl'))\n log(\" Saved scaler_vae\")\nexcept Exception as e:\n log(f\" Scaler_vae save failed: {e}\")\n\ntry:\n if 'scaler_lstm' in dir():\n joblib.dump(scaler_lstm, os.path.join(SAVE_DIR, 'scaler_lstm.pkl'))\n log(\" Saved scaler_lstm\")\nexcept Exception as e:\n log(f\" Scaler_lstm save failed: {e}\")\n\n# Preprocessing artifacts\ntry:\n joblib.dump(train_medians.to_dict(), os.path.join(SAVE_DIR, 'train_medians.pkl'))\n log(\" Saved train medians\")\nexcept Exception as e:\n log(f\" Train medians save failed: {e}\")\n\ntry:\n joblib.dump(clip_bounds, os.path.join(SAVE_DIR, 'clip_bounds.pkl'))\n log(\" Saved clip bounds\")\nexcept Exception as e:\n log(f\" Clip bounds save failed: {e}\")\n\n# VAE/IF normalization bounds\ntry:\n if 'vae_p5' in dir() and 'vae_p95' in dir():\n np.save(os.path.join(SAVE_DIR, 'vae_bounds.npy'),\n np.array([vae_p5, vae_p95]))\n log(\" Saved VAE bounds\")\nexcept Exception as e:\n log(f\" VAE bounds save failed: {e}\")\n\ntry:\n if 'iso_min' in dir() and 'iso_max' in dir():\n np.save(os.path.join(SAVE_DIR, 'iso_bounds.npy'),\n np.array([iso_min, iso_max]))\n log(\" Saved IF bounds\")\nexcept Exception as e:\n log(f\" IF bounds save failed: {e}\")\n\n# Feature list and threshold\ntry:\n with open(os.path.join(SAVE_DIR, 'feature_cols.txt'), 'w') as f:\n f.write('\\n'.join(FEATURE_COLS))\n log(f\" Saved feature list ({len(FEATURE_COLS)} features)\")\nexcept Exception as e:\n log(f\" Feature list save failed: {e}\")\n\ntry:\n with open(os.path.join(SAVE_DIR, 'optimal_threshold.txt'), 'w') as f:\n f.write(str(optimal_threshold))\n log(f\" Saved optimal threshold ({optimal_threshold:.4f})\")\nexcept Exception as e:\n log(f\" Threshold save failed: {e}\")\n\n# Results\ntry:\n results_df.to_csv(os.path.join(SAVE_DIR, 'evaluation_results.csv'), index=False)\n log(\" Saved evaluation results CSV\")\nexcept Exception as e:\n log(f\" Results save failed: {e}\")\n\nlog(f\"SECTION 17 complete — took {time.time()-t0:.1f}s\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# ============================================================\n# SECTION 18: FINAL SUMMARY\n# ============================================================\nlog(\"=\" * 60)\nlog(\"FINAL SUMMARY\")\nlog(\"=\" * 60)\n\ntotal_time = time.time() - NOTEBOOK_START\nprint(f\"\\nTotal runtime: {int(total_time)//3600:02d}h\"\n f\"{(int(total_time)%3600)//60:02d}m\"\n f\"{int(total_time)%60:02d}s\")\n\nprint(f\"\\nDataset: {len(X_train):,} train / {len(X_val):,} val / {len(X_test):,} test\")\nprint(f\"Features: {len(FEATURE_COLS)} \"\n f\"({len(BASE_FEATURES)} base + {len(TEMPORAL_FEATURES)} temporal)\")\nprint(f\"Optimal ensemble threshold: {optimal_threshold:.4f}\\n\")\n\nprint(\"┌\" + \"─\"*73 + \"┐\")\nprint(\"│{:^73}│\".format(\"TEST SET RESULTS\"))\nprint(\"├\" + \"─\"*17 + \"┬\" + \"─\"*8 + \"┬\" + \"─\"*8 +\n \"┬\" + \"─\"*8 + \"┬\" + \"─\"*7 + \"┬\" + \"─\"*7 + \"┬\" + \"─\"*7 + \"┤\")\nprint(\"│{:<17}│{:^8}│{:^8}│{:^8}│{:^7}│{:^7}│{:^7}│\".format(\n \" Model\", \"PR-AUC\", \"ROC-AUC\", \"F2\", \"Prec\", \"Rec\", \"F1\"))\nprint(\"├\" + \"─\"*17 + \"┼\" + \"─\"*8 + \"┼\" + \"─\"*8 +\n \"┼\" + \"─\"*8 + \"┼\" + \"─\"*7 + \"┼\" + \"─\"*7 + \"┼\" + \"─\"*7 + \"┤\")\n\nfor _, row in results_df.sort_values('PR_AUC', ascending=False).iterrows():\n marker = \" ◄\" if row['Model'] == 'Ensemble' else \" \"\n print(\"│ {:<15s}│{:^8.4f}│{:^8.4f}│{:^8.4f}│{:^7.3f}│{:^7.3f}│{:^7.3f}│{}\".format(\n row['Model'], row['PR_AUC'], row['ROC_AUC'], row['F2'],\n row['Precision'], row['Recall'], row['F1'], marker))\n\nprint(\"└\" + \"─\"*17 + \"┴\" + \"─\"*8 + \"┴\" + \"─\"*8 +\n \"┴\" + \"─\"*8 + \"┴\" + \"─\"*7 + \"┴\" + \"─\"*7 + \"┴\" + \"─\"*7 + \"┘\")\n\nprint(f\"\\nModels saved to: {SAVE_DIR}\")\nprint(f\"Plots saved to: {PLOT_DIR}\")\nprint(f\"Results CSV: /kaggle/working/evaluation_results.csv\")\nprint(\"\\n\" + \"=\" * 60)\nprint(\"TRAINING COMPLETE\")\nprint(\"=\" * 60)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}