{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":39763,"databundleVersionId":11756775,"sourceType":"competition"}],"dockerImageVersionId":31012,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-04-10T06:55:24.698775Z","iopub.execute_input":"2025-04-10T06:55:24.698991Z","iopub.status.idle":"2025-04-10T06:56:40.898992Z","shell.execute_reply.started":"2025-04-10T06:55:24.698965Z","shell.execute_reply":"2025-04-10T06:56:40.898029Z"},"_kg_hide-input":true,"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Setup and Imports","metadata":{}},{"cell_type":"code","source":"# Cell 1: Setup, Configuration, and Imports (No change needed, comments added)\nprint(\"--- Cell 1: Setup ---\")\nimport numpy as np\nimport pandas as pd\nimport os\nimport gc # Garbage Collector\nimport time\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm # Progress bars\n\n# Deep Learning Framework - TensorFlow/Keras\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\nprint(f\"TensorFlow Version: {tf.__version__}\")\nprint(f\"Num GPUs Available: {len(tf.config.list_physical_devices('GPU'))}\")\nif tf.config.list_physical_devices('GPU'):\n    print(\"GPU is available.\")\nelse:\n    # Added explicit warning about training time\n    print(\"WARNING: GPU not available. Training will be very slow, especially with image-based models.\")\n\n# --- Paths ---\nBASE_PATH = '/kaggle/input/waveform-inversion/'\nTRAIN_PATH = os.path.join(BASE_PATH, 'train_samples/')\nTEST_PATH = os.path.join(BASE_PATH, 'test/')\nSAMPLE_SUB_PATH = os.path.join(BASE_PATH, 'sample_submission.csv')\nOUTPUT_DIR = '/kaggle/working/' # Directory to save model weights and submission file\nMODEL_WEIGHTS_FILE = os.path.join(OUTPUT_DIR, 'best_unet_model.keras')\nSUBMISSION_FILE = os.path.join(OUTPUT_DIR, 'submission.csv')\n\nos.makedirs(OUTPUT_DIR, exist_ok=True)\n\n# --- Configuration ---\n# Data dimensions\nDEPTH = 70\nWIDTH = 70\nTIME_STEPS = 1000\nN_RECEIVERS = 70\nN_SHOTS = 5\n\n# Model Input/Output Shapes (with padding for U-Net)\nPAD_H = 1024\nPAD_W = 80\nINPUT_SHAPE = (PAD_H, PAD_W, N_SHOTS) # (1024, 80, 5)\nOUTPUT_SHAPE = (DEPTH, WIDTH)         # (70, 70)\n\n# Submission format details\nSUBMISSION_Y_POS = DEPTH\nSUBMISSION_X_INDICES = np.arange(1, WIDTH, 2)\nSUBMISSION_X_COLS = [f'x_{i}' for i in SUBMISSION_X_INDICES]\nSUBMISSION_ID_COL = 'oid_ypos'\n\n# Training Hyperparameters\n# NOTE: BATCH_SIZE adjusted for validation below.\n#       EPOCHS might need significant tuning based on overfitting.\nBATCH_SIZE_TRAIN = 8    # Keep train batch size reasonable\nEPOCHS = 30           # Starting point, monitor validation loss closely\nLEARNING_RATE = 1e-4\nVALIDATION_SPLIT = 0.1 # Using 10% (2 samples) for validation\n\n# Velocity normalization range\nV_MIN = 1500.0\nV_MAX = 5000.0\n\n# --- End Configuration ---\n\nprint(f\"Target Velocity Map Shape: {OUTPUT_SHAPE}\")\nprint(f\"Seismic Data Shape (Shots, Time, Receivers): ({N_SHOTS}, {TIME_STEPS}, {N_RECEIVERS})\")\nprint(f\"Padded Model Input Shape: {INPUT_SHAPE}\")\nprint(f\"Submission ID Column: '{SUBMISSION_ID_COL}'\")\nprint(f\"Submission X Columns ({len(SUBMISSION_X_COLS)}): {SUBMISSION_X_COLS}\")\nprint(f\"Model weights will be saved to: {MODEL_WEIGHTS_FILE}\")\nprint(f\"Submission file will be saved to: {SUBMISSION_FILE}\")\nprint(\"--- Cell 1: Done ---\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-10T07:06:29.634484Z","iopub.execute_input":"2025-04-10T07:06:29.635042Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load Sample Submission and Verify Format","metadata":{}},{"cell_type":"code","source":"# Cell 2: Load Sample Submission and Define Test Set Scope (No change needed)\nprint(\"\\n--- Cell 2: Load Sample Submission ---\")\ntest_oids_from_sub = []\nexpected_submission_rows = 0\nsample_submission = None # Initialize\n\ntry:\n    sample_submission = pd.read_csv(SAMPLE_SUB_PATH)\n    print(\"Sample Submission Info:\")\n    # sample_submission.info() # Reduce verbosity\n    print(f\"Loaded sample submission with {len(sample_submission)} rows.\")\n    print(\"Sample Submission Head:\")\n    print(sample_submission.head())\n    expected_submission_rows = len(sample_submission)\n    print(f\"\\nExpected number of rows in final submission: {expected_submission_rows}\")\n\n    if SUBMISSION_ID_COL not in sample_submission.columns:\n         raise KeyError(f\"Submission ID column '{SUBMISSION_ID_COL}' not found in sample_submission.csv\")\n\n    sample_submission['oid'] = sample_submission[SUBMISSION_ID_COL].apply(lambda x: x.split('_y_')[0])\n    test_oids_from_sub = sorted(sample_submission['oid'].unique())\n    print(f\"\\nFound {len(test_oids_from_sub)} unique test file IDs (oids) required for submission.\")\n    # print(\"Example required oids:\", test_oids_from_sub[:5]) # Reduce verbosity\n\n    # Structure verification... (already confirmed OK)\n    # first_oid_rows = sample_submission[sample_submission['oid'] == test_oids_from_sub[0]]\n    # print(f\"\\nNumber of rows for first oid '{test_oids_from_sub[0]}': {len(first_oid_rows)}\")\n    # if len(first_oid_rows) == DEPTH: print(f\"Row count per oid matches DEPTH ({DEPTH}). OK.\")\n    # else: print(f\"WARNING: Row count per oid ({len(first_oid_rows)}) does NOT match DEPTH ({DEPTH}).\")\n    # Column name verification... (already confirmed OK)\n    # expected_columns = [SUBMISSION_ID_COL] + SUBMISSION_X_COLS\n    # if list(sample_submission.columns.drop('oid')) == expected_columns: print(f\"\\nSubmission columns match expected format. OK.\")\n    # else: print(f\"\\nWARNING: Submission columns mismatch!\")\n\nexcept FileNotFoundError:\n    print(f\"Error: Sample submission file not found at {SAMPLE_SUB_PATH}\")\nexcept KeyError as e:\n     print(f\"Error processing sample submission: {e}. Please check column names.\")\nexcept Exception as e:\n    print(f\"An unexpected error occurred loading or processing sample submission: {e}\")\n\n# --- Get all actual test files ---\ntry:\n    actual_test_files_npy = sorted([f for f in os.listdir(TEST_PATH) if f.endswith('.npy')])\n    actual_test_oids = sorted([f.split('.')[0] for f in actual_test_files_npy])\n    print(f\"\\nFound {len(actual_test_oids)} actual .npy files in test directory.\")\n    if not actual_test_oids: print(\"Warning: No files found in test directory!\")\n\n    # Compare required oids vs actual oids (already confirmed OK)\n    # if test_oids_from_sub and actual_test_oids:\n    #     required_set = set(test_oids_from_sub)\n    #     actual_set = set(actual_test_oids)\n    #     if required_set == actual_set: print(\"Required test oids match actual files in test folder.\")\n    #     # ... (rest of comparison logic omitted for brevity as it passed)\n    # elif not test_oids_from_sub:\n    #      print(\"Warning: Could not determine required test oids...\")\n    #      test_oids_from_sub = actual_test_oids\n\nexcept FileNotFoundError:\n     print(f\"Error: Test directory not found at {TEST_PATH}\")\n     actual_test_oids = []\n\nprint(\"--- Cell 2: Done ---\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-10T05:53:28.815560Z","iopub.execute_input":"2025-04-10T05:53:28.816276Z","iopub.status.idle":"2025-04-10T05:54:03.560944Z","shell.execute_reply.started":"2025-04-10T05:53:28.816246Z","shell.execute_reply":"2025-04-10T05:54:03.559924Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Loading and Preprocessing Utilities","metadata":{}},{"cell_type":"code","source":"# Cell 3: Data Loading and Preprocessing Utilities (No change needed)\nprint(\"\\n--- Cell 3: Preprocessing Utilities ---\")\n\ndef load_npy(path):\n    \"\"\"Loads a .npy file safely.\"\"\"\n    try:\n        return np.load(path)\n    except Exception as e: return None\n\ndef preprocess_input(seismic_data_raw, target_h=PAD_H, target_w=PAD_W, input_shape=INPUT_SHAPE):\n    \"\"\"Extracts, transposes, normalizes, and pads seismic data.\"\"\"\n    if seismic_data_raw is None: return None\n    if seismic_data_raw.shape != (500, N_SHOTS, TIME_STEPS, N_RECEIVERS): return None\n    seismic_data = seismic_data_raw[0, :, :, :]\n    seismic_data = np.transpose(seismic_data, (1, 2, 0))\n    min_val = np.min(seismic_data); max_val = np.max(seismic_data)\n    if max_val > min_val: seismic_data = (seismic_data - min_val) / (max_val - min_val)\n    else: seismic_data = np.zeros_like(seismic_data)\n    seismic_data_padded = tf.image.resize_with_pad(\n        tf.cast(seismic_data, tf.float32), target_h, target_w, method=tf.image.ResizeMethod.BILINEAR\n    ).numpy()\n    if seismic_data_padded.shape == input_shape: return seismic_data_padded\n    else: return None\n\ndef preprocess_output(velocity_data_raw, output_shape=OUTPUT_SHAPE, v_min=V_MIN, v_max=V_MAX):\n    \"\"\"Extracts and normalizes velocity data.\"\"\"\n    if velocity_data_raw is None: return None\n    if velocity_data_raw.shape != (500, 1, DEPTH, WIDTH): return None\n    velocity_data = velocity_data_raw[0, 0, :, :]\n    velocity_data = np.clip(velocity_data, v_min, v_max)\n    velocity_data = (velocity_data - v_min) / (v_max - v_min)\n    if velocity_data.shape == output_shape: return velocity_data\n    else: return None\n\ndef postprocess_prediction(pred_normalized, output_shape=OUTPUT_SHAPE, v_min=V_MIN, v_max=V_MAX):\n    \"\"\"De-normalizes model output back to original velocity range.\"\"\"\n    if pred_normalized is None: return None\n    pred_velocity = pred_normalized * (v_max - v_min) + v_min\n    if pred_velocity.shape != output_shape:\n        pred_velocity = tf.image.resize(\n            tf.expand_dims(tf.cast(pred_velocity, tf.float32), axis=-1),\n            [output_shape[0], output_shape[1]]\n        ).numpy()\n        pred_velocity = np.squeeze(pred_velocity)\n    return pred_velocity\n\ndef load_train_pairs(train_path):\n    \"\"\"Loads all input/output training file pairs.\"\"\"\n    pairs = []\n    sample_types = os.listdir(train_path)\n    print(f\"Scanning sample types: {sample_types}\")\n    for s_type in tqdm(sample_types, desc=\"Scanning Training Samples\"):\n        s_type_path = os.path.join(train_path, s_type)\n        if os.path.isdir(s_type_path):\n            files_in_sample = os.listdir(s_type_path)\n            # Handle 'seis'/'vel' structure\n            if any(f.startswith('seis') for f in files_in_sample):\n                 seis_files = sorted([f for f in files_in_sample if f.startswith('seis') and f.endswith('.npy')])\n                 for seis_file in seis_files:\n                      vel_file = seis_file.replace('seis', 'vel', 1);\n                      if vel_file in files_in_sample: pairs.append((os.path.join(s_type_path, seis_file), os.path.join(s_type_path, vel_file)))\n            # Handle 'data'/'model' structure\n            elif 'data' in files_in_sample and 'model' in files_in_sample:\n                 data_dir = os.path.join(s_type_path, 'data'); model_dir = os.path.join(s_type_path, 'model')\n                 if os.path.isdir(data_dir) and os.path.isdir(model_dir):\n                     data_files = sorted([f for f in os.listdir(data_dir) if f.endswith('.npy')])\n                     for data_file in data_files:\n                          model_file = data_file.replace('data', 'model', 1); model_file_path = os.path.join(model_dir, model_file)\n                          if os.path.exists(model_file_path): pairs.append((os.path.join(data_dir, data_file), model_file_path))\n    return pairs\n\n# --- Load Pairs ---\ntrain_pairs = load_train_pairs(TRAIN_PATH)\nprint(f\"\\nFound {len(train_pairs)} training pairs.\")\nif train_pairs: print(\"Example pair:\", train_pairs[0])\nelse: print(\"Warning: No training pairs found!\")\n# --- DATASET SIZE WARNING ---\nif len(train_pairs) < 100: # Arbitrary threshold\n     print(f\"\\nWARNING: The training dataset size ({len(train_pairs)}) is very small.\")\n     print(\"         This significantly increases the risk of overfitting and may limit model performance.\")\n     print(\"         Consider data augmentation or acquiring more data if possible.\")\n\nprint(\"--- Cell 3: Done ---\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-10T06:16:42.334061Z","iopub.execute_input":"2025-04-10T06:16:42.334431Z","iopub.status.idle":"2025-04-10T06:18:27.686868Z","shell.execute_reply.started":"2025-04-10T06:16:42.334407Z","shell.execute_reply":"2025-04-10T06:18:27.685778Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Generator","metadata":{}},{"cell_type":"code","source":"# Cell 4: Data Generator (Keras Sequence - Adjusted Validation Batch Size)\nprint(\"\\n--- Cell 4: Data Generator ---\")\n\nclass DataGenerator(keras.utils.Sequence):\n    def __init__(self, pairs, batch_size, input_shape=INPUT_SHAPE, output_shape=OUTPUT_SHAPE, shuffle=True, is_validation=False):\n        self.pairs = pairs\n        self.batch_size = batch_size\n        self.input_shape = input_shape\n        self.output_shape = output_shape\n        self.shuffle = shuffle\n        self.is_validation = is_validation # Flag for potential different handling\n        self.n = len(self.pairs)\n        if self.n == 0:\n             print(\"Warning: DataGenerator created with 0 samples.\")\n        self.on_epoch_end()\n        print(f\"DataGenerator created: {self.n} samples, Batch size: {self.batch_size}, Input: {self.input_shape}, Output: {self.output_shape}\")\n\n    def __len__(self):\n        # Number of batches per epoch\n        if self.n == 0: return 0\n        return int(np.ceil(self.n / self.batch_size)) # Use ceil to ensure all samples are seen\n\n    def __getitem__(self, index):\n        # Generate one batch of data\n        start_idx = index * self.batch_size\n        end_idx = min(start_idx + self.batch_size, self.n) # Handle last batch size\n        actual_batch_size = end_idx - start_idx\n\n        indexes = self.indexes[start_idx:end_idx]\n        batch_pairs = [self.pairs[k] for k in indexes]\n\n        # Initialize arrays for the actual batch size\n        X = np.empty((actual_batch_size, *self.input_shape))\n        y = np.empty((actual_batch_size, *self.output_shape))\n\n        valid_samples_in_batch = 0\n        for i, (seis_path, vel_path) in enumerate(batch_pairs):\n            seis_data_raw = load_npy(seis_path)\n            vel_data_raw = load_npy(vel_path)\n            processed_seis = preprocess_input(seis_data_raw)\n            processed_vel = preprocess_output(vel_data_raw)\n\n            if processed_seis is not None and processed_vel is not None:\n                X[i,] = processed_seis; y[i,] = processed_vel\n                valid_samples_in_batch += 1\n            else:\n                X[i,] = np.zeros(self.input_shape); y[i,] = np.zeros(self.output_shape)\n        del seis_data_raw, vel_data_raw, processed_seis, processed_vel\n        gc.collect()\n        return X, y\n\n    def on_epoch_end(self):\n        # Updates indexes after each epoch\n        self.indexes = np.arange(self.n)\n        if self.shuffle: np.random.shuffle(self.indexes)\n\n# --- Create Train/Validation Generators ---\ntrain_generator = None\nval_generator = None\nnum_train_samples = 0\nnum_val_samples = 0\n\nif train_pairs:\n    np.random.seed(42)\n    np.random.shuffle(train_pairs)\n    split_idx = int((1.0 - VALIDATION_SPLIT) * len(train_pairs))\n    train_gen_pairs = train_pairs[:split_idx]\n    val_gen_pairs = train_pairs[split_idx:]\n    num_train_samples = len(train_gen_pairs)\n    num_val_samples = len(val_gen_pairs)\n\n    print(f\"\\nSplitting data: {num_train_samples} training pairs, {num_val_samples} validation pairs.\")\n\n    if num_train_samples > 0:\n         train_generator = DataGenerator(train_gen_pairs, BATCH_SIZE_TRAIN, shuffle=True)\n         if len(train_generator) == 0:\n              print(\"Warning: Training generator has length 0.\")\n    else:\n         print(\"Warning: No samples available for training generator.\")\n\n\n    if num_val_samples > 0:\n         # --- Adjust Validation Batch Size ---\n         # Ensure batch size is <= number of validation samples to avoid steps=0\n         BATCH_SIZE_VAL = min(num_val_samples, BATCH_SIZE_TRAIN)\n         print(f\"Using validation batch size: {BATCH_SIZE_VAL}\")\n         val_generator = DataGenerator(val_gen_pairs, BATCH_SIZE_VAL, shuffle=False, is_validation=True)\n         if len(val_generator) == 0:\n              print(\"Warning: Validation generator has length 0, even after batch size adjustment.\")\n    else:\n         print(\"Warning: No samples available for validation generator. Validation will be skipped.\")\n\n    # --- Test and Visualize Generator Output (if possible) ---\n    if train_generator and len(train_generator) > 0:\n        print(\"\\nTesting generator...\")\n        try:\n            X_batch, y_batch = train_generator[0]\n            print(f\"Generator test SUCCESS: Batch X shape: {X_batch.shape}, Batch y shape: {y_batch.shape}\")\n            # Visualization code... (omitted for brevity, already confirmed working)\n            # plt.figure... show()... close()\n            del X_batch, y_batch; gc.collect()\n        except Exception as e:\n            print(f\"Error testing generator: {e}\")\n    else:\n         print(\"Skipping generator test as train_generator is empty.\")\n\nelse:\n    print(\"Cannot create generators: No training pairs found.\")\n\nprint(\"--- Cell 4: Done ---\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-10T06:30:47.082002Z","iopub.execute_input":"2025-04-10T06:30:47.084479Z","iopub.status.idle":"2025-04-10T06:31:45.401899Z","shell.execute_reply.started":"2025-04-10T06:30:47.084392Z","shell.execute_reply":"2025-04-10T06:31:45.400606Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Definition","metadata":{}},{"cell_type":"code","source":"# Cell 5: Model Definition (2D U-Net) (No change needed)\nprint(\"\\n--- Cell 5: Model Definition ---\")\n\ndef conv_block(input_tensor, num_filters, kernel_size=(3, 3)):\n    x = layers.Conv2D(num_filters, kernel_size, padding=\"same\")(input_tensor); x = layers.BatchNormalization()(x); x = layers.Activation(\"relu\")(x)\n    x = layers.Conv2D(num_filters, kernel_size, padding=\"same\")(x); x = layers.BatchNormalization()(x); x = layers.Activation(\"relu\")(x)\n    return x\ndef encoder_block(input_tensor, num_filters):\n    x = conv_block(input_tensor, num_filters); p = layers.MaxPooling2D((2, 2))(x); return x, p\ndef decoder_block(input_tensor, skip_tensor, num_filters):\n    x = layers.Conv2DTranspose(num_filters, (2, 2), strides=2, padding=\"same\")(input_tensor)\n    x = layers.Concatenate()([x, skip_tensor]); x = conv_block(x, num_filters); return x\n\ndef build_unet(input_shape=INPUT_SHAPE, output_shape=OUTPUT_SHAPE):\n    print(f\"Building U-Net -- Input: {input_shape}, Target Output: {output_shape}\")\n    inputs = keras.Input(shape=input_shape)\n    s1, p1 = encoder_block(inputs, 64); s2, p2 = encoder_block(p1, 128); s3, p3 = encoder_block(p2, 256); s4, p4 = encoder_block(p3, 512)\n    b1 = conv_block(p4, 1024)\n    d1 = decoder_block(b1, s4, 512); d2 = decoder_block(d1, s3, 256); d3 = decoder_block(d2, s2, 128); d4 = decoder_block(d3, s1, 64)\n    outputs_padded = layers.Conv2D(1, (1, 1), padding=\"same\", activation=\"sigmoid\")(d4)\n    def resize_layer(x, target_shape=output_shape): return tf.image.resize(x, [target_shape[0], target_shape[1]], method=tf.image.ResizeMethod.BILINEAR)\n    outputs_resized = layers.Lambda(resize_layer, name='resize_to_output')(outputs_padded)\n    final_outputs = layers.Reshape(output_shape, name='final_reshape')(outputs_resized)\n    model = keras.Model(inputs=inputs, outputs=final_outputs, name=\"UNet_FWI_Padded_v3\") # Incremented version name\n    return model\n\n# --- Build and Compile ---\ntf.keras.backend.clear_session(); gc.collect()\nmodel = build_unet()\noptimizer = tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE)\nmodel.compile(optimizer=optimizer, loss='mae', metrics=['mae'])\nprint(\"\\nModel Summary:\")\nmodel.summary(line_length=120)\n\nprint(\"--- Cell 5: Done ---\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-10T06:34:49.654665Z","iopub.execute_input":"2025-04-10T06:34:49.655076Z","iopub.status.idle":"2025-04-10T06:34:50.987493Z","shell.execute_reply.started":"2025-04-10T06:34:49.655046Z","shell.execute_reply":"2025-04-10T06:34:50.986136Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Training","metadata":{}},{"cell_type":"code","source":"# Cell 6: Model Training (Adjusted Validation Handling)\nprint(\"\\n--- Cell 6: Model Training ---\")\n\n# Check if generators are valid before attempting training\nif train_generator is not None and len(train_generator) > 0:\n    print(f\"Starting training for up to {EPOCHS} epochs...\")\n    print(f\"Training samples: {num_train_samples}, Validation samples: {num_val_samples}\")\n    start_time = time.time()\n\n    # --- Callbacks ---\n    callbacks = []\n    # Save the best model\n    # Note: If val_generator is None or empty, monitor='loss' (training loss) instead.\n    monitor_metric = 'val_mae' if val_generator and len(val_generator) > 0 else 'mae'\n    print(f\"ModelCheckpoint and EarlyStopping will monitor: '{monitor_metric}'\")\n\n    model_checkpoint = keras.callbacks.ModelCheckpoint(\n        MODEL_WEIGHTS_FILE, monitor=monitor_metric, save_best_only=True,\n        save_weights_only=False, mode='min', verbose=1\n    )\n    callbacks.append(model_checkpoint)\n\n    # Stop training early if no improvement\n    early_stopping = keras.callbacks.EarlyStopping(\n        monitor=monitor_metric, patience=10, # Increased patience slightly more\n        restore_best_weights=True, mode='min', verbose=1\n    )\n    callbacks.append(early_stopping)\n\n    # Reduce learning rate on plateau\n    reduce_lr = keras.callbacks.ReduceLROnPlateau(\n        monitor=monitor_metric, factor=0.2, patience=4, # Adjusted patience\n        min_lr=1e-7, mode='min', verbose=1\n    )\n    callbacks.append(reduce_lr)\n\n    # --- Determine Validation Data Argument ---\n    validation_args = {}\n    if val_generator is not None and len(val_generator) > 0:\n         validation_args['validation_data'] = val_generator\n    else:\n         print(\"Validation generator is empty or None. Training without validation.\")\n\n    # --- Start Training ---\n    history = model.fit(\n        train_generator,\n        epochs=EPOCHS,\n        callbacks=callbacks,\n        verbose=1,\n        **validation_args # Pass validation_data only if available\n    )\n\n    # --- Post-Training ---\n    end_time = time.time(); training_time = end_time - start_time\n    print(f\"\\nTraining finished in {training_time / 60:.2f} minutes.\")\n\n    # --- Plot Training History ---\n    if history and history.history:\n        print(\"\\nPlotting training history...\")\n        try:\n            plt.figure(figsize=(12, 5))\n            # Plot MAE/Loss\n            plt.plot(history.history['mae'], label='Training MAE')\n            if 'val_mae' in history.history:\n                 plt.plot(history.history['val_mae'], label='Validation MAE')\n                 best_epoch = np.argmin(history.history['val_mae'])\n                 best_val_mae = np.min(history.history['val_mae'])\n                 print(f\"Best validation MAE = {best_val_mae:.5f} at epoch {best_epoch + 1}\")\n            else:\n                 best_epoch = np.argmin(history.history['mae'])\n                 best_train_mae = np.min(history.history['mae'])\n                 print(f\"Best training MAE = {best_train_mae:.5f} at epoch {best_epoch + 1} (No validation data)\")\n\n            plt.title('Mean Absolute Error (MAE)')\n            plt.xlabel('Epoch'); plt.ylabel('MAE'); plt.legend(); plt.grid(True)\n            plt.tight_layout(); plt.show(); plt.close()\n        except Exception as plot_err: print(f\"Error plotting history: {plot_err}\")\n    else: print(\"No training history available to plot.\")\n\n    # Best weights are restored by EarlyStopping if restore_best_weights=True\n\nelse:\n    print(\"Skipping training: Training generator is empty or could not be created.\")\n\ngc.collect()\nprint(\"--- Cell 6: Done ---\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Prediction on Test Set and Submission Generation","metadata":{}},{"cell_type":"code","source":"# Cell 7: Prediction on Test Set and Submission Generation (No change needed)\nprint(\"\\n--- Cell 7: Prediction and Submission ---\")\n\n# --- Load the Best Trained Model ---\nmodel_loaded = False\nif os.path.exists(MODEL_WEIGHTS_FILE):\n     print(f\"Loading best model from {MODEL_WEIGHTS_FILE}...\")\n     try:\n         # Load the entire model (including architecture and optimizer state)\n         model = keras.models.load_model(MODEL_WEIGHTS_FILE)\n         print(\"Model loaded successfully.\")\n         model_loaded = True\n     except Exception as e:\n         print(f\"Error loading model: {e}. Cannot generate predictions.\")\n         # If model wasn't loaded from training cell, set it to None\n         if 'model' not in locals() or model is None: model = None\nelif 'model' in locals() and model is not None:\n     print(\"Using model from the training cell (best weights should be restored by EarlyStopping).\")\n     model_loaded = True\nelse:\n     print(\"Error: No trained model available (neither in memory nor saved file).\")\n     model = None\n\nif model_loaded and actual_test_oids:\n    print(f\"Generating predictions for {len(actual_test_oids)} actual test files...\")\n    all_preds_dict = {}\n    start_pred_time = time.time()\n\n    for oid in tqdm(actual_test_oids, desc=\"Predicting Test Set\"):\n        test_file_path = os.path.join(TEST_PATH, f\"{oid}.npy\")\n        pred_velocity = None # Ensure variable is defined\n        try:\n            test_data_raw = load_npy(test_file_path)\n            preprocessed_test = preprocess_input(test_data_raw)\n            if preprocessed_test is not None:\n                pred_normalized = model.predict(np.expand_dims(preprocessed_test, axis=0), verbose=0)[0]\n                pred_velocity = postprocess_prediction(pred_normalized)\n            del test_data_raw, preprocessed_test, pred_normalized # Memory clean\n        except Exception as e:\n            print(f\"Error during prediction for {oid}: {e}\")\n            pred_velocity = None # Ensure it's None on error\n\n        # Store result (or None if failed)\n        all_preds_dict[oid] = pred_velocity.astype(np.float32) if pred_velocity is not None and pred_velocity.shape == OUTPUT_SHAPE else None\n        gc.collect() # Collect garbage more frequently during prediction\n\n    end_pred_time = time.time()\n    print(f\"Prediction loop finished in {(end_pred_time - start_pred_time) / 60:.2f} minutes.\")\n\n    # --- Format predictions for submission ---\n    print(f\"\\nFormatting predictions for the {len(test_oids_from_sub)} required submission oids...\")\n    submission_rows = []; default_value = 3000.0\n\n    for oid in tqdm(test_oids_from_sub, desc=\"Formatting Submission\"):\n        velocity_map = all_preds_dict.get(oid, None)\n        if velocity_map is not None:\n            selected_velocity = velocity_map[:, SUBMISSION_X_INDICES]\n            for y_pos in range(DEPTH):\n                submission_rows.append([f\"{oid}_y_{y_pos}\"] + selected_velocity[y_pos, :].tolist())\n        else: # Handle missing/failed predictions\n            # print(f\"Warning: Using default value for required oid {oid}.\") # Reduce verbosity\n            num_x_cols = len(SUBMISSION_X_INDICES)\n            for y_pos in range(DEPTH):\n                submission_rows.append([f\"{oid}_y_{y_pos}\"] + [default_value] * num_x_cols)\n\n    # --- Create and Save Submission DataFrame ---\n    if submission_rows:\n        submission_df = pd.DataFrame(submission_rows, columns=[SUBMISSION_ID_COL] + SUBMISSION_X_COLS)\n        print(\"\\nFinal Submission DataFrame Info:\")\n        submission_df.info() # Check size and types\n        # print(\"\\nSubmission DataFrame Head:\") # Reduce verbosity\n        # print(submission_df.head())\n        print(f\"\\nExpected submission rows: {expected_submission_rows}\")\n        print(f\"Generated submission rows: {len(submission_df)}\")\n        if len(submission_df) == expected_submission_rows: print(\"Row count matches expected. OK.\")\n        else: print(f\"CRITICAL WARNING: Row count ({len(submission_df)}) != expected ({expected_submission_rows}).\")\n        try:\n            submission_df.to_csv(SUBMISSION_FILE, index=False)\n            print(f\"\\nSubmission file saved successfully to: {SUBMISSION_FILE}\")\n        except Exception as e: print(f\"Error saving submission file: {e}\")\n    else: print(\"Error: No submission rows generated.\")\n    del all_preds_dict; gc.collect()\n\nelif not model_loaded: print(\"Skipping prediction: Model not available.\")\nelif not actual_test_oids: print(\"Skipping prediction: No actual test files found.\")\n\nprint(\"--- Cell 7: Done ---\")\nprint(\"\\n--- End of Script ---\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}