{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":98450,"databundleVersionId":11749951,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\n# Load the CSV files\ntrain_df = pd.read_csv(\"/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-2025/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-2025/test.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-16T18:19:39.106758Z","iopub.execute_input":"2025-05-16T18:19:39.107016Z","iopub.status.idle":"2025-05-16T18:19:41.248646Z","shell.execute_reply.started":"2025-05-16T18:19:39.106996Z","shell.execute_reply":"2025-05-16T18:19:41.247777Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Display dataset info\nprint(\"Train shape :\", train_df.shape)\nprint(\"Test shape : \",test_df.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-16T18:19:41.250397Z","iopub.execute_input":"2025-05-16T18:19:41.250754Z","iopub.status.idle":"2025-05-16T18:19:41.255568Z","shell.execute_reply.started":"2025-05-16T18:19:41.250721Z","shell.execute_reply":"2025-05-16T18:19:41.254607Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-16T18:19:41.256527Z","iopub.execute_input":"2025-05-16T18:19:41.256847Z","iopub.status.idle":"2025-05-16T18:19:41.318358Z","shell.execute_reply.started":"2025-05-16T18:19:41.256822Z","shell.execute_reply":"2025-05-16T18:19:41.317407Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df['label'].hist(bins=100)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-16T18:19:41.319308Z","iopub.execute_input":"2025-05-16T18:19:41.319743Z","iopub.status.idle":"2025-05-16T18:19:41.918446Z","shell.execute_reply.started":"2025-05-16T18:19:41.319705Z","shell.execute_reply":"2025-05-16T18:19:41.917616Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport os\n\nnpy_dir = \"/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-2025/ot/ot\"\n\nsample_id = train_df['id'].iloc[0]\nsample_path = os.path.join(npy_dir,sample_id)\n\ndata = np.load(sample_path)\nprint(\"Shape of the sample is : \",data.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-16T18:19:41.920260Z","iopub.execute_input":"2025-05-16T18:19:41.920512Z","iopub.status.idle":"2025-05-16T18:19:41.957638Z","shell.execute_reply.started":"2025-05-16T18:19:41.920490Z","shell.execute_reply":"2025-05-16T18:19:41.956773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# View a few bands\nplt.figure(figsize=(12, 4))\nfor i, band in enumerate([0, 10, 30, 50, 70, 90]):\n    plt.subplot(1, 6, i+1)\n    plt.imshow(data[:, :, band], cmap='viridis')\n    plt.title(f'Band {band}')\n    plt.axis('off')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-16T18:19:41.958609Z","iopub.execute_input":"2025-05-16T18:19:41.958860Z","iopub.status.idle":"2025-05-16T18:19:42.768594Z","shell.execute_reply.started":"2025-05-16T18:19:41.958832Z","shell.execute_reply":"2025-05-16T18:19:42.767429Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Final code:","metadata":{}},{"cell_type":"markdown","source":"### Below is the full compiled code for XGBoost model along with parameters that gave best results.","metadata":{}},{"cell_type":"markdown","source":"#### **RUN THE BELOW CODE FOR FINAL SUBMISSION FILE . MAKE SURE GPU IS ON (GPU IS NEEDED FOR ACCELERATION)**\n<BR>\nIt takes some time to run the cell","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_absolute_error\nimport xgboost as xgb\n\n# --- Configuration ---\nBASE_DIR    = \"/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-2025\"\nNPY_DIR     = os.path.join(BASE_DIR, \"ot/ot\")\nTRAIN_CSV   = os.path.join(BASE_DIR, \"train.csv\")\nTEST_CSV    = os.path.join(BASE_DIR, \"test.csv\")\nSUBMISSION  = \"submission_xgb2.csv\"\n\nIMG_SHAPE   = (128, 128, 125)\nTARGET_SIZE = np.prod(IMG_SHAPE)\n\n\ndef load_and_flatten(path):\n    \"\"\"\n    Load a .npy file (or raw .npy if header corrupt), flatten to 1D,\n    pad by repeating last value if too short, or truncate if too long.\n    \"\"\"\n    try:\n        arr = np.load(path)\n        flat = arr.ravel()\n    except Exception:\n        flat = np.fromfile(path, dtype=np.float32)\n\n    # pad/truncate to TARGET_SIZE\n    if flat.size < TARGET_SIZE:\n        if flat.size == 0:\n            flat = np.zeros(TARGET_SIZE, dtype=np.float32)\n        else:\n            pad_vals = np.full(TARGET_SIZE - flat.size, flat[-1], dtype=np.float32)\n            flat = np.concatenate([flat, pad_vals])\n    else:\n        flat = flat[:TARGET_SIZE]\n\n    return flat\n\n\ndef extract_features(df):\n    \"\"\"\n    For each row in df (with 'id'), load the patch, fix shape, and compute\n    mean reflectance for each of the 125 bands.\n    Returns an (n_samples, 125) array.\n    \"\"\"\n    features = []\n    for fn in df['id']:\n        path = os.path.join(NPY_DIR, fn)\n        flat = load_and_flatten(path)\n        # reshape and compute band means\n        cube = flat.reshape(IMG_SHAPE)\n        band_means = cube.mean(axis=(0, 1))\n        features.append(band_means)\n    return np.vstack(features)\n\n\n# --- 1) Load CSVs ---\ntrain_df = pd.read_csv(TRAIN_CSV)\ntest_df  = pd.read_csv(TEST_CSV)\n\n# --- 2) Build feature matrices ---\nprint(\"Extracting features for training set...\")\nX = extract_features(train_df)    # shape (n_train, 125)\ny = train_df['label'].values\n\nprint(\"Extracting features for test set...\")\nX_test = extract_features(test_df) # shape (n_test, 125)\n\n# --- 3) Train‐validation split ---\nX_train, X_val, y_train, y_val = train_test_split(\n    X, y, test_size=0.1, random_state=42\n)\n\n# --- 4) XGBoost Regressor setup ---\nxgb_model = xgb.XGBRegressor(\n    n_estimators=800,\n    learning_rate=0.08,\n    max_depth=8,\n    subsample=0.8,\n    colsample_bytree=0.8,\n    random_state=42,\n    tree_method='gpu_hist'  # or 'hist' if no GPU\n)\n\n# --- 5) Train with early stopping ---\nxgb_model.fit(\n    X_train, y_train,\n    eval_set=[(X_train, y_train), (X_val, y_val)],\n    eval_metric='mae',\n    early_stopping_rounds=20,\n    verbose=True\n)\n\n# --- 6) Validation performance ---\ny_pred_val = xgb_model.predict(X_val)\nval_mae = mean_absolute_error(y_val, y_pred_val)\nprint(f\"Validation MAE: {val_mae:.4f}\")\n\n# --- 7) Predict on test set & save submission ---\ny_pred_test = xgb_model.predict(X_test)\ny_pred_test = np.clip(np.round(y_pred_test), 1, 100).astype(int)\n\nsubmission_df = pd.DataFrame({\n    \"id\": test_df[\"id\"],\n    \"label\": y_pred_test\n})\nsubmission_df.to_csv(SUBMISSION, index=False)\nprint(f\"✅ Submission saved to {SUBMISSION}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-21T09:55:29.520887Z","iopub.execute_input":"2025-05-21T09:55:29.521106Z","iopub.status.idle":"2025-05-21T09:57:17.308733Z","shell.execute_reply.started":"2025-05-21T09:55:29.521084Z","shell.execute_reply":"2025-05-21T09:57:17.307998Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"# Other approaches I experimented with","metadata":{}},{"cell_type":"markdown","source":"## 1.Transformer + ANN","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-16T18:19:42.769271Z","iopub.execute_input":"2025-05-16T18:19:42.769693Z","iopub.status.idle":"2025-05-16T18:19:57.765532Z","shell.execute_reply.started":"2025-05-16T18:19:42.769666Z","shell.execute_reply":"2025-05-16T18:19:57.764591Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Configuration ---\nBASE_DIR = \"/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-2025\"\nNPY_DIR = os.path.join(BASE_DIR, \"ot/ot\")\nTRAIN_CSV = os.path.join(BASE_DIR, \"train.csv\")\nTEST_CSV = os.path.join(BASE_DIR, \"test.csv\")\nSUBMISSION_CSV = \"submission.csv\"\n\nIMG_SHAPE = (128, 128, 125)\nBATCH_SIZE = 16\nEPOCHS = 20\nAUTOTUNE = tf.data.AUTOTUNE","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-16T18:19:57.766444Z","iopub.execute_input":"2025-05-16T18:19:57.767258Z","iopub.status.idle":"2025-05-16T18:19:57.772905Z","shell.execute_reply.started":"2025-05-16T18:19:57.767231Z","shell.execute_reply":"2025-05-16T18:19:57.771961Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Utility: pad or crop to target shape ---\ndef fix_shape(arr):\n    target = IMG_SHAPE\n    fixed = np.zeros(target, dtype=np.float32)\n    # compute minimal overlap\n    mins = np.minimum(arr.shape, target)\n    fixed[:mins[0], :mins[1], :mins[2]] = arr[:mins[0], :mins[1], :mins[2]]\n    return fixed\n\n\n# --- Data loading functions for tf.data ---\ndef _load_npy(path):\n    arr = np.load(path.decode())\n    arr = fix_shape(arr)\n    arr = arr.astype(np.float32) / 255.0\n    # add channel\n    return arr[..., np.newaxis]\n\n\ndef parse_train(path, label):\n    x = _load_npy(path)\n    return x, label\n\n\ndef parse_test(path):\n    x = _load_npy(path)\n    return x\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-16T18:19:57.774001Z","iopub.execute_input":"2025-05-16T18:19:57.774584Z","iopub.status.idle":"2025-05-16T18:19:57.816680Z","shell.execute_reply.started":"2025-05-16T18:19:57.774528Z","shell.execute_reply":"2025-05-16T18:19:57.815774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Build tf.data pipelines ---\n# Load metadata\ntrain_df = pd.read_csv(TRAIN_CSV)\ntest_df = pd.read_csv(TEST_CSV)\n\n# File paths\ntrain_paths = train_df['id'].apply(lambda x: os.path.join(NPY_DIR, x)).values\ntrain_labels = train_df['label'].values.astype(np.float32)\n\ntest_paths = test_df['id'].apply(lambda x: os.path.join(NPY_DIR, x)).values\n\n# Training dataset\ntrain_ds = tf.data.Dataset.from_tensor_slices((train_paths, train_labels))\ntrain_ds = (train_ds\n            .shuffle(len(train_paths))\n            .map(lambda p, y: tf.py_function(parse_train, [p, y], [tf.float32, tf.float32]), num_parallel_calls=AUTOTUNE)\n            .map(lambda x, y: (tf.ensure_shape(x, (*IMG_SHAPE, 1)), tf.ensure_shape(y, [])))\n            .batch(BATCH_SIZE)\n            .prefetch(AUTOTUNE))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-16T18:19:57.821384Z","iopub.execute_input":"2025-05-16T18:19:57.821668Z","iopub.status.idle":"2025-05-16T18:19:59.032271Z","shell.execute_reply.started":"2025-05-16T18:19:57.821648Z","shell.execute_reply":"2025-05-16T18:19:59.031360Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Validation split\nval_size = int(0.1 * len(train_paths))\nval_ds = train_ds.take(val_size)\ntrain_ds = train_ds.skip(val_size)\n\n# Test dataset\ntest_ds = tf.data.Dataset.from_tensor_slices(test_paths)\ntest_ds = (test_ds\n           .map(lambda p: tf.py_function(parse_test, [p], tf.float32), num_parallel_calls=AUTOTUNE)\n           .map(lambda x: tf.ensure_shape(x, (*IMG_SHAPE, 1)))\n           .batch(BATCH_SIZE)\n           .prefetch(AUTOTUNE))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-16T18:20:10.263479Z","iopub.execute_input":"2025-05-16T18:20:10.264136Z","iopub.status.idle":"2025-05-16T18:20:10.320847Z","shell.execute_reply.started":"2025-05-16T18:20:10.264108Z","shell.execute_reply":"2025-05-16T18:20:10.320216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Transformer Model Definition ---\ndef transformer_encoder(inputs, head_size, num_heads, ff_dim, dropout=0.1):\n    # Multi-head Self-Attention\n    x = tf.keras.layers.LayerNormalization(epsilon=1e-6)(inputs)\n    x = tf.keras.layers.MultiHeadAttention(key_dim=head_size, num_heads=num_heads, dropout=dropout)(x, x)\n    x = tf.keras.layers.Add()([x, inputs])\n    # Feed-forward\n    y = tf.keras.layers.LayerNormalization(epsilon=1e-6)(x)\n    y = tf.keras.layers.Dense(ff_dim, activation=\"relu\")(y)\n    y = tf.keras.layers.Dense(inputs.shape[-1])(y)\n    return tf.keras.layers.Add()([y, x])\n\n\ndef build_transformer_model(input_shape):\n    inputs = tf.keras.Input(shape=input_shape)\n    # flatten spatial dims into sequence\n    seq = tf.keras.layers.Reshape((input_shape[0] * input_shape[1], input_shape[2]))(inputs)\n    # transformer blocks\n    x = transformer_encoder(seq, head_size=32, num_heads=4, ff_dim=128)\n    x = transformer_encoder(x, head_size=32, num_heads=4, ff_dim=128)\n    x = tf.keras.layers.GlobalAveragePooling1D()(x)\n    # regression head\n    x = tf.keras.layers.Dense(256, activation=\"relu\")(x)\n    x = tf.keras.layers.Dropout(0.3)(x)\n    outputs = tf.keras.layers.Dense(1)(x)\n    return tf.keras.Model(inputs, outputs)\n\n\nmodel = build_transformer_model((*IMG_SHAPE, 1))\nmodel.compile(optimizer=\"adam\", loss=\"mse\", metrics=[\"mae\"])\nmodel.summary()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-16T18:20:19.977038Z","iopub.execute_input":"2025-05-16T18:20:19.977303Z","iopub.status.idle":"2025-05-16T18:20:21.592933Z","shell.execute_reply.started":"2025-05-16T18:20:19.977285Z","shell.execute_reply":"2025-05-16T18:20:21.592317Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Train ---\nhistory = model.fit(train_ds, validation_data=val_ds, epochs=EPOCHS)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# WORKING GAVE GOOD RESULTS\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras import layers, Model, optimizers\nfrom sklearn.model_selection import train_test_split\n\n# --- Configuration ---\nBASE_DIR    = \"/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-2025\"\nNPY_DIR     = os.path.join(BASE_DIR, \"ot/ot\")\nTRAIN_CSV   = os.path.join(BASE_DIR, \"train.csv\")\nTEST_CSV    = os.path.join(BASE_DIR, \"test.csv\")\nSUBMISSION  = \"submission.csv\"\n\nIMG_SHAPE   = (128, 128, 125)\nTARGET_SIZE = np.prod(IMG_SHAPE)\nBATCH_SIZE  = 4    # adjust to your GPU memory\nEPOCHS      = 20\nAUTOTUNE    = tf.data.AUTOTUNE\n\n\n# --- 1) Robust loader + NaN handling + per-sample min-max ---\ndef load_and_process(path_bytes):\n    path = path_bytes.numpy().decode()\n    # try np.load; else fallback to raw read\n    try:\n        arr = np.load(path)\n    except Exception:\n        arr = np.fromfile(path, dtype=np.float32)\n    else:\n        arr = arr.ravel()\n    # pad / truncate\n    if arr.size < TARGET_SIZE:\n        arr = np.pad(arr, (0, TARGET_SIZE - arr.size), mode=\"constant\")\n    else:\n        arr = arr[:TARGET_SIZE]\n    # reshape\n    cube = arr.reshape(IMG_SHAPE).astype(np.float32)\n    # replace NaNs & infs\n    cube = np.nan_to_num(cube, nan=0.0, posinf=0.0, neginf=0.0)\n    # per-sample min-max\n    mn, mx = cube.min(), cube.max()\n    cube = (cube - mn) / ( (mx - mn) + 1e-6 )\n    # add channel axis\n    return cube[..., np.newaxis]\n\n\ndef tf_parse_train(path, label):\n    x = tf.py_function(load_and_process, [path], tf.float32)\n    x.set_shape((*IMG_SHAPE, 1))\n    return x, tf.cast(label, tf.float32)\n\n\ndef tf_parse_test(path):\n    x = tf.py_function(load_and_process, [path], tf.float32)\n    x.set_shape((*IMG_SHAPE, 1))\n    return x\n\n\n# --- 2) Prepare tf.data pipelines ---\ntrain_df = pd.read_csv(TRAIN_CSV)\ntest_df  = pd.read_csv(TEST_CSV)\n\ntrain_paths  = train_df[\"id\"].apply(lambda f: os.path.join(NPY_DIR, f)).values\ntrain_labels = train_df[\"label\"].values.astype(np.float32)\ntest_paths   = test_df[\"id\"].apply(lambda f: os.path.join(NPY_DIR, f)).values\n\n# full dataset\nfull_ds = tf.data.Dataset.from_tensor_slices((train_paths, train_labels))\nfull_ds = full_ds.shuffle(len(train_paths), reshuffle_each_iteration=True)\nfull_ds = full_ds.map(tf_parse_train, num_parallel_calls=AUTOTUNE)\n\n# split 10% for validation\nval_count = int(0.1 * len(train_paths))\nval_ds   = full_ds.take(val_count).batch(BATCH_SIZE).prefetch(AUTOTUNE)\ntrain_ds = full_ds.skip(val_count).batch(BATCH_SIZE).prefetch(AUTOTUNE)\n\ntest_ds = (\n    tf.data.Dataset.from_tensor_slices(test_paths)\n      .map(tf_parse_test, num_parallel_calls=AUTOTUNE)\n      .batch(BATCH_SIZE)\n      .prefetch(AUTOTUNE)\n)\n\n\n# --- 3) Model: 3D-Conv ↓ → Transformer → Regression head ---\ndef transformer_block(x, head_size, num_heads, ff_dim, dropout=0.1):\n    attn = layers.MultiHeadAttention(key_dim=head_size,\n                                     num_heads=num_heads,\n                                     dropout=dropout)(x, x)\n    x = layers.Add()([x, attn])\n    x = layers.LayerNormalization()(x)\n    ff = layers.Dense(ff_dim, activation=\"relu\")(x)\n    ff = layers.Dense(x.shape[-1])(ff)\n    return layers.Add()([x, ff])\n\ndef build_model():\n    inp = layers.Input((*IMG_SHAPE, 1))\n    # downsample via 3D conv\n    x = layers.Conv3D(16, 3, strides=2, padding=\"same\", activation=\"relu\")(inp)  # 64×64×63×16\n    x = layers.Conv3D(32, 3, strides=2, padding=\"same\", activation=\"relu\")(x)    # 32×32×32×32\n    # flatten spatial dims into sequence\n    b,h,w,d,c = x.shape\n    x = layers.Reshape((h*w, d*c))(x)\n    # two transformer blocks\n    x = transformer_block(x, head_size=16, num_heads=2, ff_dim=64)\n    x = transformer_block(x, head_size=16, num_heads=2, ff_dim=64)\n    # regression head\n    x = layers.GlobalAveragePooling1D()(x)\n    x = layers.Dense(128, activation=\"relu\")(x)\n    x = layers.Dropout(0.3)(x)\n    out = layers.Dense(1, activation=\"linear\")(x)\n    return Model(inp, out)\n\nmodel = build_model()\nopt = optimizers.Adam(learning_rate=1e-4, clipnorm=1.0)\nmodel.compile(optimizer=opt, loss=\"mse\", metrics=[\"mae\"])\nmodel.summary()\n\n\n# --- 4) Train without NaNs ---\nmodel.fit(train_ds, validation_data=val_ds, epochs=EPOCHS)\n\n\n# --- 5) Predict & Save Submission ---\npreds = model.predict(test_ds).flatten()\npreds = np.clip(np.round(preds), 1, 100).astype(int)\n\npd.DataFrame({\n    \"ID\": test_df[\"id\"],\n    \"label\": preds\n}).to_csv(SUBMISSION, index=False)\n\nprint(\"✅ Done — submission.csv created.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-16T18:31:19.060616Z","iopub.execute_input":"2025-05-16T18:31:19.060916Z","iopub.status.idle":"2025-05-16T18:49:16.018182Z","shell.execute_reply.started":"2025-05-16T18:31:19.060890Z","shell.execute_reply":"2025-05-16T18:49:16.017368Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.Conv3D + ANN","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras import layers, Model, optimizers\nfrom sklearn.model_selection import train_test_split\n\n# --- Configuration ---\nBASE_DIR    = \"/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-2025\"\nNPY_DIR     = os.path.join(BASE_DIR, \"ot/ot\")\nTRAIN_CSV   = os.path.join(BASE_DIR, \"train.csv\")\nTEST_CSV    = os.path.join(BASE_DIR, \"test.csv\")\nSUBMISSION  = \"submission.csv\"\n\nIMG_SHAPE   = (128, 128, 125)\nTARGET_SIZE = np.prod(IMG_SHAPE)\nBATCH_SIZE  = 4\nEPOCHS      = 20\nAUTOTUNE    = tf.data.AUTOTUNE\n\n\n# --- 1) Robust loader + NaN handling + per-sample min-max ---\ndef load_and_process(path_bytes):\n    path = path_bytes.numpy().decode()\n    try:\n        arr = np.load(path)\n    except Exception:\n        arr = np.fromfile(path, dtype=np.float32)\n    else:\n        arr = arr.ravel()\n    if arr.size < TARGET_SIZE:\n        arr = np.pad(arr, (0, TARGET_SIZE - arr.size), mode=\"constant\")\n    else:\n        arr = arr[:TARGET_SIZE]\n    cube = arr.reshape(IMG_SHAPE).astype(np.float32)\n    cube = np.nan_to_num(cube, nan=0.0, posinf=0.0, neginf=0.0)\n    mn, mx = cube.min(), cube.max()\n    cube = (cube - mn) / ((mx - mn) + 1e-6)\n    return cube[..., np.newaxis]\n\n\ndef tf_parse_train(path, label):\n    x = tf.py_function(load_and_process, [path], tf.float32)\n    x.set_shape((*IMG_SHAPE, 1))\n    return x, tf.cast(label, tf.float32)\n\n\ndef tf_parse_test(path):\n    x = tf.py_function(load_and_process, [path], tf.float32)\n    x.set_shape((*IMG_SHAPE, 1))\n    return x\n\n\n# --- Data pipelines ---\ntrain_df = pd.read_csv(TRAIN_CSV)\ntest_df  = pd.read_csv(TEST_CSV)\n\ntrain_paths  = train_df[\"id\"].apply(lambda f: os.path.join(NPY_DIR, f)).values\ntrain_labels = train_df[\"label\"].values.astype(np.float32)\ntest_paths   = test_df[\"id\"].apply(lambda f: os.path.join(NPY_DIR, f)).values\n\nfull_ds = tf.data.Dataset.from_tensor_slices((train_paths, train_labels))\nfull_ds = full_ds.shuffle(len(train_paths), reshuffle_each_iteration=True)\nfull_ds = full_ds.map(tf_parse_train, num_parallel_calls=AUTOTUNE)\n\nval_count = int(0.1 * len(train_paths))\nval_ds   = full_ds.take(val_count).batch(BATCH_SIZE).prefetch(AUTOTUNE)\ntrain_ds = full_ds.skip(val_count).batch(BATCH_SIZE).prefetch(AUTOTUNE)\n\ntest_ds = (\n    tf.data.Dataset.from_tensor_slices(test_paths)\n      .map(tf_parse_test, num_parallel_calls=AUTOTUNE)\n      .batch(BATCH_SIZE)\n      .prefetch(AUTOTUNE)\n)\n\n\n# --- Model: 3D-Conv ↓ → 2D-Conv layers → Transformer → Regression ---\ndef transformer_block(x, head_size, num_heads, ff_dim, dropout=0.1):\n    attn = layers.MultiHeadAttention(key_dim=head_size,\n                                     num_heads=num_heads,\n                                     dropout=dropout)(x, x)\n    x = layers.Add()([x, attn])\n    x = layers.LayerNormalization()(x)\n    ff = layers.Dense(ff_dim, activation=\"relu\")(x)\n    ff = layers.Dense(x.shape[-1])(ff)\n    return layers.Add()([x, ff])\n\ndef build_model():\n    inp = layers.Input((*IMG_SHAPE, 1))\n    # 3D downsampling\n    x = layers.Conv3D(16, 3, strides=2, padding=\"same\", activation=\"relu\")(inp)\n    x = layers.Conv3D(32, 3, strides=2, padding=\"same\", activation=\"relu\")(x)\n    # 2D conv: collapse spectral into channels\n    shape = tf.keras.backend.int_shape(x)\n    # shape: (batch, h, w, d, c)\n    h, w, d, c = shape[1], shape[2], shape[3], shape[4]\n    x2d = layers.Reshape((h, w, d * c))(x)\n    # add deep 2D conv layers\n    x2d = layers.Conv2D(64, 3, padding=\"same\", activation=\"relu\")(x2d)\n    x2d = layers.Conv2D(64, 3, padding=\"same\", activation=\"relu\")(x2d)\n    # prepare sequence for transformer\n    seq_len = h * w\n    feat_dim = 64\n    x_seq = layers.Reshape((seq_len, feat_dim))(x2d)\n    # transformer blocks\n    x_seq = transformer_block(x_seq, head_size=16, num_heads=2, ff_dim=64)\n    x_seq = transformer_block(x_seq, head_size=16, num_heads=2, ff_dim=64)\n    # regression head\n    x_out = layers.GlobalAveragePooling1D()(x_seq)\n    x_out = layers.Dense(128, activation=\"relu\")(x_out)\n    x_out = layers.Dropout(0.3)(x_out)\n    out = layers.Dense(1, activation=\"linear\")(x_out)\n    return Model(inp, out)\n\nmodel = build_model()\nopt = optimizers.Adam(learning_rate=1e-4, clipnorm=1.0)\nmodel.compile(optimizer=opt, loss=\"mse\", metrics=[\"mae\"])\nmodel.summary()\n\n# --- Train & Submit ---\nmodel.fit(train_ds, validation_data=val_ds, epochs=EPOCHS)\npreds = model.predict(test_ds).flatten()\npreds = np.clip(np.round(preds), 1, 100).astype(int)\n\npd.DataFrame({\"ID\": test_df[\"id\"], \"label\": preds}).to_csv(SUBMISSION, index=False)\nprint(\"Saved submission:\", SUBMISSION)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-17T03:34:19.402926Z","iopub.execute_input":"2025-05-17T03:34:19.403134Z","iopub.status.idle":"2025-05-17T03:51:48.375445Z","shell.execute_reply.started":"2025-05-17T03:34:19.403117Z","shell.execute_reply":"2025-05-17T03:51:48.374709Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.XGBOOST","metadata":{}},{"cell_type":"markdown","source":"This is the model before hyperparameter tuning","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_absolute_error\nimport xgboost as xgb\n\n# --- Configuration ---\nBASE_DIR    = \"/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-2025\"\nNPY_DIR     = os.path.join(BASE_DIR, \"ot/ot\")\nTRAIN_CSV   = os.path.join(BASE_DIR, \"train.csv\")\nTEST_CSV    = os.path.join(BASE_DIR, \"test.csv\")\nSUBMISSION  = \"submission_xgb.csv\"\n\nIMG_SHAPE   = (128, 128, 125)\nTARGET_SIZE = np.prod(IMG_SHAPE)\n\n\ndef load_and_flatten(path):\n    \"\"\"\n    Load a .npy file (or raw .npy if header corrupt), flatten to 1D,\n    pad by repeating last value if too short, or truncate if too long.\n    \"\"\"\n    try:\n        arr = np.load(path)\n        flat = arr.ravel()\n    except Exception:\n        flat = np.fromfile(path, dtype=np.float32)\n\n    # pad/truncate to TARGET_SIZE\n    if flat.size < TARGET_SIZE:\n        if flat.size == 0:\n            # completely missing? fill with zeros\n            flat = np.zeros(TARGET_SIZE, dtype=np.float32)\n        else:\n            # repeat last value\n            pad_vals = np.full(TARGET_SIZE - flat.size, flat[-1], dtype=np.float32)\n            flat = np.concatenate([flat, pad_vals])\n    else:\n        flat = flat[:TARGET_SIZE]\n\n    return flat\n\n\ndef extract_features(df):\n    \"\"\"\n    For each row in df (with 'id'), load the patch, fix shape, and compute\n    mean reflectance for each of the 125 bands.\n    Returns an (n_samples, 125) array.\n    \"\"\"\n    features = []\n    for fn in df['id']:\n        path = os.path.join(NPY_DIR, fn)\n        flat = load_and_flatten(path)\n        # reshape and compute band means\n        cube = flat.reshape(IMG_SHAPE)\n        band_means = cube.mean(axis=(0, 1))\n        features.append(band_means)\n    return np.vstack(features)\n\n\n# --- 1) Load CSVs ---\ntrain_df = pd.read_csv(TRAIN_CSV)\ntest_df  = pd.read_csv(TEST_CSV)\n\n# --- 2) Build feature matrices ---\nprint(\"Extracting features for training set...\")\nX = extract_features(train_df)    # shape (n_train, 125)\ny = train_df['label'].values\n\nprint(\"Extracting features for test set...\")\nX_test = extract_features(test_df) # shape (n_test, 125)\n\n# --- 3) Train‐validation split ---\nX_train, X_val, y_train, y_val = train_test_split(\n    X, y, test_size=0.1, random_state=42\n)\n\n# --- 4) XGBoost Regressor setup ---\nxgb_model = xgb.XGBRegressor(\n    n_estimators=500,\n    learning_rate=0.05,\n    max_depth=6,\n    subsample=0.8,\n    colsample_bytree=0.8,\n    random_state=42,\n    tree_method='gpu_hist'  # or 'hist' if no GPU\n)\n\n# --- 5) Train with early stopping ---\nxgb_model.fit(\n    X_train, y_train,\n    eval_set=[(X_train, y_train), (X_val, y_val)],\n    eval_metric='mae',\n    early_stopping_rounds=20,\n    verbose=True\n)\n\n# --- 6) Validation performance ---\ny_pred_val = xgb_model.predict(X_val)\nval_mae = mean_absolute_error(y_val, y_pred_val)\nprint(f\"Validation MAE: {val_mae:.4f}\")\n\n# --- 7) Predict on test set & save submission ---\ny_pred_test = xgb_model.predict(X_test)\ny_pred_test = np.clip(np.round(y_pred_test), 1, 100).astype(int)\n\nsubmission_df = pd.DataFrame({\n    \"id\": test_df[\"id\"],\n    \"label\": y_pred_test\n})\nsubmission_df.to_csv(SUBMISSION, index=False)\nprint(f\"✅ Submission saved to {SUBMISSION}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-17T05:13:35.586086Z","iopub.execute_input":"2025-05-17T05:13:35.586659Z","iopub.status.idle":"2025-05-17T05:15:00.104033Z","shell.execute_reply.started":"2025-05-17T05:13:35.586631Z","shell.execute_reply":"2025-05-17T05:15:00.103300Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4.3D CNN + TRANSFORMER","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras import layers, Model, optimizers\nfrom tensorflow.keras import mixed_precision\nfrom sklearn.model_selection import train_test_split\n\n# -----------------------------------------------------------------------------\n# 0) Mixed Precision & Strategy\n# -----------------------------------------------------------------------------\nmixed_precision.set_global_policy('mixed_float16')\nstrategy = tf.distribute.MirroredStrategy()\nprint(f\"Using {strategy.num_replicas_in_sync} GPUs\")\n\n# -----------------------------------------------------------------------------\n# 1) Configuration\n# -----------------------------------------------------------------------------\nBASE_DIR   = \"/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-2025\"\nNPY_DIR    = os.path.join(BASE_DIR, \"ot/ot\")\nTRAIN_CSV  = os.path.join(BASE_DIR, \"train.csv\")\nTEST_CSV   = os.path.join(BASE_DIR, \"test.csv\")\nSUBMISSION = \"submission_ensemble.csv\"\n\nIMG_SHAPE   = (128, 128, 125)\nTARGET_SIZE = np.prod(IMG_SHAPE)\nBATCH_SIZE  = 4       # per GPU\nEPOCHS      = 20\nLR          = 1e-4\nAUTOTUNE    = tf.data.AUTOTUNE\n\n# -----------------------------------------------------------------------------\n# 2) NaN-safe loader + normalization\n# -----------------------------------------------------------------------------\ndef load_and_normalize(path_bytes):\n    path = path_bytes.numpy().decode('utf-8')\n    try:\n        arr = np.load(path)\n        flat = arr.ravel()\n    except Exception:\n        flat = np.fromfile(path, dtype=np.float32)\n    # pad/truncate\n    if flat.size < TARGET_SIZE:\n        if flat.size == 0:\n            flat = np.zeros(TARGET_SIZE, dtype=np.float32)\n        else:\n            pad = np.full(TARGET_SIZE - flat.size, flat[-1], dtype=np.float32)\n            flat = np.concatenate([flat, pad])\n    else:\n        flat = flat[:TARGET_SIZE]\n    cube = flat.reshape(IMG_SHAPE).astype(np.float32)\n    cube = np.nan_to_num(cube, nan=0.0, posinf=0.0, neginf=0.0)\n    mn, mx = cube.min(), cube.max()\n    cube = (cube - mn) / ((mx - mn) + 1e-6)\n    return cube\n\ndef tf_process(path, label=None):\n    cube = tf.py_function(load_and_normalize, [path], tf.float32)\n    cube.set_shape(IMG_SHAPE)\n    cnn_in = cube[..., tf.newaxis]\n    seq_in = tf.reshape(cube, (IMG_SHAPE[0]*IMG_SHAPE[1], IMG_SHAPE[2]))\n    if label is None:\n        return (cnn_in, seq_in)\n    return (cnn_in, seq_in), tf.cast(label, tf.float32)\n\n# -----------------------------------------------------------------------------\n# 3) Load CSV & Split\n# -----------------------------------------------------------------------------\ntrain_df = pd.read_csv(TRAIN_CSV)\ntest_df  = pd.read_csv(TEST_CSV)\n\npaths = train_df['id'].apply(lambda f: os.path.join(NPY_DIR, f)).values\nlabels = train_df['label'].values.astype(np.float32)\n\np_train, p_val, y_train, y_val = train_test_split(\n    paths, labels, test_size=0.1, random_state=42\n)\n\n# Build datasets\ntrain_ds = (\n    tf.data.Dataset\n      .from_tensor_slices((p_train, y_train))\n      .shuffle(len(p_train))\n      .map(lambda p,y: tf_process(p,y), num_parallel_calls=AUTOTUNE)\n      .batch(BATCH_SIZE)\n      .prefetch(AUTOTUNE)\n)\n\nval_ds = (\n    tf.data.Dataset\n      .from_tensor_slices((p_val, y_val))\n      .map(lambda p,y: tf_process(p,y), num_parallel_calls=AUTOTUNE)\n      .batch(BATCH_SIZE)\n      .prefetch(AUTOTUNE)\n)\n\ntest_paths = test_df['id'].apply(lambda f: os.path.join(NPY_DIR, f)).values\ntest_ds = (\n    tf.data.Dataset\n      .from_tensor_slices(test_paths)\n      .map(lambda p: tf_process(p), num_parallel_calls=AUTOTUNE)\n      .batch(BATCH_SIZE)\n      .prefetch(AUTOTUNE)\n)\n\n# -----------------------------------------------------------------------------\n# 4) Model Definition (unchanged Big Model)\n# -----------------------------------------------------------------------------\nwith strategy.scope():\n    def inception3d(x, f1, f3, f5, proj):\n        p1 = layers.Conv3D(f1,1,padding='same',activation='relu')(x)\n        p2 = layers.Conv3D(proj,1,padding='same',activation='relu')(x)\n        p2 = layers.Conv3D(f3,3,padding='same',activation='relu')(p2)\n        p3 = layers.Conv3D(proj,1,padding='same',activation='relu')(x)\n        p3 = layers.Conv3D(f5,5,padding='same',activation='relu')(p3)\n        p4 = layers.MaxPool3D(3,strides=1,padding='same')(x)\n        p4 = layers.Conv3D(f5,1,padding='same',activation='relu')(p4)\n        return layers.Concatenate()([p1,p2,p3,p4])\n\n    def transformer_block(seq, head_size, num_heads, ff_dim, dropout=0.1):\n        attn = layers.MultiHeadAttention(key_dim=head_size,\n                                         num_heads=num_heads,\n                                         dropout=dropout)(seq, seq)\n        x = layers.Add()([seq, attn])\n        x = layers.LayerNormalization()(x)\n        ff = layers.Dense(ff_dim, activation='relu')(x)\n        ff = layers.Dense(x.shape[-1], dtype='float32')(ff)\n        return layers.Add()([x, ff])\n\n    inp_cnn = layers.Input((*IMG_SHAPE,1))\n    inp_seq = layers.Input((IMG_SHAPE[0]*IMG_SHAPE[1], IMG_SHAPE[2]))\n\n    # CNN branch\n    x = layers.Conv3D(64,7,strides=2,padding='same',activation='relu')(inp_cnn)\n    x = layers.MaxPool3D(3,strides=2,padding='same')(x)\n    x = inception3d(x,32,64,16,32)\n    x = inception3d(x,64,128,32,64)\n    x = layers.GlobalAveragePooling3D()(x)\n    out_cnn = layers.Dense(128, activation='relu')(x)\n\n    # Transformer branch\n    y = transformer_block(inp_seq,32,4,128)\n    y = transformer_block(y,32,4,128)\n    y = layers.GlobalAveragePooling1D()(y)\n    out_trans = layers.Dense(128, activation='relu')(y)\n\n    # Merge + head\n    merged = layers.Concatenate()([out_cnn, out_trans])\n    z = layers.Dense(256, activation='relu')(merged)\n    z = layers.Dropout(0.4)(z)\n    z = layers.Dense(64, activation='relu')(z)\n    out = layers.Dense(1, activation='linear', dtype='float32')(z)\n\n    model = Model([inp_cnn, inp_seq], out)\n    model.compile(\n        optimizer=optimizers.Adam(LR, clipnorm=1.0),\n        loss='mse', metrics=['mae']\n    )\n\nmodel.summary()\n\n# -----------------------------------------------------------------------------\n# 5) Train & Predict\n# -----------------------------------------------------------------------------\nmodel.fit(train_ds, validation_data=val_ds, epochs=EPOCHS)\n\npreds = model.predict(test_ds).flatten()\npreds = np.clip(np.round(preds), 1, 100).astype(int)\n\npd.DataFrame({'id': test_df['id'], 'label': preds}) \\\n  .to_csv(SUBMISSION, index=False)\n\nprint(\"✅ Saved:\", SUBMISSION)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}