{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[]}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🌍 MODIS Evapotranspiration Prediction using XGBoost\n\n## Overview\nThis notebook demonstrates an end-to-end machine learning workflow on the MODIS Evapotranspiration dataset, including:\n\n- Data Cleaning\n- Exploratory Data Analysis (EDA)\n- Feature Engineering\n- Data Visualization\n- XGBoost Regression\n- Model Evaluation\n- Time-Series Forecasting\n\n**Technologies:** Python, Pandas, NumPy, Matplotlib, Seaborn, Scikit-learn, XGBoost","metadata":{"id":"7KN-mRT6Y6ZJ"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Set professional business styling for the dashboard\nplt.style.use('seaborn-v0_8-whitegrid')\nsns.set_context(\"talk\")","metadata":{"id":"rwCEtyH2ZU94","trusted":true,"execution":{"iopub.status.busy":"2026-07-07T19:19:12.423681Z","iopub.execute_input":"2026-07-07T19:19:12.424043Z","iopub.status.idle":"2026-07-07T19:19:12.429653Z","shell.execute_reply.started":"2026-07-07T19:19:12.424013Z","shell.execute_reply":"2026-07-07T19:19:12.428828Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    print(dirname)\n    for filename in filenames:\n        print(\"   \", filename)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-07T19:19:12.431361Z","iopub.execute_input":"2026-07-07T19:19:12.431688Z","iopub.status.idle":"2026-07-07T19:19:12.450547Z","shell.execute_reply.started":"2026-07-07T19:19:12.431659Z","shell.execute_reply":"2026-07-07T19:19:12.449441Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1. Data Loading & Cleaning\nWe load the dataset, handle corrupted values, and ensure all relevant statistical boundaries are properly typed for numeric analysis.","metadata":{"id":"KzHeurGNZbsz"}},{"cell_type":"code","source":"print(\"Loading Data...\")\n# Note: Ensure this path matches your Kaggle/Colab input directory!\nimport os\nimport pandas as pd\n\nLOCAL_PATH = \"../data/raw/MOD16A2GF-006-Statistics.csv\"\nKAGGLE_PATH = \"/kaggle/input/datasets/ganeshkarthik016/modis-evapotranspiration-italy/MOD16A2GF-006-Statistics.csv\"\n\nif os.path.exists(LOCAL_PATH):\n    DATA_PATH = LOCAL_PATH\nelif os.path.exists(KAGGLE_PATH):\n    DATA_PATH = KAGGLE_PATH\nelse:\n    raise FileNotFoundError(\"Dataset not found.\")\n\nmodis = pd.read_csv(DATA_PATH)\n\n# Clean corrupted values denoted by '########'\nmodis = modis.replace(\"########\", np.nan)\n\n# Convert all relevant summary statistics to numeric floats\nnumeric_cols = ['Mean', 'Minimum', 'Maximum', 'Lower Quartile', 'Upper Quartile']\nfor col in numeric_cols:\n    if col in modis.columns:\n        modis[col] = pd.to_numeric(modis[col], errors='coerce')\n\nprint(f\"Initial Data Shape: {modis.shape}\")","metadata":{"id":"ce3nSvb_Zcnm","outputId":"4b2645c3-3e4e-4b2f-fdef-e196512c0ade","trusted":true,"execution":{"iopub.status.busy":"2026-07-07T19:19:12.462925Z","iopub.execute_input":"2026-07-07T19:19:12.463306Z","iopub.status.idle":"2026-07-07T19:19:12.508671Z","shell.execute_reply.started":"2026-07-07T19:19:12.463277Z","shell.execute_reply":"2026-07-07T19:19:12.507788Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Temporal Feature Engineering\nTo perform Time-Series analysis, we must extract seasonal identifiers (Month and Year) from the raw timestamp string. We then filter out any incomplete records.","metadata":{"id":"Ejbz8SQiZfdP"}},{"cell_type":"code","source":"# Extract Date, Month, and Year for trend analysis\nif 'Date' in modis.columns:\n    modis['Date'] = pd.to_datetime(modis['Date'], errors='coerce')\n    modis['Month'] = modis['Date'].dt.month\n    modis['Year'] = modis['Date'].dt.year\n\n# Drop rows with missing target data or missing dates\nmodis = modis.dropna(subset=['Mean', 'Date'])\n\nprint(f\"Cleaned Data Shape: {modis.shape}\")\nprint(\"Data Cleaned! Generating Insights Dashboard...\\n\")","metadata":{"id":"9x9PJvTEZiUq","outputId":"9d97adcf-946c-423a-fca1-eb9309445f7a","trusted":true,"execution":{"iopub.status.busy":"2026-07-07T19:19:12.510223Z","iopub.execute_input":"2026-07-07T19:19:12.510518Z","iopub.status.idle":"2026-07-07T19:19:12.531410Z","shell.execute_reply.started":"2026-07-07T19:19:12.510491Z","shell.execute_reply":"2026-07-07T19:19:12.530246Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Automated Insights Dashboard\nThis dashboard visualizes two key metrics:\n1. **Month-over-Month Seasonality:** Identifying which months experience the highest average evapotranspiration across all recorded years.\n2. **Long-Term Trend & Variance:** Tracking the ET over a 15+ year timeline, utilizing the Interquartile Range (IQR) to show the variance/spread of the data at any given point.","metadata":{"id":"S0cSorgtZkxn"}},{"cell_type":"code","source":"# Create a Multi-plot Business Dashboard\nfig, axes = plt.subplots(2, 1, figsize=(14, 14))\n\n# --- PLOT 1: Month-over-Month Seasonality ---\n# Group by Month to see the average behavior across all years\nmonthly_stats = modis.groupby('Month')['Mean'].mean().reset_index()\n\n# UPDATED: Added hue='Month' and legend=False to fix the Seaborn warning\nsns.barplot(data=monthly_stats, x='Month', y='Mean',\n            hue='Month', palette='viridis', legend=False,\n            ax=axes[0], edgecolor='black')\n\naxes[0].set_title(\"Seasonal Trend: Average Evapotranspiration by Month\", fontsize=16, weight='bold')\naxes[0].set_xlabel(\"Month (1 = Jan, 12 = Dec)\")\naxes[0].set_ylabel(\"Average ET (Mean)\")\n\n# --- PLOT 2: Long-Term Time Series with Variance Bounds ---\n# Sort chronologically for accurate line plotting\nmodis_sorted = modis.sort_values('Date')\n\n# Plot the primary Mean trendline\naxes[1].plot(modis_sorted['Date'], modis_sorted['Mean'],\n             color='darkblue', linewidth=2, label='Mean ET')\n\n# Add the shaded region to show the spread (variance) of the data\nif 'Lower Quartile' in modis.columns and 'Upper Quartile' in modis.columns:\n    axes[1].fill_between(modis_sorted['Date'],\n                         modis_sorted['Lower Quartile'],\n                         modis_sorted['Upper Quartile'],\n                         color='blue', alpha=0.2, label='Interquartile Range (IQR)')\n\naxes[1].set_title(\"Long-Term Evapotranspiration Tracking (With Variance Bounds)\", fontsize=16, weight='bold')\naxes[1].set_xlabel(\"Year\")\naxes[1].set_ylabel(\"Evapotranspiration\")\naxes[1].legend(loc='upper right')\n\nplt.tight_layout(pad=4.0)\nplt.show()","metadata":{"id":"bFFDDNRbZpZo","outputId":"05b87836-3310-4052-cb4b-40cb042d881f","trusted":true,"execution":{"iopub.status.busy":"2026-07-07T19:19:12.538544Z","iopub.execute_input":"2026-07-07T19:19:12.539279Z","iopub.status.idle":"2026-07-07T19:19:13.554224Z","shell.execute_reply.started":"2026-07-07T19:19:12.539243Z","shell.execute_reply":"2026-07-07T19:19:13.553218Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\nplt.figure(figsize=(12,8))\n\nnumeric_cols = modis.select_dtypes(include='number')\n\nsns.heatmap(\n    numeric_cols.corr(),\n    annot=True,\n    cmap='coolwarm',\n    fmt=\".2f\"\n)\n\nplt.title(\"Correlation Heatmap\")\nplt.tight_layout()\n\nplt.savefig(\"correlation_heatmap.png\", dpi=300)\nplt.show()","metadata":{"id":"loNxsaVt1Hw5","outputId":"14895a99-71ad-42c7-e60f-f2ee4cbd0cbf","trusted":true,"execution":{"iopub.status.busy":"2026-07-07T19:19:13.555789Z","iopub.execute_input":"2026-07-07T19:19:13.556193Z","iopub.status.idle":"2026-07-07T19:19:15.416563Z","shell.execute_reply.started":"2026-07-07T19:19:13.556153Z","shell.execute_reply":"2026-07-07T19:19:15.415522Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing = modis.isnull().sum()\nmissing = missing[missing > 0]\n\nif len(missing) > 0:\n    plt.figure(figsize=(8,5))\n    missing.sort_values().plot(kind=\"bar\")\n    plt.title(\"Missing Values by Column\")\n    plt.ylabel(\"Count\")\n    plt.tight_layout()\n    plt.savefig(\"missing_values.png\", dpi=300)\n    plt.show()\nelse:\n    print(\"✅ No missing values found in the dataset.\")","metadata":{"id":"B7Ep6fyK1I0s","outputId":"71462e74-83ca-4d84-e724-c8adee999a0a","trusted":true,"execution":{"iopub.status.busy":"2026-07-07T19:19:15.417820Z","iopub.execute_input":"2026-07-07T19:19:15.418233Z","iopub.status.idle":"2026-07-07T19:19:15.429314Z","shell.execute_reply.started":"2026-07-07T19:19:15.418180Z","shell.execute_reply":"2026-07-07T19:19:15.428211Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10,5))\n\nsns.histplot(modis[\"Mean\"], bins=30, kde=True)\n\nplt.title(\"Distribution of Mean Evapotranspiration\")\n\nplt.xlabel(\"Mean ET\")\n\nplt.tight_layout()\n\nplt.savefig(\"distribution.png\", dpi=300)\n\nplt.show()","metadata":{"id":"cQ3TOjWR1Mcg","outputId":"90a950c8-a68e-40e2-c12a-971660ffe642","trusted":true,"execution":{"iopub.status.busy":"2026-07-07T19:19:15.431896Z","iopub.execute_input":"2026-07-07T19:19:15.432389Z","iopub.status.idle":"2026-07-07T19:19:16.155801Z","shell.execute_reply.started":"2026-07-07T19:19:15.432257Z","shell.execute_reply":"2026-07-07T19:19:16.154769Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10,5))\n\nsns.boxplot(x=modis[\"Mean\"])\n\nplt.title(\"Outlier Detection using Boxplot\")\n\nplt.tight_layout()\n\nplt.savefig(\"boxplot.png\", dpi=300)\n\nplt.show()","metadata":{"id":"KeWrKy0e1Q0K","outputId":"01bd9012-78e9-4c7e-d8ef-0100e5dac350","trusted":true,"execution":{"iopub.status.busy":"2026-07-07T19:19:16.156885Z","iopub.execute_input":"2026-07-07T19:19:16.157214Z","iopub.status.idle":"2026-07-07T19:19:16.597491Z","shell.execute_reply.started":"2026-07-07T19:19:16.157183Z","shell.execute_reply":"2026-07-07T19:19:16.596576Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly = modis.groupby(\"Month\")[\"Mean\"].mean()\n\nplt.figure(figsize=(10,5))\n\nplt.plot(\n    monthly.index,\n    monthly.values,\n    marker='o',\n    linewidth=3\n)\n\nplt.title(\"Average Monthly Evapotranspiration\")\n\nplt.xlabel(\"Month\")\n\nplt.ylabel(\"Mean ET\")\n\nplt.grid(True)\n\nplt.savefig(\"monthly_trend.png\", dpi=300)\n\nplt.show()","metadata":{"id":"PMse9b2m1VMo","outputId":"f36345a4-015b-4955-dc38-65c941de9498","trusted":true,"execution":{"iopub.status.busy":"2026-07-07T19:19:16.598632Z","iopub.execute_input":"2026-07-07T19:19:16.599495Z","iopub.status.idle":"2026-07-07T19:19:17.151076Z","shell.execute_reply.started":"2026-07-07T19:19:16.599457Z","shell.execute_reply":"2026-07-07T19:19:17.150052Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nos.makedirs(\"data/processed\", exist_ok=True)\n\nmodis.to_csv(\"data/processed/cleaned_MOD16A2GF_Statistics.csv\", index=False)\n\nprint(\"✅ Cleaned dataset saved!\")","metadata":{"id":"_tT6hsknxhs5","outputId":"8436869d-c970-430c-dc70-64853ed1b8f2","trusted":true,"execution":{"iopub.status.busy":"2026-07-07T19:19:17.152323Z","iopub.execute_input":"2026-07-07T19:19:17.152647Z","iopub.status.idle":"2026-07-07T19:19:17.208970Z","shell.execute_reply.started":"2026-07-07T19:19:17.152616Z","shell.execute_reply":"2026-07-07T19:19:17.207970Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Phase 2: Time-Series Forecasting (XGBoost)","metadata":{"id":"aTtAb8ARbzwN"}},{"cell_type":"code","source":"from xgboost import XGBRegressor\nfrom sklearn.metrics import mean_squared_error, r2_score","metadata":{"id":"zurZgJafbzct","trusted":true,"execution":{"iopub.status.busy":"2026-07-07T19:19:17.262326Z","iopub.execute_input":"2026-07-07T19:19:17.262638Z","iopub.status.idle":"2026-07-07T19:19:17.845811Z","shell.execute_reply.started":"2026-07-07T19:19:17.262606Z","shell.execute_reply":"2026-07-07T19:19:17.844682Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. Prepare the Data (Strictly avoiding target leakage)\nforecast_data = modis[['Year', 'Month', 'Mean']].dropna()\n\n# Sort chronologically to ensure our train/test split respects time\nforecast_data = forecast_data.sort_values(['Year', 'Month'])\n\nX = forecast_data[['Year', 'Month']]\ny = forecast_data['Mean']\n\n# 2. Chronological Train-Test Split (80% Past to Train, 20% Future to Test)\nsplit_index = int(len(forecast_data) * 0.8)\n\nX_train, X_test = X.iloc[:split_index], X.iloc[split_index:]\ny_train, y_test = y.iloc[:split_index], y.iloc[split_index:]\n\n# 3. Initialize and Train XGBoost\nprint(\"Training XGBoost Forecasting Model...\")\nxgb_forecast = XGBRegressor(\n    n_estimators=200,\n    learning_rate=0.05,\n    max_depth=4,\n    random_state=42\n)\nxgb_forecast.fit(X_train, y_train)","metadata":{"id":"B3gqTe89b4KI","outputId":"7190ce4b-dfef-4f58-917d-9806a41dc14d","trusted":true,"execution":{"iopub.status.busy":"2026-07-07T19:19:17.848603Z","iopub.execute_input":"2026-07-07T19:19:17.849083Z","iopub.status.idle":"2026-07-07T19:19:18.009617Z","shell.execute_reply.started":"2026-07-07T19:19:17.849055Z","shell.execute_reply":"2026-07-07T19:19:18.008633Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4. Generate Predictions & Calculate Metrics\npreds = xgb_forecast.predict(X_test)\nrmse = np.sqrt(mean_squared_error(y_test, preds))\nr2 = r2_score(y_test, preds)\n\nprint(f\"\\n--- FORECASTING RESULTS ---\")\nprint(f\"RMSE : {rmse:.4f}\")\nprint(f\"R²   : {r2:.4f}\\n\")","metadata":{"id":"Fe4Qp5MQb_8X","outputId":"956ebf84-7b0d-4ac5-ca6f-e60e288fcd5d","trusted":true,"execution":{"iopub.status.busy":"2026-07-07T19:19:18.010965Z","iopub.execute_input":"2026-07-07T19:19:18.011307Z","iopub.status.idle":"2026-07-07T19:19:18.030062Z","shell.execute_reply.started":"2026-07-07T19:19:18.011278Z","shell.execute_reply":"2026-07-07T19:19:18.029124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\n# Reset index for plotting purposes\nplt.plot(y_test.reset_index(drop=True), label='Actual ET (Future Data)', color='darkblue', alpha=0.7, linewidth=2)\nplt.plot(preds, label='XGBoost Forecast', color='coral', linestyle='--', linewidth=2)\n\nplt.title(\"XGBoost Time-Series Forecast: Actual vs. Predicted Evapotranspiration\", fontsize=16, weight='bold')\nplt.xlabel(\"Time Steps (Test Data Timeline)\")\nplt.ylabel(\"Evapotranspiration (Mean)\")\nplt.legend(loc='best')\nplt.tight_layout()\nplt.show()","metadata":{"id":"nMYbn4QjcCNa","outputId":"4e227386-c106-4cf2-95a3-d029598ff327","trusted":true,"execution":{"iopub.status.busy":"2026-07-07T19:19:18.031616Z","iopub.execute_input":"2026-07-07T19:19:18.032225Z","iopub.status.idle":"2026-07-07T19:19:18.323633Z","shell.execute_reply.started":"2026-07-07T19:19:18.032180Z","shell.execute_reply":"2026-07-07T19:19:18.322668Z"}},"outputs":[],"execution_count":null}]}