{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":98450,"databundleVersionId":11749951,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install kornia","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:41.693823Z","iopub.execute_input":"2025-05-19T02:18:41.694138Z","iopub.status.idle":"2025-05-19T02:18:44.738598Z","shell.execute_reply.started":"2025-05-19T02:18:41.694118Z","shell.execute_reply":"2025-05-19T02:18:44.737817Z"}},"outputs":[{"name":"stdout","text":"Requirement already satisfied: kornia in /usr/local/lib/python3.11/dist-packages (0.8.0)\nRequirement already satisfied: kornia_rs>=0.1.0 in /usr/local/lib/python3.11/dist-packages (from kornia) (0.1.8)\nRequirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from kornia) (24.2)\nRequirement already satisfied: torch>=1.9.1 in /usr/local/lib/python3.11/dist-packages (from kornia) (2.5.1+cu124)\nRequirement already satisfied: filelock in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (3.18.0)\nRequirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (4.13.1)\nRequirement already satisfied: networkx in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (3.4.2)\nRequirement already satisfied: jinja2 in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (3.1.6)\nRequirement already satisfied: fsspec in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (2025.3.2)\nRequirement already satisfied: nvidia-cuda-nvrtc-cu12==12.4.127 in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (12.4.127)\nRequirement already satisfied: nvidia-cuda-runtime-cu12==12.4.127 in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (12.4.127)\nRequirement already satisfied: nvidia-cuda-cupti-cu12==12.4.127 in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (12.4.127)\nRequirement already satisfied: nvidia-cudnn-cu12==9.1.0.70 in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (9.1.0.70)\nRequirement already satisfied: nvidia-cublas-cu12==12.4.5.8 in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (12.4.5.8)\nRequirement already satisfied: nvidia-cufft-cu12==11.2.1.3 in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (11.2.1.3)\nRequirement already satisfied: nvidia-curand-cu12==10.3.5.147 in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (10.3.5.147)\nRequirement already satisfied: nvidia-cusolver-cu12==11.6.1.9 in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (11.6.1.9)\nRequirement already satisfied: nvidia-cusparse-cu12==12.3.1.170 in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (12.3.1.170)\nRequirement already satisfied: nvidia-nccl-cu12==2.21.5 in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (2.21.5)\nRequirement already satisfied: nvidia-nvtx-cu12==12.4.127 in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (12.4.127)\nRequirement already satisfied: nvidia-nvjitlink-cu12==12.4.127 in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (12.4.127)\nRequirement already satisfied: triton==3.1.0 in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (3.1.0)\nRequirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.11/dist-packages (from torch>=1.9.1->kornia) (1.13.1)\nRequirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from sympy==1.13.1->torch>=1.9.1->kornia) (1.3.0)\nRequirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/dist-packages (from jinja2->torch>=1.9.1->kornia) (3.0.2)\n","output_type":"stream"}],"execution_count":61},{"cell_type":"code","source":"import os\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torchvision.transforms as T\nfrom torch.utils.data import Dataset, DataLoader\nimport kornia.augmentation as K\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\nfrom einops import rearrange, repeat","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:44.740117Z","iopub.execute_input":"2025-05-19T02:18:44.740391Z","iopub.status.idle":"2025-05-19T02:18:44.746079Z","shell.execute_reply.started":"2025-05-19T02:18:44.740368Z","shell.execute_reply":"2025-05-19T02:18:44.745273Z"}},"outputs":[],"execution_count":62},{"cell_type":"markdown","source":"# Create data loader","metadata":{}},{"cell_type":"code","source":"DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n\nclass HyperspectralDataset(Dataset):\n    def __init__(self, df, base_path, patch_size=64, augment=False, num_bands=100):\n        self.df = df\n        self.base_path = base_path\n        self.patch_size = patch_size\n        self.augment = augment\n        self.num_bands = num_bands\n        self.transform = nn.Sequential(\n            K.RandomHorizontalFlip(p=0.3),     \n            K.RandomVerticalFlip(p=0.3),\n            K.RandomAffine(degrees=5, translate=(0.05, 0.05), scale=(0.95, 1.05), p=0.5),\n            K.RandomCrop((patch_size, patch_size), padding=4, p=0.5)\n        )\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = f\"{self.base_path}/{row['id']}\"\n\n        try:\n            img = np.load(img_path)\n\n            if len(img.shape) == 2:\n                img = np.repeat(img[:, :, np.newaxis], self.num_bands, axis=2)\n            elif len(img.shape) == 3:\n                if img.shape[2] > self.num_bands:\n                    img = img[:, :, :self.num_bands]\n                elif img.shape[2] < self.num_bands:\n                    pad_width = ((0, 0), (0, 0), (0, self.num_bands - img.shape[2]))\n                    img = np.pad(img, pad_width, mode='constant')\n\n            img = img.astype(np.float32) / 65535.0  # Normalize image\n\n            img = torch.tensor(img, dtype=torch.float32).permute(2, 0, 1)  # Convert to [C, H, W]\n\n            if self.augment:\n                img = self.transform(img.unsqueeze(0)).squeeze(0)\n\n            if img.shape[1] != self.patch_size or img.shape[2] != self.patch_size:\n                img = F.interpolate(img.unsqueeze(0), size=(self.patch_size, self.patch_size), mode='bilinear').squeeze(0)\n\n            label = torch.tensor(row['label'], dtype=torch.long)  \n\n            if label > 0:\n                label = label - 1\n\n            return img, label\n\n        except Exception as e:\n            print(f\"Error loading {img_path}: {str(e)}\")\n            dummy_img = torch.zeros(self.num_bands, self.patch_size, self.patch_size)\n            dummy_label = torch.tensor(0, dtype=torch.long)  \n            return dummy_img, dummy_label\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:44.746985Z","iopub.execute_input":"2025-05-19T02:18:44.74722Z","iopub.status.idle":"2025-05-19T02:18:44.764372Z","shell.execute_reply.started":"2025-05-19T02:18:44.747205Z","shell.execute_reply":"2025-05-19T02:18:44.763813Z"}},"outputs":[],"execution_count":63},{"cell_type":"markdown","source":"# Explore data loader","metadata":{}},{"cell_type":"code","source":"# train_df = pd.read_csv('/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-2025/train.csv')\n# base_path = '/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-2025/ot/ot'\n\n# dataset = HyperspectralDataset(train_df, base_path)\n# loader = DataLoader(dataset, batch_size=2,shuffle=False)\n\n# images, labels = next(iter(loader))\n\n# print(\"shape images:\", images.shape)\n# print(\"labels:\", labels)\n\n# img0, label0 = dataset[0]\n\n# print(\"image 0 shape\",img0.shape)\n# print(\"label 0:\", label0)\n\n# plt.figure(figsize=(8,8))\n# plt.imshow(img0[0,:,:], cmap=\"gray\")\n# plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:44.766097Z","iopub.execute_input":"2025-05-19T02:18:44.766712Z","iopub.status.idle":"2025-05-19T02:18:44.78197Z","shell.execute_reply.started":"2025-05-19T02:18:44.766692Z","shell.execute_reply":"2025-05-19T02:18:44.781211Z"}},"outputs":[],"execution_count":64},{"cell_type":"markdown","source":"# Spectral Former","metadata":{}},{"cell_type":"code","source":"class Residual(nn.Module):\n    def __init__(self, fn):\n        super().__init__()\n        self.fn = fn\n    def forward(self, x, **kwargs):\n        return self.fn(x, **kwargs) + x\n\nclass PreNorm(nn.Module):\n    def __init__(self, dim, fn):\n        super().__init__()\n        self.norm = nn.LayerNorm(dim)\n        self.fn = fn\n    def forward(self, x, **kwargs):\n        return self.fn(self.norm(x), **kwargs)\n\nclass FeedForward(nn.Module):\n    def __init__(self, dim, hidden_dim, dropout = 0.):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(dim, hidden_dim),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim, dim),\n            nn.Dropout(dropout)\n        )\n    def forward(self, x):\n        return self.net(x)\n\nclass Attention(nn.Module):\n    def __init__(self, dim, heads, dim_head, dropout):\n        super().__init__()\n        inner_dim = dim_head * heads\n        self.heads = heads\n        self.scale = dim_head ** -0.5\n\n        self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False)\n        self.to_out = nn.Sequential(\n            nn.Linear(inner_dim, dim),\n            nn.Dropout(dropout)\n        )\n    def forward(self, x, mask = None):\n        # x:[b,n,dim]\n        b, n, _, h = *x.shape, self.heads\n\n        # get qkv tuple:([b,n,head_num*head_dim],[...],[...])\n        qkv = self.to_qkv(x).chunk(3, dim = -1)\n        # split q,k,v from [b,n,head_num*head_dim] -> [b,head_num,n,head_dim]\n        q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = h), qkv)\n\n        # transpose(k) * q / sqrt(head_dim) -> [b,head_num,n,n]\n        dots = torch.einsum('bhid,bhjd->bhij', q, k) * self.scale\n        mask_value = -torch.finfo(dots.dtype).max\n\n        # mask value: -inf\n        if mask is not None:\n            mask = F.pad(mask.flatten(1), (1, 0), value = True)\n            assert mask.shape[-1] == dots.shape[-1], 'mask has incorrect dimensions'\n            mask = mask[:, None, :] * mask[:, :, None]\n            dots.masked_fill_(~mask, mask_value)\n            del mask\n\n        # softmax normalization -> attention matrix\n        attn = dots.softmax(dim=-1)\n        # value * attention matrix -> output\n        out = torch.einsum('bhij,bhjd->bhid', attn, v)\n        # cat all output -> [b, n, head_num*head_dim]\n        out = rearrange(out, 'b h n d -> b n (h d)')\n        out = self.to_out(out)\n        return out\n\nclass Transformer(nn.Module):\n    def __init__(self, dim, depth, heads, dim_head, mlp_head, dropout, num_channel, mode):\n        super().__init__()\n        \n        self.layers = nn.ModuleList([])\n        for _ in range(depth):\n            self.layers.append(nn.ModuleList([\n                Residual(PreNorm(dim, Attention(dim, heads = heads, dim_head = dim_head, dropout = dropout))),\n                Residual(PreNorm(dim, FeedForward(dim, mlp_head, dropout = dropout)))\n            ]))\n\n        self.mode = mode\n        self.skipcat = nn.ModuleList([])\n        for _ in range(depth-2):\n            self.skipcat.append(nn.Conv2d(num_channel+1, num_channel+1, [1, 2], 1, 0))\n\n    def forward(self, x, mask = None):\n        if self.mode == 'ViT':\n            for attn, ff in self.layers:\n                x = attn(x, mask = mask)\n                x = ff(x)\n        elif self.mode == 'CAF':\n            last_output = []\n            nl = 0\n            for attn, ff in self.layers:           \n                last_output.append(x)\n                if nl > 1:             \n                    x = self.skipcat[nl-2](torch.cat([x.unsqueeze(3), last_output[nl-2].unsqueeze(3)], dim=3)).squeeze(3)\n                x = attn(x, mask = mask)\n                x = ff(x)\n                nl += 1\n\n        return x\n\nclass ViT(nn.Module):\n    def __init__(self, image_size, near_band, num_patches, num_classes, dim, depth, heads, mlp_dim, pool='cls', channels=1, dim_head = 16, dropout=0., emb_dropout=0., mode='ViT'):\n        super().__init__()\n\n        patch_dim = image_size ** 2 * near_band\n        \n        self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, dim))\n        self.patch_to_embedding = nn.Linear(patch_dim, dim)\n        self.cls_token = nn.Parameter(torch.randn(1, 1, dim))\n\n        self.dropout = nn.Dropout(emb_dropout)\n        self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim, dropout, num_patches, mode)\n\n        self.pool = pool\n        self.to_latent = nn.Identity()\n\n        self.mlp_head = nn.Sequential(\n            nn.LayerNorm(dim),\n            nn.Linear(dim, 1),        # saldrá un escalar\n            nn.Sigmoid()              # 0 <–> 1\n        )\n    def forward(self, x, mask = None):\n       \n        # patchs[batch, patch_num, patch_size*patch_size*c]  [batch,200,145*145]\n        # x = rearrange(x, 'b c h w -> b c (h w)')\n\n        ## embedding every patch vector to embedding size: [batch, patch_num, embedding_size]\n        x = self.patch_to_embedding(x) #[b,n,dim]\n        b, n, _ = x.shape\n\n        # add position embedding\n        cls_tokens = repeat(self.cls_token, '() n d -> b n d', b = b) #[b,1,dim]\n        x = torch.cat((cls_tokens, x), dim = 1) #[b,n+1,dim]\n        x += self.pos_embedding[:, :(n + 1)]\n        x = self.dropout(x)\n\n        # transformer: x[b,n + 1,dim] -> x[b,n + 1,dim]\n        x = self.transformer(x, mask)\n\n        # classification: using cls_token output\n        x = self.to_latent(x[:,0])\n\n        # MLP classification layer\n        raw = self.mlp_head(x)         # en (0,1)\n        scaled = raw * 100             # ahora en (0,100)\n        return scaled.squeeze(-1) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:44.782749Z","iopub.execute_input":"2025-05-19T02:18:44.783028Z","iopub.status.idle":"2025-05-19T02:18:44.804384Z","shell.execute_reply.started":"2025-05-19T02:18:44.782999Z","shell.execute_reply":"2025-05-19T02:18:44.803742Z"}},"outputs":[],"execution_count":65},{"cell_type":"markdown","source":"# create model","metadata":{}},{"cell_type":"code","source":"# ——— Función para crear el modelo Spectral ViT ———\ndef create_spectral_transformer(\n    image_size: int,\n    near_band: int,\n    num_patches: int,\n    num_classes: int,\n    dim: int,\n    depth: int,\n    heads: int,\n    mlp_dim: int,\n    pool: str = 'cls',\n    channels: int = 1,\n    dim_head: int = 16,\n    dropout: float = 0.,\n    emb_dropout: float = 0.,\n    mode: str = 'ViT'\n) -> nn.Module:\n    \"\"\"\n    Construye y devuelve un modelo ViT para datos hiperespectrales.\n    \n    Parámetros:\n      - image_size: tamaño (H=W) de cada parche de entrada.\n      - near_band: número de bandas que entran en el patch embedding.\n      - num_patches: cantidad total de patches por imagen.\n      - num_classes: número de clases para clasificación final.\n      - dim: dimensión del embedding en todo el Transformer.\n      - depth: número de bloques (layers) del Transformer.\n      - heads: número de cabezas de atención multi-head.\n      - mlp_dim: dimensión interna del MLP en cada bloque.\n      - pool: 'cls' o 'mean', cómo agregas la salida antes de la MLP final.\n      - channels: canales de entrada (1 para cada vector de bandas).\n      - dim_head: dimensión de cada cabeza de atención.\n      - dropout: dropout dentro de atención y MLPs.\n      - emb_dropout: dropout tras sumar la pos_embedding.\n      - mode: 'ViT' (transformer estándar) o 'CAF' (skipped-cat).\n    \"\"\"\n    model = ViT(\n        image_size=image_size,\n        near_band=near_band,\n        num_patches=num_patches,\n        num_classes=num_classes,\n        dim=dim,\n        depth=depth,\n        heads=heads,\n        mlp_dim=mlp_dim,\n        pool=pool,\n        channels=channels,\n        dim_head=dim_head,\n        dropout=dropout,\n        emb_dropout=emb_dropout,\n        mode=mode\n    )\n    return model\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:44.804994Z","iopub.execute_input":"2025-05-19T02:18:44.805161Z","iopub.status.idle":"2025-05-19T02:18:44.8222Z","shell.execute_reply.started":"2025-05-19T02:18:44.805148Z","shell.execute_reply":"2025-05-19T02:18:44.821573Z"}},"outputs":[],"execution_count":66},{"cell_type":"markdown","source":"# create dataloader","metadata":{}},{"cell_type":"code","source":"# ——— Función para crear el DataLoader hiperespectral ———\ndef create_hyperspectral_dataloader(\n    df,\n    base_path: str,\n    patch_size: int = 128,\n    num_bands: int = 125,\n    augment: bool = False,\n    batch_size: int = 16,\n    shuffle: bool = True,\n    num_workers: int = 4,\n    drop_last: bool = False\n) -> DataLoader:\n    \"\"\"\n    Construye y devuelve un DataLoader para datos hiperespectrales.\n    \n    Parámetros:\n      - df: pandas.DataFrame con columnas ['id', 'label'].\n      - base_path: ruta donde están los archivos .npy.\n      - patch_size: tamaño H×W al que recortas/interpolas cada imagen.\n      - num_bands: número de bandas espectrales por muestra.\n      - augment: si aplicas o no data augmentation.\n      - batch_size: tamaño de batch.\n      - shuffle: true para barajar cada época.\n      - num_workers: número de procesos para cargar datos.\n      - drop_last: descarta el último batch si es incompleto.\n    \"\"\"\n    dataset = HyperspectralDataset(\n        df=df,\n        base_path=base_path,\n        patch_size=patch_size,\n        augment=augment,\n        num_bands=num_bands\n    )\n    loader = DataLoader(\n        dataset,\n        batch_size=batch_size,\n        shuffle=shuffle,\n        num_workers=num_workers,\n        drop_last=drop_last\n    )\n    return loader\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:44.823089Z","iopub.execute_input":"2025-05-19T02:18:44.823359Z","iopub.status.idle":"2025-05-19T02:18:44.840862Z","shell.execute_reply.started":"2025-05-19T02:18:44.823337Z","shell.execute_reply":"2025-05-19T02:18:44.840238Z"}},"outputs":[],"execution_count":67},{"cell_type":"markdown","source":"# Simple fordward","metadata":{}},{"cell_type":"code","source":"# # original imagen size\n# IMAGE_SIZE = 128\n\n# #data loader\n# patch_size = 16\n# num_bands = 125\n# batch_size = 32\n\n# #calculo de num_patches\n# num_patches = (IMAGE_SIZE//patch_size)**2\n\n# train_df = pd.read_csv('/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-2025/train.csv')\n# base_path = '/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-2025/ot/ot'\n\n\n# loader = create_hyperspectral_dataloader(\n#     df=train_df,\n#     base_path=base_path,\n#     patch_size=patch_size,\n#     num_bands=num_bands,\n#     augment=True,\n#     batch_size=batch_size\n# )\n\n# model = create_spectral_transformer(\n#     image_size=patch_size,\n#     near_band=num_bands,\n#     num_patches=num_patches,       # p.ej. si divides 128×128 en parches de 16×16: (128/16)**2 = 64\n#     num_classes=100,\n#     dim=512,\n#     depth=6,\n#     heads=8,\n#     mlp_dim=1024,\n#     pool='cls',\n#     mode='ViT'\n# ).to(DEVICE)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:44.841554Z","iopub.execute_input":"2025-05-19T02:18:44.841791Z","iopub.status.idle":"2025-05-19T02:18:44.857586Z","shell.execute_reply.started":"2025-05-19T02:18:44.841768Z","shell.execute_reply":"2025-05-19T02:18:44.856959Z"}},"outputs":[],"execution_count":68},{"cell_type":"code","source":"# # Definir optimizador y función de pérdida\n# optimizer = optim.Adam(model.parameters(), lr=1e-4)\n# criterion = nn.CrossEntropyLoss()\n\n# model.train()\n# running_loss = 0.0\n\n# for batch_idx, (images, labels) in enumerate(loader):\n#     images, labels = images.to(DEVICE), labels.to(DEVICE)\n\n#     # Patchify igual que en validación\n#     patches = rearrange(\n#         images,\n#         'b c (h p_h) (w p_w) -> b (h w) (p_h p_w c)',\n#         p_h=patch_size,\n#         p_w=patch_size\n#     )\n\n#     # Forward + loss\n#     logits = model(patches)  \n#     loss   = criterion(logits, labels)\n\n#     # Backward + óptimo\n#     optimizer.zero_grad()\n#     loss.backward()\n#     optimizer.step()\n\n#     running_loss += loss.item()\n#     if batch_idx % 10 == 0:\n#         print(f\"Batch {batch_idx} — loss: {loss.item():.4f}\")\n\n# # Reporte final de la época\n# avg_loss = running_loss / len(loader)\n# print(f\"Época completada — loss promedio: {avg_loss:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:44.858271Z","iopub.execute_input":"2025-05-19T02:18:44.858532Z","iopub.status.idle":"2025-05-19T02:18:44.873816Z","shell.execute_reply.started":"2025-05-19T02:18:44.858495Z","shell.execute_reply":"2025-05-19T02:18:44.873195Z"}},"outputs":[],"execution_count":69},{"cell_type":"markdown","source":"# build dataloaders","metadata":{}},{"cell_type":"code","source":"def build_dataloaders(\n    train_df,\n    val_df,\n    base_path,\n    patch_size,\n    num_bands,\n    batch_size,\n    augment,\n    num_workers=4,\n    drop_last=False\n):\n    \"\"\"\n    Construye y devuelve dos DataLoaders (train y validation)\n    para datos hiperespectrales.\n\n    Parámetros:\n      - train_df: pandas.DataFrame con columnas ['id','label'] para entrenamiento.\n      - val_df: pandas.DataFrame con columnas ['id','label'] para validación.\n      - base_path: ruta donde están los archivos .npy.\n      - patch_size: tamaño H×W de cada parche.\n      - num_bands: número de bandas espectrales.\n      - batch_size: tamaño de batch.\n      - augment: si aplicar data augmentation en train.\n      - num_workers: hilos para DataLoader.\n      - drop_last: descartar último batch incompleto.\n    \"\"\"\n    train_loader = create_hyperspectral_dataloader(\n        df=train_df,\n        base_path=base_path,\n        patch_size=patch_size,\n        num_bands=num_bands,\n        augment=augment,\n        batch_size=batch_size,\n        shuffle=True,\n        num_workers=num_workers,\n        drop_last=drop_last\n    )\n\n    val_loader = create_hyperspectral_dataloader(\n        df=val_df,\n        base_path=base_path,\n        patch_size=patch_size,\n        num_bands=num_bands,\n        augment=False,\n        batch_size=batch_size,\n        shuffle=False,\n        num_workers=num_workers,\n        drop_last=drop_last\n    )\n\n    return train_loader, val_loader","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:44.875755Z","iopub.execute_input":"2025-05-19T02:18:44.876006Z","iopub.status.idle":"2025-05-19T02:18:44.886705Z","shell.execute_reply.started":"2025-05-19T02:18:44.875989Z","shell.execute_reply":"2025-05-19T02:18:44.886039Z"}},"outputs":[],"execution_count":70},{"cell_type":"markdown","source":"# import and login wandb","metadata":{}},{"cell_type":"code","source":"%%capture\n! pip install --upgrade wandb","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:44.887365Z","iopub.execute_input":"2025-05-19T02:18:44.887566Z","iopub.status.idle":"2025-05-19T02:18:48.133404Z","shell.execute_reply.started":"2025-05-19T02:18:44.887538Z","shell.execute_reply":"2025-05-19T02:18:48.132606Z"}},"outputs":[],"execution_count":71},{"cell_type":"code","source":"import wandb\nwandb.login(key=\"98ffa71ad6bc0bc82603b546951d6460076caded\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:48.134678Z","iopub.execute_input":"2025-05-19T02:18:48.135384Z","iopub.status.idle":"2025-05-19T02:18:48.566707Z","shell.execute_reply.started":"2025-05-19T02:18:48.135356Z","shell.execute_reply":"2025-05-19T02:18:48.566072Z"}},"outputs":[{"name":"stderr","text":"\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n","output_type":"stream"},{"execution_count":72,"output_type":"execute_result","data":{"text/plain":"True"},"metadata":{}}],"execution_count":72},{"cell_type":"code","source":"def train():\n    #run wandb\n    wandb.init()\n    config = wandb.config\n    \n    #device\n    DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    #original image size\n    IMAGE_SIZE = 128\n    \n    #num clases\n    NUM_CLASES=100\n\n    #definition df train an test\n    df = pd.read_csv('/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-2025/train.csv')\n    base_path = '/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-2025/ot/ot'\n\n    #split 20% validation\n    train_df, val_df = train_test_split(\n        df,\n        test_size = 0.2,\n        random_state = 42\n    )\n\n    # create loaders\n    train_loader, val_loader = build_dataloaders(\n        train_df = train_df,\n        val_df = val_df,\n        base_path = base_path,\n        patch_size = config.patch_size,\n        num_bands = config.num_bands,\n        batch_size = config.batch_size,\n        augment=config.augment\n    )\n\n    # calc num patches\n    num_patches = (IMAGE_SIZE//config.patch_size)**2\n\n    # create model\n    model = create_spectral_transformer(\n        image_size=config.patch_size,\n        near_band=config.num_bands,\n        num_patches=num_patches,\n        num_classes=NUM_CLASES,\n        dim=config.dim,\n        depth=config.depth,\n        heads=config.heads,\n        mlp_dim=config.mlp_dim,\n        pool=config.pool,\n        mode=config.mode\n    ).to(DEVICE)\n\n    #select optimize\n    if config.optimizer == 'adam':\n        optimizer = optim.Adam(model.parameters(), lr=config.lr)\n    else:\n        optimizer = optim.SGD(model.parameters(), lr=config.lr)\n\n    # loss MSE\n    criterion = nn.MSELoss()\n\n    # eraly sttoping\n    best_val_mse = float('inf')\n    patience = 10\n    model_no_inprove = 0\n    num_epochs = 50\n\n    # output file\n    run_name = wandb.run.name\n    output_dir = \"/kaggle/working/models\"\n    os.makedirs(output_dir, exist_ok=True)\n    name_file = f\"{run_name}.pth\"\n    file_path = os.path.join(output_dir, name_file)\n\n    #trainin\n    for epoch in range(num_epochs):\n        \n        model.train()\n        train_loss = 0.0\n\n        for images, labels in train_loader:\n            #obtain images and labels\n            images = images.to(DEVICE)\n            labels = labels.float().to(DEVICE)\n\n            #patches\n            patches = rearrange(\n                images,\n                'b c (h p_h) (w p_w) -> b (h w) (p_h p_w c)',\n                p_h=config.patch_size,\n                p_w=config.patch_size\n            )\n\n            preds = model(patches)\n            loss  = criterion(preds, labels)\n\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            train_loss += loss.item()\n\n        train_mse = train_loss / len(train_loader)\n\n        # ---- Validation ----\n        model.eval()\n        val_loss = 0.0\n        with torch.no_grad():\n            for images, labels in val_loader:\n                images = images.to(DEVICE)\n                labels= labels.float().to(DEVICE)\n\n                patches = rearrange(\n                    images,\n                    'b c (h p_h) (w p_w) -> b (h w) (p_h p_w c)',\n                    p_h=config.patch_size,\n                    p_w=config.patch_size\n                )\n                preds = model(patches)\n                val_loss += criterion(preds, labels).item()\n\n        val_mse = val_loss / len(val_loader)\n\n        # ---- Logging ----\n        wandb.log({\n            'epoch': epoch+1,\n            'train_mse': train_mse,\n            'val_mse':   val_mse\n        })\n        \n        print(f\"Epoch {epoch+1}/{num_epochs} — train_mse: {train_mse:.4f}, val_mse: {val_mse:.4f}\")\n\n        # ---- Early Stopping Check ----\n        if val_mse < best_val_mse:\n            best_val_mse = val_mse\n            model_no_inprove = 0\n            # Guardar checkpoint\n            torch.save(model.state_dict(), file_path)\n            # Registrar artifact en W&B\n            wandb.save(name_file)\n        else:\n            model_no_inprove += 1\n            if model_no_inprove >= patience:\n                print(f\"No mejora tras {patience} épocas — deteniendo temprano.\")\n                break\n\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:48.567464Z","iopub.execute_input":"2025-05-19T02:18:48.567822Z","iopub.status.idle":"2025-05-19T02:18:48.579721Z","shell.execute_reply.started":"2025-05-19T02:18:48.567802Z","shell.execute_reply":"2025-05-19T02:18:48.5789Z"}},"outputs":[],"execution_count":73},{"cell_type":"markdown","source":"# init sweep and run wandb","metadata":{}},{"cell_type":"code","source":"wandb.agent(\n    sweep_id=\"eorwdes3\",\n    project=\"challenge opt\",\n    entity=\"ai-uis\",\n    function=train\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:48.58068Z","iopub.execute_input":"2025-05-19T02:18:48.581558Z","iopub.status.idle":"2025-05-19T02:18:48.598505Z","shell.execute_reply.started":"2025-05-19T02:18:48.581535Z","shell.execute_reply":"2025-05-19T02:18:48.597899Z"}},"outputs":[],"execution_count":74},{"cell_type":"code","source":"# !rm -rf \"/kaggle/working/models\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:48.599119Z","iopub.execute_input":"2025-05-19T02:18:48.599293Z","iopub.status.idle":"2025-05-19T02:18:48.613896Z","shell.execute_reply.started":"2025-05-19T02:18:48.599269Z","shell.execute_reply":"2025-05-19T02:18:48.613221Z"}},"outputs":[],"execution_count":75},{"cell_type":"markdown","source":"## Make submision","metadata":{}},{"cell_type":"code","source":"SWEEP_NAME = \"dutiful-sweep-404\"\n\nPATH_MODELS = \"/kaggle/working/models/\"\nPATH_PTH = PATH_MODELS+SWEEP_NAME+\".pth\"\n\nPATCH_SIZE  = 16\nNUM_BANDS   = 110\nORIG_SIZE   = 128\nNUM_CLASSES = 1     \n\nDIM         = 256\nDEPTH       = 4\nHEADS       = 16\nMLP_DIM     = 1024\n\nPOOL        = \"mean\"\nMODE        = 'ViT'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:48.614439Z","iopub.execute_input":"2025-05-19T02:18:48.614645Z","iopub.status.idle":"2025-05-19T02:18:48.640671Z","shell.execute_reply.started":"2025-05-19T02:18:48.61463Z","shell.execute_reply":"2025-05-19T02:18:48.639665Z"}},"outputs":[],"execution_count":76},{"cell_type":"code","source":"DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n\nmodel = create_spectral_transformer(\n    image_size   = PATCH_SIZE,        \n    near_band    = NUM_BANDS,          \n    num_patches  = (ORIG_SIZE//PATCH_SIZE)**2,\n    num_classes  = NUM_CLASSES,                  \n    dim          = DIM,\n    depth        = DEPTH,\n    heads        = HEADS,\n    mlp_dim      = MLP_DIM,\n    pool         = POOL,               \n    mode         = MODE\n).to(DEVICE)\n\nmodel.load_state_dict(torch.load(PATH_PTH, map_location=DEVICE))\nmodel.eval()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:48.64166Z","iopub.execute_input":"2025-05-19T02:18:48.641994Z","iopub.status.idle":"2025-05-19T02:18:49.007437Z","shell.execute_reply.started":"2025-05-19T02:18:48.641967Z","shell.execute_reply":"2025-05-19T02:18:49.00668Z"}},"outputs":[{"name":"stderr","text":"/tmp/ipykernel_31/1041277808.py:17: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n  model.load_state_dict(torch.load(PATH_PTH, map_location=DEVICE))\n","output_type":"stream"},{"execution_count":77,"output_type":"execute_result","data":{"text/plain":"ViT(\n  (patch_to_embedding): Linear(in_features=28160, out_features=256, bias=True)\n  (dropout): Dropout(p=0.0, inplace=False)\n  (transformer): Transformer(\n    (layers): ModuleList(\n      (0-3): 4 x ModuleList(\n        (0): Residual(\n          (fn): PreNorm(\n            (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n            (fn): Attention(\n              (to_qkv): Linear(in_features=256, out_features=768, bias=False)\n              (to_out): Sequential(\n                (0): Linear(in_features=256, out_features=256, bias=True)\n                (1): Dropout(p=0.0, inplace=False)\n              )\n            )\n          )\n        )\n        (1): Residual(\n          (fn): PreNorm(\n            (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n            (fn): FeedForward(\n              (net): Sequential(\n                (0): Linear(in_features=256, out_features=1024, bias=True)\n                (1): GELU(approximate='none')\n                (2): Dropout(p=0.0, inplace=False)\n                (3): Linear(in_features=1024, out_features=256, bias=True)\n                (4): Dropout(p=0.0, inplace=False)\n              )\n            )\n          )\n        )\n      )\n    )\n    (skipcat): ModuleList(\n      (0-1): 2 x Conv2d(65, 65, kernel_size=(1, 2), stride=(1, 1))\n    )\n  )\n  (to_latent): Identity()\n  (mlp_head): Sequential(\n    (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n    (1): Linear(in_features=256, out_features=1, bias=True)\n    (2): Sigmoid()\n  )\n)"},"metadata":{}}],"execution_count":77},{"cell_type":"code","source":"class TestHyperspectralDataset(Dataset):\n    def __init__(self, test_csv, base_path, patch_size=64, num_bands=100):\n        self.df = pd.read_csv(test_csv)\n        self.base_path = base_path\n        self.patch_size = patch_size\n        self.num_bands = num_bands\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = os.path.join(self.base_path, row['id'])\n        \n        try:\n            img = np.load(img_path)\n            \n            if len(img.shape) == 2:\n                img = np.repeat(img[:, :, np.newaxis], self.num_bands, axis=2)\n            elif len(img.shape) == 3:\n                if img.shape[2] > self.num_bands:\n                    img = img[:, :, :self.num_bands] \n                elif img.shape[2] < self.num_bands:\n                    pad_width = ((0, 0), (0, 0), (0, self.num_bands - img.shape[2]))\n                    img = np.pad(img, pad_width, mode='constant')\n            \n            normalized_img = np.zeros_like(img)\n            for band in range(img.shape[2]):\n                band_data = img[:, :, band]\n                if np.max(band_data) > 0:  \n                    normalized_img[:, :, band] = (band_data - np.min(band_data)) / (np.max(band_data) - np.min(band_data))\n            \n            img_tensor = torch.tensor(normalized_img, dtype=torch.float32).permute(2, 0, 1)\n            \n            if img_tensor.shape[1] != self.patch_size or img_tensor.shape[2] != self.patch_size:\n                img_tensor = F.interpolate(img_tensor.unsqueeze(0), \n                                         size=(self.patch_size, self.patch_size),\n                                         mode='bilinear').squeeze(0)\n            \n            return img_tensor, row['id']\n        \n        except Exception as e:\n            print(f\"Error loading {img_path}: {str(e)}\")\n            dummy_img = torch.zeros(self.num_bands, self.patch_size, self.patch_size)\n            return dummy_img, row['id']\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:49.008295Z","iopub.execute_input":"2025-05-19T02:18:49.008609Z","iopub.status.idle":"2025-05-19T02:18:49.017483Z","shell.execute_reply.started":"2025-05-19T02:18:49.008582Z","shell.execute_reply":"2025-05-19T02:18:49.016784Z"}},"outputs":[],"execution_count":78},{"cell_type":"markdown","source":"## Make submision","metadata":{}},{"cell_type":"code","source":"test_csv_path = '/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-2025/test.csv'\nbase_path = '/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-2025/ot/ot'\n\ntest_dataset = TestHyperspectralDataset(test_csv_path, base_path, num_bands=NUM_BANDS)\ntest_loader = DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\n\npredictions = []\nids = []\n\nmodel.eval()\nwith torch.no_grad():\n    for imgs, img_ids in test_loader:\n        imgs = imgs.to(DEVICE)  # [B, C, H, W]\n\n        patches = rearrange(\n            imgs,\n            'b c (h p_h) (w p_w) -> b (h w) (p_h p_w c)',\n            p_h=PATCH_SIZE,\n            p_w=PATCH_SIZE\n        )\n\n        # 2) Forward\n        preds = model(patches)\n        \n        preds = preds.cpu().tolist()\n\n        predictions.extend(preds)\n        ids.extend(img_ids)\n\n\noutput_dir = \"/kaggle/working/submissions\"\nos.makedirs(output_dir, exist_ok=True)\npath_submission = output_dir+\"/\"+SWEEP_NAME+\".csv\"\n\n# 4) Armado del submission\nsubmission_df = pd.DataFrame({\n    \"ID\":     ids,\n    \"TARGET\": predictions\n})\nsubmission_df.to_csv(path_submission, index=False)\nprint(submission_df.head(30))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:18:49.018233Z","iopub.execute_input":"2025-05-19T02:18:49.018512Z","iopub.status.idle":"2025-05-19T02:19:05.205854Z","shell.execute_reply.started":"2025-05-19T02:18:49.018486Z","shell.execute_reply":"2025-05-19T02:19:05.205022Z"}},"outputs":[{"name":"stdout","text":"                ID     TARGET\n0   sample1957.npy  47.850380\n1   sample1369.npy  47.850838\n2    sample782.npy  47.851048\n3   sample2255.npy  47.852268\n4   sample2160.npy  47.851116\n5   sample2140.npy  47.850761\n6    sample134.npy  47.849735\n7   sample1835.npy  47.851135\n8   sample2681.npy  47.851067\n9    sample777.npy  47.851528\n10  sample1636.npy  47.850563\n11  sample2060.npy  47.850849\n12  sample2509.npy  47.850372\n13  sample2534.npy  47.850990\n14    sample56.npy  47.851135\n15  sample1924.npy  47.851696\n16   sample712.npy  47.851055\n17   sample763.npy  47.847466\n18  sample1159.npy  47.850697\n19  sample2403.npy  47.852085\n20  sample1774.npy  47.850838\n21   sample364.npy  47.850353\n22  sample2015.npy  47.851246\n23   sample155.npy  47.851101\n24  sample1856.npy  47.851147\n25  sample1764.npy  47.851440\n26  sample1543.npy  47.851597\n27   sample132.npy  47.851078\n28  sample1941.npy  47.851135\n29   sample196.npy  47.851246\n","output_type":"stream"}],"execution_count":79},{"cell_type":"code","source":"# !rm /kaggle/working/models/flowing-sweep-24.pth","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T02:19:05.206807Z","iopub.execute_input":"2025-05-19T02:19:05.207123Z","iopub.status.idle":"2025-05-19T02:19:05.211295Z","shell.execute_reply.started":"2025-05-19T02:19:05.207097Z","shell.execute_reply":"2025-05-19T02:19:05.21054Z"}},"outputs":[],"execution_count":80}]}