{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"machine_shape":"hm","gpuType":"T4"},"accelerator":"GPU","kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":9797223,"sourceType":"datasetVersion","datasetId":982170},{"sourceId":266728918,"sourceType":"kernelVersion"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nsys.path.append(\"../input/tez-lib/\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T10:20:39.045803Z","iopub.execute_input":"2025-10-11T10:20:39.046107Z","iopub.status.idle":"2025-10-11T10:20:39.049809Z","shell.execute_reply.started":"2025-10-11T10:20:39.046087Z","shell.execute_reply":"2025-10-11T10:20:39.049072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random\nimport numpy as np\nimport torch\nimport os\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_everything(42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T10:20:39.051091Z","iopub.execute_input":"2025-10-11T10:20:39.051364Z","iopub.status.idle":"2025-10-11T10:20:39.071124Z","shell.execute_reply.started":"2025-10-11T10:20:39.051346Z","shell.execute_reply":"2025-10-11T10:20:39.070409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch.nn as nn\n\nfrom tez import Tez, TezConfig\nimport tez\nimport albumentations\nimport pandas as pd\nimport cv2\nimport numpy as np\nimport timm\nimport torch.nn as nn\nfrom sklearn import metrics\nimport torch\nfrom tez.callbacks import EarlyStopping\nfrom tqdm import tqdm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T10:20:39.071905Z","iopub.execute_input":"2025-10-11T10:20:39.072154Z","iopub.status.idle":"2025-10-11T10:20:39.084000Z","shell.execute_reply.started":"2025-10-11T10:20:39.072130Z","shell.execute_reply":"2025-10-11T10:20:39.083339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class args:\n    batch_size = 8\n    image_size = 384\n    epochs = 1\n    fold = 0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T10:20:39.084758Z","iopub.execute_input":"2025-10-11T10:20:39.084984Z","iopub.status.idle":"2025-10-11T10:20:39.098421Z","shell.execute_reply.started":"2025-10-11T10:20:39.084969Z","shell.execute_reply":"2025-10-11T10:20:39.097841Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CustomDataset:\n    def __init__(self, image_paths, dense_features, targets, augmentations):\n        self.image_paths = image_paths\n        self.dense_features = dense_features\n        self.targets = targets\n        self.augmentations = augmentations\n        \n    def __len__(self):\n        return len(self.image_paths)\n    \n    def __getitem__(self, item):\n        image = cv2.imread(self.image_paths[item])\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        if self.augmentations is not None:\n            augmented = self.augmentations(image=image)\n            image = augmented[\"image\"]\n            \n        image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n        \n        features = self.dense_features[item, :]\n        targets = self.targets[item]\n        \n        return {\n            \"image\": torch.tensor(image, dtype=torch.float),\n            # \"features\": torch.tensor(features, dtype=torch.float),\n            \"features\": torch.tensor(features, dtype=torch.float),\n            \"targets\": torch.tensor(targets, dtype=torch.float),\n        }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T10:20:39.100102Z","iopub.execute_input":"2025-10-11T10:20:39.100325Z","iopub.status.idle":"2025-10-11T10:20:39.115314Z","shell.execute_reply.started":"2025-10-11T10:20:39.100310Z","shell.execute_reply":"2025-10-11T10:20:39.114594Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CustomModel(nn.Module):\n    def __init__(self):\n        super().__init__()        \n        self.model = timm.create_model(\"tf_efficientnet_b3_ns\", pretrained=True, in_chans=3)\n\n        \n        self.dropout = nn.Dropout(0.5)\n        self.out = nn.Linear(1000, 1)\n        \n        self.step_scheduler_after = \"epoch\"\n\n\n    def monitor_metrics(self, outputs, targets, loss):\n        valid_binaryloss = loss\n        if str(valid_binaryloss) == 'nan':\n            valid_binaryloss = float('inf')\n        return {\"binaryloss\": valid_binaryloss}\n\n    def optimizer_scheduler(self):\n        opt = torch.optim.AdamW(self.parameters(), lr=2.5e-05, weight_decay=0.01)\n        sch = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(\n            opt, T_0=10, T_mult=1, eta_min=1e-6, last_epoch=-1\n        )\n        return opt,sch\n\n    def forward(self, image, features, targets=None):\n\n        x = self.model(image)\n        x = self.dropout(x)\n        x = self.out(x)\n\n        if targets is not None:\n            loss = nn.BCEWithLogitsLoss()(x, targets.view(-1, 1).type_as(x))\n            metrics = self.monitor_metrics(x, targets, loss)\n            return x, loss, metrics\n        return x, 0, {}\n\n# # # === PATCH E: stronger backbone + metadata head ===\n# import timm, torch\n# import torch.nn as nn\n# import torch.optim as optim\n\n# class CustomModel(nn.Module):\n#     def __init__(self, model_name=\"tf_efficientnet_b3_ns\", n_meta=0, pretrained=True):\n#         super().__init__()\n#         self.backbone = timm.create_model(model_name, pretrained=pretrained, num_classes=0, global_pool=\"avg\")\n#         in_feats = self.backbone.num_features\n\n#         self.use_meta = n_meta > 0\n#         if self.use_meta:\n#             self.meta = nn.Sequential(\n#                 nn.Linear(n_meta, 32),\n#                 nn.ReLU(inplace=True),\n#                 nn.Dropout(0.2),\n#             )\n#             in_feats += 32\n\n#         self.head = nn.Sequential(\n#             nn.Dropout(0.3),\n#             nn.Linear(in_feats, 1)\n#         )\n\n#         # Tez expects a loss function on the model\n#         self.loss_fn = nn.BCEWithLogitsLoss()   # you can overwrite with pos_weight later\n\n#     def forward(self, image=None, features=None, targets=None, **kwargs):\n#         \"\"\"\n#         Tez passes batch via kwargs: image, features, targets.\n#         Support both your older (x_img/x_meta) style and Tez style.\n#         \"\"\"\n#         # Fallbacks for any legacy names, if present\n#         x_img = kwargs.get(\"x_img\", image)\n#         x_meta = kwargs.get(\"x_meta\", features)\n\n#         f = self.backbone(x_img)                   # [B, C]\n#         if self.use_meta and x_meta is not None:\n#             m = self.meta(x_meta)                  # [B, 32]\n#             f = torch.cat([f, m], dim=1)\n\n#         logits = self.head(f).view(-1)             # [B]\n\n#         loss = None\n#         if targets is not None:\n#             loss = self.loss_fn(logits, targets.float())\n\n#         return logits, loss\n\n#     def optimizer_scheduler(self, *args, **kwargs):\n#         \"\"\"\n#         Tez-compat: some versions call with (train_len, epochs),\n#         others call with no args. Handle both.\n#         \"\"\"\n#         import math\n#         epochs = kwargs.get(\"epochs\", None)\n#         if epochs is None and len(args) >= 2:\n#             # legacy: (train_len, epochs)\n#             epochs = args[1]\n#         if epochs is None:\n#             epochs = 10  # safe default; Tez may override lr per-epoch anyway\n    \n#         optimizer = optim.AdamW(self.parameters(), lr=1e-3, weight_decay=1e-4)\n#         scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=max(int(epochs), 1))\n#         return optimizer, scheduler\n    \n#     # Optional shims for older Tez variants that look for these names:\n#     def optimizer(self):\n#         return optim.AdamW(self.parameters(), lr=1e-3, weight_decay=1e-4)\n    \n#     def scheduler(self, optimizer):\n#         return optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T10:59:43.622344Z","iopub.execute_input":"2025-10-11T10:59:43.622645Z","iopub.status.idle":"2025-10-11T10:59:43.631444Z","shell.execute_reply.started":"2025-10-11T10:59:43.622621Z","shell.execute_reply":"2025-10-11T10:59:43.630577Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_aug = albumentations.Compose(\n    [\n        albumentations.LongestMaxSize(args.image_size, p=1),\n        albumentations.PadIfNeeded(args.image_size,args.image_size, p=1,border_mode=0),\n        \n       albumentations.HorizontalFlip(p=0.5),\n       albumentations.VerticalFlip(p=0.1),\n       albumentations.Rotate(limit=180, p=0.5),\n       albumentations.ShiftScaleRotate(\n                shift_limit=0.1, scale_limit=0.1, rotate_limit=45, p=0.5\n            ),\n        \n        albumentations.HueSaturationValue(\n            hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5\n        ),\n        albumentations.RandomBrightnessContrast(\n            brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5\n        ),\n        albumentations.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225],\n            max_pixel_value=255.0,\n            p=1.0,\n        ),\n    ],\n    p=1.0,\n)\n\nvalid_aug = albumentations.Compose(\n    [\n        albumentations.LongestMaxSize(args.image_size, p=1),\n        albumentations.PadIfNeeded(args.image_size,args.image_size, p=1,border_mode=0),\n        albumentations.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225],\n            max_pixel_value=255.0,\n            p=1.0,\n        ),\n    ],\n    p=1.0,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T10:56:19.888781Z","iopub.execute_input":"2025-10-11T10:56:19.889029Z","iopub.status.idle":"2025-10-11T10:56:19.902526Z","shell.execute_reply.started":"2025-10-11T10:56:19.889012Z","shell.execute_reply":"2025-10-11T10:56:19.901805Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/creating-folds-siim-isic-melanoma/train_5folds.csv\")\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T10:56:20.318398Z","iopub.execute_input":"2025-10-11T10:56:20.319250Z","iopub.status.idle":"2025-10-11T10:56:20.391961Z","shell.execute_reply.started":"2025-10-11T10:56:20.319193Z","shell.execute_reply":"2025-10-11T10:56:20.391162Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedGroupKFold\n\nif (\"kfold\" not in df.columns) or (df[\"kfold\"].nunique() < 2):\n    df = df.sample(frac=1.0, random_state=42).reset_index(drop=True)\n    sgkf = StratifiedGroupKFold(n_splits=5, shuffle=True, random_state=42)\n    df[\"kfold\"] = -1\n    for f, (tr, va) in enumerate(sgkf.split(df, df[\"target\"], groups=df[\"patient_id\"])):\n        df.loc[va, \"kfold\"] = f\n\nmeta_cat = [\"sex\", \"anatom_site_general_challenge\"]\ndf = pd.get_dummies(df, columns=meta_cat, dummy_na=True)\n\nmeta_onehot_cols = [c for c in df.columns if c.startswith(\"sex_\") or c.startswith(\"anatom_site_general_challenge_\")]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T10:56:21.297726Z","iopub.execute_input":"2025-10-11T10:56:21.298008Z","iopub.status.idle":"2025-10-11T10:56:21.320629Z","shell.execute_reply.started":"2025-10-11T10:56:21.297985Z","shell.execute_reply":"2025-10-11T10:56:21.319725Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in [0,1,2]:\n    print(f\"training fold: {i} start\")\n    args.fold = i\n    df_train = df[df.kfold != i].reset_index(drop=True)\n    df_valid = df[df.kfold == i].reset_index(drop=True)\n\n    N_pos = int((df_train.target == 1).sum())\n    N_neg = int((df_train.target == 0).sum())\n    pos_weight = torch.tensor([N_neg / max(1, N_pos)], dtype=torch.float32)\n    \n    age_mean = df_train[\"age_approx\"].mean()\n    age_std  = df_train[\"age_approx\"].std() + 1e-6\n    df_train.loc[:, \"age_approx\"] = (df_train[\"age_approx\"] - age_mean) / age_std\n    df_valid.loc[:, \"age_approx\"] = (df_valid[\"age_approx\"] - age_mean) / age_std\n\n    if True:  \n        pos_idx = df_train.index[df_train.target == 1]\n        neg_idx = df_train.index[df_train.target == 0]\n        rep = max(1, len(neg_idx) // max(1, len(pos_idx)))\n        new_idx = list(neg_idx) + list(np.repeat(pos_idx, rep))\n        df_train = df_train.loc[new_idx].sample(frac=1.0, random_state=42).reset_index(drop=True)\n\n    dense_features = [\"age_approx\"] + meta_onehot_cols\n\n    df_train[\"age_approx\"] = df_train[\"age_approx\"].fillna(0.0)\n    df_valid[\"age_approx\"] = df_valid[\"age_approx\"].fillna(0.0)\n    \n    for c in dense_features:\n        df_train[c] = pd.to_numeric(df_train[c], errors=\"coerce\").fillna(0.0)\n        df_valid[c] = pd.to_numeric(df_valid[c], errors=\"coerce\").fillna(0.0)\n    \n    X_train_meta = np.asarray(df_train[dense_features].values, dtype=np.float32)\n    X_valid_meta = np.asarray(df_valid[dense_features].values, dtype=np.float32)\n\n    train_img_paths = [f\"/kaggle/input/siim-isic-melanoma-classification/jpeg/train/{x}.jpg\"\n                   for x in df_train[\"image_name\"].values]\n    valid_img_paths = [f\"/kaggle/input/siim-isic-melanoma-classification/jpeg/train/{x}.jpg\"\n                   for x in df_valid[\"image_name\"].values]\n\n    train_dataset = CustomDataset(\n        image_paths=train_img_paths,\n        dense_features=X_train_meta,\n        targets=df_train.target.values,\n        augmentations=train_aug,\n    )\n    \n    valid_dataset = CustomDataset(\n        image_paths=valid_img_paths,\n        dense_features=X_valid_meta,\n        targets=df_valid.target.values,\n        augmentations=valid_aug,\n    )\n\n    n_meta = len(dense_features)\n    model = CustomModel() \n    model = Tez(model)\n    \n    config = TezConfig(\n    training_batch_size=args.batch_size,\n    validation_batch_size=2 * args.batch_size,\n    epochs=args.epochs,\n    step_scheduler_after=\"epoch\",\n    step_scheduler_metric=\"valid_binaryloss\",\n    fp16=True,\n    )\n    \n    es = EarlyStopping(\n        monitor=\"valid_binaryloss\",\n        model_path=f\"model_f{i}.bin\",\n        patience=4,\n        mode=\"min\",\n        save_weights_only=True,\n    )\n    model.fit(train_dataset, valid_dataset=valid_dataset, callbacks=[es], config=config)\n    print(f\"training fold: {i} complete\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T11:01:10.313305Z","iopub.execute_input":"2025-10-11T11:01:10.313902Z","iopub.status.idle":"2025-10-11T11:01:42.514958Z","shell.execute_reply.started":"2025-10-11T11:01:10.313877Z","shell.execute_reply":"2025-10-11T11:01:42.513399Z"}},"outputs":[],"execution_count":null}]}