{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"},{"sourceId":1339680,"sourceType":"datasetVersion","datasetId":756214},{"sourceId":1339691,"sourceType":"datasetVersion","datasetId":756247},{"sourceId":1339694,"sourceType":"datasetVersion","datasetId":756315},{"sourceId":1353805,"sourceType":"datasetVersion","datasetId":762181},{"sourceId":1353810,"sourceType":"datasetVersion","datasetId":762191},{"sourceId":1353811,"sourceType":"datasetVersion","datasetId":762203},{"sourceId":1444814,"sourceType":"datasetVersion","datasetId":846815}],"dockerImageVersionId":30919,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Setup and Configuration","metadata":{}},{"cell_type":"code","source":"import os\nos.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-18T11:28:23.055307Z","iopub.execute_input":"2025-04-18T11:28:23.055595Z","iopub.status.idle":"2025-04-18T11:28:23.059359Z","shell.execute_reply.started":"2025-04-18T11:28:23.055559Z","shell.execute_reply":"2025-04-18T11:28:23.058566Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import time\n\nNOTEBOOK_START_TIME = time.time()\n# Set maximum runtime in seconds (11 hours 25 minutes to leave buffer)\nMAX_RUNTIME_SECONDS = 11 * 3600 + 25 * 60\nprint(f\"Notebook execution started at: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(NOTEBOOK_START_TIME))}\")\nprint(f\"Maximum training runtime set to: {MAX_RUNTIME_SECONDS / 3600:.2f} hours ({MAX_RUNTIME_SECONDS} seconds)\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!nvidia-smi","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-18T11:28:23.060713Z","iopub.execute_input":"2025-04-18T11:28:23.061127Z","iopub.status.idle":"2025-04-18T11:28:23.299011Z","shell.execute_reply.started":"2025-04-18T11:28:23.061072Z","shell.execute_reply":"2025-04-18T11:28:23.297940Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import subprocess\nimport sys\n\ndef install_package(package, upgrade=False):\n \"\"\"Installs a package, suppressing stdout and stderr.\"\"\"\n try:\n command = [sys.executable, \"-m\", \"pip\", \"install\"]\n if upgrade:\n command.append(\"--upgrade\")\n command.append(package)\n\n # Redirect stdout and stderr to /dev/null (or equivalent)\n with open('/dev/null', 'w') as devnull:\n subprocess.check_call(command, stdout=devnull, stderr=devnull)\n print(f\"Successfully installed/upgraded: {package}\") # Inform user\n except subprocess.CalledProcessError as e:\n print(f\"Error installing {package}: {e}\", file=sys.stderr)\n except Exception as e: # Catch other potential exceptions\n print(f\"An unexpected error occurred: {e}\", file=sys.stderr)\n\n# Example usage: (replace with your package list)\ninstall_package(\"git+https://github.com/ildoonet/pytorch-gradual-warmup-lr.git\")\ninstall_package(\"geffnet\")\ninstall_package(\"albumentations\", upgrade=True) # Example with upgrade\ninstall_package(\"wandb\")\ninstall_package(\"opencv-python\")\ninstall_package(\"pytz\")\ninstall_package(\"timm\", upgrade=True)\n# Add near your other install commands\ninstall_package(\"grad-cam\") \ninstall_package(\"ttach\") # pytorch-grad-cam sometimes uses this\n\nprint(\"All packages installed.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-18T11:28:23.300880Z","iopub.execute_input":"2025-04-18T11:28:23.301140Z","iopub.status.idle":"2025-04-18T11:29:06.630217Z","shell.execute_reply.started":"2025-04-18T11:28:23.301106Z","shell.execute_reply":"2025-04-18T11:29:06.629329Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Standard Libraries\nimport os\nimport time\nimport warnings\nimport logging\nimport subprocess\nimport traceback\nfrom datetime import datetime\nfrom tqdm import tqdm\nfrom tqdm.notebook import tqdm as tqdm_notebook\n\n# Data Handling and Visualization\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport PIL.Image\nimport cv2\nimport re\n# Machine Learning and Metrics\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.metrics import (\n roc_auc_score, f1_score, accuracy_score, precision_score, recall_score,\n classification_report, confusion_matrix, roc_curve, precision_recall_curve\n)\nfrom sklearn.calibration import calibration_curve\nfrom scipy.stats import wilcoxon\n\n# Deep Learning Frameworks\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import (\n TensorDataset, DataLoader, Dataset, RandomSampler, SubsetRandomSampler,\n SequentialSampler, WeightedRandomSampler\n)\nfrom torch.amp import autocast as amp_autocast, GradScaler\nimport torchvision\nimport torchvision.transforms as transforms\nimport math\n# Model Architectures and Utilities\nimport timm\nfrom timm import create_model\nimport geffnet\nfrom transformers import ViTFeatureExtractor, ViTModel, ViTConfig, SwinConfig, SwinModel\n\n# Learning Rate Schedulers\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, ReduceLROnPlateau\nfrom warmup_scheduler import GradualWarmupScheduler\n# Distributed Training\nfrom accelerate import Accelerator, notebook_launcher\nfrom torch.utils.data.distributed import DistributedSampler\n\n# Metrics and Evaluation\nfrom torchmetrics.classification import (\n MulticlassAccuracy, MulticlassF1Score, MulticlassAUROC, MulticlassConfusionMatrix,\n BinaryAUROC\n)\nfrom torchmetrics.functional.classification import binary_accuracy, binary_f1_score\n\n# Image Augmentation\nimport albumentations as A\n\nfrom skimage.segmentation import slic\n\n# Timezone Handling\nimport pytz\nfrom typing import Dict, Optional, Union\n\n# Experiment Tracking\nimport wandb\n%matplotlib inline\ndevice = torch.device('cuda')\nwarnings.filterwarnings(\"ignore\")\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\nos.environ['PYTHONWARNINGS'] = 'ignore'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-18T11:29:06.631473Z","iopub.execute_input":"2025-04-18T11:29:06.631700Z","iopub.status.idle":"2025-04-18T11:29:34.221321Z","shell.execute_reply.started":"2025-04-18T11:29:06.631679Z","shell.execute_reply":"2025-04-18T11:29:34.220599Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nmy_secret = user_secrets.get_secret(\"wandb_api_key\") \nwandb.login(key=my_secret)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-18T11:29:34.222057Z","iopub.execute_input":"2025-04-18T11:29:34.222581Z","iopub.status.idle":"2025-04-18T11:29:41.208130Z","shell.execute_reply.started":"2025-04-18T11:29:34.222556Z","shell.execute_reply":"2025-04-18T11:29:41.207438Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Parameter Configuration","metadata":{}},{"cell_type":"code","source":"import re\n\n# --- Core Run Configuration ---\n# SET THE CONFIGURATION FOR THIS SPECIFIC RUN:\nmodel_type = 'swin_only' # Options: 'efficientnetv2', 'hybrid_swin', 'swin_only'\n# Use the torch.hub names now for V2\ncnn_backbone_name = 'efficientnet_v2_m' # e.g., 'efficientnet_v2_s', 'efficientnet_v2_m', 'efficientnet_v2_l'\n# Use the TIMM name for Swin\ntransformer_backbone_name = 'swin_base_patch4_window12_384.ms_in22k_ft_in1k' # Keep this for hybrid\nuse_amp = True\nuse_external = True\nuse_meta = True\nDEBUG = False\ntarget_total_epochs = 30 \n# --- Standard Settings ---\nnum_workers = 4\ninit_lr = 1e-3 # Base LR\n\n# --- REMOVED Pretrained File Settings ---\n# model_dir = '../input/melanoma-winning-models'\n# pretrained_type = ...\n# i_fold = ...\n# model_file = ...\n\nprint(f\"Run Config: Model={model_type}, CNN={cnn_backbone_name}, UseMeta={use_meta}, UseExt={use_external}\")\n\n# --- Helper Function ---\ndef get_v2_size_map(v2_name_str):\n if 'v2_s' in v2_name_str: return 's'\n if 'v2_m' in v2_name_str: return 'm'\n if 'v2_l' in v2_name_str: return 'l'\n print(f\"Warning: Could not map V2 size from '{v2_name_str}'. Defaulting to 'm'.\")\n return 'm'\n\n# --- Determine V2 Size and TARGET Image Size ---\ncnn_size_tag = get_v2_size_map(cnn_backbone_name)\ntarget_image_sizes_map = {'s': 384, 'm': 448, 'l': 480}\nimage_size = 384\n# --- CORRECTED LINE ---\nif model_type in ['hybrid_swin', 'swin_only'] and '384' in transformer_backbone_name:\n# --- END CORRECTION ---\n print(f\"NOTE: Forcing image_size to 384 for Swin model compatibility.\")\n image_size = 384\nprint(f\"CNN Size Tag: {cnn_size_tag}, Target Image Size: {image_size}x{image_size}\")\n\n# --- Generate kernel_type ---\nkernel_base = f\"{cnn_backbone_name.replace('_','').replace('efficientnet','')}\" # e.g., effnetv2m\nif model_type == 'hybrid_swin': kernel_base += f\"_{transformer_backbone_name.split('_')[0]}\"\nkernel_type = f\"{kernel_base}_{image_size}{'_ext' if use_external else ''}{'_meta' if use_meta else ''}\"\nprint(f\"Generated kernel_type: {kernel_type}\")\n\n# --- Determine Data Directory Size ---\navailable_data_sizes = [256, 384, 512]; data_dir_size = 384 if image_size <= 384 else 512\nprint(f\"Selected Data Directory Size: {data_dir_size}x{data_dir_size}\")\n\n# --- Set Data Directory Paths ---\ndata_dir = f'../input/jpeg-melanoma-{data_dir_size}x{data_dir_size}'\ndata_dir2 = f'../input/jpeg-isic2019-{data_dir_size}x{data_dir_size}' if use_external else None\nprint(f\"Comp Data Dir: {data_dir}\" + (f\", Ext Data Dir: {data_dir2}\" if data_dir2 else \"\"))\n\n# --- Dynamic Variables Note ---\nprint(\"Note: Batch size, accum steps, epoch phases are set dynamically.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:50.417989Z","iopub.execute_input":"2025-04-12T19:32:50.418516Z","iopub.status.idle":"2025-04-12T19:32:50.428369Z","shell.execute_reply.started":"2025-04-12T19:32:50.418494Z","shell.execute_reply":"2025-04-12T19:32:50.427493Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Scaling Factors","metadata":{}},{"cell_type":"code","source":"def configure_dynamic_parameters(model_type, cnn_backbone_name, use_meta, use_external):\n \"\"\"\n Calculates base scaling factors based ONLY on CNN size ('s', 'm', 'l').\n Adaptations based on meta, external data, or hybrid type are REMOVED\n to ensure consistent LR/Regularization multipliers across runs for purity.\n Returns factors that will be applied later (e.g., in LLRD, WD, Dropout init).\n \"\"\"\n cnn_size_tag = get_v2_size_map(cnn_backbone_name)\n print(f\"Configuring BASE scaling factors based on CNN size ONLY for: type='{model_type}', cnn='{cnn_backbone_name}' ({cnn_size_tag}), meta={use_meta}, external={use_external}\")\n print(\" NOTE: Multipliers for meta, external, hybrid are DISABLED for purity.\")\n\n # --- Base Scales based ONLY on CNN Size ---\n # Still apply a slight adjustment based on model scale, as this is common practice.\n size_to_b_map = {'s': 3, 'm': 5, 'l': 7}\n equiv_b_version = size_to_b_map.get(cnn_size_tag, 5)\n # Base LR factor: Slightly lower for S, slightly higher for L relative to M (applied to base_lr later)\n lr_base_scale = { 3: 0.9, 5: 1.0, 7: 1.1 } # Adjusted slightly: s: 0.9, m: 1.0, l: 1.1\n # Base Regularization factor: Slightly higher for S, slightly lower for L (applied to WD/Dropout later)\n reg_base_scale = { 3: 1.05, 5: 1.0, 7: 0.95 } # Adjusted slightly: s: 1.05, m: 1.0, l: 0.95\n \n current_base_lr_factor = lr_base_scale.get(equiv_b_version, 1.0)\n current_base_reg_factor = reg_base_scale.get(equiv_b_version, 1.0)\n print(f\" Base factors based on CNN size '{cnn_size_tag}': LR Factor={current_base_lr_factor:.2f}, Reg Factor={current_base_reg_factor:.2f}\")\n\n # --- Combine (Now only uses base factors) ---\n final_lr_factor = current_base_lr_factor # * 1.0 (removed multiplier)\n final_reg_factor = current_base_reg_factor # * 1.0 (removed multiplier)\n\n # Clamp to reasonable bounds (still useful for the base factors)\n final_lr_factor = max(0.5, min(final_lr_factor, 1.5)) # Clamp LR factor bounds if needed\n final_reg_factor = max(0.5, min(final_reg_factor, 1.5)) # Clamp Reg factor bounds if needed\n\n # Rename variables for clarity (these factors modify base LR/WD/Dropout later)\n lr_adaptation_factor = round(final_lr_factor, 6)\n regularization_factor = round(final_reg_factor, 6)\n\n print(f\" ==> Final Factors (Based ONLY on CNN Size): LR Adapt Factor={lr_adaptation_factor}, Reg Factor={regularization_factor}\")\n\n # Return only the two factors\n return lr_adaptation_factor, regularization_factor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:50.429676Z","iopub.execute_input":"2025-04-12T19:32:50.429923Z","iopub.status.idle":"2025-04-12T19:32:50.446564Z","shell.execute_reply.started":"2025-04-12T19:32:50.429898Z","shell.execute_reply":"2025-04-12T19:32:50.445744Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Preparation","metadata":{}},{"cell_type":"code","source":"# Load test data\ndf_test = pd.read_csv(os.path.join(data_dir, 'test.csv'))\ndf_test['filepath'] = df_test['image_name'].apply(lambda x: os.path.join(data_dir, 'test', f'{x}.jpg'))\nprint(\"Test Data Loaded - Shape:\", df_test.shape)\nprint(\"Test Data Sample:\\n\", df_test.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:50.447376Z","iopub.execute_input":"2025-04-12T19:32:50.447645Z","iopub.status.idle":"2025-04-12T19:32:50.530078Z","shell.execute_reply.started":"2025-04-12T19:32:50.447624Z","shell.execute_reply":"2025-04-12T19:32:50.529289Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load train data and filter\ndf_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))\nprint(\"Initial Train Data Shape:\", df_train.shape)\ndf_train = df_train[df_train['tfrecord'] != -1].reset_index(drop=True)\nprint(\"Train Data Shape after tfrecord filter:\", df_train.shape)\ndf_train['is_ext'] = 0\ndf_train['filepath'] = df_train['image_name'].apply(lambda x: os.path.join(data_dir, 'train', f'{x}.jpg'))\n\n# Clean diagnosis labels\ndf_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('seborrheic keratosis', 'BKL'))\ndf_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('lichenoid keratosis', 'BKL'))\ndf_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('solar lentigo', 'BKL'))\ndf_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('lentigo NOS', 'BKL'))\ndf_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('cafe-au-lait macule', 'unknown'))\ndf_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('atypical melanocytic proliferation', 'unknown'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:50.530815Z","iopub.execute_input":"2025-04-12T19:32:50.531131Z","iopub.status.idle":"2025-04-12T19:32:50.736369Z","shell.execute_reply.started":"2025-04-12T19:32:50.531107Z","shell.execute_reply":"2025-04-12T19:32:50.735539Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Add external data if enabled\nif use_external:\n df_train2 = pd.read_csv(os.path.join(data_dir2, 'train.csv'))\n print(\"External Train Data Shape:\", df_train2.shape)\n df_train2 = df_train2[df_train2['tfrecord'] >= 0].reset_index(drop=True)\n df_train2['is_ext'] = 1\n df_train2['filepath'] = df_train2['image_name'].apply(lambda x: os.path.join(data_dir2, 'train', f'{x}.jpg'))\n df_train2['diagnosis'] = df_train2['diagnosis'].apply(lambda x: x.replace('NV', 'nevus'))\n df_train2['diagnosis'] = df_train2['diagnosis'].apply(lambda x: x.replace('MEL', 'melanoma'))\n print(\"External Data Diagnosis Unique:\", df_train2['diagnosis'].unique())\n \n # Combine datasets\n df_train = pd.concat([df_train, df_train2]).reset_index(drop=True)\n print(\"Combined Train Data Shape:\", df_train.shape)\n \n# Add assertion to ensure data isn't empty\nassert not df_train.empty, \"Error: df_train is empty after preparation!\"\nassert 'diagnosis' in df_train.columns, \"Error: 'diagnosis' column missing in df_train!\"\n\n# Map diagnosis to target indices\ndiagnosis2idx = {d: idx for idx, d in enumerate(sorted(df_train.diagnosis.unique()))}\ndf_train['target'] = df_train['diagnosis'].map(diagnosis2idx)\nmel_idx = diagnosis2idx['melanoma']\nprint(\"Diagnosis to Index Mapping:\", diagnosis2idx)\nprint(\"Target Value Counts:\\n\", df_train['target'].value_counts())\n\n# Dynamically set out_dim\nout_dim = len(df_train['target'].unique())\nprint(f\"Number of unique classes (out_dim): {out_dim}\")\nprint(f\"Melanoma index (mel_idx): {mel_idx}\")\n\n# Final assertions to verify critical variables\nassert mel_idx in df_train['target'].values, f\"Error: mel_idx ({mel_idx}) not found in target values!\"\nassert out_dim > 1, \"Error: out_dim is 1 or less, indicating no class variation!\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:50.737048Z","iopub.execute_input":"2025-04-12T19:32:50.737250Z","iopub.status.idle":"2025-04-12T19:32:50.921110Z","shell.execute_reply.started":"2025-04-12T19:32:50.737231Z","shell.execute_reply":"2025-04-12T19:32:50.920299Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Class Distribution","metadata":{}},{"cell_type":"code","source":"# Class distribution\nclass_counts = df_train['diagnosis'].value_counts()\ntotal_samples = len(df_train)\nclass_percentages = (class_counts / total_samples) * 100\n\nplt.figure(figsize=(12, 6))\nsns.barplot(x=class_counts.index, y=class_counts.values, palette='viridis')\nplt.title('Distribution of Diagnosis Classes', fontsize=16)\nplt.xlabel('Diagnosis', fontsize=14)\nplt.ylabel('Count', fontsize=14)\nplt.xticks(rotation=45)\nfor i, count in enumerate(class_counts):\n plt.text(i, count + 0.5, f'{class_percentages[i]:.2f}%', ha='center', fontsize=12)\nplt.tight_layout()\nplt.savefig('class_distribution.png', dpi=300)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:51.495801Z","iopub.execute_input":"2025-04-12T19:32:51.496055Z","iopub.status.idle":"2025-04-12T19:32:52.235701Z","shell.execute_reply.started":"2025-04-12T19:32:51.496034Z","shell.execute_reply":"2025-04-12T19:32:52.234679Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preprocess Meta Data","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm # Ensure this import is present\n\nif use_meta:\n # One-hot encoding of anatom_site_general_challenge feature\n print(\"One-hot encoding 'anatom_site_general_challenge'...\")\n concat = pd.concat([df_train['anatom_site_general_challenge'], df_test['anatom_site_general_challenge']], ignore_index=True)\n dummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')\n df_train = pd.concat([df_train, dummies.iloc[:df_train.shape[0]]], axis=1)\n df_test = pd.concat([df_test, dummies.iloc[df_train.shape[0]:].reset_index(drop=True)], axis=1)\n \n # Sex features\n print(\"Encoding 'sex' feature...\")\n df_train['sex'] = df_train['sex'].map({'male': 1, 'female': 0})\n df_test['sex'] = df_test['sex'].map({'male': 1, 'female': 0})\n df_train['sex'] = df_train['sex'].fillna(-1)\n df_test['sex'] = df_test['sex'].fillna(-1)\n \n # Age features\n print(\"Normalizing 'age_approx' feature...\")\n df_train['age_approx'] /= 90\n df_test['age_approx'] /= 90\n df_train['age_approx'] = df_train['age_approx'].fillna(0)\n df_test['age_approx'] = df_test['age_approx'].fillna(0)\n \n # Patient ID features\n print(\"Handling 'patient_id' feature...\")\n df_train['patient_id'] = df_train['patient_id'].fillna(0)\n \n # n_images per user\n print(\"Calculating 'n_images' per patient...\")\n df_train['n_images'] = df_train.patient_id.map(df_train.groupby(['patient_id']).image_name.count())\n df_test['n_images'] = df_test.patient_id.map(df_test.groupby(['patient_id']).image_name.count())\n df_train.loc[df_train['patient_id'] == -1, 'n_images'] = 1\n df_train['n_images'] = np.log1p(df_train['n_images'].values)\n df_test['n_images'] = np.log1p(df_test['n_images'].values)\n \n # Image size\n print(\"Calculating image sizes...\")\n train_images = df_train['filepath'].values\n train_sizes = np.zeros(train_images.shape[0])\n for i, img_path in enumerate(tqdm(train_images, desc=\"Processing training images\", unit=\"image\")):\n train_sizes[i] = os.path.getsize(img_path)\n df_train['image_size'] = np.log(train_sizes)\n \n test_images = df_test['filepath'].values\n test_sizes = np.zeros(test_images.shape[0])\n for i, img_path in enumerate(tqdm(test_images, desc=\"Processing test images\", unit=\"image\")):\n test_sizes[i] = os.path.getsize(img_path)\n df_test['image_size'] = np.log(test_sizes)\n # Improved age normalization\n mean_age = df_train['age_approx'].mean()\n std_age = df_train['age_approx'].std()\n df_train['age_approx'] = (df_train['age_approx'].fillna(mean_age) - mean_age) / std_age\n df_test['age_approx'] = (df_test['age_approx'].fillna(mean_age) - mean_age) / std_age\n \n # Log-transformed features standardization\n df_train['n_images'] = (df_train['n_images'] - df_train['n_images'].mean()) / df_train['n_images'].std()\n df_test['n_images'] = (df_test['n_images'] - df_train['n_images'].mean()) / df_train['n_images'].std()\n # Meta features\n meta_features = ['sex', 'age_approx', 'n_images', 'image_size'] + [col for col in df_train.columns if col.startswith('site_')]\n n_meta_features = len(meta_features)\n print(f\"Meta features created: {meta_features}\")\nelse:\n n_meta_features = 0\n print(\"Meta features disabled.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:52.237637Z","iopub.execute_input":"2025-04-12T19:32:52.237874Z","iopub.status.idle":"2025-04-12T19:32:52.249599Z","shell.execute_reply.started":"2025-04-12T19:32:52.237854Z","shell.execute_reply":"2025-04-12T19:32:52.248649Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_meta_features","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:52.250534Z","iopub.execute_input":"2025-04-12T19:32:52.250844Z","iopub.status.idle":"2025-04-12T19:32:52.270598Z","shell.execute_reply.started":"2025-04-12T19:32:52.250812Z","shell.execute_reply":"2025-04-12T19:32:52.269684Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Display a random sample of 5 rows\nprint(df_train.sample(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:52.271434Z","iopub.execute_input":"2025-04-12T19:32:52.271747Z","iopub.status.idle":"2025-04-12T19:32:52.290039Z","shell.execute_reply.started":"2025-04-12T19:32:52.271717Z","shell.execute_reply":"2025-04-12T19:32:52.289211Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Define Dataset","metadata":{}},{"cell_type":"code","source":"class SIIMISICDataset(Dataset):\n def __init__(self, csv, split, mode, transform=None):\n\n self.csv = csv.reset_index(drop=True)\n self.split = split\n self.mode = mode\n self.transform = transform\n\n def __len__(self):\n return self.csv.shape[0]\n\n def __getitem__(self, index):\n row = self.csv.iloc[index]\n \n image = cv2.imread(row.filepath)\n image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n\n if self.transform is not None:\n res = self.transform(image=image)\n image = res['image'].astype(np.float32)\n else:\n image = image.astype(np.float32)\n\n image = image.transpose(2, 0, 1)\n\n if use_meta:\n data = (torch.tensor(image).float(), torch.tensor(self.csv.iloc[index][meta_features]).float())\n else:\n data = torch.tensor(image).float()\n\n if self.mode == 'test':\n return data\n else:\n return data, torch.tensor(self.csv.iloc[index].target).long()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:52.290844Z","iopub.execute_input":"2025-04-12T19:32:52.291145Z","iopub.status.idle":"2025-04-12T19:32:52.306128Z","shell.execute_reply.started":"2025-04-12T19:32:52.291124Z","shell.execute_reply":"2025-04-12T19:32:52.305328Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Augmentations","metadata":{}},{"cell_type":"code","source":"import albumentations as A # Use A alias if used elsewhere, or albumentations directly\n\nprint(f\"Defining standard training and validation transforms using image_size: {image_size}\")\n\ntransforms_train = A.Compose([\n A.Transpose(p=0.5),\n A.VerticalFlip(p=0.5),\n A.HorizontalFlip(p=0.5),\n # A.RandomBrightness(limit=0.2, p=0.75), # Older version uses this\n # A.RandomContrast(limit=0.2, p=0.75), # Older version uses this\n A.RandomBrightnessContrast(limit=0.2, p=0.75), # Newer version combines these\n\n A.OneOf([\n A.MotionBlur(blur_limit=5),\n A.MedianBlur(blur_limit=5),\n A.GaussianBlur(blur_limit=5),\n A.GaussNoise(var_limit=(5.0, 30.0)),\n ], p=0.7),\n\n A.OneOf([\n A.OpticalDistortion(distort_limit=1.0),\n A.GridDistortion(num_steps=5, distort_limit=1.),\n A.ElasticTransform(alpha=3),\n ], p=0.7),\n\n A.CLAHE(clip_limit=4.0, p=0.7),\n A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=10, p=0.5),\n A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=15, border_mode=0, p=0.85),\n A.Resize(image_size, image_size), # Ensure image_size is defined before this cell\n # A.Cutout(max_h_size=int(image_size * 0.375), max_w_size=int(image_size * 0.375), num_holes=1, p=0.7), # Older Cutout\n A.CoarseDropout(max_holes=1, max_height=int(image_size * 0.375), max_width=int(image_size * 0.375),\n min_holes=1, min_height=int(image_size * 0.1), min_width=int(image_size * 0.1), # Define min sizes\n fill_value=0, p=0.7), # Newer CoarseDropout is preferred replacement for Cutout\n A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)) # Standard normalization\n])\n\ntransforms_val = A.Compose([\n A.Resize(image_size, image_size),\n A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225))\n])\n\nprint(\"Standard transforms_train and transforms_val defined.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:52.307037Z","iopub.execute_input":"2025-04-12T19:32:52.307306Z","iopub.status.idle":"2025-04-12T19:32:52.342689Z","shell.execute_reply.started":"2025-04-12T19:32:52.307272Z","shell.execute_reply":"2025-04-12T19:32:52.341965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Minimal transform for original image (just resize and normalize)\ntransforms_original = A.Compose([\n A.Resize(image_size, image_size), # Match augmented image size\n A.Normalize()\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:52.343481Z","iopub.execute_input":"2025-04-12T19:32:52.343755Z","iopub.status.idle":"2025-04-12T19:32:52.348543Z","shell.execute_reply.started":"2025-04-12T19:32:52.343727Z","shell.execute_reply":"2025-04-12T19:32:52.347738Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Pastikan df_train sudah ada dan memiliki kolom 'filepath'\n# Contoh load jika belum ada (SESUAIKAN PATH)\nif 'df_train' not in locals():\n print(\"Warning: 'df_train' not found. Attempting to load example data structure.\")\n # Ganti path ini sesuai lokasi data Anda jika menjalankan di luar Kaggle\n data_dir_example = '../input/jpeg-melanoma-384x384' # Ganti jika perlu\n train_csv_path = os.path.join(data_dir_example, 'train.csv')\n if os.path.exists(train_csv_path):\n df_train = pd.read_csv(train_csv_path)\n # Buat kolom filepath jika belum ada\n if 'filepath' not in df_train.columns:\n image_folder = os.path.join(data_dir_example, 'train')\n df_train['filepath'] = df_train['image_name'].apply(lambda x: os.path.join(image_folder, f'{x}.jpg'))\n df_train = df_train.dropna(subset=['filepath']).reset_index(drop=True) # Hapus baris jika path tidak valid\n print(f\"Loaded df_train with {len(df_train)} samples.\")\n else:\n print(f\"Error: Train CSV not found at {train_csv_path}. Cannot proceed with visualization.\")\n df_train = pd.DataFrame() # Buat dataframe kosong\n\nimport cv2\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport albumentations as A\nimport random\nimport os\nimport pandas as pd\nimport sys\n\n# --- Konfigurasi & Prasyarat (Sama seperti sebelumnya) ---\n# ... (pastikan image_size, transforms_train, df_train didefinisikan) ...\n\n# --- Fungsi Denormalisasi (Sama seperti sebelumnya) ---\ndef denormalize(img_array, mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)):\n \"\"\"Reverses the normalization applied by A.Normalize for visualization.\"\"\"\n std_array = np.array(std).reshape(1, 1, 3)\n mean_array = np.array(mean).reshape(1, 1, 3)\n std_array[std_array == 0] = 1e-6 \n img_denorm = (img_array * std_array) + mean_array\n img_denorm = np.clip(img_denorm, 0, 1)\n return img_denorm\n\n# --- Visualisasi ---\nNUM_AUG_EXAMPLES = 2 # Anda bisa tetap 4 atau sesuaikan\nFIG_WIDTH_PER_PLOT = 6 # Lebar per subplot\nFIG_HEIGHT = 7 # Tinggi figure keseluruhan (lebih tinggi)\n\nif 'df_train' in locals() and not df_train.empty:\n try:\n sample_row = df_train.sample(1).iloc[0]\n image_path = sample_row['filepath']\n image_name = sample_row.get('image_name', os.path.basename(image_path))\n\n print(f\"Loading image: {image_path}\")\n original_img_bgr = cv2.imread(image_path)\n\n if original_img_bgr is None:\n print(f\"Error: Could not read image at {image_path}\", file=sys.stderr)\n else:\n original_img_rgb = cv2.cvtColor(original_img_bgr, cv2.COLOR_BGR2RGB)\n original_img_resized = cv2.resize(original_img_rgb, (image_size, image_size))\n original_img_display = original_img_resized / 255.0\n\n # Buat figure dengan ukuran yang disesuaikan\n fig, axes = plt.subplots(1, NUM_AUG_EXAMPLES + 1,\n figsize=(FIG_WIDTH_PER_PLOT * (NUM_AUG_EXAMPLES + 1), FIG_HEIGHT))\n plt.suptitle(f\"Augmentation Examples for: {image_name}\", fontsize=18, y=0.98) # Sesuaikan posisi Y\n\n # Tampilkan gambar asli\n axes[0].imshow(original_img_display)\n axes[0].set_title(\"Original (Resized)\", fontsize=12)\n axes[0].axis('off')\n\n # Terapkan augmentasi dan tampilkan\n for i in range(NUM_AUG_EXAMPLES):\n augmented_data = transforms_train(image=original_img_rgb)\n augmented_img_normalized = augmented_data['image']\n augmented_img_display = denormalize(augmented_img_normalized)\n\n axes[i+1].imshow(augmented_img_display)\n\n # Generate caption (sama seperti sebelumnya)\n potential_transforms = []\n if random.random() < 0.5: potential_transforms.append(\"Flip H\")\n if random.random() < 0.5: potential_transforms.append(\"Flip V\")\n if random.random() < 0.5: potential_transforms.append(\"Transpose\")\n if random.random() < 0.85: potential_transforms.append(\"Shift/Scale/Rotate\")\n if random.random() < 0.75: potential_transforms.append(\"Bright/Contrast\") # Persingkat\n if random.random() < 0.5: potential_transforms.append(\"HSV\")\n if random.random() < 0.7: potential_transforms.append(\"CLAHE\")\n if random.random() < 0.7: potential_transforms.append(\"Blur/Noise\")\n if random.random() < 0.7: potential_transforms.append(\"Distortion\")\n if random.random() < 0.7: potential_transforms.append(\"Dropout\")\n\n caption = f\"Augmented {i+1}\"\n if potential_transforms:\n # Batasi jumlah item di caption jika terlalu panjang\n caption += \"\\n(\" + \", \".join(potential_transforms[:4]) + (\"...\" if len(potential_transforms)>4 else \"\") + \")\"\n else:\n caption += \"\\n(No major transforms)\"\n\n axes[i+1].set_title(caption, fontsize=10) # Ukuran font judul subplot\n axes[i+1].axis('off')\n\n # Gunakan subplots_adjust untuk spasi\n # Sesuaikan nilai wspace (horizontal), hspace (vertical), top, bottom, left, right\n plt.subplots_adjust(wspace=0.1, hspace=0.1, top=0.85, bottom=0.05, left=0.05, right=0.95)\n\n plt.show()\n\n except FileNotFoundError:\n print(f\"Error: File not found at {image_path}. Ensure dataset path is correct.\", file=sys.stderr)\n except Exception as e:\n print(f\"An error occurred during visualization: {e}\", file=sys.stderr)\n import traceback\n traceback.print_exc()\nelse:\n print(\"Skipping visualization because df_train is not loaded or is empty.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:52.349426Z","iopub.execute_input":"2025-04-12T19:32:52.349708Z","iopub.status.idle":"2025-04-12T19:32:53.681221Z","shell.execute_reply.started":"2025-04-12T19:32:52.349671Z","shell.execute_reply":"2025-04-12T19:32:53.679003Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Create datasets\n# df_show = df_train.sample(min(1000, len(df_train))) # Sample dataset\n# dataset_original = SIIMISICDataset(df_show, 'train', 'train', transform=transforms_original) # Original images\n# dataset_augmented = SIIMISICDataset(df_show, 'train', 'train', transform=transforms_train) # Augmented images\n\n# # Reverse mapping for labels (assume diagnosis2idx is defined earlier)\n# idx2diagnosis = {v: k for k, v in diagnosis2idx.items()}\n\n# # Display original and augmented images side by side\n# from pylab import rcParams\n# rcParams['figure.figsize'] = 20, 10\n\n# for i in range(2): # Show 2 rows\n# f, axarr = plt.subplots(2, 5) # 2 rows: original (top), augmented (bottom); 5 columns\n# for p in range(5):\n# idx = np.random.randint(0, len(dataset_original)) # Same index for both datasets\n \n# # Original image\n# img_original, label_tensor = dataset_original[idx]\n# if use_meta:\n# img_original = img_original[0] # Extract image tensor if metadata is used\n# label_idx = label_tensor.item()\n# label_name = idx2diagnosis[label_idx]\n \n# # Augmented image\n# img_augmented, _ = dataset_augmented[idx] # Same index, ignore label since it’s identical\n# if use_meta:\n# img_augmented = img_augmented[0]\n\n# # Plot original (top row)\n# axarr[0, p].imshow(img_original.transpose(0, 1).transpose(1, 2).squeeze())\n# axarr[0, p].set_title(f\"Original: {label_name}\")\n# axarr[0, p].axis('off')\n\n# # Plot augmented (bottom row)\n# axarr[1, p].imshow(img_augmented.transpose(0, 1).transpose(1, 2).squeeze())\n# axarr[1, p].set_title(f\"Augmented: {label_name}\")\n# axarr[1, p].axis('off')\n\n# plt.tight_layout()\n# plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:53.682243Z","iopub.execute_input":"2025-04-12T19:32:53.682568Z","iopub.status.idle":"2025-04-12T19:32:53.686577Z","shell.execute_reply.started":"2025-04-12T19:32:53.682537Z","shell.execute_reply":"2025-04-12T19:32:53.685801Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Pretrained Model Helper","metadata":{}},{"cell_type":"code","source":"# --- Torchvision EfficientNet Implementation (Corrected Syntax) ---\nimport torch\nimport torch.nn as nn\nfrom torch import Tensor\nfrom dataclasses import dataclass\nfrom functools import partial\nfrom typing import Any, Callable, Dict, List, Optional, Sequence, Union, Tuple\nimport math\nimport copy\n\n# --- Essential Dependencies from Torchvision ---\n# Make sure torchvision is installed (`pip install torchvision`)\ntry:\n from torchvision.ops import StochasticDepth\n from torchvision.ops.misc import Conv2dNormActivation, SqueezeExcitation\n from torchvision.models._api import WeightsEnum, Weights\n from torchvision.models._utils import handle_legacy_interface, _ovewrite_named_param\n from torch.hub import load_state_dict_from_url\n print(\"Successfully imported required components from torchvision.\")\nexcept ImportError as e:\n print(f\"ERROR: Failed to import components from torchvision: {e}\")\n print(\"Please ensure torchvision is installed and compatible.\")\n # Raising an error might be better than continuing with non-functional dummies\n raise ImportError(\"Torchvision components required for EfficientNet definition are missing.\") from e\n\n# Utility function (ensure this is defined correctly)\ndef _make_divisible(v: float, divisor: int, min_value: Optional[int] = None) -> int:\n if min_value is None: min_value = divisor\n new_v = max(min_value, int(v + divisor / 2) // divisor * divisor)\n if new_v < 0.9 * v: new_v += divisor\n return new_v\n\n# --- Core Configuration Classes ---\n@dataclass\nclass _MBConvConfig:\n expand_ratio: float\n kernel: int\n stride: int\n input_channels: int\n out_channels: int\n num_layers: int\n block: Callable[..., nn.Module]\n\n @staticmethod\n def adjust_channels(channels: int, width_mult: float, min_value: Optional[int]=None) -> int:\n return _make_divisible(channels * width_mult, 8, min_value)\n\nclass MBConvConfig(_MBConvConfig):\n def __init__(self, expand_ratio: float, kernel: int, stride: int, input_channels: int,\n out_channels: int, num_layers: int, width_mult: float = 1.0,\n depth_mult: float = 1.0, block: Optional[Callable[..., nn.Module]] = None):\n input_channels = self.adjust_channels(input_channels, width_mult)\n out_channels = self.adjust_channels(out_channels, width_mult)\n num_layers = self.adjust_depth(num_layers, depth_mult)\n if block is None:\n block = MBConv\n super().__init__(expand_ratio, kernel, stride, input_channels, out_channels, num_layers, block)\n\n @staticmethod\n def adjust_depth(num_layers: int, depth_mult: float):\n return int(math.ceil(num_layers * depth_mult))\n\nclass FusedMBConvConfig(_MBConvConfig):\n def __init__(self, expand_ratio: float, kernel: int, stride: int, input_channels: int,\n out_channels: int, num_layers: int, block: Optional[Callable[..., nn.Module]] = None):\n if block is None:\n block = FusedMBConv\n super().__init__(expand_ratio, kernel, stride, input_channels, out_channels, num_layers, block)\n\n# --- Convolutional Block Classes ---\nclass MBConv(nn.Module):\n def __init__(self, cnf: MBConvConfig, stochastic_depth_prob: float,\n norm_layer: Callable[..., nn.Module], se_layer: Callable[..., nn.Module] = SqueezeExcitation):\n super().__init__()\n if not (1 <= cnf.stride <= 2): raise ValueError(\"illegal stride value\")\n\n self.use_res_connect = cnf.stride == 1 and cnf.input_channels == cnf.out_channels\n layers: List[nn.Module] = []\n activation_layer = nn.SiLU\n\n # Expand phase\n expanded_channels = cnf.adjust_channels(cnf.input_channels, cnf.expand_ratio)\n if expanded_channels != cnf.input_channels:\n layers.append(Conv2dNormActivation(cnf.input_channels, expanded_channels, kernel_size=1,\n norm_layer=norm_layer, activation_layer=activation_layer))\n # Depthwise convolution phase\n layers.append(Conv2dNormActivation(expanded_channels, expanded_channels, kernel_size=cnf.kernel,\n stride=cnf.stride, groups=expanded_channels, norm_layer=norm_layer,\n activation_layer=activation_layer))\n # Squeeze and excitation layer\n squeeze_channels = max(1, cnf.input_channels // 4)\n layers.append(se_layer(expanded_channels, squeeze_channels, activation=partial(nn.SiLU, inplace=True)))\n # Pointwise convolution phase\n layers.append(Conv2dNormActivation(expanded_channels, cnf.out_channels, kernel_size=1,\n norm_layer=norm_layer, activation_layer=None))\n\n self.block = nn.Sequential(*layers)\n self.stochastic_depth = StochasticDepth(stochastic_depth_prob, \"row\")\n self.out_channels = cnf.out_channels\n\n def forward(self, input: Tensor) -> Tensor:\n result = self.block(input)\n if self.use_res_connect:\n result = self.stochastic_depth(result)\n result += input\n return result\n\nclass FusedMBConv(nn.Module):\n def __init__(self, cnf: FusedMBConvConfig, stochastic_depth_prob: float,\n norm_layer: Callable[..., nn.Module]):\n super().__init__()\n if not (1 <= cnf.stride <= 2): raise ValueError(\"illegal stride value\")\n\n self.use_res_connect = cnf.stride == 1 and cnf.input_channels == cnf.out_channels\n layers: List[nn.Module] = []\n activation_layer = nn.SiLU\n\n expanded_channels = cnf.adjust_channels(cnf.input_channels, cnf.expand_ratio)\n if expanded_channels != cnf.input_channels:\n # Fused expand\n layers.append(Conv2dNormActivation(cnf.input_channels, expanded_channels, kernel_size=cnf.kernel,\n stride=cnf.stride, norm_layer=norm_layer, activation_layer=activation_layer))\n # Project\n layers.append(Conv2dNormActivation(expanded_channels, cnf.out_channels, kernel_size=1,\n norm_layer=norm_layer, activation_layer=None))\n else:\n # Single convolution\n layers.append(Conv2dNormActivation(cnf.input_channels, cnf.out_channels, kernel_size=cnf.kernel,\n stride=cnf.stride, norm_layer=norm_layer, activation_layer=activation_layer))\n\n self.block = nn.Sequential(*layers)\n self.stochastic_depth = StochasticDepth(stochastic_depth_prob, \"row\")\n self.out_channels = cnf.out_channels\n\n def forward(self, input: Tensor) -> Tensor:\n result = self.block(input)\n if self.use_res_connect:\n result = self.stochastic_depth(result)\n result += input\n return result\n\n# --- EfficientNet Main Class (Using Torchvision Structure) ---\nclass EfficientNet(nn.Module):\n def __init__(\n self,\n inverted_residual_setting: Sequence[Union[MBConvConfig, FusedMBConvConfig]],\n dropout: float,\n stochastic_depth_prob: float = 0.2,\n num_classes: int = 1000,\n norm_layer: Optional[Callable[..., nn.Module]] = None,\n last_channel: Optional[int] = None,\n ) -> None:\n super().__init__()\n if not inverted_residual_setting: raise ValueError(\"Setting should not be empty\")\n if not (isinstance(inverted_residual_setting, Sequence) and all([isinstance(s, _MBConvConfig) for s in inverted_residual_setting])):\n raise TypeError(\"Setting should be List[_MBConvConfig]\")\n\n norm_layer = norm_layer or nn.BatchNorm2d\n layers: List[nn.Module] = []\n\n # Building first layer\n firstconv_output_channels = inverted_residual_setting[0].input_channels\n layers.append(Conv2dNormActivation(3, firstconv_output_channels, kernel_size=3, stride=2,\n norm_layer=norm_layer, activation_layer=nn.SiLU))\n\n # Building inverted residual blocks\n total_stage_blocks = sum(cnf.num_layers for cnf in inverted_residual_setting)\n stage_block_id = 0\n for cnf in inverted_residual_setting:\n stage: List[nn.Module] = []\n for _ in range(cnf.num_layers):\n block_cnf = copy.copy(cnf)\n if stage: # if not the first block in stage\n block_cnf.input_channels = block_cnf.out_channels\n block_cnf.stride = 1\n sd_prob = stochastic_depth_prob * float(stage_block_id) / total_stage_blocks\n stage.append(block_cnf.block(block_cnf, sd_prob, norm_layer))\n stage_block_id += 1\n layers.append(nn.Sequential(*stage))\n\n # Building last several layers\n lastconv_input_channels = inverted_residual_setting[-1].out_channels\n lastconv_output_channels = last_channel if last_channel is not None else 4 * lastconv_input_channels\n layers.append(Conv2dNormActivation(lastconv_input_channels, lastconv_output_channels, kernel_size=1,\n norm_layer=norm_layer, activation_layer=nn.SiLU))\n\n self.features = nn.Sequential(*layers)\n self.avgpool = nn.AdaptiveAvgPool2d(1)\n self.classifier = nn.Sequential(\n nn.Dropout(p=dropout, inplace=True),\n nn.Linear(lastconv_output_channels, num_classes),\n )\n\n # Weight initialization (standard torchvision practice)\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n nn.init.kaiming_normal_(m.weight, mode=\"fan_out\", nonlinearity=\"relu\")\n if m.bias is not None: nn.init.zeros_(m.bias)\n elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):\n nn.init.ones_(m.weight); nn.init.zeros_(m.bias)\n elif isinstance(m, nn.Linear):\n init_range = 1.0 / math.sqrt(m.out_features)\n nn.init.uniform_(m.weight, -init_range, init_range)\n nn.init.zeros_(m.bias)\n\n def forward(self, x: Tensor) -> Tensor:\n x = self.features(x)\n x = self.avgpool(x)\n x = torch.flatten(x, 1)\n x = self.classifier(x)\n return x\n\n# --- Configuration Function ---\ndef _efficientnet_conf(arch: str, **kwargs: Any) -> Tuple[Sequence[Union[MBConvConfig, FusedMBConvConfig]], Optional[int]]:\n # Defines layer configurations for each variant\n if arch.startswith(\"efficientnet_b\"):\n b_conf=partial(MBConvConfig,width_mult=kwargs.pop(\"width_mult\"),depth_mult=kwargs.pop(\"depth_mult\")); setting=[b_conf(1,3,1,32,16,1), b_conf(6,3,2,16,24,2), b_conf(6,5,2,24,40,2), b_conf(6,3,2,40,80,3), b_conf(6,5,1,80,112,3), b_conf(6,5,2,112,192,4), b_conf(6,3,1,192,320,1)]; last_ch=None\n elif arch == \"efficientnet_v2_s\": setting=[FusedMBConvConfig(1,3,1,24,24,2), FusedMBConvConfig(4,3,2,24,48,4), FusedMBConvConfig(4,3,2,48,64,4), MBConvConfig(4,3,2,64,128,6), MBConvConfig(6,3,1,128,160,9), MBConvConfig(6,3,2,160,256,15)]; last_ch=1280\n elif arch == \"efficientnet_v2_m\": setting=[FusedMBConvConfig(1,3,1,24,24,3), FusedMBConvConfig(4,3,2,24,48,5), FusedMBConvConfig(4,3,2,48,80,5), MBConvConfig(4,3,2,80,160,7), MBConvConfig(6,3,1,160,176,14), MBConvConfig(6,3,2,176,304,18), MBConvConfig(6,3,1,304,512,5)]; last_ch=1280\n elif arch == \"efficientnet_v2_l\": setting=[FusedMBConvConfig(1,3,1,32,32,4), FusedMBConvConfig(4,3,2,32,64,7), FusedMBConvConfig(4,3,2,64,96,7), MBConvConfig(4,3,2,96,192,10), MBConvConfig(6,3,1,192,224,19), MBConvConfig(6,3,2,224,384,25), MBConvConfig(6,3,1,384,640,7)]; last_ch=1280\n else: raise ValueError(f\"Unsupported model arch: {arch}\")\n return setting, last_ch\n\n# --- Define Weights Classes (Simplified - URLs assume availability) ---\n_COMMON_META_V2 = {\"categories\": [\"placeholder\"], \"min_size\": (33, 33)} # Need actual categories\nclass EfficientNet_V2_S_Weights(WeightsEnum):\n IMAGENET1K_V1=Weights(url=\"https://download.pytorch.org/models/efficientnet_v2_s-dd5fe13b.pth\", transforms=None, meta={**_COMMON_META_V2, \"num_params\": 21458488}) # Transforms defined elsewhere\n DEFAULT = IMAGENET1K_V1\nclass EfficientNet_V2_M_Weights(WeightsEnum):\n IMAGENET1K_V1=Weights(url=\"https://download.pytorch.org/models/efficientnet_v2_m-dc08266a.pth\", transforms=None, meta={**_COMMON_META_V2, \"num_params\": 54139356})\n DEFAULT = IMAGENET1K_V1\nclass EfficientNet_V2_L_Weights(WeightsEnum):\n IMAGENET1K_V1=Weights(url=\"https://download.pytorch.org/models/efficientnet_v2_l-59c71312.pth\", transforms=None, meta={**_COMMON_META_V2, \"num_params\": 118515272})\n DEFAULT = IMAGENET1K_V1\n# Model builder function (CORRECTED for num_classes handling)\ndef _efficientnet_model_builder(\n arch: str,\n inverted_residual_setting: Sequence[Union[MBConvConfig, FusedMBConvConfig]],\n dropout: float,\n last_channel: Optional[int],\n weights: Optional[WeightsEnum], # Pass weights object directly\n progress: bool,\n **kwargs: Any,\n) -> EfficientNet:\n\n # --- CORRECTED num_classes Logic ---\n # If loading weights, ALWAYS build the initial structure with the original\n # number of classes the weights were trained on (1000 for ImageNet1k).\n # Ignore kwargs['num_classes'] at this stage if weights are provided.\n if weights is not None:\n num_classes_for_build = 1000 # Hardcode for standard ImageNet1k weights\n print(f\"Building initial structure for {arch} with {num_classes_for_build} classes to load weights.\")\n # If not loading weights, use kwargs['num_classes'] if provided, else default.\n elif \"num_classes\" in kwargs:\n num_classes_for_build = kwargs[\"num_classes\"]\n print(f\"Building structure for {arch} with {num_classes_for_build} classes (random init).\")\n else:\n num_classes_for_build = 1000 # Default if no weights and no override\n print(f\"Building structure for {arch} with default {num_classes_for_build} classes (random init).\")\n # --- END CORRECTION ---\n\n # Clean kwargs before passing to EfficientNet constructor to avoid conflicts\n build_kwargs = kwargs.copy()\n if \"num_classes\" in build_kwargs: # Remove num_classes if it exists in kwargs\n del build_kwargs[\"num_classes\"]\n\n # Build model structure\n model = EfficientNet(\n inverted_residual_setting,\n dropout,\n last_channel=last_channel,\n num_classes=num_classes_for_build, # Use the correctly determined num_classes\n **build_kwargs # Pass other kwargs like norm_layer\n )\n\n if weights is not None:\n # Load state dict using the weights object\n print(f\"Loading state_dict from weights for {arch}...\")\n try:\n model.load_state_dict(weights.get_state_dict(progress=progress, check_hash=True))\n print(\"State dict loaded successfully.\")\n except Exception as e:\n print(f\"ERROR loading state_dict: {e}\")\n print(\"Continuing without pretrained weights for this backbone.\")\n # Optionally: re-initialize model randomly? Or just proceed?\n\n return model\n\n# --- Keep the helper functions efficientnet_v2_s/m/l as they were in the previous block ---\n# They correctly pass the weights object to the corrected builder.\n\n@handle_legacy_interface(weights=(\"pretrained\", EfficientNet_V2_S_Weights.IMAGENET1K_V1))\ndef efficientnet_v2_s(*, weights: Optional[EfficientNet_V2_S_Weights] = None, progress: bool = True, **kwargs: Any) -> EfficientNet:\n weights = EfficientNet_V2_S_Weights.verify(weights)\n inv_res_setting, last_ch = _efficientnet_conf(\"efficientnet_v2_s\")\n model = _efficientnet_model_builder(\"efficientnet_v2_s\", inv_res_setting, kwargs.pop(\"dropout\", 0.2), last_ch, weights, progress, norm_layer=partial(nn.BatchNorm2d, eps=1e-03), **kwargs)\n return model\n\n@handle_legacy_interface(weights=(\"pretrained\", EfficientNet_V2_M_Weights.IMAGENET1K_V1))\ndef efficientnet_v2_m(*, weights: Optional[EfficientNet_V2_M_Weights] = None, progress: bool = True, **kwargs: Any) -> EfficientNet:\n weights = EfficientNet_V2_M_Weights.verify(weights)\n inv_res_setting, last_ch = _efficientnet_conf(\"efficientnet_v2_m\")\n model = _efficientnet_model_builder(\"efficientnet_v2_m\", inv_res_setting, kwargs.pop(\"dropout\", 0.3), last_ch, weights, progress, norm_layer=partial(nn.BatchNorm2d, eps=1e-03), **kwargs)\n return model\n\n@handle_legacy_interface(weights=(\"pretrained\", EfficientNet_V2_L_Weights.IMAGENET1K_V1))\ndef efficientnet_v2_l(*, weights: Optional[EfficientNet_V2_L_Weights] = None, progress: bool = True, **kwargs: Any) -> EfficientNet:\n weights = EfficientNet_V2_L_Weights.verify(weights)\n inv_res_setting, last_ch = _efficientnet_conf(\"efficientnet_v2_l\")\n model = _efficientnet_model_builder(\"efficientnet_v2_l\", inv_res_setting, kwargs.pop(\"dropout\", 0.4), last_ch, weights, progress, norm_layer=partial(nn.BatchNorm2d, eps=1e-03), **kwargs)\n return model\n\n\nprint(\"Torchvision EfficientNet V2 classes and helpers defined (Corrected Syntax).\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:53.687297Z","iopub.execute_input":"2025-04-12T19:32:53.687594Z","iopub.status.idle":"2025-04-12T19:32:53.753239Z","shell.execute_reply.started":"2025-04-12T19:32:53.687560Z","shell.execute_reply":"2025-04-12T19:32:53.752326Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Architecture","metadata":{}},{"cell_type":"code","source":"class MetadataProcessor(nn.Module): \n def __init__(self, n_meta_features, output_dim=128, hidden_factor=2, dropout_p=0.3): \n super().__init__()\n hidden_dim = max(output_dim * hidden_factor, n_meta_features // 2)\n self.mlp = nn.Sequential(\n nn.Linear(n_meta_features, hidden_dim),\n nn.BatchNorm1d(hidden_dim),\n nn.SiLU(),\n nn.Dropout(p=dropout_p),\n nn.Linear(hidden_dim, output_dim), \n nn.BatchNorm1d(output_dim),\n nn.SiLU()\n )\n print(f\"Initialized MetadataProcessor (MLP): Input={n_meta_features} -> Hidden={hidden_dim} -> Output={output_dim}\")\n\n def forward(self, meta):\n return self.mlp(meta)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class EffNetV2Model(nn.Module):\n def __init__(self, backbone_name, out_dim, n_meta_features=0, pretrained=True, regularization_factor=1.0):\n super().__init__()\n self.n_meta_features = n_meta_features\n self.backbone_name = backbone_name\n self.out_dim = out_dim\n self.regularization_factor = regularization_factor\n # --- Load CNN Backbone ---\n print(f\"Instantiating EffNetV2Model ({backbone_name})...\")\n model_fn_map={'efficientnet_v2_s':efficientnet_v2_s, 'efficientnet_v2_m':efficientnet_v2_m, 'efficientnet_v2_l':efficientnet_v2_l}\n if backbone_name not in model_fn_map: raise ValueError(f\"Unsupported CNN: {backbone_name}\")\n self.cnn_backbone = model_fn_map[backbone_name](weights='DEFAULT' if pretrained else None)\n print(f\" Loaded {backbone_name}. Pretrained={pretrained}\")\n if isinstance(self.cnn_backbone.classifier, nn.Sequential) and isinstance(self.cnn_backbone.classifier[1], nn.Linear):\n self.cnn_features_dim = self.cnn_backbone.classifier[1].in_features\n self.cnn_backbone.classifier = nn.Identity()\n print(f\" CNN features: {self.cnn_features_dim}. Classifier removed.\")\n else: raise AttributeError(f\"Unexpected classifier structure {backbone_name}\")\n # Define BASE initial rates\n BASE_INITIAL_CLASSIFIER_DROPOUT = 0.40\n self.initial_classifier_dropout = min(0.9, max(0.05, BASE_INITIAL_CLASSIFIER_DROPOUT * self.regularization_factor))\n print(f\" Regularization Factor: {self.regularization_factor:.4f}\")\n print(f\" Base Classifier Dropout: {BASE_INITIAL_CLASSIFIER_DROPOUT:.2f} -> Effective Initial: {self.initial_classifier_dropout:.4f}\")\n self.image_dropout = nn.Dropout(0.0) \n self.classifier_dropout = nn.Dropout(self.initial_classifier_dropout)\n meta_pathway_dim = 0\n self.meta_processor = None \n if n_meta_features > 0:\n meta_output_dim = 128\n self.meta_processor = MetadataProcessor(n_meta_features, output_dim=meta_output_dim, dropout_p=0.3)\n meta_pathway_dim = meta_output_dim\n print(f\" - Meta Pathway Enabled ({n_meta_features} -> {meta_pathway_dim}) using MetadataProcessor (MLP Internals)\")\n else:\n print(\" - Meta Pathway: Disabled\")\n # --- Final Classifier ---\n self.myfc = nn.Linear(self.cnn_features_dim + meta_pathway_dim, out_dim)\n self.current_epoch = 0\n\n def set_epoch(self, epoch): \n self.current_epoch = epoch\n max_epoch_anneal = 25.0 \n min_dropout_mult = 0.3 \n decay_power = 1.5\n progress = min(epoch / max_epoch_anneal, 1.0)\n current_mult = 1.0 - (1.0 - min_dropout_mult) * (progress**decay_power)\n self.classifier_dropout.p = max(0.01, self.initial_classifier_dropout * current_mult)\n def forward(self, x, x_meta=None):\n x_features = self.cnn_backbone(x)\n if self.n_meta_features > 0 and self.meta_processor is not None:\n if x_meta is None: raise ValueError(\"x_meta required\");\n proc_meta = self.meta_processor(x_meta.to(x_features.device)) \n combined = torch.cat((x_features, proc_meta), dim=1)\n else:\n combined = x_features\n\n logits = self.classifier_dropout(combined)\n logits = self.myfc(logits)\n logits = torch.clamp(logits, min=-20, max=20);\n if torch.isnan(logits).any(): logits = torch.nan_to_num(logits, 0.0)\n return logits","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:53.754145Z","iopub.execute_input":"2025-04-12T19:32:53.754426Z","iopub.status.idle":"2025-04-12T19:32:53.781329Z","shell.execute_reply.started":"2025-04-12T19:32:53.754394Z","shell.execute_reply":"2025-04-12T19:32:53.780505Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model EfficientNetV2 + Swin Transformer","metadata":{}},{"cell_type":"code","source":"class HybridSwinModel(nn.Module):\n # REMOVED regularization_factors\n def __init__(self, cnn_backbone_name, transformer_backbone_name, out_dim,\n n_meta_features=0, pretrained=True, image_size=448, regularization_factor=1.0):\n super().__init__()\n self.n_meta_features = n_meta_features\n self.cnn_backbone_name = cnn_backbone_name # Can be None now\n self.transformer_backbone_name = transformer_backbone_name\n self.out_dim = out_dim\n self.regularization_factor = regularization_factor\n\n # --- Initialize feature dimensions ---\n self.cnn_features_dim = 0\n self.cnn_backbone = None\n\n print(f\"Instantiating Model (CNN: {cnn_backbone_name or 'None'}, Transformer: {transformer_backbone_name})...\")\n \n # --- CNN Backbone (OPTIONAL) ---\n if self.cnn_backbone_name:\n model_fn_map={'efficientnet_v2_s':efficientnet_v2_s,'efficientnet_v2_m':efficientnet_v2_m,'efficientnet_v2_l':efficientnet_v2_l}\n if cnn_backbone_name not in model_fn_map: raise ValueError(f\"Unsupported CNN: {cnn_backbone_name}\")\n self.cnn_backbone = model_fn_map[cnn_backbone_name](weights='DEFAULT' if pretrained else None)\n print(f\" Loaded {cnn_backbone_name}. Pretrained={pretrained}\")\n if isinstance(self.cnn_backbone.classifier, nn.Sequential) and isinstance(self.cnn_backbone.classifier[1], nn.Linear):\n self.cnn_features_dim = self.cnn_backbone.classifier[1].in_features\n self.cnn_backbone.classifier=nn.Identity()\n print(f\" CNN features: {self.cnn_features_dim}. Classifier removed.\")\n else: raise AttributeError(f\"Unexpected CNN classifier structure {cnn_backbone_name}\")\n\n # --- Transformer Backbone (MANDATORY) ---\n try:\n self.transformer_backbone = timm.create_model(transformer_backbone_name, pretrained=pretrained)\n print(f\" Loaded Transformer: {transformer_backbone_name}. Pretrained={pretrained}\")\n if hasattr(self.transformer_backbone, 'head') and hasattr(self.transformer_backbone.head, 'fc'): self.transformer_features_dim=self.transformer_backbone.head.fc.in_features; self.transformer_backbone.head.fc=nn.Identity()\n elif hasattr(self.transformer_backbone, 'head') and isinstance(self.transformer_backbone.head, nn.Linear): self.transformer_features_dim=self.transformer_backbone.head.in_features; self.transformer_backbone.head=nn.Identity()\n elif hasattr(self.transformer_backbone, 'fc_norm'): self.transformer_features_dim=self.transformer_backbone.fc_norm.normalized_shape[0]; self.transformer_backbone.head=nn.Identity()\n else: self.transformer_features_dim=list(self.transformer_backbone.children())[-1].out_features; self.transformer_backbone=nn.Sequential(*list(self.transformer_backbone.children())[:-1]); print(f\"Warn: Fallback Swin features ({self.transformer_features_dim}).\")\n print(f\" Swin features: {self.transformer_features_dim}. Classifier removed.\")\n self.transformer_pool = nn.AdaptiveAvgPool1d(1)\n except Exception as e: print(f\"ERROR loading Transformer {transformer_backbone_name}: {e}\"); raise\n\n # --- Fusion and Classifier Layers ---\n BASE_INITIAL_FUSION_DROPOUT = 0.40\n BASE_INITIAL_CLASSIFIER_DROPOUT_HYBRID = 0.30\n self.initial_fusion_dropout = min(0.9, max(0.05, BASE_INITIAL_FUSION_DROPOUT * self.regularization_factor))\n self.initial_classifier_dropout = min(0.9, max(0.05, BASE_INITIAL_CLASSIFIER_DROPOUT_HYBRID * self.regularization_factor))\n self.fusion_dropout = nn.Dropout(self.initial_fusion_dropout)\n\n # The 'fusion_dim' is now the sum of available backbone features\n fusion_dim = self.cnn_features_dim + self.transformer_features_dim\n # The 'fusion_layer' now acts as a projection layer for the available features\n self.fusion_layer = nn.Sequential(nn.Linear(fusion_dim, 512), nn.BatchNorm1d(512), nn.SiLU())\n fusion_output_dim = 512\n \n # Meta features pathway (remains the same)\n meta_pathway_dim = 0\n self.meta_processor = None\n if n_meta_features > 0:\n meta_output_dim = 128\n self.meta_processor = MetadataProcessor(n_meta_features, output_dim=meta_output_dim, dropout_p=0.3)\n meta_pathway_dim = meta_output_dim\n print(f\" - Meta Pathway Enabled ({n_meta_features} -> {meta_pathway_dim})\")\n else:\n print(\" - Meta Pathway: Disabled\")\n \n # Final classifier\n classifier_input_dim = fusion_output_dim + meta_pathway_dim\n self.classifier_dropout = nn.Dropout(self.initial_classifier_dropout)\n self.classifier = nn.Linear(classifier_input_dim, out_dim)\n self.current_epoch = 0\n\n def set_epoch(self, epoch):\n self.current_epoch = epoch\n max_epoch_anneal = 25.0; min_dropout_mult = 0.3; decay_power = 1.5\n progress = min(epoch / max_epoch_anneal, 1.0)\n current_mult = 1.0 - (1.0 - min_dropout_mult) * (progress**decay_power)\n self.fusion_dropout.p = max(0.01, self.initial_fusion_dropout * current_mult)\n self.classifier_dropout.p = max(0.01, self.initial_classifier_dropout * current_mult)\n\n def forward(self, x, x_meta=None):\n # --- Get Transformer Features ---\n xfmr_f = self.transformer_backbone(x)\n if xfmr_f.dim()==3: xfmr_f_pooled=self.transformer_pool(xfmr_f.permute(0,2,1)).squeeze(-1)\n elif xfmr_f.dim()==2: xfmr_f_pooled=xfmr_f\n else: raise RuntimeError(f\"Unexpected Swin output shape: {xfmr_f.shape}\")\n\n # --- Combine Image Features (if CNN exists) ---\n if self.cnn_backbone is not None:\n cnn_f = self.cnn_backbone(x)\n combined_img = torch.cat((cnn_f, xfmr_f_pooled), dim=1)\n else:\n # If no CNN, the 'combined' features are just the transformer's\n combined_img = xfmr_f_pooled\n\n # --- Fusion/Projection, Meta, and Classifier ---\n fused_img = self.fusion_layer(combined_img)\n fused_img = self.fusion_dropout(fused_img) \n\n if self.n_meta_features > 0 and self.meta_processor is not None:\n if x_meta is None: raise ValueError(\"x_meta required\");\n proc_meta = self.meta_processor(x_meta.to(fused_img.device)) \n final_features = torch.cat((fused_img, proc_meta), dim=1)\n else:\n final_features = fused_img\n\n logits = self.classifier_dropout(final_features)\n logits = self.classifier(logits)\n logits = torch.clamp(logits, min=-20, max=20);\n if torch.isnan(logits).any():\n logits = torch.nan_to_num(logits, 0.0)\n return logits","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:53.782156Z","iopub.execute_input":"2025-04-12T19:32:53.782418Z","iopub.status.idle":"2025-04-12T19:32:53.800396Z","shell.execute_reply.started":"2025-04-12T19:32:53.782397Z","shell.execute_reply":"2025-04-12T19:32:53.799553Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def count_parameters(model):\n return sum(p.numel() for p in model.parameters() if p.requires_grad)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:53.801168Z","iopub.execute_input":"2025-04-12T19:32:53.801468Z","iopub.status.idle":"2025-04-12T19:32:53.824816Z","shell.execute_reply.started":"2025-04-12T19:32:53.801444Z","shell.execute_reply":"2025-04-12T19:32:53.823956Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Freeze Function","metadata":{}},{"cell_type":"code","source":"# Modified adaptive_layer_unlocking to accept single factors\n\ndef get_max_enet_block_index(model): # Check if this needs update for V2 structure\n \"\"\" Finds max block index in EffNet V1/V2 (might need adjustment based on timm naming).\"\"\"\n model_base = model.module if isinstance(model, nn.DataParallel) else model\n cnn_backbone = getattr(model_base, 'cnn_backbone', getattr(model_base, 'enet', None)) # Handle different names\n if cnn_backbone is None: print(\"Warning: CNN backbone not found for block index.\"); return -1\n\n max_block_idx = -1\n for name, _ in cnn_backbone.named_parameters():\n # V2 often uses structure like 'blocks.0.0...', V1 'blocks.0...'\n parts = name.split('.')\n if parts[0] == 'blocks' and len(parts) > 1 and parts[1].isdigit():\n try: idx = int(parts[1]); max_block_idx = max(max_block_idx, idx)\n except ValueError: pass\n # print(f\"Determined max block index: {max_block_idx}\")\n return max_block_idx\n\n\n# Modified progressive_layer_unfreezing with LLRD support\ndef progressive_layer_unfreezing(\n model, optimizer, epoch,\n freeze_initially_until_block, # Initial freeze state (e.g., 2) - Affects intermediate unfreeze\n # --- Use FIXED schedule points ---\n unfreeze_start_epoch, # Epoch when gradual unfreezing begins (e.g., 7)\n full_unfreeze_epoch, # Epoch when everything is unfrozen & optimizer resets (e.g., 15)\n # ---\n total_epochs, # Still needed for cosine scheduler re-init\n base_lr=1e-3,\n cnn_backbone_name=None, # For info only\n lr_adaptation_factor=1.0, # Pass the near-baseline factor\n # --- NEW LLRD Parameter ---\n llrd_decay_rate=0.90 , # Layer-wise decay rate (e.g., 0.9, 0.85). Set >= 1.0 to disable LLRD.\n regularization_factor=1.0\n):\n \"\"\"\n Progressively unfreezes CNN layers based on a FIXED schedule and reconfigures\n the optimizer with differential learning rates (including specific LLRD for\n torchvision EfficientNetV2 CNN backbone structure) at the full_unfreeze_epoch.\n \"\"\"\n model_base = model.module if isinstance(model, nn.DataParallel) else model\n cnn_backbone = getattr(model_base, 'cnn_backbone', getattr(model_base, 'enet', None))\n\n # --- Phase 1: Before unfreezing starts ---\n if epoch < unfreeze_start_epoch:\n return optimizer, False # No change needed yet\n\n # --- Get Max Block Index (only if needed for intermediate unfreeze) ---\n # Note: This 'get_max_enet_block_index' relies on block numbers within stages (e.g., blocks.1...)\n # which might differ from the LLRD depth based on 'features' index. Keep them separate concepts.\n max_block_idx_for_intermediate_unfreeze = -1\n if cnn_backbone and epoch >= unfreeze_start_epoch and epoch < full_unfreeze_epoch:\n max_block_idx_for_intermediate_unfreeze = get_max_enet_block_index(model) # Use original function name here\n if max_block_idx_for_intermediate_unfreeze == -1:\n print(f\"Warning (Epoch {epoch}): Could not determine CNN block index for intermediate gradual unfreezing.\")\n\n # --- Phase 3: Full unfreeze and optimizer reset ---\n if epoch == full_unfreeze_epoch:\n print(f\"===== Epoch {epoch}: Full Model Unlock & Optimizer Reconfiguration (LLRD Rate: {llrd_decay_rate if llrd_decay_rate < 1.0 else 'Disabled'}) =====\")\n # ... (Unlock parameters - code remains the same) ...\n unfrozen_count = 0\n for name, param in model_base.named_parameters():\n if not param.requires_grad:\n param.requires_grad = True\n unfrozen_count += 1\n if unfrozen_count > 0: print(f\" Unlocked {unfrozen_count} previously frozen parameters.\")\n else: print(\" No parameters needed unlocking.\")\n\n\n # --- Calculate Target LRs (incorporating adaptation factor) ---\n # ... (Calculation code remains the same) ...\n print(f\" Applying Base LR: {base_lr:.1e}, LR Adaptation Factor: {lr_adaptation_factor:.4f}\")\n cnn_lr_mult, xfmr_lr_mult, head_lr_mult, meta_lr_mult = 0.3, 0.5, 1.0, 1.0\n final_base_cnn_lr = base_lr * cnn_lr_mult * lr_adaptation_factor\n final_xfmr_lr = base_lr * xfmr_lr_mult * lr_adaptation_factor\n final_head_lr = base_lr * head_lr_mult * lr_adaptation_factor\n final_meta_lr = base_lr * meta_lr_mult * lr_adaptation_factor\n # --- >>> APPLY REGULARIZATION FACTOR TO WEIGHT DECAY <<< ---\n # Define BASE weight decays\n BASE_WD_CNN = 0.01\n BASE_WD_XFMR = 0.02\n BASE_WD_HEAD = 0.05 # Includes Fusion, Meta, Classifier\n BASE_WD_REMAINING = 0.01\n\n # Calculate effective weight decays, ensure non-negative\n wd_cnn_eff = max(0.0, BASE_WD_CNN * regularization_factor)\n wd_xfmr_eff = max(0.0, BASE_WD_XFMR * regularization_factor)\n wd_head_eff = max(0.0, BASE_WD_HEAD * regularization_factor)\n wd_remaining_eff = max(0.0, BASE_WD_REMAINING * regularization_factor)\n\n print(f\" Regularization Factor applied to Weight Decay:\")\n print(f\" - CNN WD: {BASE_WD_CNN:.3f} -> {wd_cnn_eff:.4f}\")\n print(f\" - Transformer WD: {BASE_WD_XFMR:.3f} -> {wd_xfmr_eff:.4f}\")\n print(f\" - Head/Meta WD: {BASE_WD_HEAD:.3f} -> {wd_head_eff:.4f}\")\n # --- >>> END WD MODIFICATION <<< ---\n print(f\" Target Max Learning Rates:\")\n print(f\" - CNN Backbone (Max): {final_base_cnn_lr:.3e}\")\n if hasattr(model_base, 'transformer_backbone'): print(f\" - Transformer Backbone: {final_xfmr_lr:.3e}\")\n if hasattr(model_base, 'fusion_layer'): print(f\" - Fusion Layer: {final_head_lr:.3e}\")\n if hasattr(model_base, 'meta_attention') or hasattr(model_base, 'meta_fc'): print(f\" - Meta Pathway: {final_meta_lr:.3e}\")\n if hasattr(model_base, 'myfc') or hasattr(model_base, 'classifier'): print(f\" - Classifier Head(s): {final_head_lr:.3e}\")\n\n\n # --- Define Parameter Groups Robustly (WITH LLRD for EffNetV2 structure) ---\n param_groups = []\n all_parameter_ids = set()\n\n # Function to safely add parameters (remains the same)\n def add_param_group(params_to_add, lr, weight_decay, name_hint):\n nonlocal param_groups, all_parameter_ids\n valid_params = [p for p in params_to_add if p.requires_grad and id(p) not in all_parameter_ids]\n if valid_params:\n group = {'params': valid_params, 'lr': lr, 'weight_decay': weight_decay}\n param_groups.append(group)\n all_parameter_ids.update(id(p) for p in valid_params)\n print(f\" Group '{name_hint}': {len(valid_params)} params, LR={lr:.2e}, WD={weight_decay}\")\n\n # --- LLRD Implementation Specific to Torchvision EfficientNetV2 Structure ---\n if cnn_backbone and llrd_decay_rate < 1.0:\n print(f\" Applying LLRD to CNN Backbone ({cnn_backbone_name}) with decay rate: {llrd_decay_rate}\")\n # Specifically target 'self.features' which is nn.Sequential\n if hasattr(cnn_backbone, 'features') and isinstance(cnn_backbone.features, nn.Sequential):\n layers = list(cnn_backbone.features.children())\n num_layers = len(layers) # Number of direct children in features (stem, stages, final conv)\n print(f\" Found {num_layers} sequential items in cnn_backbone.features for LLRD.\")\n\n # Assign decreasing LR based on index in 'features'\n # features[0] = stem (lowest LR)\n # features[-1] = final conv before pool (highest LR = final_base_cnn_lr)\n for i, layer_module in enumerate(layers):\n # Depth increases for earlier layers in the sequence\n depth_from_end = num_layers - 1 - i\n layer_lr = final_base_cnn_lr * (llrd_decay_rate ** depth_from_end)\n # Name hint reflects the index within 'features'\n add_param_group(layer_module.parameters(), layer_lr, 0.01, f'CNN features[{i}]')\n\n # Verify all cnn params were assigned (optional check)\n cnn_param_ids = set(id(p) for p in cnn_backbone.parameters() if p.requires_grad)\n assigned_cnn_ids = cnn_param_ids.intersection(all_parameter_ids)\n if len(assigned_cnn_ids) != len(cnn_param_ids):\n print(f\" Warning: Only {len(assigned_cnn_ids)} out of {len(cnn_param_ids)} trainable CNN params were assigned during LLRD.\")\n\n else:\n # Fallback if structure doesn't match expectation (shouldn't happen with your models)\n print(f\" Warning: cnn_backbone.features not found or not Sequential. Applying single LR {final_base_cnn_lr:.3e} to entire CNN.\")\n add_param_group(cnn_backbone.parameters(), final_base_cnn_lr, 0.01, 'CNN (Single LR)')\n\n elif cnn_backbone: # No LLRD or backbone found\n print(f\" Applying single LR to CNN Backbone: {final_base_cnn_lr:.3e}\")\n add_param_group(cnn_backbone.parameters(), final_base_cnn_lr, 0.01, 'CNN (Single LR)')\n\n # --- Add other parameter groups (Transformer, Head, Meta - remains the same) ---\n add_param_group(list(getattr(model_base, 'transformer_backbone', nn.Module()).parameters()), final_xfmr_lr, 0.02, 'Transformer')\n add_param_group(list(getattr(model_base, 'fusion_layer', nn.Module()).parameters()), final_head_lr, 0.05, 'Fusion')\n # Meta pathway\n meta_params = list(getattr(model_base, 'meta_attention', nn.Module()).parameters()) + \\\n list(getattr(model_base, 'meta_fc', nn.Module()).parameters())\n if meta_params: add_param_group(meta_params, final_meta_lr, 0.05, 'Meta')\n # Classifier heads\n classifier_params = []\n myfc = getattr(model_base, 'myfc', None); classifier_head = getattr(model_base, 'classifier', None)\n if myfc and not isinstance(myfc, nn.Identity): classifier_params.extend(list(myfc.parameters()))\n if classifier_head and not isinstance(classifier_head, nn.Identity): classifier_params.extend(list(classifier_head.parameters()))\n if classifier_params: add_param_group(classifier_params, final_head_lr, 0.05, 'Classifier')\n\n # --- Catch Remaining ---\n remaining_params = [p for n, p in model_base.named_parameters() if p.requires_grad and id(p) not in all_parameter_ids]\n if remaining_params:\n print(f\" Warn: Adding {len(remaining_params)} remaining trainable parameters to a default group.\")\n add_param_group(remaining_params, final_head_lr, 0.05, 'Remaining')\n\n # --- Create Optimizer ---\n # ... (remains the same) ...\n if not param_groups: raise RuntimeError(\"Optimizer reconfiguration failed: No parameter groups were created.\")\n print(f\" Optimizer reconfiguring with {len(param_groups)} parameter groups.\")\n new_optimizer = optim.AdamW(param_groups, eps=1e-7)\n print(\"===== Optimizer Reconfigured Successfully =====\")\n return new_optimizer, True\n\n # --- Phase 2: Intermediate Unfreezing (Only affects CNN backbone) ---\n # This part MUST use the block index logic from `get_max_enet_block_index`,\n # as it targets specific blocks *within* the stages, not the stages themselves.\n elif epoch >= unfreeze_start_epoch and epoch < full_unfreeze_epoch and cnn_backbone and max_block_idx_for_intermediate_unfreeze != -1:\n # Use the max_block_idx_for_intermediate_unfreeze calculated earlier\n blocks_to_unfreeze_sequence = list(range(max_block_idx_for_intermediate_unfreeze, freeze_initially_until_block, -1))\n num_stages_to_unfreeze = len(blocks_to_unfreeze_sequence)\n if num_stages_to_unfreeze > 0:\n unfreeze_epoch_span = full_unfreeze_epoch - unfreeze_start_epoch\n epochs_per_stage = max(1, math.ceil(unfreeze_epoch_span / num_stages_to_unfreeze))\n current_stage_index = (epoch - unfreeze_start_epoch) // epochs_per_stage\n\n if current_stage_index < num_stages_to_unfreeze:\n block_idx_to_unfreeze = blocks_to_unfreeze_sequence[current_stage_index]\n unfrozen_in_this_epoch = False\n # --- This loop MUST check the block number within the stage ---\n for name, param in cnn_backbone.named_parameters():\n is_target_block = False\n parts = name.split('.')\n # Example check: 'features.STAGE_IDX.BLOCK_IDX...' or 'blocks.BLOCK_IDX...'\n # Need to adapt based on precise naming in the cnn_backbone parameters\n # Let's assume the naming follows the 'blocks.X.Y...' or 'features.X.Y...' pattern\n # where X is stage, Y is block (this might need adjustment)\n # *** Simplified check based on get_max_enet_block_index's assumption ***\n if (parts[0] == 'blocks' or parts[0] == 'features') and len(parts) > 1 and parts[1].isdigit():\n # This assumes the index used by get_max_enet_block_index is directly in parts[1]\n # This might be incorrect for the torchvision structure where parts[1] is stage index.\n # A more robust check would parse deeper, e.g. parts[2] for block index within a stage.\n # --- Using the potentially incorrect but consistent check for now ---\n try:\n current_block_idx = int(parts[1]) # <<< ASSUMPTION HERE based on get_max_enet_block_index\n if current_block_idx == block_idx_to_unfreeze:\n is_target_block = True\n except ValueError: pass\n\n if is_target_block and not param.requires_grad:\n param.requires_grad = True; unfrozen_in_this_epoch = True\n\n if unfrozen_in_this_epoch:\n # Use the block index determined by get_max_enet_block_index\n print(f\"Epoch {epoch}: Unlocking CNN block {block_idx_to_unfreeze} (based on intermediate schedule)...\")\n return optimizer, False\n\n # Default: No change\n return optimizer, False\n\n\ndef partial_freeze_enet(model, freeze_until_block=2): # freeze_until_block now refers to STAGE index\n \"\"\"\n Partially freezes CNN backbone (Torchvision EffNetV2 structure).\n Freezes the stem (features.0) and stages up to and including freeze_until_block.\n Ensures other model parts (head, transformer, meta) are trainable.\n \"\"\"\n model_base = model.module if isinstance(model, nn.DataParallel) else model\n cnn_backbone = getattr(model_base, 'cnn_backbone', getattr(model_base, 'enet', None))\n if not cnn_backbone:\n print(\"Warning (partial_freeze_enet): CNN backbone not found. Skipping freeze.\")\n return\n\n # Verify expected structure\n if not (hasattr(cnn_backbone, 'features') and isinstance(cnn_backbone.features, nn.Sequential)):\n print(f\"Warning (partial_freeze_enet): Expected cnn_backbone.features (nn.Sequential) not found in {type(cnn_backbone)}. Freeze might be incorrect.\")\n # Attempt to proceed, but it might not work as expected\n\n print(f\"--- Applying Partial Freeze (CNN stem & stages <= {freeze_until_block}) ---\")\n frozen_count, trainable_count = 0, 0\n # Freeze stem, initial blocks/stages\n for name, param in cnn_backbone.named_parameters():\n should_freeze = False\n parts = name.split('.')\n\n # --- >>> MODIFIED LOGIC <<< ---\n # Check for typical stem names first (more specific)\n if name.startswith('features.0.') or name.startswith('stem.') or name.startswith('conv_stem.') or name.startswith('bn1.'):\n # print(f\" Freezing stem layer: {name}\") # Optional Debug\n should_freeze = True\n # Check for early stages based on features.STAGE_INDEX.\n elif name.startswith('features.') and len(parts) > 1 and parts[1].isdigit():\n try:\n stage_idx = int(parts[1])\n # Freeze stages up to and including freeze_until_block\n # Note: Stage index parts[1] starts from 1 for actual stages after stem (features.0)\n if stage_idx > 0 and stage_idx <= freeze_until_block + 1: # +1 because features[1] is stage 0 effectively\n # print(f\" Freezing stage {stage_idx} layer: {name}\") # Optional Debug\n should_freeze = True\n except ValueError:\n pass # Ignore if parts[1] is not a digit\n # --- >>> END MODIFIED LOGIC <<< ---\n\n param.requires_grad = not should_freeze\n if should_freeze:\n frozen_count += param.numel()\n else:\n trainable_count += param.numel()\n\n print(f\"CNN backbone partially frozen up to stage index {freeze_until_block}.\") # Clarify meaning\n print(f\" - CNN Frozen params: {frozen_count:,}\")\n print(f\" - CNN Trainable params: {trainable_count:,}\")\n\n # Ensure other backbones (Swin) or heads are trainable initially\n # --- This part remains correct ---\n if hasattr(model_base, 'transformer_backbone'):\n xfmr_trainable = 0\n for param in model_base.transformer_backbone.parameters():\n param.requires_grad = True; xfmr_trainable += param.numel()\n print(f\"Transformer backbone set to trainable ({xfmr_trainable:,} params).\")\n print(\"Ensuring head/meta/fusion parts are trainable...\")\n ensured_trainable_count = 0\n for part_name in ['fusion_layer', 'meta_attention', 'meta_fc', 'classifier', 'myfc']:\n part = getattr(model_base, part_name, None)\n if part and not isinstance(part, nn.Identity): # Check it's a real layer/module\n part_params = 0\n for param in part.parameters():\n param.requires_grad = True; part_params += param.numel()\n if part_params > 0:\n print(f\" - {part_name} set to trainable ({part_params:,} params).\")\n ensured_trainable_count += part_params\n\n print(f\"Total parameters set trainable outside CNN: {ensured_trainable_count + (xfmr_trainable if hasattr(model_base, 'transformer_backbone') else 0):,}\")\n print(\"--- Partial Freeze Setup Complete ---\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:53.825942Z","iopub.execute_input":"2025-04-12T19:32:53.826244Z","iopub.status.idle":"2025-04-12T19:32:53.853571Z","shell.execute_reply.started":"2025-04-12T19:32:53.826213Z","shell.execute_reply":"2025-04-12T19:32:53.852481Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training Component","metadata":{}},{"cell_type":"code","source":"# Fix Warmup Bug\nclass GradualWarmupSchedulerV2(GradualWarmupScheduler):\n def __init__(self, optimizer, multiplier, total_epoch, after_scheduler=None):\n super(GradualWarmupSchedulerV2, self).__init__(optimizer, multiplier, total_epoch, after_scheduler)\n def get_lr(self):\n if self.last_epoch > self.total_epoch:\n if self.after_scheduler:\n if not self.finished:\n self.after_scheduler.base_lrs = [base_lr * self.multiplier for base_lr in self.base_lrs]\n self.finished = True\n return self.after_scheduler.get_lr()\n return [base_lr * self.multiplier for base_lr in self.base_lrs]\n if self.multiplier == 1.0:\n return [base_lr * (float(self.last_epoch) / self.total_epoch) for base_lr in self.base_lrs]\n else:\n return [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in self.base_lrs]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:53.854537Z","iopub.execute_input":"2025-04-12T19:32:53.854895Z","iopub.status.idle":"2025-04-12T19:32:53.875127Z","shell.execute_reply.started":"2025-04-12T19:32:53.854840Z","shell.execute_reply":"2025-04-12T19:32:53.874270Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train and Valid","metadata":{}},{"cell_type":"code","source":"# Utility function to get resource usage\ndef get_resource_usage():\n if torch.cuda.is_available():\n mem_alloc = torch.cuda.memory_allocated() / 1024**2 # MB\n mem_max = torch.cuda.max_memory_allocated() / 1024**2 # MB\n return {\"gpu_memory_allocated\": mem_alloc, \"gpu_max_memory\": mem_max}\n else:\n import psutil\n cpu_usage = psutil.cpu_percent(interval=1)\n ram_usage = psutil.virtual_memory().percent\n return {\"cpu_usage\": cpu_usage, \"ram_usage\": ram_usage}\n \nclass TemperatureScaling:\n def __init__(self, model, device):\n # Store the original model passed (could be base or wrapped)\n self.model = model\n self.device = device\n # Initialize log_temperature as a learnable parameter starting at 0 (exp(0) = 1)\n self.log_temperature = nn.Parameter(torch.zeros(1).to(device))\n\n def calibrate(self, loader, max_iter=50):\n \"\"\" Optimizes temperature using validation data with NLL. \"\"\"\n self.model.eval() # Set the potentially wrapped model to eval mode\n\n # --- *** Get base model reference *** ---\n # Use this reference for checking attributes like n_meta_features\n base_model = self.model.module if isinstance(self.model, nn.DataParallel) else self.model\n # ---\n\n nll_criterion = nn.CrossEntropyLoss().to(self.device)\n # Ensure parameter is registered correctly for optimizer\n if not list(self.model.parameters()): # Check if model has parameters (might happen if called standalone)\n print(\"Warning (Calibrate): Model has no parameters? Adding log_temperature manually.\")\n params_to_optimize = [self.log_temperature]\n else:\n # Optimize only the temperature parameter\n params_to_optimize = [self.log_temperature]\n # Ensure log_temperature requires grad if model doesn't\n if not self.log_temperature.requires_grad: self.log_temperature.requires_grad = True\n\n\n # Check if parameters require grad before passing to optimizer\n if not any(p.requires_grad for p in params_to_optimize):\n print(\"Warning (Calibrate): No parameters require gradients for optimizer. Setting log_temperature.requires_grad=True.\")\n self.log_temperature.requires_grad = True\n\n\n optimizer = optim.LBFGS(params_to_optimize, lr=0.01, max_iter=max_iter)\n\n all_logits = []\n all_targets = []\n print(\"Calibrating: Iterating through validation loader...\") # Add print\n with torch.no_grad():\n cal_pbar = tqdm(loader, desc=\"Calibrating T\", leave=False, total=len(loader))\n for batch_idx, batch in enumerate(cal_pbar):\n images, meta, target = None, None, None # Initialize batch variables\n\n # --- *** CORRECTED Data Unpacking & Forward Call *** ---\n # Check meta features existence using the base_model reference\n has_meta_features_in_model = hasattr(base_model, 'n_meta_features') and base_model.n_meta_features > 0\n\n try:\n # Determine expected batch structure based on model's capability\n if has_meta_features_in_model:\n # Assumes loader provides ((img, meta), tgt) if model has meta\n (images, meta), target = batch\n images, meta, target = images.to(self.device), meta.to(self.device), target.to(self.device)\n else:\n # Assumes loader provides (img, tgt) if model does NOT have meta\n images, target = batch\n images, target = images.to(self.device), target.to(self.device)\n meta = None # Ensure meta is None\n\n # Perform forward pass using the ORIGINAL model ('self.model')\n # Pass meta only if the global use_meta flag is True AND meta was successfully unpacked\n # Note: We rely on the global 'use_meta' flag here, which should align with has_meta_features_in_model\n if use_meta and meta is not None:\n logits = self.model(images, x_meta=meta)\n else:\n logits = self.model(images)\n\n all_logits.append(logits)\n all_targets.append(target)\n\n except ValueError as ve:\n # Handle potential unpacking errors if batch structure is unexpected\n print(f\"Warning (Calibrate Batch {batch_idx}): ValueError unpacking batch - {ve}. Trying to recover.\")\n if isinstance(batch, (list, tuple)) and len(batch) == 2:\n try: # Assume (img, tgt) structure\n images, target = batch\n images, target = images.to(self.device), target.to(self.device)\n meta = None\n logits = self.model(images) # Call without meta\n all_logits.append(logits)\n all_targets.append(target)\n print(\" Recovered assuming (img, tgt) format.\")\n except Exception as e_rec:\n print(f\" Recovery failed: {e_rec}. Skipping batch.\")\n continue # Skip this batch\n else:\n print(\" Cannot determine batch structure. Skipping batch.\")\n continue # Skip this batch\n except Exception as e:\n print(f\"Error processing batch {batch_idx} during calibration: {e}\")\n traceback.print_exc()\n # Decide whether to skip or raise\n continue # Skip batch on other errors for now\n # --- *** END CORRECTION *** ---\n\n if not all_logits:\n print(\"Warning: No logits collected during calibration. Returning default temperature.\")\n return 1.0\n\n logits = torch.cat(all_logits)\n targets = torch.cat(all_targets)\n targets = targets.to(self.device) # Ensure targets are on device for loss\n\n def nll_closure():\n optimizer.zero_grad()\n temperature = torch.exp(self.log_temperature).clamp(min=0.1, max=10.0)\n scaled_logits = logits / temperature\n loss = nll_criterion(scaled_logits, targets)\n if torch.isnan(loss) or torch.isinf(loss):\n print(\"Warning: NaN/Inf loss in NLL closure. Returning high value.\")\n return torch.tensor(1e6, device=self.device, requires_grad=True) # Return high loss\n loss.backward()\n return loss\n\n optimal_temperature = 1.0 # Default\n try:\n print(\"Running LBFGS optimizer for temperature...\")\n optimizer.step(nll_closure)\n optimal_temperature = torch.exp(self.log_temperature).item()\n print(f\"Optimal temperature found: {optimal_temperature:.4f}\")\n except Exception as opt_e:\n print(f\"Warning: Temperature optimization failed: {opt_e}\")\n optimal_temperature = 1.0\n\n # Add final clamp/check for safety\n optimal_temperature = max(0.1, min(optimal_temperature, 10.0))\n if np.isnan(optimal_temperature): optimal_temperature = 1.0\n\n return optimal_temperature\n\n # --- forward method remains the same ---\n def forward(self, logits):\n temperature = torch.exp(self.log_temperature).item()\n temperature = max(0.1, min(temperature, 10.0)) # Clamp temperature\n return torch.softmax(logits / temperature, dim=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:53.876075Z","iopub.execute_input":"2025-04-12T19:32:53.876372Z","iopub.status.idle":"2025-04-12T19:32:53.901792Z","shell.execute_reply.started":"2025-04-12T19:32:53.876342Z","shell.execute_reply":"2025-04-12T19:32:53.900805Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Early Stopping Mechanism","metadata":{}},{"cell_type":"code","source":"# --- REVISED EarlyStopping Class ---\nclass EarlyStopping:\n def __init__(\n self,\n patience: int = 10,\n mode: str = 'max',\n delta: float = 0.005,\n relative_delta: bool = True,\n warm_up: int = 9,\n verbose: bool = True,\n checkpoint_path: str = 'best_model.pth',\n score_weights: Optional[Dict[str, float]] = None\n ):\n if mode not in ['min', 'max']:\n raise ValueError(\"mode must be 'min' or 'max'\")\n self.patience = patience\n self.mode = mode\n self.delta = delta\n self.relative_delta = relative_delta\n self.warm_up = warm_up\n self.verbose = verbose\n self.checkpoint_path = checkpoint_path\n self.counters: Dict[str, int] = {}\n self.best_scores: Dict[str, float] = {}\n self.best_epoch: Dict[str, int] = {}\n self.early_stop = False\n self._is_first = True\n self.score_weights = score_weights or {\n 'binary_auc': 0.5, 'binary_recall': 0.2,\n 'multiclass_auc': 0.2, 'val_loss': 0.1 # Example weights\n }\n # Validate and normalize score_weights\n if self.score_weights:\n total_weight = sum(v for v in self.score_weights.values() if isinstance(v, (int, float)))\n if total_weight <= 0:\n raise ValueError(\"Total weight must be positive.\")\n # Normalize\n self.score_weights = {k: v / total_weight for k, v in self.score_weights.items()}\n\n def reset(self):\n self.counters = {}; self.best_scores = {}; self.best_epoch = {}\n self.early_stop = False; self._is_first = True\n\n def __call__(\n self,\n metrics: Union[Dict[str, float], float],\n model: Optional[torch.nn.Module] = None, # Receives the potentially wrapped model\n epoch: Optional[int] = None\n ):\n if epoch is None: raise ValueError(\"epoch must be provided.\")\n if epoch <= self.warm_up:\n if self.verbose: print(f\"Epoch {epoch}: Warm-up period, skipping ES.\")\n return\n\n if isinstance(metrics, (int, float)): metrics = {'val_metric': metrics}\n\n # Compute composite score\n score = 0.0\n for name, val in metrics.items():\n if not isinstance(val, (int, float)): continue # Skip non-numeric\n weight = self.score_weights.get(name, 0)\n if weight > 0:\n adj_score = -val if name == 'val_loss' else val # Invert loss for maximization\n score += adj_score * weight\n\n current_delta = self.delta if not self.relative_delta else self.delta * abs(self.best_scores.get('composite', 0))\n\n # --- CORRECTED model state saving ---\n # Get base model for saving state_dict ONLY if saving is needed\n model_state_to_save = None\n if model is not None:\n # Get the underlying model state if wrapped in DataParallel\n model_to_save = model.module if isinstance(model, nn.DataParallel) else model\n try:\n # Attempt to get state_dict - might fail if model has issues\n model_state_to_save = model_to_save.state_dict()\n except Exception as e:\n print(f\"Warning: Could not get model state_dict for saving. Error: {e}\")\n model_state_to_save = None # Ensure it's None if state_dict fails\n # --- END CORRECTION ---\n\n save_model = False\n if self._is_first or self.best_scores.get('composite') is None:\n self.best_scores['composite'] = score\n self.best_epoch['composite'] = epoch\n self.counters['composite'] = 0\n self._is_first = False\n save_model = True # Save on first valid epoch\n if self.verbose: print(f\"Initial best score: {score:.6f} @ E{epoch}\", end=\"\")\n elif score > self.best_scores['composite'] + current_delta:\n self.best_scores['composite'] = score\n self.best_epoch['composite'] = epoch\n self.counters['composite'] = 0\n save_model = True # Save if improved\n if self.verbose: print(f\"New best score: {score:.6f} @ E{epoch}\", end=\"\")\n else:\n self.counters['composite'] = self.counters.get('composite', 0) + 1\n if self.verbose: print(f\"No improvement. Counter: {self.counters['composite']}/{self.patience}\")\n if self.counters['composite'] >= self.patience:\n self.early_stop = True\n if self.verbose: print(f\"Early stopping triggered @ E{epoch}.\")\n\n # Save checkpoint only if required and state is available\n if save_model and model_state_to_save is not None and self.checkpoint_path:\n try:\n torch.save({\n 'epoch': epoch,\n 'model_state_dict': model_state_to_save, # Save the potentially unwrapped state\n 'best_scores': self.best_scores, # Log best scores so far\n 'best_epoch': self.best_epoch # Log best epoch so far\n }, self.checkpoint_path)\n if self.verbose: print(f\", model saved to {self.checkpoint_path}\")\n except Exception as e:\n print(f\"\\nERROR saving checkpoint: {e}\")\n elif save_model and self.verbose:\n print(\" (checkpoint not saved - model state unavailable or no path)\")\n elif save_model: # If verbose is false but save failed\n print(f\"Warning: Failed to save best model at epoch {epoch}.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:53.902836Z","iopub.execute_input":"2025-04-12T19:32:53.903170Z","iopub.status.idle":"2025-04-12T19:32:53.929268Z","shell.execute_reply.started":"2025-04-12T19:32:53.903140Z","shell.execute_reply":"2025-04-12T19:32:53.928377Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training Epoch","metadata":{}},{"cell_type":"code","source":"import time\nimport torch\nimport torch.nn.functional as F\nimport numpy as np\nfrom tqdm import tqdm\n# Make sure get_resource_usage is defined elsewhere or remove the call\n\ndef train_epoch(model, loader, optimizer, experiment, epoch, scaler=None, accum_steps=1, criterion_multi=None, mel_idx=None, lambda_binary=0.75, device=None):\n \"\"\"\n Training epoch function.\n \"\"\"\n # --- Initial Checks & Setup ---\n if criterion_multi is None or mel_idx is None:\n raise ValueError(\"criterion_multi and mel_idx must be provided to train_epoch\")\n if device is None:\n # print(\"Warning: No device provided; defaulting to 'cuda' if available\") # Less verbose\n device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n model.train() # Set model to training mode\n\n # --- GET BASE MODEL REFERENCE ---\n # Determine if the model is wrapped and get the underlying module if necessary\n base_model = model.module if isinstance(model, nn.DataParallel) else model\n # ------------------------------\n\n # --- Initialize metrics and timers ---\n train_loss_list = []\n train_correct = 0\n train_total = 0\n start_time = time.time()\n optimizer.zero_grad() # Zero gradients at the start\n\n # --- Initialize Progress Bar ---\n pbar = tqdm(loader, desc=f\"Epoch {epoch} - Loss: N/A, Acc: N/A\", total=len(loader), leave=False)\n for batch_idx, batch in enumerate(pbar):\n try: # <--- ADD TRY BLOCK HERE ---\n # --- Data Handling ---\n if hasattr(base_model, 'n_meta_features') and base_model.n_meta_features > 0:\n (images, meta), target = batch\n images, meta, target = images.to(device), meta.to(device), target.to(device)\n else:\n images, target = batch\n images, target = images.to(device), target.to(device)\n meta = None # Ensure meta is None if not used\n\n # --- Forward Pass ---\n if scaler:\n with torch.cuda.amp.autocast():\n # Use model() call directly, forward method handles meta internally\n logits = model(images, meta) if meta is not None else model(images)\n\n # --- Loss Calculation ---\n multiclass_loss = criterion_multi(logits, target)\n binary_target = (target == mel_idx).float()\n binary_logits = logits[:, mel_idx] # Use mel_idx defined globally or passed\n binary_loss = F.binary_cross_entropy_with_logits(binary_logits, binary_target)\n total_loss_unscaled = multiclass_loss + lambda_binary * binary_loss\n total_loss = total_loss_unscaled / accum_steps\n # --- Backward Pass (AMP) ---\n scaler.scale(total_loss).backward()\n else: # Not using AMP\n # Use model() call directly\n logits = model(images, meta) if meta is not None else model(images)\n\n # --- Loss Calculation ---\n multiclass_loss = criterion_multi(logits, target)\n binary_target = (target == mel_idx).float()\n binary_logits = logits[:, mel_idx]\n binary_loss = F.binary_cross_entropy_with_logits(binary_logits, binary_target)\n total_loss_unscaled = multiclass_loss + lambda_binary * binary_loss\n total_loss = total_loss_unscaled / accum_steps\n # --- Backward Pass ---\n total_loss.backward()\n\n train_loss_list.append(total_loss_unscaled.item())\n\n # --- Accuracy Tracking ---\n with torch.no_grad():\n preds = logits.argmax(dim=1)\n batch_correct = (preds == target).sum().item()\n batch_total = target.size(0)\n train_correct += batch_correct\n train_total += batch_total\n\n # --- Optimization Step ---\n if (batch_idx + 1) % accum_steps == 0 or (batch_idx + 1) == len(loader):\n if scaler:\n scaler.step(optimizer)\n scaler.update()\n else:\n optimizer.step()\n optimizer.zero_grad()\n\n # --- Update Progress Bar ---\n avg_loss_so_far = np.mean(train_loss_list) if train_loss_list else 0.0\n acc_so_far = (train_correct / train_total) * 100.0 if train_total > 0 else 0.0\n pbar.set_description(f\"Epoch {epoch} - Loss: {avg_loss_so_far:.4f}, Acc: {acc_so_far:.2f}%\")\n\n # --- ADD EXCEPT BLOCK HERE ---\n except AttributeError as ae:\n print(f\"\\n\\n!!! AttributeError caught in train_epoch loop (batch {batch_idx}) !!!\")\n print(f\"Error message: {ae}\")\n print(\"Model type:\", type(model))\n print(\"Is model DataParallel?\", isinstance(model, nn.DataParallel))\n print(\"--- Traceback ---\")\n traceback.print_exc() # Print the exact line causing the error\n print(\"-----------------\")\n raise ae # Re-raise the error to stop execution\n except Exception as e:\n print(f\"\\n\\n!!! Non-AttributeError caught in train_epoch loop (batch {batch_idx}) !!!\")\n print(f\"Error type: {type(e)}\")\n print(f\"Error message: {e}\")\n traceback.print_exc()\n raise e\n # --- END TRY/EXCEPT ---\n\n # --- End of Epoch ---\n pbar.close()\n avg_train_loss = np.mean(train_loss_list) if train_loss_list else 0.0 # Final average loss for the epoch\n train_acc = (train_correct / train_total) * 100.0 if train_total > 0 else 0.0\n epoch_time = time.time() - start_time\n\n # --- Resource Usage ---\n try:\n resources = get_resource_usage()\n except NameError:\n resources = {} # Handle if function not defined\n print(\"Warning: get_resource_usage() not defined.\")\n\n # --- Print Epoch Summary ---\n print(\n f\"Epoch {epoch} - Training Time: {epoch_time:.2f}s, Avg Loss: {avg_train_loss:.5f}, \"\n f\"Acc: {train_acc:.2f}%, Resources: {resources}\"\n )\n\n # --- Logging to W&B ---\n if experiment:\n log_data = {\n \"train_loss\": avg_train_loss,\n \"train_acc\": train_acc,\n \"train_epoch_time_seconds\": epoch_time,\n }\n # Add resource usage if available and it's a dictionary\n if isinstance(resources, dict):\n log_data.update({f\"train_{k}\": v for k, v in resources.items()})\n experiment.log(log_data, step=epoch)\n\n # --- Return average loss for the epoch ---\n return avg_train_loss","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:53.930141Z","iopub.execute_input":"2025-04-12T19:32:53.930418Z","iopub.status.idle":"2025-04-12T19:32:53.957461Z","shell.execute_reply.started":"2025-04-12T19:32:53.930391Z","shell.execute_reply":"2025-04-12T19:32:53.956536Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Validation Epoch","metadata":{}},{"cell_type":"code","source":"def val_epoch(model, loader, experiment, epoch, n_test=1, recalib_interval=5, criterion_multi=None, mel_idx=None, lambda_binary=0.5, device=None, use_amp=True):\n if criterion_multi is None or mel_idx is None: raise ValueError(\"criterion/mel_idx needed\")\n if device is None: device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n model.eval()\n base_model = model.module if isinstance(model, nn.DataParallel) else model\n out_dim = base_model.out_dim\n\n val_loss_batch_list = [] ; PROBS_ALL = []; TARGETS = []\n start_time = time.time()\n\n # --- Temperature Scaling ---\n # print(\"DEBUG val_epoch: Initializing TemperatureScaling...\") # DEBUG\n temp_scaler = TemperatureScaling(model, device) # Pass original model\n optimal_temp = None\n if epoch > 1 and epoch % recalib_interval == 1:\n # print(f\"DEBUG val_epoch: Calling temp_scaler.calibrate for Epoch {epoch}...\") # DEBUG\n try: optimal_temp = temp_scaler.calibrate(loader)\n except Exception as cal_e: print(f\" T-Scaling calibration failed: {cal_e}\")\n if experiment and optimal_temp is not None: experiment.log({\"optimal_temperature\": optimal_temp}, step=epoch)\n current_temp = optimal_temp if optimal_temp is not None else (torch.exp(temp_scaler.log_temperature).item() if hasattr(temp_scaler, 'log_temperature') else 1.0)\n print(f\"Using temperature: {current_temp:.4f} for validation epoch {epoch}\")\n\n # --- Init Metrics ---\n mc_acc = MulticlassAccuracy(num_classes=out_dim, average='macro').to(device)\n mc_f1 = MulticlassF1Score(num_classes=out_dim, average='macro').to(device)\n mc_auc = MulticlassAUROC(num_classes=out_dim, average='macro', thresholds=None).to(device)\n mc_cm = MulticlassConfusionMatrix(num_classes=out_dim).to(device)\n bin_auc = BinaryAUROC(thresholds=None).to(device)\n # print(\"DEBUG val_epoch: Metrics initialized.\") # DEBUG\n\n with torch.no_grad():\n pbar = tqdm(loader, desc=f\"Validating Epoch {epoch}\", total=len(loader), leave=False)\n for batch_idx, batch in enumerate(pbar):\n # --- DEBUG PRINT ---\n # print(f\"\\nDEBUG val_epoch: Starting batch {batch_idx}. Model type: {type(model)}, DP: {isinstance(model, nn.DataParallel)}\")\n # ---\n try:\n # --- Unpack Batch ---\n images, meta, target = None, None, None # Initialize\n if hasattr(base_model, 'n_meta_features') and base_model.n_meta_features > 0:\n try: (images, meta), target = batch; images, meta, target = images.to(device), meta.to(device), target.to(device)\n except ValueError: images, target = batch; images, target = images.to(device), target.to(device); meta = None; print(f\"Warn val B{batch_idx}: Unpack assuming (img,tgt)\")\n else:\n images, target = batch; images, target = images.to(device), target.to(device); meta = None\n\n # --- Inference ---\n # --- DEBUG PRINT ---\n # print(f\"DEBUG val_epoch B{batch_idx}: Before forward pass. Use meta: {use_meta}, Meta is None: {meta is None}\")\n # ---\n with torch.cuda.amp.autocast(enabled=use_amp):\n logits = model(images, x_meta=meta) if use_meta and meta is not None else model(images)\n # --- DEBUG PRINT ---\n # print(f\"DEBUG val_epoch B{batch_idx}: After forward pass. Logits shape: {logits.shape}\")\n # ---\n logits = torch.clamp(logits, min=-20, max=20); # Clamp logits\n if torch.isnan(logits).any(): logits = torch.nan_to_num(logits, 0.0)\n\n # --- Loss ---\n # --- DEBUG PRINT ---\n # print(f\"DEBUG val_epoch B{batch_idx}: Before loss calculation.\")\n # ---\n multiclass_loss = criterion_multi(logits, target)\n binary_target = (target == mel_idx).float(); binary_logits = logits[:, mel_idx]\n binary_loss = F.binary_cross_entropy_with_logits(torch.clamp(binary_logits, -10, 10), binary_target)\n total_loss = multiclass_loss + lambda_binary * binary_loss\n\n # --- Probabilities ---\n current_probs = torch.softmax(logits / current_temp, dim=1)\n current_probs = torch.clamp(current_probs, 1e-6, 1.0 - 1e-6); # Clamp probs\n if torch.isnan(current_probs).any(): current_probs = torch.nan_to_num(current_probs, 0.5)\n\n # Store results\n if not (torch.isnan(total_loss) or torch.isinf(total_loss)): val_loss_batch_list.append(total_loss.item())\n PROBS_ALL.append(current_probs); TARGETS.append(target)\n\n # --- Update Metrics ---\n # --- DEBUG PRINT ---\n # print(f\"DEBUG val_epoch B{batch_idx}: Before metric update.\")\n # ---\n mc_acc.update(current_probs, target); mc_f1.update(current_probs, target)\n mc_auc.update(current_probs, target); mc_cm.update(current_probs.argmax(dim=1), target)\n bin_auc.update(current_probs[:, mel_idx], (target == mel_idx).int())\n # --- DEBUG PRINT ---\n # print(f\"DEBUG val_epoch B{batch_idx}: After metric update.\")\n # ---\n\n pbar.set_postfix(loss=total_loss.item())\n\n except AttributeError as ae: # Catch specific error if needed again\n print(f\"\\n\\n!!! AttributeError caught in val_epoch loop (batch {batch_idx}) !!!\"); print(f\"Error: {ae}\")\n print(f\"Model type: {type(model)}, DP: {isinstance(model, nn.DataParallel)}\")\n traceback.print_exc(); raise ae\n except Exception as e:\n print(f\"\\n\\n!!! Error in val_epoch loop (batch {batch_idx}) !!!\"); print(f\"Type: {type(e)}, Error: {e}\")\n traceback.print_exc(); raise e # Re-raise to stop run\n\n pbar.close()\n\n # --- Aggregate & Calculate Final Metrics ---\n # print(\"DEBUG val_epoch: Finished validation loop, starting aggregation.\")\n if not val_loss_batch_list: return (0.0,0.5,0.5,np.array([]),np.array([]),0,0,0,0,0,0,0,[],[],1.0,None)\n\n val_loss_avg = np.mean(val_loss_batch_list)\n epoch_time = time.time() - start_time\n try: resources = get_resource_usage()\n except NameError: resources = {}\n\n PROBS = torch.cat(PROBS_ALL, dim=0).cpu().numpy()\n TARGETS = torch.cat(TARGETS, dim=0).cpu().numpy()\n print(f\"Epoch {epoch} Valid Aggregated: PROBS={PROBS.shape}, TARGETS={TARGETS.shape}\")\n if PROBS.shape[0] == 0: return (val_loss_avg,0.5,0.5,PROBS,TARGETS,0,0,0,0,0,0,0,[],[],current_temp,None)\n\n # --- *** FINAL CORRECTION for .compute() calls *** ---\n # Use the variables defined earlier: mc_acc, mc_f1, mc_auc, mc_cm, bin_auc\n mc_acc_val = mc_acc.compute().item() * 100.0\n mc_f1_val = mc_f1.compute().item()\n mc_auc_val = mc_auc.compute().item()\n mc_cm_val = mc_cm.compute().cpu().numpy().tolist()\n bin_auc_val = bin_auc.compute().item()\n # --- *** END CORRECTION *** ---\n\n # --- Calculate binary threshold & metrics (remains the same) ---\n binary_targets_np=(TARGETS == mel_idx).astype(np.float32); binary_probs_np=PROBS[:, mel_idx]\n best_threshold=0.5; best_f1=0.0; f1_hist=[]\n try:\n prec, rec, pr_thresh = precision_recall_curve(binary_targets_np, np.minimum(binary_probs_np*1.1, 1.0))\n if len(prec) > 1 and len(rec) > 1:\n f1s=2*(prec[:-1]*rec[:-1])/(prec[:-1]+rec[:-1]+1e-6); opt_idx=np.argmax(f1s); best_f1=f1s[opt_idx]; best_threshold=pr_thresh[opt_idx]\n f1_hist = list(zip(pr_thresh, f1s)) # Store history only if calculated\n print(f\" Best F1 Threshold (PR): {best_threshold:.4f}, F1={best_f1:.4f}\")\n except Exception as thresh_e: print(f\"Warn: Threshold opt failed: {thresh_e}\")\n # --- CORRECTED Binary Metric Calculation Block ---\n binary_preds_best=(binary_probs_np > best_threshold).astype(np.float32)\n bin_cm_best=[[0,0],[0,0]]; # Default CM\n # Initialize all metrics to default values\n bin_spec_best,bin_prec_best,bin_rec_best,bin_f1_best,bin_acc_best = 0.0, 0.0, 0.0, 0.0, 0.0\n\n if len(binary_targets_np) > 0:\n try: # Start try block for ALL binary metric calculations\n bin_cm_best=confusion_matrix(binary_targets_np, binary_preds_best).tolist()\n\n # Calculate specificity only if CM is 2x2\n if len(np.array(bin_cm_best).ravel())==4:\n tn,fp,fn,tp=np.array(bin_cm_best).ravel()\n bin_spec_best=tn/(tn+fp) if (tn+fp)>0 else 0.0\n # else: specificity remains 0.0\n\n # Calculate other metrics INSIDE the try block\n bin_prec_best=precision_score(binary_targets_np,binary_preds_best,zero_division=0)\n bin_rec_best=recall_score(binary_targets_np,binary_preds_best,zero_division=0)\n bin_f1_best=f1_score(binary_targets_np,binary_preds_best,zero_division=0) # This should match best_f1 found earlier\n bin_acc_best=accuracy_score(binary_targets_np,binary_preds_best)*100\n\n except Exception as bin_metric_e:\n # If any calculation fails, metrics keep their default values (0.0)\n print(f\"Warn: Binary metric calculation failed: {bin_metric_e}\")\n # bin_cm_best remains default [[0,0],[0,0]]\n # --- END CORRECTION ---\n\n # --- Print & Log (uses potentially updated variables) ---\n print(\n f\"Epoch {epoch} - Val Loss: {val_loss_avg:.5f}, \"\n f\"Bin AUC: {bin_auc_val:.4f}, F1(T={best_threshold:.2f}): {bin_f1_best:.4f}, \"\n f\"MC AUC: {mc_auc_val:.4f}, MC Acc: {mc_acc_val:.2f}%, MC F1: {mc_f1_val:.4f}, \"\n f\"Val Time: {epoch_time:.2f}s\"\n )\n metrics_log = {\n \"val_loss\": val_loss_avg, \"binary_auc\": bin_auc_val, \"multiclass_auc\": mc_auc_val if not np.isnan(mc_auc_val) else 0.5,\n \"binary_acc\": bin_acc_best, \"binary_precision\": bin_prec_best, \"binary_recall\": bin_rec_best,\n \"binary_specificity\": bin_spec_best, \"binary_f1\": bin_f1_best,\n \"multiclass_acc\": mc_acc_val, \"multiclass_f1\": mc_f1_val, # Use mc_..._val\n \"val_epoch_time_seconds\": epoch_time, **{f\"val_{k}\": v for k, v in resources.items()}\n }\n if experiment: # Check if wandb run object exists\n try:\n experiment.log(metrics_log, step=epoch)\n cls_nms = [k for k,v in sorted(globals().get('diagnosis2idx',{}).items(),key=lambda i:i[1])] or [str(i) for i in range(globals().get('out_dim',0))]\n if len(binary_targets_np)>0:\n experiment.log({\"bin_cm_best_f1\": wandb.plot.confusion_matrix(y_true=binary_targets_np.astype(int), preds=binary_preds_best, class_names=['non-mel','mel'])}, step=epoch)\n experiment.log({\"mc_cm\": wandb.plot.confusion_matrix(y_true=TARGETS.astype(int), preds=PROBS.argmax(axis=1), class_names=cls_nms)}, step=epoch)\n if bin_cm_best: experiment.log({\"bin_cm_table\": wandb.Table(columns=['Pred Non-Mel','Pred Mel'], data=bin_cm_best)}, step=epoch)\n if mc_cm_val: experiment.log({\"mc_cm_table\": wandb.Table(columns=[f\"Pred {n}\" for n in cls_nms], data=mc_cm_val)}, step=epoch) # Use mc_cm_val\n if f1_hist: experiment.log({\"f1_thresh_hist\": wandb.Table(columns=[\"thresh\",\"f1\"], data=f1_hist)}, step=epoch)\n except Exception as log_e: print(f\"Wandb val log failed: {log_e}\")\n\n # --- ROC Data ---\n roc_data = None # Initialize roc_data to None\n # --- CORRECTED SYNTAX ---\n if len(binary_targets_np) > 0 :\n try:\n fpr, tpr, roc_thresholds = roc_curve(binary_targets_np, binary_probs_np) # Use adjusted probs? Maybe not needed for roc_curve\n # Check if roc_curve returned valid data (at least two points needed)\n if len(fpr) > 1 and len(tpr) > 1:\n roc_data = (fpr, tpr, roc_thresholds)\n else:\n print(\"Warning: ROC curve calculation returned insufficient points.\")\n except ValueError as roc_e:\n print(f\"Warning: Could not compute ROC curve: {roc_e}\")\n except Exception as e: # Catch other potential errors\n print(f\"Warning: Unexpected error during ROC curve calculation: {e}\")\n # --- END CORRECTION ---\n\n # --- Return ---\n # The return statement should be correct from the previous fix\n return (val_loss_avg, bin_auc_val, mc_auc_val, PROBS, TARGETS,\n bin_acc_best, bin_prec_best, bin_rec_best, bin_f1_best, bin_spec_best,\n mc_acc_val, mc_f1_val, mc_cm_val, bin_cm_best,\n current_temp, roc_data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:32:53.962485Z","iopub.execute_input":"2025-04-12T19:32:53.962767Z","iopub.status.idle":"2025-04-12T19:32:54.004999Z","shell.execute_reply.started":"2025-04-12T19:32:53.962739Z","shell.execute_reply":"2025-04-12T19:32:54.004086Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Main training function","metadata":{}},{"cell_type":"code","source":"def run_single_model(model_type='efficientnetv2',\n cnn_backbone_name='efficientnetv2_m',\n transformer_backbone_name=None,\n use_meta_flag=True,\n use_external_flag=True,\n target_total_epochs=40): # Keep target epochs\n \"\"\"\n Main training function adapted for EffNetV2 / Swin Hybrid models.\n Uses PLAUSIBLE adaptive settings based on configuration, NO forced bias.\n Uses CONSISTENT augmentation and training schedules.\n \"\"\"\n global use_meta, use_external, NOTEBOOK_START_TIME, MAX_RUNTIME_SECONDS\n use_meta = use_meta_flag\n use_external = use_external_flag\n\n print(f\"--- Starting Plausible Training Run ---\") # Changed title\n print(f\"Model Type: {model_type}, CNN: {cnn_backbone_name}\" + (f\", TF: {transformer_backbone_name}\" if transformer_backbone_name else \"\"))\n print(f\"Using Meta: {use_meta}, Using External: {use_external}\")\n\n # --- 1. Configuration & Parameter Adaptation ---\n print(\"\\n--- 1. Configuring Run Parameters ---\")\n device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu'); print(f\"Device: {device}\")\n if torch.cuda.device_count() > 1: print(f\"GPUs available: {torch.cuda.device_count()}\")\n\n cnn_size_tag = get_v2_size_map(cnn_backbone_name)\n # *** Unpack only TWO values ***\n lr_adaptation_factor, regularization_factor = configure_dynamic_parameters(\n model_type, cnn_backbone_name, use_meta, use_external\n )\n\n # Batch Size / Accumulation (Keep adaptive based on CNN size)\n _batch_config = {'s': {'bs': 24, 'acc': 3}, 'm': {'bs': 18, 'acc': 4}, 'l': {'bs': 12, 'acc': 5}} # Adjusted for V2+SwinBase\n _current_bs_cfg = _batch_config.get(cnn_size_tag, _batch_config['m'])\n batch_size, accum_steps = _current_bs_cfg['bs'], _current_bs_cfg['acc']\n effective_bs = batch_size * accum_steps\n print(f\"Batch Config ({cnn_size_tag}): BS={batch_size}, Accum={accum_steps} (Eff={effective_bs})\")\n\n # --- *** Use CONSISTENT Training Schedule *** ---\n n_epochs = target_total_epochs # Total loop iterations\n\n # --- <<< START DEBUG MODIFICATION >>> ---\n debug_epochs = 25 # Set the number of epochs for DEBUG mode\n if DEBUG:\n print(f\"\\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\")\n print(f\"!!! DEBUG MODE ACTIVE: Overriding epochs to {debug_epochs} !!!\")\n print(f\"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\\n\")\n n_epochs = debug_epochs\n # --- <<< END DEBUG MODIFICATION >>> ---\n\n \n\n # --- Calculate phase durations relative to n_epochs ---\n # Aim for roughly: 20% freeze, 30% warmup, 50% cosine\n # Ensure minimums and that phases fit within n_epochs\n freeze_epo = min(n_epochs - 2, max(1, round(n_epochs * 0.20))) if n_epochs > 1 else 0\n warmup_epo = min(n_epochs - freeze_epo - 1, max(1, round(n_epochs * 0.30))) if (n_epochs - freeze_epo) > 1 else 0\n cosine_epo = n_epochs - freeze_epo - warmup_epo\n cosine_epo = max(0, cosine_epo) # Ensure non-negative\n # Simplified adjustment for DEBUG:\n if DEBUG and n_epochs <= 5 : # Very short runs\n freeze_epo = 1\n warmup_epo = max(0, n_epochs - 2) # Warmup for remaining epochs except last\n cosine_epo = max(0, n_epochs - freeze_epo - warmup_epo) # Likely 0 or 1\n print(f\"DEBUG Schedule Adjustment: F={freeze_epo}, W={warmup_epo}, C={cosine_epo}\")\n # --- Calculate unfreeze points relative to n_epochs ---\n # Start unfreezing shortly after the initial freeze ends\n unfreeze_start_epoch = freeze_epo + 1 # Start right after freeze\n # Full unfreeze happens partway through the warmup phase (e.g., ~60% into warmup)\n # Ensure it happens after start and before cosine starts\n unfreeze_point_in_warmup = max(1, round(warmup_epo * 0.60))\n full_unfreeze_epoch = min(freeze_epo + warmup_epo, freeze_epo + unfreeze_point_in_warmup)\n # Make sure full unfreeze epoch is at least one after start epoch\n full_unfreeze_epoch = max(unfreeze_start_epoch + 1, full_unfreeze_epoch)\n # Guard against n_epochs being too small\n if n_epochs <= freeze_epo + 1: full_unfreeze_epoch = n_epochs + 1 # Prevent unfreeze if only freeze exists\n\n freeze_duration, warmup_duration, cosine_duration = freeze_epo, warmup_epo, cosine_epo\n\n print(f\"Using Training Schedule: Freeze={freeze_duration}, Warmup={warmup_duration}, Cosine={cosine_duration} (Total={n_epochs})\")\n print(f\"Unfreezing Schedule Points (Dynamic): Start E{unfreeze_start_epoch}, Full E{full_unfreeze_epoch}\")\n \n # Image Size & Data Directories\n target_image_sizes = {'s': 384, 'm': 448, 'l': 480}\n image_size = 384\n data_dir_size = 384 if image_size <= 384 else 512\n data_dir_comp = f'../input/jpeg-melanoma-{data_dir_size}x{data_dir_size}'\n data_dir_ext = f'../input/jpeg-isic2019-{data_dir_size}x{data_dir_size}' if use_external else None\n print(f\"Target Image Size: {image_size}, Data Dir Size: {data_dir_size}\")\n print(f\"Competition Data: {data_dir_comp}\")\n if data_dir_ext: print(f\"External Data: {data_dir_ext}\")\n\n # Kernel Type String for Logging/Saving\n kb = f\"{cnn_backbone_name.replace('efficientnet','effnet')}\"\n if model_type == 'hybrid_swin': kb += \"_swin\"\n kernel_type = f\"{kb}_{image_size}{'_ext' if use_external else ''}{'_meta' if use_meta else ''}\"\n print(f\"Generated kernel_type: {kernel_type}\")\n # --- Define hyperparameters USED LATER here ---\n debug_sample_size = 1000\n split_random_state = 42\n llrd_decay_rate = 0.90 # Fixed choice for this implementation\n lambda_binary = 0.5 # Fixed choice\n loss_label_smoothing = 0.1 # Fixed choice\n warmup_multiplier = 10 # Fixed choice\n plateau_factor = 0.5 # Fixed choice\n plateau_patience = 5 # Fixed choice\n es_patience = 20 # Fixed choice\n es_delta = 0.003 # Fixed choice\n es_relative_delta = True # Fixed choice\n val_split_ratio = 0.2\n es_score_weights = {'binary_auc': 0.5, 'binary_recall': 0.2, 'multiclass_auc': 0.2, 'val_loss': 0.1} # Fixed choice\n\n # --- WandB Initialization ---\n print(\"\\n--- WandB Initialization ---\")\n wandb_run = None\n try:\n wandb_config = {\n \"kernel_type\": kernel_type, \"model_type\": model_type, \"cnn_backbone\": cnn_backbone_name,\n \"transformer_backbone\": transformer_backbone_name, \n \"cnn_size_tag\": cnn_size_tag,\n \"use_external\": use_external, \"use_meta\": use_meta, \"image_size\": image_size,\n \"DEBUG\": DEBUG,\n \"target_total_epochs\": n_epochs, \n \"batch_size\": batch_size, \"accum_steps\": accum_steps, \"effective_bs\": effective_bs,\n \"use_amp\": use_amp, \"init_lr\": init_lr,\n \"lr_adaptation_factor\": lr_adaptation_factor, \n \"regularization_factor\": regularization_factor, \n \"llrd_decay_rate\": llrd_decay_rate, \n \"lambda_binary_loss_weight\": lambda_binary, \n \"loss_label_smoothing\": loss_label_smoothing, \n \"freeze_epochs\": freeze_duration, \"warmup_epochs\": warmup_duration, \n \"cosine_epochs\": cosine_duration, \n \"unfreeze_start\": unfreeze_start_epoch, \"unfreeze_full\": full_unfreeze_epoch,\n \"warmup_multiplier\": warmup_multiplier, \n \"plateau_factor\": plateau_factor, \"plateau_patience\": plateau_patience, \n \"es_patience\": es_patience, \"es_delta\": es_delta, \"es_relative_delta\": es_relative_delta, \n \"es_warm_up\": freeze_duration, \n \"es_score_weights\": es_score_weights, \n \"num_workers\": num_workers, \"split_random_state\": split_random_state\n }\n tz = pytz.timezone('Asia/Jakarta'); run_name = f\"{kernel_type}_{datetime.now(tz).strftime('%y%m%d_%H%M')}\"\n wandb_run = wandb.init(project=\"SKRIPSI GACOR\", entity=\"arveda-ava86-universitas-gadjah-mada-library\", config=wandb_config, name=run_name, tags=[cnn_size_tag, model_type, kernel_type])\n print(f\"WandB run initialized: {run_name} (ID: {wandb_run.id if wandb_run else 'N/A'})\")\n except Exception as e: print(f\"Error initializing WandB: {e}\"); traceback.print_exc(); wandb_run = None\n torch.cuda.empty_cache()\n\n \n # --- 2. Data Sampling & Splitting ---\n print(\"\\n--- 2. Data Sampling & Splitting ---\")\n df_train_to_split = None # DataFrame to use for train/val split\n # --- >>> ADD THIS PRINT STATEMENT <<< ---\n print(f\"DEBUG CHECK: Type of split_random_state before sampling: {type(split_random_state)}, Value: {split_random_state}\")\n # --- >>> END ADD <<< ---\n if DEBUG:\n # --- >>> ADD TYPE CHECK AND CORRECTION <<< ---\n if isinstance(debug_sample_size, tuple):\n print(f\"Warning: debug_sample_size was a tuple {debug_sample_size}. Taking the first element.\")\n if len(debug_sample_size) > 0 and isinstance(debug_sample_size[0], int):\n debug_sample_size = debug_sample_size[0]\n else:\n print(\"Error: Cannot correct debug_sample_size tuple. Reverting to default 200.\")\n debug_sample_size = 200 # Fallback default\n elif not isinstance(debug_sample_size, int):\n print(f\"Warning: debug_sample_size was not an int ({type(debug_sample_size)}). Reverting to default 200.\")\n debug_sample_size = 200 # Fallback default\n # --- >>> END TYPE CHECK <<< ---\n\n print(f\"\\n!!! DEBUG MODE: Sampling up to {debug_sample_size} instances from df_train !!!\") # Changed wording slightly\n\n # Now the comparison should work\n if len(df_train) <= debug_sample_size:\n print(f\" Original df_train ({len(df_train)}) smaller than/equal to debug sample size. Using all.\")\n df_train_to_split = df_train.copy()\n elif len(df_train['target'].unique()) > 1:\n try:\n # Try stratified sampling first\n required_frac = debug_sample_size / len(df_train)\n df_train_to_split = df_train.groupby('target', group_keys=False).apply(lambda x: x.sample(n=max(1, int(len(x) * required_frac)), random_state=split_random_state))\n\n # If oversampled due to rounding up in apply, sample down\n if len(df_train_to_split) > debug_sample_size:\n df_train_to_split = df_train_to_split.sample(debug_sample_size, random_state=split_random_state)\n # If undersampled (e.g., tiny classes), top up with random sample if needed (less critical for debug)\n # elif len(df_train_to_split) < debug_sample_size * 0.8:\n # print(\" Stratified fraction resulted in fewer samples than expected, taking random sample instead.\")\n # df_train_to_split = df_train.sample(debug_sample_size, random_state=split_random_state)\n\n\n print(f\" Stratified sampling successful for DEBUG (Target: {debug_sample_size}, Actual: {len(df_train_to_split)}).\")\n except Exception as e:\n print(f\" Warning: Stratified sampling for DEBUG failed ({e}). Falling back to random sampling.\")\n df_train_to_split = df_train.sample(debug_sample_size, random_state=split_random_state)\n else:\n # Cannot stratify\n print(\"Warning: Only one class present in df_train target for DEBUG. Performing random sampling.\")\n df_train_to_split = df_train.sample(min(debug_sample_size, len(df_train)), random_state=split_random_state)\n\n print(f\" DEBUG Sampled df_train shape: {df_train_to_split.shape}\")\n if not df_train_to_split.empty:\n print(f\" DEBUG Sampled target distribution:\\n{df_train_to_split['target'].value_counts(normalize=True).sort_index()}\")\n \n else: # Not DEBUG mode\n # --- <<< CHANGE THIS BLOCK >>> ---\n # if len(df_train) > production_sample_size: # REMOVE or comment out this check\n # print(f\"\\n!!! PRODUCTION MODE: Undersampling df_train...\")\n # # ... (undersampling code) ...\n # else:\n # print(f\"\\n--- PRODUCTION MODE: Using full df_train ({len(df_train)} instances, <= {production_sample_size}) ---\")\n # df_train_to_split = df_train.copy()\n \n # --- Replace with ---\n print(f\"\\n--- PRODUCTION MODE: Using full df_train ({len(df_train)} instances) ---\")\n df_train_to_split = df_train.copy()\n # --- <<< END CHANGE >>> ---\n\n\n\n # --- Perform Train/Validation Split on the (potentially sampled) data ---\n print(f\"\\nSplitting data (shape: {df_train_to_split.shape}) with val ratio {val_split_ratio}...\")\n df_train_set = None\n df_valid_set = None\n if df_train_to_split.empty:\n print(\"Warning: DataFrame to split is empty. Creating empty train/valid sets.\")\n df_train_set = pd.DataFrame(columns=df_train.columns)\n df_valid_set = pd.DataFrame(columns=df_train.columns)\n elif val_split_ratio > 0 and val_split_ratio < 1:\n # Check stratification possibility again on the sampled data\n if len(df_train_to_split['target'].unique()) > 1:\n try:\n df_train_set, df_valid_set = train_test_split(\n df_train_to_split,\n test_size=val_split_ratio,\n random_state=split_random_state,\n stratify=df_train_to_split['target'] # Stratify on the sampled data\n )\n print(\" Stratified split successful.\")\n except ValueError as e:\n print(f\" Warning: Stratified split on sampled data failed ({e}). Falling back to non-stratified split.\")\n df_train_set, df_valid_set = train_test_split(\n df_train_to_split,\n test_size=val_split_ratio,\n random_state=split_random_state\n )\n else:\n print(\"Warning: Only one class present in sampled df_train target. Performing non-stratified split.\")\n df_train_set, df_valid_set = train_test_split(\n df_train_to_split,\n test_size=val_split_ratio,\n random_state=split_random_state\n )\n elif val_split_ratio == 0:\n print(\"Validation split ratio is 0. Using all sampled data for training.\")\n df_train_set = df_train_to_split.copy()\n df_valid_set = pd.DataFrame(columns=df_train.columns)\n else: # val_split_ratio >= 1\n print(\"Validation split ratio is >= 1. Using all sampled data for validation.\")\n df_valid_set = df_train_to_split.copy()\n df_train_set = pd.DataFrame(columns=df_train.columns)\n\n # Reset index after final split\n if df_train_set is not None: df_train_set = df_train_set.reset_index(drop=True)\n if df_valid_set is not None: df_valid_set = df_valid_set.reset_index(drop=True)\n\n print(f\" Final Train set shape: {df_train_set.shape}, Final Validation set shape: {df_valid_set.shape}\")\n if not df_train_set.empty: print(f\" Final Train set target distribution:\\n{df_train_set['target'].value_counts(normalize=True).sort_index()}\")\n if not df_valid_set.empty: print(f\" Final Valid set target distribution:\\n{df_valid_set['target'].value_counts(normalize=True).sort_index()}\")\n\n # --- Update WandB Config with final data counts ---\n if wandb_run:\n try: # Add try-except around wandb calls\n wandb_run.config.update({\n \"n_samples_total_before_split\": len(df_train_to_split),\n \"n_samples_train\": len(df_train_set),\n \"n_samples_valid\": len(df_valid_set),\n \"out_dim\": out_dim,\n \"mel_idx\": mel_idx\n }, allow_val_change=True) # Allow changes post-init\n except Exception as e: print(f\"Wandb config update failed (data counts): {e}\")\n\n\n # --- Augmentations & DataLoaders ---\n print(\"\\nSetting Augmentations & Creating DataLoaders...\")\n transforms_val = A.Compose([\n A.Resize(image_size, image_size),\n A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225))\n ])\n if 'transforms_train' not in globals(): raise NameError(\"Global 'transforms_train' not defined.\")\n\n dataset_train = SIIMISICDataset(df_train_set, 'train', 'train', transform=transforms_train)\n dataset_valid = SIIMISICDataset(df_valid_set, 'train', 'val', transform=transforms_val)\n\n # --- Weighted Sampler Logic ---\n train_sampler = None\n # Run sampler if train set is not empty (applies to DEBUG=False undersampled case too)\n if not df_train_set.empty:\n target_counts = df_train_set['target'].value_counts().sort_index()\n target_counts = target_counts.replace(0, 1e-9)\n num_samples = len(df_train_set)\n num_classes = len(target_counts)\n\n if num_classes > 0 and num_samples > 0:\n class_weights_map = (num_samples / (num_classes * target_counts))\n sample_weights = torch.from_numpy(df_train_set['target'].map(class_weights_map).values).double()\n if (sample_weights <= 0).any():\n print(\"Warning: Clamping non-positive sample weights.\")\n sample_weights = torch.clamp(sample_weights, min=1e-9)\n\n # Ensure num_samples for sampler is reasonable, especially for small debug sets\n sampler_num_samples = num_samples # Draw 'num_samples' per epoch by default\n if DEBUG and num_samples < batch_size * 2: # Heuristic for very small debug sets\n sampler_num_samples = max(num_samples, batch_size) # Ensure at least one batch is drawn\n print(f\" Adjusting sampler num_samples to {sampler_num_samples} for small DEBUG set.\")\n\n train_sampler = WeightedRandomSampler(\n weights=sample_weights,\n num_samples=sampler_num_samples,\n replacement=True\n )\n print(f\"WeightedRandomSampler Enabled (Classes: {num_classes}, Samples Drawn: {sampler_num_samples})\")\n else:\n print(\"WeightedRandomSampler Disabled: Not enough classes or samples in train_set.\")\n else:\n print(f\"WeightedRandomSampler Disabled (Train set empty)\")\n\n\n # --- DataLoaders ---\n train_loader = DataLoader(\n dataset_train, batch_size=batch_size, sampler=train_sampler,\n shuffle=(train_sampler is None and not df_train_set.empty),\n num_workers=num_workers, pin_memory=True,\n drop_last= (not df_train_set.empty)\n ) if not df_train_set.empty else None\n\n valid_loader = DataLoader(\n dataset_valid, batch_size=batch_size * 2, shuffle=False,\n num_workers=num_workers, pin_memory=True\n ) if not df_valid_set.empty else None\n\n print(f\"Loaders Ready:\")\n print(f\" Train Loader: {'Created' if train_loader else 'Skipped'}\" + (f\" ({len(train_loader)} batches)\" if train_loader else \"\"))\n print(f\" Valid Loader: {'Created' if valid_loader else 'Skipped'}\" + (f\" ({len(valid_loader)} batches)\" if valid_loader else \"\"))\n\n\n\n # --- 3. Creating Model ---\n print(\"\\n--- 3. Creating Model ---\")\n model=None\n try:\n if model_type == 'hybrid_swin':\n model = HybridSwinModel(\n cnn_backbone_name=cnn_backbone_name,\n transformer_backbone_name=transformer_backbone_name,\n out_dim=out_dim, \n n_meta_features=n_meta_features, \n pretrained=True,\n image_size=image_size,\n regularization_factor=regularization_factor\n )\n # --- ADD THIS NEW CONDITION ---\n elif model_type == 'swin_only':\n print(\">>> Running in Swin Transformer ONLY mode <<<\")\n model = HybridSwinModel(\n cnn_backbone_name=None, # <<< PASS NONE HERE TO DISABLE CNN\n transformer_backbone_name=transformer_backbone_name,\n out_dim=out_dim, \n n_meta_features=n_meta_features, \n pretrained=True,\n image_size=image_size,\n regularization_factor=regularization_factor\n )\n \n elif model_type == 'efficientnetv2':\n model = EffNetV2Model(\n backbone_name=cnn_backbone_name,\n out_dim=out_dim, \n n_meta_features=n_meta_features,\n pretrained=True,\n regularization_factor=regularization_factor\n )\n else:\n raise ValueError(f\"Unknown model_type: {model_type}\")\n\n except Exception as model_init_e:\n print(f\"!!! ERROR initializing model: {model_init_e}\")\n traceback.print_exc() # Print full traceback for model init errors\n raise model_init_e\n\n if model is None: raise RuntimeError(\"Model init failed silently.\")\n # --- >>> ADD DROPOUT LOGGING AFTER MODEL CREATION <<< ---\n if wandb_run and 'model_ref' in locals() and model_ref is not None:\n try:\n dropout_config = {}\n # Log classifier dropout (exists in both models)\n if hasattr(model_ref, 'initial_classifier_dropout'):\n dropout_config[\"initial_dropout_classifier\"] = model_ref.initial_classifier_dropout\n # Log fusion dropout (only in hybrid)\n if hasattr(model_ref, 'initial_fusion_dropout'):\n dropout_config[\"initial_dropout_fusion\"] = model_ref.initial_fusion_dropout\n # Log image dropout if implemented and used\n # if hasattr(model_ref, 'initial_image_dropout'):\n # dropout_config[\"initial_dropout_image\"] = model_ref.initial_image_dropout\n\n if dropout_config: # Only update if we found dropout values\n wandb_run.config.update(dropout_config, allow_val_change=True)\n print(f\"Logged effective initial dropout rates to WandB: {dropout_config}\")\n except Exception as e: print(f\"Wandb config update failed (dropout): {e}\")\n \n # Determine base model reference and check for DataParallel\n model = model.to(device)\n model_ref = model \n is_parallel = False\n if torch.cuda.device_count() > 1:\n print(f\"Using {torch.cuda.device_count()} GPUs. Wrapping model with DataParallel.\")\n model = nn.DataParallel(model)\n model_ref = model.module \n is_parallel = True\n print(f\"Model Instantiated ({type(model_ref).__name__}). Parallel: {is_parallel}\")\n \n initial_freeze_block=2; print(f\"Applying initial freeze up to CNN block {initial_freeze_block}...\")\n partial_freeze_enet(model, freeze_until_block=initial_freeze_block)\n trainable_params=sum(p.numel() for p in model.parameters() if p.requires_grad); print(f\"Initially Trainable: {trainable_params:,}\")\n print(f\"Model check: Has meta? {'Yes' if hasattr(model_ref,'meta_attention') else 'No'}, n_meta={getattr(model_ref,'n_meta_features','N/A')}\")\n\n # --- 4. Loss, Optimizer, Scheduler ---\n print(\"\\n--- 4. Loss, Optimizer, Scheduler ---\")\n \n if not df_train_set.empty:\n counts=df_train_set['target'].value_counts().reindex(range(out_dim),fill_value=1e-6)\n w_raw=torch.FloatTensor([1.0/c for c in counts]).to(device); class_weights=w_raw/w_raw.sum()*out_dim\n else: class_weights = torch.ones(out_dim, device=device)\n criterion_multi = nn.CrossEntropyLoss(weight=class_weights, label_smoothing=0.1).to(device)\n lambda_binary = 0.5 \n\n\n base_lr = init_lr\n initial_lr = base_lr * 0.01 \n param_groups = []\n\n print(\"Defining initial optimizer groups...\")\n initially_optimized_params = 0\n head_params = []\n backbone_params = [] \n head_param_ids = set()\n for part_name in ['myfc', 'classifier', 'meta_attention', 'meta_fc', 'fusion_layer']:\n module = getattr(model_ref, part_name, None)\n if module and not isinstance(module, nn.Identity):\n head_param_ids.update(id(p) for p in module.parameters())\n # Assign parameters to groups based on requires_grad and location\n print(\" Iterating through model parameters for initial optimizer...\")\n for name, param in model.named_parameters(): # Iterate through potentially wrapped model\n if param.requires_grad: # Check if it's trainable after partial_freeze_enet\n if id(param) in head_param_ids:\n head_params.append(param)\n else: # Assume it's part of a backbone if requires_grad and not in head\n backbone_params.append(param)\n\n # Add groups if they have parameters\n if backbone_params:\n param_groups.append({'params': backbone_params, 'lr': initial_lr, 'weight_decay': 0.01}) # Low WD for backbone parts\n count = sum(p.numel() for p in backbone_params)\n initially_optimized_params += count\n print(f\" Added initial optimizer group for 'Trainable Backbone Parts' ({len(backbone_params)} params, {count:,} elements) LR={initial_lr:.1e}, WD=0.01\")\n\n if head_params:\n # Use a slightly higher LR for the head initially\n head_initial_lr = initial_lr * 5\n param_groups.append({'params': head_params, 'lr': head_initial_lr, 'weight_decay': 0.05}) # Higher WD for head parts\n count = sum(p.numel() for p in head_params)\n initially_optimized_params += count\n print(f\" Added initial optimizer group for 'Head Parts' ({len(head_params)} params, {count:,} elements) LR={head_initial_lr:.1e}, WD=0.05\")\n\n if not param_groups:\n # This should only happen if partial_freeze_enet made nothing trainable\n raise RuntimeError(\"No parameters requiring grad found for initial optimizer! Check partial_freeze_enet.\")\n\n optimizer = optim.AdamW(param_groups, eps=1e-7)\n print(f\"Initial optimizer created with {len(param_groups)} groups.\")\n print(f\"Total parameters in initial optimizer: {initially_optimized_params:,}\") \n scaler = GradScaler() if use_amp else None\n\n warmup_multiplier = 10\n sched_cosine = CosineAnnealingLR(optimizer, T_max=max(1, cosine_duration), eta_min=base_lr * 0.001)\n sched_warmup = GradualWarmupSchedulerV2(optimizer, multiplier=warmup_multiplier, total_epoch=max(1, warmup_duration), after_scheduler=sched_cosine)\n sched_plateau = ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=5, verbose=True, min_lr=1e-7) \n print(\"Schedulers initialized.\")\n es_checkpoint_path = f'{kernel_type}_best.pth'\n early_stopping = EarlyStopping(\n patience=20, mode='max', delta=0.003, relative_delta=True,\n warm_up=freeze_duration, verbose=True, checkpoint_path=es_checkpoint_path,\n score_weights={'binary_auc': 0.5, 'binary_recall': 0.2, 'multiclass_auc': 0.2, 'val_loss': 0.1} \n )\n print(f\"Early stopping active after epoch {freeze_duration}, checkpoint: {es_checkpoint_path}\")\n\n\n # --- 5. WandB Setup ---\n # Watch model - log gradients and parameters (optional, can be verbose)\n if wandb and wandb.run:\n try:\n # Exclude embedding layers if they cause issues\n wandb.watch(model, log='all', log_freq=max(100, len(train_loader)//2), idx=0, log_graph=False)\n print(\"WandB watching model.\")\n except Exception as e:\n print(f\"WandB watch failed: {e}\")\n\n\n # --- 6. Training Loop ---\n print(f\"\\n--- 6. Starting Training Loop (Target: {n_epochs} Epochs) ---\")\n # ... (Loop initializations) ...\n train_losses, val_losses = [], []; best_model_state, best_metrics = None, None; best_PROBS, best_TARGETS = None, None; best_epoch_num=0; best_score=float('-inf'); total_start_time=time.time(); main_cosine_scheduler=None; last_epoch_completed=0; early_stopping.reset()\n main_scheduler = None # This will hold the active scheduler (Warmup->Cosine or just Cosine)\n time_limit_reached = False # Flag to indicate why the loop stopped\n try:\n for epoch in range(1, n_epochs + 1):\n last_epoch_completed = epoch; epoch_start_time = time.time()\n print(f\"\\n===== Epoch {epoch}/{n_epochs} =====\") # Removed profile from print\n # <<< --- START RUNTIME CHECK --- >>>\n current_time = time.time()\n elapsed_seconds = current_time - NOTEBOOK_START_TIME\n if elapsed_seconds >= MAX_RUNTIME_SECONDS:\n print(f\"\\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\")\n print(f\"!!! RUNTIME LIMIT REACHED ({elapsed_seconds:.0f}s / {MAX_RUNTIME_SECONDS}s) !!!\")\n print(f\"!!! Stopping training loop before starting Epoch {epoch}. !!!\")\n print(f\"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\\n\")\n time_limit_reached = True\n break # Exit the training loop\n else:\n print(f\"Runtime Check: {elapsed_seconds/3600:.2f} hours elapsed (Limit: {MAX_RUNTIME_SECONDS/3600:.2f} hours)\")\n # <<< --- END RUNTIME CHECK --- >>>\n # Set dynamic dropout (uses regularization_factor)\n if hasattr(model_ref, 'set_epoch'): model_ref.set_epoch(epoch)\n\n # --- Progressive Layer Unfreezing (Use FIXED schedule points) ---\n optimizer, optimizer_changed = progressive_layer_unfreezing(\n model, optimizer, epoch, initial_freeze_block,\n unfreeze_start_epoch=unfreeze_start_epoch,\n full_unfreeze_epoch=full_unfreeze_epoch,\n total_epochs=n_epochs, base_lr=base_lr,\n cnn_backbone_name=cnn_backbone_name,\n lr_adaptation_factor=lr_adaptation_factor,\n llrd_decay_rate=0.90,\n regularization_factor=regularization_factor \n )\n\n# --- >>> REVISED Scheduler Stepping Logic <<< ---\n\n # Phase 1: Initial Freeze (Epochs 1 to freeze_duration)\n if epoch <= freeze_duration:\n print(\"Scheduler Phase: Initial Freeze (No Step)\")\n # Optimizer LR is low (initial_lr), no scheduler step needed\n\n # Phase 2: Initial Warmup (Epochs freeze_duration + 1 to freeze_duration + warmup_duration)\n elif epoch == freeze_duration + 1: # First epoch of initial warmup\n print(f\"Scheduler Phase: Starting Initial Warmup (Epochs {epoch} to {freeze_duration + warmup_duration})\")\n # Initialize the first main_scheduler (WarmupV2 -> Cosine)\n sched_cosine_init = CosineAnnealingLR(optimizer, T_max=max(1, cosine_duration), eta_min=base_lr * 0.001)\n sched_warmup_init = GradualWarmupSchedulerV2(optimizer, multiplier=warmup_multiplier, total_epoch=max(1, warmup_duration), after_scheduler=sched_cosine_init)\n main_scheduler = sched_warmup_init\n main_scheduler.step() # Step for the current epoch (epoch 1 of warmup)\n elif epoch <= freeze_duration + warmup_duration:\n if main_scheduler: main_scheduler.step() # Continue stepping initial warmup/cosine\n else: print(\"Error: Initial scheduler missing during warmup phase!\")\n\n # Phase 3: Post-Warmup / Post-Unfreeze Reset\n else: # epoch > freeze_duration + warmup_duration\n if optimizer_changed: # This happens *exactly* at full_unfreeze_epoch\n print(\"Optimizer Changed! Resetting FULL scheduler sequence.\")\n remaining_epochs_total = n_epochs - epoch + 1\n # Define a *new, shorter* warmup + cosine for the remaining epochs\n # Example: 5 epochs warmup, rest cosine. Adjust as needed.\n new_warmup_epochs = min(5, max(1, remaining_epochs_total // 4))\n new_cosine_epochs = remaining_epochs_total - new_warmup_epochs\n print(f\" New schedule phase: Warmup={new_warmup_epochs}, Cosine={new_cosine_epochs}\")\n\n if new_warmup_epochs + new_cosine_epochs != remaining_epochs_total:\n print(\" Warning: New schedule epoch calculation mismatch.\")\n new_cosine_epochs = max(0, remaining_epochs_total - new_warmup_epochs) # Ensure non-negative\n\n # Create NEW schedulers based on the *new* optimizer state\n # Use the LRs set by progressive_layer_unfreezing\n sched_cosine_reset = CosineAnnealingLR(optimizer, T_max=max(1, new_cosine_epochs), eta_min=base_lr * 0.0001) # Lower final LR maybe?\n sched_warmup_reset = GradualWarmupSchedulerV2(optimizer, multiplier=warmup_multiplier, total_epoch=max(1, new_warmup_epochs), after_scheduler=sched_cosine_reset)\n main_scheduler = sched_warmup_reset # OVERWRITE main_scheduler\n\n # Reset Plateau scheduler with the new optimizer\n sched_plateau = ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=5, verbose=True, min_lr=1e-7); sched_plateau._reset()\n print(\" Plateau scheduler reset.\")\n\n main_scheduler.step() # Step the *new* scheduler for the current epoch\n\n elif main_scheduler: # Optimizer didn't change (already past full_unfreeze_epoch)\n print(\"Scheduler Phase: Stepping Existing Post-Warmup Scheduler\")\n main_scheduler.step() # Step the active scheduler (could be warmup_reset or cosine_reset)\n else:\n # This case should ideally not be reached if logic is correct\n print(\"Error: Main scheduler missing in post-warmup/post-reset phase!\")\n # Fallback: Create a simple cosine scheduler for remaining epochs\n remaining_at_fallback = n_epochs - epoch + 1\n main_scheduler = CosineAnnealingLR(optimizer, T_max=max(1, remaining_at_fallback), eta_min=base_lr * 0.001)\n main_scheduler.step()\n\n # --- >>> END REVISED Scheduler Stepping Logic <<< ---\n # Log LR\n current_lr = optimizer.param_groups[0]['lr']\n print(f\"Current LR: {current_lr:.2e}\")\n if wandb and wandb.run: wandb.log({\"learning_rate\": current_lr}, step=epoch)\n\n # --- Train ---\n if len(train_loader) > 0:\n train_loss = train_epoch(\n model, train_loader, optimizer, wandb, epoch, scaler=scaler,\n accum_steps=accum_steps, criterion_multi=criterion_multi,\n mel_idx=mel_idx, lambda_binary=lambda_binary, device=device\n )\n if train_loss is not None: train_losses.append(train_loss)\n else:\n print(\"Skipping train epoch: train_loader is empty.\")\n train_loss = None # Signify no training loss\n\n # --- Validate ---\n if len(valid_loader) > 0:\n val_results = val_epoch(\n model, valid_loader, wandb, epoch, n_test=1, # n_test=1 assumes no TTA\n criterion_multi=criterion_multi, mel_idx=mel_idx,\n lambda_binary=lambda_binary, device=device, use_amp=use_amp\n )\n # Unpack all 15 results\n (val_loss, bin_auc, mc_auc, PROBS, TARGETS, bin_acc, bin_prec,\n bin_rec, bin_f1, bin_spec, mc_acc, mc_f1, mc_cm, bin_cm, temp, roc) = val_results\n val_losses.append(val_loss)\n\n # --- Early Stopping & Best Model Check ---\n # Normalize loss for ES (higher score is better)\n normalized_val_loss = max(0, 1.0 - min(val_loss / 5.0, 1.0)) # Adjust divisor if loss range differs\n metrics_for_es = {\n 'binary_auc': bin_auc if not np.isnan(bin_auc) else 0.0,\n 'val_loss': normalized_val_loss,\n 'binary_recall': bin_rec if not np.isnan(bin_rec) else 0.0,\n 'multiclass_auc': mc_auc if not np.isnan(mc_auc) else 0.0,\n }\n early_stopping(metrics_for_es, model, epoch) # Check ES\n\n composite_score = sum(\n early_stopping.score_weights.get(m, 0) * metrics_for_es.get(m, 0)\n for m in early_stopping.score_weights\n )\n\n # Log comprehensive validation metrics\n if wandb and wandb.run:\n wandb.log({\n \"composite_score\": composite_score, \"val_loss_raw\": val_loss,\n **metrics_for_es, # Include metrics used for ES score\n 'binary_f1': bin_f1, 'binary_acc': bin_acc, 'binary_precision': bin_prec,\n 'binary_specificity': bin_spec, 'multiclass_acc': mc_acc, 'multiclass_f1': mc_f1,\n 'temperature': temp\n }, step=epoch)\n\n # Save best model state if score improved\n if composite_score > best_score:\n best_score = composite_score\n best_model_state = model_ref.state_dict() # Get state from base model\n best_epoch_num = epoch\n best_metrics = { # Store detailed metrics for the best epoch\n 'epoch': epoch, 'composite_score': composite_score, 'val_loss_raw': val_loss,\n 'binary_auc': bin_auc, 'multiclass_auc': mc_auc, 'binary_recall': bin_rec,\n 'binary_f1': bin_f1, 'binary_acc': bin_acc, 'binary_precision': bin_prec,\n 'binary_specificity': bin_spec, 'multiclass_acc': mc_acc, 'multiclass_f1': mc_f1,\n 'temperature': temp, 'binary_cm': bin_cm, 'multiclass_cm': mc_cm\n }\n best_PROBS = PROBS\n best_TARGETS = TARGETS\n print(f\"*** New Best Score: {best_score:.6f} at Epoch {epoch} ***\")\n # Save temporary best model (overwritten each time)\n torch.save({'epoch': best_epoch_num, 'model_state_dict': best_model_state},\n early_stopping.checkpoint_path) # Use ES path\n\n # --- Plateau Scheduler Step ---\n # Step based on composite score during cosine phase\n if epoch > freeze_duration + warmup_duration:\n if hasattr(sched_plateau, 'optimizer') and sched_plateau.optimizer is optimizer:\n sched_plateau.step(composite_score)\n elif optimizer_changed:\n pass # Optimizer was reset, plateau scheduler already updated\n else: # Fallback if optimizer reference somehow differs\n print(\"Warn: Recreating Plateau scheduler due to optimizer mismatch.\")\n sched_plateau=ReduceLROnPlateau(optimizer,mode='max',factor=0.5,patience=5,verbose=True,min_lr=1e-7)\n sched_plateau._reset()\n sched_plateau.step(composite_score)\n\n else: # No validation data\n print(\"Skipping validation epoch: valid_loader is empty.\")\n # Cannot check early stopping or update best model\n\n # --- Check Early Stopping Trigger ---\n if early_stopping.early_stop:\n print(f\"EARLY STOPPING triggered after epoch {epoch}.\")\n break\n\n print(f\"Epoch {epoch} completed in {time.time() - epoch_start_time:.2f}s.\")\n torch.cuda.empty_cache() # Clear cache at end of epoch\n # --- End MAIN TRAINING LOOP ---\n\n except KeyboardInterrupt:\n print(\"\\nTraining interrupted by user.\")\n except Exception as e:\n print(f\"\\nAn error occurred during training loop: {e}\")\n logging.error(traceback.format_exc()) # Log detailed error\n finally:\n # --- Final Operations ---\n total_time = time.time() - total_start_time\n loop_stop_reason = \"Runtime Limit\" if time_limit_reached else (\"Early Stopping\" if early_stopping.early_stop else \"Completed Epochs\")\n print(f\"\\n===== Training Finished / Stopped (Epoch {last_epoch_completed}/{n_epochs}, Reason: {loop_stop_reason}) =====\") # <--- Modified print\n print(f\"Total time: {total_time:.2f}s\")\n\n # Reload best model state if early stopping happened and we have a best epoch recorded\n if early_stopping.early_stop and best_epoch_num > 0:\n best_model_path_es = early_stopping.checkpoint_path\n if os.path.exists(best_model_path_es):\n print(f\"Reloading best model state from Early Stopping checkpoint (Epoch {best_epoch_num})...\")\n try:\n checkpoint = torch.load(best_model_path_es)\n # Load into the base model structure first\n temp_model = model_ref.__class__( # Re-instantiate base model\n backbone_name=cnn_backbone_name, # Or use appropriate args for hybrid\n # Add other necessary args based on model type...\n transformer_backbone_name=transformer_backbone_name if model_type=='hybrid_swin' else None,\n out_dim=out_dim, n_meta_features=n_meta_features, image_size=image_size,\n pretrained=False # Don't need pretrained weights here\n )\n temp_model.load_state_dict(checkpoint['model_state_dict'])\n best_model_state = temp_model.state_dict() # Get the clean state_dict\n print(\" Successfully reloaded best model state.\")\n except Exception as load_err:\n print(f\" Warning: Failed to reload best model state from {best_model_path_es}. Error: {load_err}\")\n # Keep the state from before loop end if loading fails\n else:\n print(f\"Warning: Early stopping triggered, but checkpoint file not found at {best_model_path_es}.\")\n\n\n # Prepare final metrics dictionary\n final_log_metrics = {}\n if best_metrics:\n print(f\"Using metrics from Best Epoch: {best_epoch_num}\")\n final_log_metrics = best_metrics.copy() # Use copy\n elif last_epoch_completed > 0 and 'val_results' in locals():\n print(f\"WARNING: No best metrics saved. Using metrics from last completed validation epoch ({last_epoch_completed}).\")\n # Reconstruct from last val_results if needed (simplified example)\n final_log_metrics = {'epoch': last_epoch_completed,\n 'val_loss_raw': val_loss, 'binary_auc': bin_auc, 'multiclass_auc': mc_auc}\n if best_model_state is None and 'model' in locals(): # Use last model state if no best saved\n best_model_state = model_ref.state_dict()\n best_epoch_num = last_epoch_completed # Mark as last epoch if using last metrics\n else:\n print(\"WARNING: No validation results available to log final metrics.\")\n\n # Add training summary stats\n final_log_metrics['total_training_time_sec'] = round(total_time, 2)\n final_log_metrics['last_epoch_completed'] = last_epoch_completed\n final_log_metrics['best_epoch_logged'] = best_epoch_num # Epoch whose metrics are being logged\n\n # Log final summary to WandB\n if wandb and wandb.run:\n summary = {f\"final_{k}\": v for k, v in final_log_metrics.items() if isinstance(v, (int, float, bool, str))}\n print(\"\\nFinal Summary Metrics:\")\n for k, v in sorted(summary.items()): print(f\" - {k}: {v}\")\n try: wandb.summary.update(summary)\n except Exception as wb_err: print(f\"Warning: Failed to update WandB summary: {wb_err}\")\n\n # --- Save Final Artifacts ---\n if best_model_state is not None and best_epoch_num > 0:\n # Use the explicitly saved ES checkpoint path as the final path now\n final_model_path = early_stopping.checkpoint_path\n # Re-save with additional info if needed, or just use the ES checkpoint\n print(f\"\\nFinal Best Model saved at: {final_model_path}\")\n # Optionally save metrics dict separately or add to checkpoint\n # torch.save({ ... 'final_metrics': final_log_metrics ... }, final_model_path) # Example if re-saving\n\n # Log Model Artifact to WandB\n if wandb and wandb.run:\n model_artifact_name = f\"model-{wandb.run.id}-final\"\n description=f\"Final model ({kernel_type}, Best E{best_epoch_num})\"\n try:\n model_artifact = wandb.Artifact(model_artifact_name, type=\"model\",\n description=description, metadata=summary)\n if os.path.exists(final_model_path):\n model_artifact.add_file(final_model_path)\n wandb.log_artifact(model_artifact, aliases=[\"best\", f\"E{best_epoch_num}\"])\n print(\"Logged FINAL model artifact to WandB.\")\n else: print(f\"Warning: Final model file {final_model_path} not found for artifact logging.\")\n except Exception as art_err: print(f\"Warning: Failed logging model artifact: {art_err}\")\n\n # and if wandb run is active\n if best_PROBS is not None and best_TARGETS is not None and \\\n wandb is not None and wandb.run is not None:\n preds_artifact_name = f\"preds-{wandb.run.id}-final\"\n preds_desc = f\"Val preds/tgts (Best E{best_epoch_num})\"\n probs_fn = f\"best_probs_E{best_epoch_num}_{wandb.run.id}.npy\"\n tgts_fn = f\"best_tgts_E{best_epoch_num}_{wandb.run.id}.npy\"\n valid_ids_fn = f\"best_valid_ids_E{best_epoch_num}_{wandb.run.id}.csv\"\n\n try:\n probs_np = np.array(best_PROBS)\n tgts_np = np.array(best_TARGETS)\n if 'df_valid_set' in locals() and df_valid_set is not None and not df_valid_set.empty:\n valid_ids_df = df_valid_set[['image_name']].copy() \n valid_ids_df.to_csv(valid_ids_fn, index=False)\n print(f\" Saved validation IDs ({len(valid_ids_df)}) to {valid_ids_fn}\")\n else:\n print(\"Warning: Could not save validation IDs (df_valid_set unavailable or empty).\")\n valid_ids_fn = None \n\n np.save(probs_fn, probs_np); np.save(tgts_fn, tgts_np)\n print(f\" Saved best predictions ({probs_np.shape}) to {probs_fn}\")\n print(f\" Saved best targets ({tgts_np.shape}) to {tgts_fn}\")\n\n pred_artifact = wandb.Artifact(preds_artifact_name, type=\"val_predictions\", description=preds_desc)\n pred_artifact.add_file(probs_fn); pred_artifact.add_file(tgts_fn)\n\n if valid_ids_fn and os.path.exists(valid_ids_fn):\n pred_artifact.add_file(valid_ids_fn)\n wandb.log_artifact(pred_artifact, aliases=[\"best_preds\", f\"E{best_epoch_num}\"])\n print(\"Logged final predictions artifact (including validation IDs).\")\n\n # Clean up local files after logging\n try:\n os.remove(probs_fn); os.remove(tgts_fn)\n if valid_ids_fn and os.path.exists(valid_ids_fn): os.remove(valid_ids_fn)\n except OSError as e: print(f\" Warning: Failed to remove temporary prediction files: {e}\")\n\n except Exception as e: print(f\"Error saving/logging prediction artifacts: {e}\")\n else:\n print(\"\\nNo best model state recorded or best epoch is 0, skipping final artifact saving.\")\n\n # Log Final CM Tables from best_metrics if available\n if best_metrics and wandb and wandb.run:\n bin_cm=best_metrics.get('bin_cm',[]); multi_cm=best_metrics.get('mc_cm',[])\n cls_nms=[k for k,v in sorted(globals().get('diagnosis2idx',{}).items(),key=lambda i:i[1])] or [str(i) for i in range(globals().get('out_dim',0))]\n try:\n if bin_cm: wandb.log({f\"binary_cm_table_P\": wandb.Table(columns=['Pred Non-Mel','Pred Mel'],data=bin_cm)}, step=n_epochs)\n if multi_cm and len(multi_cm)==len(cls_nms): wandb.log({f\"multiclass_cm_table_P\": wandb.Table(columns=[f\"Pred {n}\" for n in cls_nms],data=multi_cm)}, step=n_epochs)\n except Exception as e: print(f\"Warn: Failed log final CMs: {e}\")\n\n # --- Function Return ---\n print(\"--- Exiting Adaptive Training Run ---\")\n # Return single factors AND the wandb_run object\n return (model, best_model_state, final_log_metrics, train_losses, val_losses,\n best_PROBS, best_TARGETS, df_valid_set, diagnosis2idx, best_epoch_num,\n kernel_type, early_stopping, model_type, cnn_backbone_name, n_meta_features,\n image_size, out_dim,\n lr_adaptation_factor, regularization_factor,\n wandb_run # <<< ADD wandb_run HERE\n ) # 20 values returned now","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:34:46.445142Z","iopub.execute_input":"2025-04-12T19:34:46.445441Z","iopub.status.idle":"2025-04-12T19:34:46.510579Z","shell.execute_reply.started":"2025-04-12T19:34:46.445420Z","shell.execute_reply":"2025-04-12T19:34:46.509754Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Main execution","metadata":{}},{"cell_type":"code","source":"import time\nif __name__ == \"__main__\":\n\n # Use the globally defined configuration directly\n print(f\"Starting run: {model_type} / {cnn_backbone_name}\" + (f\" / {transformer_backbone_name}\" if model_type == 'hybrid_swin' else \"\"))\n start_time = time.time()\n\n # --- Call the modified function using GLOBAL variables ---\n (model, best_model_state, final_metrics, train_losses, val_losses,\n best_PROBS, best_TARGETS,\n df_valid_set, diagnosis2idx,\n best_epoch_num, kernel_type_ret, early_stopping,\n model_type_ret, cnn_backbone_ret, n_meta_features_ret,\n image_size_ret, out_dim_ret,\n lr_adaptation_factor_ret, regularization_factor_ret,\n wandb_run_ret\n ) = run_single_model(\n model_type=model_type, # Use global\n cnn_backbone_name=cnn_backbone_name, # Use global\n # --- CORRECTED LINE ---\n transformer_backbone_name=transformer_backbone_name if model_type in ['hybrid_swin', 'swin_only'] else None,\n # --- END CORRECTION ---\n use_meta_flag=use_meta, # Use global\n use_external_flag=use_external, # Use global\n target_total_epochs=target_total_epochs # Use global\n # DEBUG flag is used internally by run_single_model based on global value\n )\n # --- END Function Call ---\n\n total_time = time.time() - start_time\n print(f\"\\nTotal Run Time (Training): {total_time:.2f}s\")\n print(\"\\nFinal Run Metrics Summary (from run_single_model):\")\n if final_metrics:\n # Print scalar metrics nicely\n scalar_metrics = {k: v for k, v in final_metrics.items() if isinstance(v, (int, float, bool, str))}\n for k, v in sorted(scalar_metrics.items()):\n print(f\" - {k}: {v}\")\n else:\n print(\" No final metrics available.\")\n\n# Grad-CAM prerequisites check\n if 'df_valid_set' in locals() and 'best_model_state' in locals():\n print(\"\\nPrerequisites for Grad-CAM seem available.\")\n else:\n print(\"\\nWarning: Prerequisites for Grad-CAM might be missing.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:34:54.379110Z","iopub.execute_input":"2025-04-12T19:34:54.379428Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\nAttempting to finish WandB run...\")\nif 'wandb_run_ret' in locals() and wandb_run_ret is not None:\n try:\n # Check if the run associated with the object is still active before finishing\n # Note: Accessing wandb.run might be None even if wandb_run_ret exists if finish was called elsewhere\n if wandb.run and wandb.run.id == wandb_run_ret.id:\n print(f\"Finishing active WandB run: {wandb_run_ret.id}\")\n wandb.finish()\n print(\"WandB run finished.\")\n elif wandb.run:\n print(f\"Warning: Another WandB run seems active ({wandb.run.id}). Not finishing the target run ({wandb_run_ret.id}).\")\n else:\n print(f\"WandB run ({wandb_run_ret.id}) already finished or object is detached.\")\n except Exception as e:\n print(f\"Error finishing WandB run: {e}\")\nelif 'wandb' in locals() and wandb.run is not None:\n # Fallback if wandb_run_ret wasn't captured but a run is somehow still active\n print(f\"Finishing potentially active global WandB run: {wandb.run.id}\")\n wandb.finish()\n print(\"WandB run finished.\")\nelse:\n print(\"No active WandB run object found to finish.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-12T19:34:16.876600Z","iopub.execute_input":"2025-04-12T19:34:16.876965Z","iopub.status.idle":"2025-04-12T19:34:16.883128Z","shell.execute_reply.started":"2025-04-12T19:34:16.876929Z","shell.execute_reply":"2025-04-12T19:34:16.882407Z"}},"outputs":[],"execution_count":null}]}