{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":12983951,"sourceType":"datasetVersion","datasetId":8218100},{"sourceId":12984138,"sourceType":"datasetVersion","datasetId":8218230},{"sourceId":12984139,"sourceType":"datasetVersion","datasetId":8218231},{"sourceId":12984142,"sourceType":"datasetVersion","datasetId":8218234},{"sourceId":12984880,"sourceType":"datasetVersion","datasetId":8218706},{"sourceId":12985020,"sourceType":"datasetVersion","datasetId":8218809},{"sourceId":13009877,"sourceType":"datasetVersion","datasetId":8236592}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n# for filename in filenames:\n# print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:07:44.277143Z","iopub.execute_input":"2025-09-01T00:07:44.277667Z","iopub.status.idle":"2025-09-01T00:07:44.5314Z","shell.execute_reply.started":"2025-09-01T00:07:44.277645Z","shell.execute_reply":"2025-09-01T00:07:44.530877Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport torch\nfrom tqdm import tqdm\nfrom datetime import datetime\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torchvision import transforms, models\nfrom torchvision.models import efficientnet_b0, efficientnet_b3\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torch.optim as optim\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.metrics import roc_auc_score, accuracy_score, classification_report\nimport cv2\nfrom PIL import Image\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-08T11:25:41.206546Z","iopub.execute_input":"2025-09-08T11:25:41.2068Z","iopub.status.idle":"2025-09-08T11:25:49.717829Z","shell.execute_reply.started":"2025-09-08T11:25:41.206778Z","shell.execute_reply":"2025-09-08T11:25:49.71727Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Handling & Preprocessing (Look Wey Shen)","metadata":{}},{"cell_type":"code","source":"# (1) Explore Dataset\n# Dataset path\ndata_path = \"/kaggle/input/siim-isic-melanoma-classification\"\n\n# List files\nprint(os.listdir(data_path))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:07:56.596582Z","iopub.execute_input":"2025-09-01T00:07:56.596993Z","iopub.status.idle":"2025-09-01T00:07:56.601846Z","shell.execute_reply.started":"2025-09-01T00:07:56.596971Z","shell.execute_reply":"2025-09-01T00:07:56.600981Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load Metadata (CSV File)\ntrain_df = pd.read_csv(f\"{data_path}/train.csv\")\ntest_df = pd.read_csv(f\"{data_path}/test.csv\")\n\nprint(f\"Training data shape: {train_df.shape}\")\nprint(f\"Test data shape: {test_df.shape}\")\nprint(\"\\nTraining data columns:\", train_df.columns.tolist())\nprint(\"\\nFirst 5 rows:\")\nprint(train_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:07:58.477749Z","iopub.execute_input":"2025-09-01T00:07:58.478009Z","iopub.status.idle":"2025-09-01T00:07:58.588252Z","shell.execute_reply.started":"2025-09-01T00:07:58.47799Z","shell.execute_reply":"2025-09-01T00:07:58.587645Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# (2) Metadata Cleaning and Preprocessing\ndef clean_metadata(df, is_train=True):\n \"\"\"Clean and preprocess metadata\"\"\"\n df_clean = df.copy()\n \n # Handle missing values in age\n if 'age_approx' in df_clean.columns:\n # Fill missing age with median\n median_age = df_clean['age_approx'].median()\n df_clean['age_approx'].fillna(median_age, inplace=True)\n \n # Normalize age (0-1 scale)\n df_clean['age_normalized'] = df_clean['age_approx'] / 100.0\n print(f\"Age missing values filled with median: {median_age}\")\n \n # Handle missing values in sex\n if 'sex' in df_clean.columns:\n # Fill missing sex with mode\n mode_sex = df_clean['sex'].mode()[0] if not df_clean['sex'].mode().empty else 'male'\n df_clean['sex'].fillna(mode_sex, inplace=True)\n \n # One-hot encode sex\n sex_dummies = pd.get_dummies(df_clean['sex'], prefix='sex')\n df_clean = pd.concat([df_clean, sex_dummies], axis=1)\n print(f\"Sex missing values filled with mode : {mode_sex}\")\n \n # Handle missing values in anatomical site\n if 'anatom_site_general_challenge' in df_clean.columns:\n # Fill missing site with 'unknown'\n df_clean['anatom_site_general_challenge'].fillna('unknown', inplace=True)\n \n # One-hot encode anatomical site\n site_dummies = pd.get_dummies(df_clean['anatom_site_general_challenge'], prefix='site')\n df_clean = pd.concat([df_clean, site_dummies], axis=1)\n print(\"Anatomical site missing values filled with 'unknown'\")\n \n # Create additional features\n if is_train and 'target' in df_clean.columns:\n # Calculate class weights for imbalanced dataset\n target_counts = df_clean['target'].value_counts()\n print(f\"\\nClass distribution:\")\n print(f\"Benign (0) : {target_counts[0]} ({target_counts[0]/len(df_clean)*100:.2f}%)\")\n print(f\"Malignant (1): {target_counts[1]} ({target_counts[1]/len(df_clean)*100:.2f}%)\")\n \n return df_clean","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:01.28229Z","iopub.execute_input":"2025-09-01T00:08:01.28256Z","iopub.status.idle":"2025-09-01T00:08:01.289836Z","shell.execute_reply.started":"2025-09-01T00:08:01.282539Z","shell.execute_reply":"2025-09-01T00:08:01.289035Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Clean training and test metadata\ntrain_clean = clean_metadata(train_df, is_train=True)\ntest_clean = clean_metadata(test_df, is_train=False)\n\nprint(f\"\\nCleaned training data shape: {train_clean.shape}\")\nprint(f\"Cleaned test data shape : {test_clean.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:04.586744Z","iopub.execute_input":"2025-09-01T00:08:04.587018Z","iopub.status.idle":"2025-09-01T00:08:04.628784Z","shell.execute_reply.started":"2025-09-01T00:08:04.586997Z","shell.execute_reply":"2025-09-01T00:08:04.628153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# (3) Patient-based Train/Validation Split\ndef create_patient_split(df, test_size=0.2, random_state=42):\n \"\"\"Create train/validation split by patient ID to avoid data leakage\"\"\"\n \n # Get unique patients\n unique_patients = df['patient_id'].unique()\n print(f\"Total unique patients: {len(unique_patients)}\")\n \n # Split patients (not individual images)\n train_patients, val_patients = train_test_split(\n unique_patients, \n test_size=test_size, \n random_state=random_state,\n stratify=None # Can't stratify by patient easily, would need more complex logic\n )\n \n # Create train/validation dataframes\n train_split = df[df['patient_id'].isin(train_patients)].copy()\n val_split = df[df['patient_id'].isin(val_patients)].copy()\n \n print(f\"Training patients : {len(train_patients)}\")\n print(f\"Validation patients : {len(val_patients)}\")\n print(f\"Training images : {len(train_split)}\")\n print(f\"Validation images : {len(val_split)}\")\n \n # Check target distribution in splits\n if 'target' in df.columns:\n print(f\"\\nTarget distribution in training split:\")\n print(train_split['target'].value_counts(normalize=True))\n print(f\"\\nTarget distribution in validation split:\")\n print(val_split['target'].value_counts(normalize=True))\n \n return train_split, val_split","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:08.8781Z","iopub.execute_input":"2025-09-01T00:08:08.878355Z","iopub.status.idle":"2025-09-01T00:08:08.88415Z","shell.execute_reply.started":"2025-09-01T00:08:08.878335Z","shell.execute_reply":"2025-09-01T00:08:08.883393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create patient-based split\ntrain_split, val_split = create_patient_split(train_clean)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:12.867199Z","iopub.execute_input":"2025-09-01T00:08:12.868034Z","iopub.status.idle":"2025-09-01T00:08:12.887606Z","shell.execute_reply.started":"2025-09-01T00:08:12.868009Z","shell.execute_reply":"2025-09-01T00:08:12.887044Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# (4) Image Preprocessing Functions\nclass ImagePreprocessor:\n def __init__(self, target_size=(224, 224), normalize=True):\n self.target_size = target_size\n self.normalize = normalize\n \n def load_and_preprocess_image(self, image_path, augment=False):\n \"\"\"Load and preprocess a single image\"\"\"\n try:\n # Load image\n image = cv2.imread(image_path)\n if image is None:\n print(f\"Warning: Could not load image {image_path}\")\n return None\n \n # Convert BGR to RGB\n image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n \n # Resize image\n image = cv2.resize(image, self.target_size)\n \n # Normalize pixel values to [0, 1]\n if self.normalize:\n image = image.astype(np.float32) / 255.0\n \n # Apply augmentation if specified\n if augment:\n image = self.apply_augmentation(image)\n \n return image\n \n except Exception as e:\n print(f\"Error processing image {image_path}: {str(e)}\")\n return None\n \n def apply_augmentation(self, image):\n \"\"\"Apply basic data augmentation\"\"\"\n # Random horizontal flip\n if np.random.random() > 0.5:\n image = cv2.flip(image, 1)\n \n # Random rotation (small angle)\n if np.random.random() > 0.5:\n angle = np.random.uniform(-15, 15)\n rows, cols = image.shape[:2]\n M = cv2.getRotationMatrix2D((cols/2, rows/2), angle, 1)\n image = cv2.warpAffine(image, M, (cols, rows))\n \n # Random brightness adjustment\n if np.random.random() > 0.5:\n brightness = np.random.uniform(0.8, 1.2)\n image = np.clip(image * brightness, 0, 1)\n \n return image\n \n def preprocess_batch(self, image_paths, augment=False, batch_size=32):\n \"\"\"Preprocess a batch of images\"\"\"\n images = []\n valid_paths = []\n \n for path in image_paths:\n img = self.load_and_preprocess_image(path, augment=augment)\n if img is not None:\n images.append(img)\n valid_paths.append(path)\n \n return np.array(images), valid_paths","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:15.48717Z","iopub.execute_input":"2025-09-01T00:08:15.487423Z","iopub.status.idle":"2025-09-01T00:08:15.495787Z","shell.execute_reply.started":"2025-09-01T00:08:15.487405Z","shell.execute_reply":"2025-09-01T00:08:15.495132Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Initialize preprocessor\npreprocessor = ImagePreprocessor(target_size=(224, 224), normalize=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:18.395992Z","iopub.execute_input":"2025-09-01T00:08:18.396643Z","iopub.status.idle":"2025-09-01T00:08:18.399954Z","shell.execute_reply.started":"2025-09-01T00:08:18.396621Z","shell.execute_reply":"2025-09-01T00:08:18.399206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# (5) Create Data Loading Functions\ndef create_image_paths(df, image_dir):\n \"\"\"Create full image paths from dataframe\"\"\"\n return [os.path.join(image_dir, f\"{img_id}.jpg\") for img_id in df['image_name']]\n\ndef save_processed_data(train_df, val_df, test_df, output_dir='processed_data'):\n \"\"\"Save processed dataframes\"\"\"\n os.makedirs(output_dir, exist_ok=True)\n \n # Save cleaned metadata\n train_df.to_csv(os.path.join(output_dir, 'train_processed.csv'), index=False)\n val_df.to_csv(os.path.join(output_dir, 'val_processed.csv'), index=False)\n test_df.to_csv(os.path.join(output_dir, 'test_processed.csv'), index=False)\n \n print(f\"Processed data saved to {output_dir}/\")\n \n # Save preprocessing summary\n with open(os.path.join(output_dir, 'preprocessing_summary.txt'), 'w') as f:\n f.write(\"SIIM-ISIC Data Preprocessing Summary\\n\")\n f.write(\"=\"*40 + \"\\n\\n\")\n f.write(f\"Training samples: {len(train_df)}\\n\")\n f.write(f\"Validation samples: {len(val_df)}\\n\")\n f.write(f\"Test samples: {len(test_df)}\\n\")\n f.write(f\"Image target size: {preprocessor.target_size}\\n\")\n f.write(f\"Normalization applied: {preprocessor.normalize}\\n\")\n \n # Feature columns\n feature_cols = [col for col in train_df.columns if col not in ['image_name', 'patient_id', 'target']]\n f.write(f\"\\nFeature columns ({len(feature_cols)}):\\n\")\n for col in feature_cols:\n f.write(f\" - {col}\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:21.545941Z","iopub.execute_input":"2025-09-01T00:08:21.546196Z","iopub.status.idle":"2025-09-01T00:08:21.552983Z","shell.execute_reply.started":"2025-09-01T00:08:21.546176Z","shell.execute_reply":"2025-09-01T00:08:21.55229Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# (6) Execute Preprocessing Pipeline\n# Create image paths\ntrain_paths = create_image_paths(train_split, f\"{data_path}/jpeg/train/\")\nval_paths = create_image_paths(val_split, f\"{data_path}/jpeg/train/\")\ntest_paths = create_image_paths(test_clean, f\"{data_path}/jpeg/test/\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:27.986521Z","iopub.execute_input":"2025-09-01T00:08:27.986823Z","iopub.status.idle":"2025-09-01T00:08:28.026013Z","shell.execute_reply.started":"2025-09-01T00:08:27.986802Z","shell.execute_reply":"2025-09-01T00:08:28.025428Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Validate that images exist\ndef validate_image_paths(paths, df, split_name):\n \"\"\"Validate that image files exist\"\"\"\n existing_paths = []\n valid_indices = []\n \n for i, path in enumerate(paths):\n if os.path.exists(path):\n existing_paths.append(path)\n valid_indices.append(i)\n \n print(f\"{split_name}: {len(existing_paths)}/{len(paths)} images found\")\n return df.iloc[valid_indices].reset_index(drop=True), existing_paths","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:29.510749Z","iopub.execute_input":"2025-09-01T00:08:29.511225Z","iopub.status.idle":"2025-09-01T00:08:29.515487Z","shell.execute_reply.started":"2025-09-01T00:08:29.511205Z","shell.execute_reply":"2025-09-01T00:08:29.514791Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Validate all splits\ntrain_final, train_paths_final = validate_image_paths(train_paths, train_split, \"Training\")\nval_final, val_paths_final = validate_image_paths(val_paths, val_split, \"Validation\")\ntest_final, test_paths_final = validate_image_paths(test_paths, test_clean, \"Test\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:32.360854Z","iopub.execute_input":"2025-09-01T00:08:32.36161Z","iopub.status.idle":"2025-09-01T00:10:41.253088Z","shell.execute_reply.started":"2025-09-01T00:08:32.361577Z","shell.execute_reply":"2025-09-01T00:10:41.252404Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save processed data\nsave_processed_data(train_final, val_final, test_final)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:10:41.254349Z","iopub.execute_input":"2025-09-01T00:10:41.25459Z","iopub.status.idle":"2025-09-01T00:10:41.505719Z","shell.execute_reply.started":"2025-09-01T00:10:41.254572Z","shell.execute_reply":"2025-09-01T00:10:41.505087Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Training samples : {len(train_final)}\")\nprint(f\"Validation samples : {len(val_final)}\")\nprint(f\"Test samples : {len(test_final)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:10:41.506363Z","iopub.execute_input":"2025-09-01T00:10:41.506598Z","iopub.status.idle":"2025-09-01T00:10:41.510949Z","shell.execute_reply.started":"2025-09-01T00:10:41.506581Z","shell.execute_reply":"2025-09-01T00:10:41.510234Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load and display a sample image\nif len(train_paths_final) > 0:\n # Load a sample image\n sample_image = preprocessor.load_and_preprocess_image(train_paths_final[0])\n if sample_image is not None:\n print(f\"Sample image shape: {sample_image.shape}\")\n print(f\"Sample image data type: {sample_image.dtype}\")\n print(f\"Sample image value range: [{sample_image.min():.3f}, {sample_image.max():.3f}]\")\n \n plt.figure(figsize=(8, 6))\n plt.imshow(sample_image)\n plt.title(\"Sample Preprocessed Image\")\n plt.axis('off')\n plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:10:41.5122Z","iopub.execute_input":"2025-09-01T00:10:41.512432Z","iopub.status.idle":"2025-09-01T00:10:42.050754Z","shell.execute_reply.started":"2025-09-01T00:10:41.512417Z","shell.execute_reply":"2025-09-01T00:10:42.049979Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Image Model (CNN) (Tan Jian Hua)","metadata":{}},{"cell_type":"code","source":"# Check if CUDA is available\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:10:54.625848Z","iopub.execute_input":"2025-09-01T00:10:54.626348Z","iopub.status.idle":"2025-09-01T00:10:54.680981Z","shell.execute_reply.started":"2025-09-01T00:10:54.626325Z","shell.execute_reply":"2025-09-01T00:10:54.680266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# STEP 2: Custom Dataset Class\n# ================================\n\nclass MelanomaDataset(Dataset):\n def __init__(self, dataframe, image_dir, transform=None, is_test=False):\n \"\"\"\n Custom dataset for melanoma classification\n \n Args:\n dataframe: pandas DataFrame with image_name and target columns\n image_dir: directory containing images\n transform: torchvision transforms\n is_test: whether this is test data (no targets)\n \"\"\"\n self.df = dataframe.reset_index(drop=True)\n self.image_dir = image_dir\n self.transform = transform\n self.is_test = is_test\n \n def __len__(self):\n return len(self.df)\n \n def __getitem__(self, idx):\n # Get image name and construct path\n image_name = self.df.loc[idx, 'image_name']\n image_path = os.path.join(self.image_dir, f\"{image_name}.jpg\")\n \n # Load image\n try:\n image = Image.open(image_path).convert('RGB')\n except Exception as e:\n print(f\"Error loading image {image_path}: {e}\")\n # Return a blank image if loading fails\n image = Image.new('RGB', (224, 224), (0, 0, 0))\n \n # Apply transforms\n if self.transform:\n image = self.transform(image)\n \n if self.is_test:\n return image, image_name\n else:\n target = torch.tensor(self.df.loc[idx, 'target'], dtype=torch.float32)\n return image, target","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:10:56.756079Z","iopub.execute_input":"2025-09-01T00:10:56.756643Z","iopub.status.idle":"2025-09-01T00:10:56.76244Z","shell.execute_reply.started":"2025-09-01T00:10:56.756623Z","shell.execute_reply":"2025-09-01T00:10:56.761651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# STEP 3: Data Transforms\n# ================================\n\ndef get_transforms(image_size=224, augment=True):\n \"\"\"Get training and validation transforms\"\"\"\n \n if augment:\n train_transform = transforms.Compose([\n transforms.Resize((image_size, image_size)),\n transforms.RandomHorizontalFlip(p=0.5),\n transforms.RandomVerticalFlip(p=0.5),\n transforms.RandomRotation(degrees=20),\n transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),\n transforms.RandomResizedCrop(image_size, scale=(0.8, 1.0)),\n transforms.ToTensor(),\n transforms.Normalize(mean=[0.485, 0.456, 0.406], \n std=[0.229, 0.224, 0.225]) # ImageNet normalization\n ])\n else:\n train_transform = transforms.Compose([\n transforms.Resize((image_size, image_size)),\n transforms.ToTensor(),\n transforms.Normalize(mean=[0.485, 0.456, 0.406], \n std=[0.229, 0.224, 0.225])\n ])\n \n val_transform = transforms.Compose([\n transforms.Resize((image_size, image_size)),\n transforms.ToTensor(),\n transforms.Normalize(mean=[0.485, 0.456, 0.406], \n std=[0.229, 0.224, 0.225])\n ])\n \n return train_transform, val_transform","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:11:00.250884Z","iopub.execute_input":"2025-09-01T00:11:00.251372Z","iopub.status.idle":"2025-09-01T00:11:00.257527Z","shell.execute_reply.started":"2025-09-01T00:11:00.251352Z","shell.execute_reply":"2025-09-01T00:11:00.256663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# STEP 4: EfficientNet Model\n# ================================\n\nclass MelanomaEfficientNet(nn.Module):\n def __init__(self, model_name='efficientnet_b0', num_classes=1, pretrained=True, dropout_rate=0.3):\n \"\"\"\n EfficientNet model for melanoma classification\n \n Args:\n model_name: which EfficientNet variant to use\n num_classes: number of output classes (1 for binary classification)\n pretrained: whether to use pretrained weights\n dropout_rate: dropout rate for regularization\n \"\"\"\n super(MelanomaEfficientNet, self).__init__()\n \n # Load pretrained EfficientNet\n if model_name == 'efficientnet_b0':\n self.backbone = efficientnet_b0(pretrained=pretrained)\n num_features = self.backbone.classifier[1].in_features\n elif model_name == 'efficientnet_b3':\n self.backbone = efficientnet_b3(pretrained=pretrained)\n num_features = self.backbone.classifier[1].in_features\n else:\n raise ValueError(f\"Unsupported model: {model_name}\")\n \n # Replace classifier\n self.backbone.classifier = nn.Sequential(\n nn.Dropout(p=dropout_rate),\n nn.Linear(num_features, 512),\n nn.ReLU(),\n nn.Dropout(p=dropout_rate),\n nn.Linear(512, num_classes)\n )\n \n def forward(self, x):\n return self.backbone(x)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:11:03.846265Z","iopub.execute_input":"2025-09-01T00:11:03.846539Z","iopub.status.idle":"2025-09-01T00:11:03.852476Z","shell.execute_reply.started":"2025-09-01T00:11:03.84652Z","shell.execute_reply":"2025-09-01T00:11:03.851736Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# STEP 5: Training Functions\n# ================================\n\ndef train_one_epoch(model, dataloader, criterion, optimizer, device, epoch):\n \"\"\"Train the model for one epoch\"\"\"\n model.train()\n running_loss = 0.0\n running_acc = 0.0\n \n pbar = tqdm(dataloader, desc=f'Epoch {epoch+1} - Training')\n for images, targets in pbar:\n images, targets = images.to(device), targets.to(device)\n \n # Zero gradients\n optimizer.zero_grad()\n \n # Forward pass\n outputs = model(images).squeeze()\n loss = criterion(outputs, targets)\n \n # Backward pass\n loss.backward()\n optimizer.step()\n \n # Statistics\n running_loss += loss.item()\n predictions = torch.sigmoid(outputs) > 0.5\n running_acc += (predictions == targets).float().mean().item()\n \n pbar.set_postfix({'Loss': f'{loss.item():.4f}'})\n \n epoch_loss = running_loss / len(dataloader)\n epoch_acc = running_acc / len(dataloader)\n \n return epoch_loss, epoch_acc\n\ndef validate_one_epoch(model, dataloader, criterion, device, epoch):\n \"\"\"Validate the model for one epoch\"\"\"\n model.eval()\n running_loss = 0.0\n all_predictions = []\n all_targets = []\n \n with torch.no_grad():\n pbar = tqdm(dataloader, desc=f'Epoch {epoch+1} - Validation')\n for images, targets in pbar:\n images, targets = images.to(device), targets.to(device)\n \n # Forward pass\n outputs = model(images).squeeze()\n loss = criterion(outputs, targets)\n \n # Statistics\n running_loss += loss.item()\n predictions = torch.sigmoid(outputs)\n \n all_predictions.extend(predictions.cpu().numpy())\n all_targets.extend(targets.cpu().numpy())\n \n pbar.set_postfix({'Loss': f'{loss.item():.4f}'})\n \n epoch_loss = running_loss / len(dataloader)\n \n # Calculate metrics\n all_predictions = np.array(all_predictions)\n all_targets = np.array(all_targets)\n \n auc_score = roc_auc_score(all_targets, all_predictions)\n acc_score = accuracy_score(all_targets, all_predictions > 0.5)\n \n return epoch_loss, auc_score, acc_score\n\ndef train_model(model, train_loader, val_loader, num_epochs=20, learning_rate=1e-4, weight_decay=1e-5, early_stopping_patience=5, early_stopping=True):\n \"\"\"Complete training loop\"\"\"\n \n # Loss function and optimizer\n criterion = nn.BCEWithLogitsLoss()\n optimizer = optim.AdamW(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', patience=3, factor=0.5, verbose=True)\n \n # Training history\n history = {\n 'train_loss': [], 'train_acc': [],\n 'val_loss': [], 'val_auc': [], 'val_acc': []\n }\n \n best_auc = 0.0\n best_model_state = None\n epochs_without_improvement = 0\n \n print(\"Starting training...\")\n if early_stopping:\n print(f\"Early stopping enabled with patience: {early_stopping_patience}\")\n print(\"=\"*50)\n \n for epoch in range(num_epochs):\n # Train\n train_loss, train_acc = train_one_epoch(model, train_loader, criterion, optimizer, device, epoch)\n \n # Validate\n val_loss, val_auc, val_acc = validate_one_epoch(model, val_loader, criterion, device, epoch)\n \n # Learning rate scheduling\n scheduler.step(val_loss)\n \n # Save best model\n if val_auc > best_auc:\n best_auc = val_auc\n best_model_state = model.state_dict().copy()\n epochs_without_improvement = 0\n print(f\"New best AUC: {best_auc:.4f}\")\n else:\n epochs_without_improvement += 1\n print(f\"No improvement for {epochs_without_improvement} epoch(s)\")\n \n # Update history\n history['train_loss'].append(train_loss)\n history['train_acc'].append(train_acc)\n history['val_loss'].append(val_loss)\n history['val_auc'].append(val_auc)\n history['val_acc'].append(val_acc)\n \n # Print epoch results\n print(f\"Epoch {epoch+1}/{num_epochs}:\")\n print(f\" Train Loss: {train_loss:.4f}, Train Acc: {train_acc:.4f}\")\n print(f\" Val Loss: {val_loss:.4f}, Val AUC: {val_auc:.4f}, Val Acc: {val_acc:.4f}\")\n print(f\" Current LR: {optimizer.param_groups[0]['lr']:.2e}\")\n\n # Early stopping check\n if early_stopping and epochs_without_improvement >= early_stopping_patience:\n print(f\"\\nEarly stopping triggered after {epoch+1} epochs!\")\n print(f\"No improvement for {early_stopping_patience} consecutive epochs.\")\n break\n \n print(\"-\" * 50)\n \n # Load best model\n model.load_state_dict(best_model_state)\n print(f\"\\nTraining completed! Best validation AUC: {best_auc:.4f}\")\n \n return model, history\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:11:06.845998Z","iopub.execute_input":"2025-09-01T00:11:06.846248Z","iopub.status.idle":"2025-09-01T00:11:06.859259Z","shell.execute_reply.started":"2025-09-01T00:11:06.84623Z","shell.execute_reply":"2025-09-01T00:11:06.858541Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# STEP 6: Visualization Functions\n# ================================\n\ndef plot_training_history(history):\n \"\"\"Plot training history\"\"\"\n fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 10))\n \n epochs = range(1, len(history['train_loss']) + 1)\n \n # Loss\n ax1.plot(epochs, history['train_loss'], 'b-', label='Training Loss')\n ax1.plot(epochs, history['val_loss'], 'r-', label='Validation Loss')\n ax1.set_title('Model Loss')\n ax1.set_xlabel('Epoch')\n ax1.set_ylabel('Loss')\n ax1.legend()\n ax1.grid(True)\n \n # Accuracy\n ax2.plot(epochs, history['train_acc'], 'b-', label='Training Accuracy')\n ax2.plot(epochs, history['val_acc'], 'r-', label='Validation Accuracy')\n ax2.set_title('Model Accuracy')\n ax2.set_xlabel('Epoch')\n ax2.set_ylabel('Accuracy')\n ax2.legend()\n ax2.grid(True)\n \n # AUC\n ax3.plot(epochs, history['val_auc'], 'g-', label='Validation AUC')\n ax3.set_title('Validation AUC')\n ax3.set_xlabel('Epoch')\n ax3.set_ylabel('AUC')\n ax3.legend()\n ax3.grid(True)\n \n # Learning rate would go here if tracked\n ax4.axis('off')\n \n plt.tight_layout()\n plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:11:12.476013Z","iopub.execute_input":"2025-09-01T00:11:12.476499Z","iopub.status.idle":"2025-09-01T00:11:12.482607Z","shell.execute_reply.started":"2025-09-01T00:11:12.476477Z","shell.execute_reply":"2025-09-01T00:11:12.482008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# Create subset (for testing)\n# ================================\ndef create_balanced_subset(df, subset_size=5000, min_positive_ratio=0.05):\n \"\"\"Create a subset that maintains better class balance\"\"\"\n \n # Separate positive and negative cases\n positive_cases = df[df['target'] == 1]\n negative_cases = df[df['target'] == 0]\n \n # Calculate how many positives we want (at least 5% of subset)\n min_positives = int(subset_size * min_positive_ratio)\n available_positives = len(positive_cases)\n \n # Use all available positives if we don't have enough\n n_positives = min(min_positives, available_positives)\n n_negatives = subset_size - n_positives\n \n print(f\"Creating balanced subset:\")\n print(f\" Available positives: {available_positives}\")\n print(f\" Using positives: {n_positives}\")\n print(f\" Using negatives: {n_negatives}\")\n \n # Sample\n if n_positives > 0:\n positive_subset = positive_cases.sample(n=n_positives, random_state=42)\n else:\n positive_subset = pd.DataFrame()\n \n negative_subset = negative_cases.sample(n=n_negatives, random_state=42)\n \n # Combine\n balanced_subset = pd.concat([positive_subset, negative_subset])\n \n print(f\"Subset created:\")\n print(f\" Total: {len(balanced_subset)}\")\n print(f\" Positive: {len(positive_subset)} ({len(positive_subset)/len(balanced_subset)*100:.1f}%)\")\n print(f\" Negative: {len(negative_subset)} ({len(negative_subset)/len(balanced_subset)*100:.1f}%)\")\n \n return balanced_subset.sample(frac=1, random_state=42).reset_index(drop=True) # Shuffle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:11:16.760585Z","iopub.execute_input":"2025-09-01T00:11:16.761048Z","iopub.status.idle":"2025-09-01T00:11:16.766802Z","shell.execute_reply.started":"2025-09-01T00:11:16.761024Z","shell.execute_reply":"2025-09-01T00:11:16.766188Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# STEP 7: Main Training Script\n# ================================\n\ndef main(use_subset=False, subset_size=5000):\n \"\"\"Main function to run the training\"\"\"\n \n # Configuration\n CONFIG = {\n 'model_name': 'efficientnet_b0', # or 'efficientnet_b3'\n 'image_size': 224, # 224 is good for b0, 300 for b3\n 'batch_size': 96, # Adjust based on GPU memory\n 'num_epochs': 20,\n 'learning_rate': 1e-4,\n 'weight_decay': 1e-5,\n 'num_workers': 4, # Adjust based on your system\n 'dropout_rate': 0.3,\n 'early_stopping': True, # Enable/disable early stopping\n 'early_stopping_patience': 5, # Stop if no improvement for 5 epochs\n 'use_subset': use_subset,\n 'subset_size': subset_size\n }\n \n # Data paths (adjust these to match your setup)\n DATA_PATH = \"/kaggle/input/siim-isic-melanoma-classification\"\n PROCESSED_DATA_PATH = \"processed_data\" # From member 1's preprocessing\n \n # Load processed data\n train_df = pd.read_csv(f\"{PROCESSED_DATA_PATH}/train_processed.csv\")\n val_df = pd.read_csv(f\"{PROCESSED_DATA_PATH}/val_processed.csv\")\n test_df = pd.read_csv(f\"{PROCESSED_DATA_PATH}/test_processed.csv\")\n\n print(\"=\"*50)\n print(\"ORIGINAL DATASET SIZES:\")\n print(f\"Training samples: {len(train_df)}\")\n print(f\"Validation samples: {len(val_df)}\")\n print(f\"Test samples: {len(test_df)}\")\n\n # CREATE SUBSET IF REQUESTED\n if use_subset:\n print(\"\\n\" + \"=\"*50)\n print(\"CREATING SUBSET FOR FAST EXPERIMENTATION:\")\n \n # Create balanced subsets\n train_df = create_balanced_subset(train_df, subset_size=subset_size, min_positive_ratio=0.05)\n val_df = create_balanced_subset(val_df, subset_size=subset_size//5, min_positive_ratio=0.05) # 20% of train size\n \n print(\"\\nFINAL SUBSET SIZES:\")\n print(f\"Training samples: {len(train_df)}\")\n print(f\"Validation samples: {len(val_df)}\")\n print(f\"Test samples: {len(test_df)} (unchanged)\")\n \n print(\"=\"*50)\n \n # Get transforms\n train_transform, val_transform = get_transforms(\n image_size=CONFIG['image_size'], \n augment=True\n )\n \n # Create datasets\n train_dataset = MelanomaDataset(\n train_df, \n f\"{DATA_PATH}/jpeg/train\",\n transform=train_transform\n )\n \n val_dataset = MelanomaDataset(\n val_df, \n f\"{DATA_PATH}/jpeg/train\",\n transform=val_transform\n )\n \n # Create data loaders\n train_loader = DataLoader(\n train_dataset, \n batch_size=CONFIG['batch_size'],\n shuffle=True, \n num_workers=CONFIG['num_workers'],\n pin_memory=True if torch.cuda.is_available() else False,\n persistent_workers=True if CONFIG['num_workers'] > 0 else False,\n prefetch_factor=2 if CONFIG['num_workers'] > 0 else 2\n )\n \n val_loader = DataLoader(\n val_dataset, \n batch_size=CONFIG['batch_size'],\n shuffle=False, \n num_workers=CONFIG['num_workers'],\n pin_memory=True if torch.cuda.is_available() else False,\n persistent_workers=True if CONFIG['num_workers'] > 0 else False,\n prefetch_factor=2 if CONFIG['num_workers'] > 0 else 2\n )\n \n print(f\"Train batches: {len(train_loader)}\")\n print(f\"Validation batches: {len(val_loader)}\")\n \n # Create model\n model = MelanomaEfficientNet(\n model_name=CONFIG['model_name'],\n num_classes=1,\n pretrained=True,\n dropout_rate=CONFIG['dropout_rate']\n ).to(device)\n \n print(f\"Model created: {CONFIG['model_name']}\")\n print(f\"Total parameters: {sum(p.numel() for p in model.parameters()):,}\")\n print(f\"Trainable parameters: {sum(p.numel() for p in model.parameters() if p.requires_grad):,}\")\n \n # Train the model\n trained_model, history = train_model(\n model, \n train_loader, \n val_loader,\n num_epochs=CONFIG['num_epochs'],\n learning_rate=CONFIG['learning_rate'],\n weight_decay=CONFIG['weight_decay']\n )\n \n # Plot training history\n plot_training_history(history)\n \n # Save the trained model with subset info\n model_filename = f\"melanoma_efficientnet_{'subset_' if use_subset else 'full_'}model.pth\"\n torch.save({\n 'model_state_dict': trained_model.state_dict(),\n 'config': CONFIG,\n 'history': history\n }, model_filename)\n \n print(f\"Model saved as '{model_filename}'\")\n \n return trained_model, history","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:11:19.76608Z","iopub.execute_input":"2025-09-01T00:11:19.766579Z","iopub.status.idle":"2025-09-01T00:11:19.777063Z","shell.execute_reply.started":"2025-09-01T00:11:19.766556Z","shell.execute_reply":"2025-09-01T00:11:19.776466Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# STEP 8: Inference Functions\n# ================================\n\ndef load_trained_model(model_path, model_name='efficientnet_b0', dropout_rate=0.3):\n \"\"\"Load a trained model\"\"\"\n checkpoint = torch.load(model_path, map_location=device)\n \n model = MelanomaEfficientNet(\n model_name=model_name,\n num_classes=1,\n pretrained=False, # We're loading our own weights\n dropout_rate=dropout_rate\n ).to(device)\n \n model.load_state_dict(checkpoint['model_state_dict'])\n model.eval()\n \n return model, checkpoint.get('config', {}), checkpoint.get('history', {})\n\ndef predict_test_set(model, test_df, data_path, transform, batch_size=32):\n \"\"\"Generate predictions for test set\"\"\"\n test_dataset = MelanomaDataset(\n test_df, \n f\"{data_path}/jpeg/test\",\n transform=transform,\n is_test=True\n )\n \n test_loader = DataLoader(\n test_dataset, \n batch_size=batch_size,\n shuffle=False, \n num_workers=2\n )\n \n model.eval()\n predictions = []\n image_names = []\n \n with torch.no_grad():\n for images, names in tqdm(test_loader, desc='Predicting'):\n images = images.to(device)\n outputs = model(images).squeeze()\n probs = torch.sigmoid(outputs)\n \n predictions.extend(probs.cpu().numpy())\n image_names.extend(names)\n \n return predictions, image_names\n\n# ================================\n# HOW TO USE IT\n# ================================\n\n# Option 1: Quick experiment with subset (30-60 minutes)\n# trained_model, history = main(use_subset=True, subset_size=3000)\n\n# Option 2: Full training (3-5 hours) \n# trained_model, history = main(use_subset=False)\n\n# Option 3: Different subset sizes\n# trained_model, history = main(use_subset=True, subset_size=8000) # Larger subset\n\n# Run the training when this script is executed\nif __name__ == \"__main__\":\n # Uncomment the line below to start training\n trained_model, training_history = main(use_subset=True, subset_size=3000)\n print(\"Setup complete! Uncomment the last line to start training.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:11:25.965921Z","iopub.execute_input":"2025-09-01T00:11:25.966381Z","iopub.status.idle":"2025-09-01T01:36:06.635522Z","shell.execute_reply.started":"2025-09-01T00:11:25.966358Z","shell.execute_reply":"2025-09-01T01:36:06.634825Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Metadata + Fusion Model (Seng Zi Jun)","metadata":{}},{"cell_type":"code","source":"# Set device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-08T11:31:15.19088Z","iopub.execute_input":"2025-09-08T11:31:15.191232Z","iopub.status.idle":"2025-09-08T11:31:15.195698Z","shell.execute_reply.started":"2025-09-08T11:31:15.191208Z","shell.execute_reply":"2025-09-08T11:31:15.195017Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======================\n# DATA LOADING & PREPARATION\n# ======================\nprint(\"Loading and preparing data...\")\n\n# Load processed data from Member 1\ntrain_df = pd.read_csv('/kaggle/input/processed-data/train_processed.csv')\nval_df = pd.read_csv('/kaggle/input/processed-data/val_processed.csv')\n\n# Add image paths\nDATA_PATH = \"/kaggle/input/siim-isic-melanoma-classification\"\ntrain_df['image_path'] = f\"{DATA_PATH}/jpeg/train/\" + train_df['image_name'] + '.jpg'\nval_df['image_path'] = f\"{DATA_PATH}/jpeg/train/\" + val_df['image_name'] + '.jpg'\n\n# Define metadata features\nfeature_columns = [\n 'age_normalized', 'sex_female', 'sex_male', \n 'site_head/neck', 'site_lower extremity', 'site_oral/genital',\n 'site_palms/soles', 'site_torso', 'site_unknown', 'site_upper extremity'\n]\n\n# Extract metadata features and labels\nX_meta_train = train_df[feature_columns].values.astype(np.float32)\ny_train = train_df['target'].values\nX_meta_val = val_df[feature_columns].values.astype(np.float32)\ny_val = val_df['target'].values\n\n# Convert to tensors\nX_meta_train_tensor = torch.FloatTensor(X_meta_train).to(device)\ny_train_tensor = torch.FloatTensor(y_train).unsqueeze(1).to(device)\nX_meta_val_tensor = torch.FloatTensor(X_meta_val).to(device)\ny_val_tensor = torch.FloatTensor(y_val).unsqueeze(1).to(device)\n\nprint(f\"Data loaded: Train {len(train_df)}, Val {len(val_df)}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-08T11:31:16.160341Z","iopub.execute_input":"2025-09-08T11:31:16.160619Z","iopub.status.idle":"2025-09-08T11:31:16.27878Z","shell.execute_reply.started":"2025-09-08T11:31:16.160595Z","shell.execute_reply":"2025-09-08T11:31:16.278145Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======================\n# METADATA MLP \n# ======================\nclass MetadataMLP(nn.Module):\n def __init__(self, input_dim, hidden_dims=[128, 64, 64], dropout_rate=0.3):\n super(MetadataMLP, self).__init__()\n \n layers = []\n prev_dim = input_dim\n \n for hidden_dim in hidden_dims:\n layers.extend([\n nn.Linear(prev_dim, hidden_dim),\n nn.ReLU(),\n nn.Dropout(dropout_rate)\n ])\n prev_dim = hidden_dim\n \n # Output layer\n layers.append(nn.Linear(prev_dim, 1))\n \n self.network = nn.Sequential(*layers)\n \n def forward(self, x):\n return self.network(x)\n \n def extract_features(self, x):\n features = x\n for layer in self.network[:-1]:\n features = layer(features)\n return features\n\nprint(\"Creating and training metadata MLP...\")\nmetadata_mlp = MetadataMLP(len(feature_columns)).to(device)\n\n# Loss, optimizer, scheduler\ncriterion = nn.BCEWithLogitsLoss()\noptimizer = optim.Adam(metadata_mlp.parameters(), lr=1e-3)\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', patience=3)\n\n# Early stopping setup\npatience = 10 \nbest_val_auc = 0.0\nbest_epoch = 0\npatience_counter = 0\nmax_epochs = 1000 \n\nprint(\"Epoch\\tTrain Loss\\tTrain Acc\\tTrain AUC\\tVal Loss\\tVal Acc\\tVal AUC\")\nprint(\"-\" * 80)\n\nfor epoch in range(max_epochs):\n # Training phase\n metadata_mlp.train()\n optimizer.zero_grad()\n outputs = metadata_mlp(X_meta_train_tensor)\n loss = criterion(outputs, y_train_tensor)\n loss.backward()\n optimizer.step()\n \n with torch.no_grad():\n train_probs = torch.sigmoid(outputs)\n train_preds = (train_probs > 0.5).float()\n train_acc = (train_preds == y_train_tensor).float().mean()\n try:\n train_auc = roc_auc_score(y_train_tensor.cpu().numpy(), train_probs.cpu().numpy())\n except:\n train_auc = 0.5\n \n # Validation phase\n metadata_mlp.eval()\n with torch.no_grad():\n val_outputs = metadata_mlp(X_meta_val_tensor)\n val_loss = criterion(val_outputs, y_val_tensor)\n val_probs = torch.sigmoid(val_outputs)\n val_preds = (val_probs > 0.5).float()\n val_acc = (val_preds == y_val_tensor).float().mean()\n try:\n val_auc = roc_auc_score(y_val_tensor.cpu().numpy(), val_probs.cpu().numpy())\n except:\n val_auc = 0.5\n \n scheduler.step(val_auc)\n \n # Print every 10 epochs\n if (epoch + 1) % 10 == 0:\n print(f\"{epoch+1:3d}\\t{loss.item():.4f}\\t\\t{train_acc.item():.4f}\\t\\t{train_auc:.4f}\\t\\t\"\n f\"{val_loss.item():.4f}\\t\\t{val_acc.item():.4f}\\t\\t{val_auc:.4f}\")\n \n # Check for improvement\n if val_auc > best_val_auc:\n best_val_auc = val_auc\n best_epoch = epoch\n patience_counter = 0\n else:\n patience_counter += 1\n \n # Early stopping\n if patience_counter >= patience:\n print(f\"\\nEarly stopping at epoch {epoch+1} (best epoch {best_epoch+1}, best val AUC {best_val_auc:.4f})\")\n break\n\nprint(\"\\nMetadata MLP training complete\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-08T11:31:28.591538Z","iopub.execute_input":"2025-09-08T11:31:28.592237Z","iopub.status.idle":"2025-09-08T11:31:28.762215Z","shell.execute_reply.started":"2025-09-08T11:31:28.592212Z","shell.execute_reply":"2025-09-08T11:31:28.761569Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======================\n# LOAD MEMBER 2's MODEL\n# ======================\nprint(\"Loading Member 2's model...\")\n\n# Define the same architecture as Member 2\nclass MelanomaEfficientNet(nn.Module):\n def __init__(self, num_classes=1, dropout_rate=0.3):\n super(MelanomaEfficientNet, self).__init__()\n \n # Load pretrained EfficientNet (same as Member 2)\n self.backbone = models.efficientnet_b0(pretrained=False)\n in_features = self.backbone.classifier[1].in_features\n \n # Replace classifier (same as Member 2)\n self.backbone.classifier = nn.Sequential(\n nn.Dropout(p=dropout_rate),\n nn.Linear(in_features, 512),\n nn.ReLU(),\n nn.Dropout(p=dropout_rate),\n nn.Linear(512, num_classes)\n )\n \n def forward(self, x):\n return self.backbone(x)\n \n def extract_features(self, x):\n \"\"\"Extract features before final classifier\"\"\"\n x = self.backbone.features(x)\n x = self.backbone.avgpool(x)\n x = torch.flatten(x, 1)\n return x\n\n# Load Member 2's model\ntry:\n member2_model = MelanomaEfficientNet()\n \n # Load weights\n checkpoint = torch.load(\n '/kaggle/input/melanoma-efficientnet-model/melanoma_efficientnet_subset_model.pth',\n map_location=device,\n weights_only=False # Allow loading potentially unsafe objects\n )\n \n # Handle both state dict and full checkpoint formats\n if 'model_state_dict' in checkpoint:\n member2_model.load_state_dict(checkpoint['model_state_dict'])\n else:\n member2_model.load_state_dict(checkpoint)\n \n member2_model = member2_model.to(device)\n member2_model.eval()\n \n print(\"Member 2's model loaded successfully\")\n \nexcept Exception as e:\n print(f\"Failed to load Member 2's model: {e}\")\n print(\"Using ImageNet pretrained model as fallback...\")\n member2_model = models.efficientnet_b0(pretrained=True)\n member2_model.classifier[1] = nn.Linear(member2_model.classifier[1].in_features, 1)\n member2_model = member2_model.to(device)\n member2_model.eval()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-08T11:31:32.522664Z","iopub.execute_input":"2025-09-08T11:31:32.523258Z","iopub.status.idle":"2025-09-08T11:31:32.72484Z","shell.execute_reply.started":"2025-09-08T11:31:32.523235Z","shell.execute_reply":"2025-09-08T11:31:32.724227Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======================\n# FUSION MODEL ARCHITECTURE\n# ======================\nclass FusionModel(nn.Module):\n def __init__(self, image_model, meta_input_dim, meta_hidden=[128, 64],\n fusion_hidden=[512, 256], dropout=0.3):\n super(FusionModel, self).__init__()\n self.image_model = image_model\n\n # 🔹 Get actual EfficientNet feature size\n with torch.no_grad():\n dummy = torch.randn(1, 3, 224, 224).to(next(image_model.parameters()).device)\n img_out_dim = image_model.extract_features(dummy).shape[1]\n\n # Metadata MLP\n layers = []\n prev_dim = meta_input_dim\n for h in meta_hidden:\n layers.extend([\n nn.Linear(prev_dim, h),\n nn.BatchNorm1d(h),\n nn.ReLU(),\n nn.Dropout(dropout)\n ])\n prev_dim = h\n self.meta_mlp = nn.Sequential(*layers)\n meta_out_dim = prev_dim\n\n # Fusion MLP\n fusion_layers = []\n prev_dim = img_out_dim + meta_out_dim\n for h in fusion_hidden:\n fusion_layers.extend([\n nn.Linear(prev_dim, h),\n nn.BatchNorm1d(h),\n nn.ReLU(),\n nn.Dropout(dropout)\n ])\n prev_dim = h\n fusion_layers.append(nn.Linear(prev_dim, 1))\n self.fusion = nn.Sequential(*fusion_layers)\n\n def forward(self, images, metadata):\n # Extract features properly\n image_features = self.image_model.extract_features(images)\n meta_features = self.meta_mlp(metadata)\n combined = torch.cat([image_features, meta_features], dim=1)\n return self.fusion(combined)\n\n# Create fusion model\nfusion_model = FusionModel(\n image_model=member2_model, \n meta_input_dim=len(feature_columns)\n).to(device)\nprint(f\"Fusion model created. Trainable parameters: {sum(p.numel() for p in fusion_model.parameters() if p.requires_grad):,}\")\n\n# ======================\n# FUSION DATASET\n# ======================\nfrom torch.utils.data import Dataset\nfrom PIL import Image\n\nclass FusionDataset(Dataset):\n def __init__(self, dataframe, feature_columns, transform=None):\n self.data = dataframe.reset_index(drop=True)\n self.feature_columns = feature_columns\n self.transform = transform\n\n def __len__(self):\n return len(self.data)\n\n def __getitem__(self, idx):\n row = self.data.iloc[idx]\n \n # Load image\n image = Image.open(row['image_path']).convert(\"RGB\")\n if self.transform:\n image = self.transform(image)\n \n # Metadata as tensor (force numeric)\n metadata = row[self.feature_columns].astype(float).values\n metadata = torch.tensor(metadata, dtype=torch.float32)\n \n # Target label\n target = torch.tensor(float(row['target']), dtype=torch.float32)\n \n return (image, metadata), target","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-08T11:36:44.39653Z","iopub.execute_input":"2025-09-08T11:36:44.397127Z","iopub.status.idle":"2025-09-08T11:36:44.434804Z","shell.execute_reply.started":"2025-09-08T11:36:44.397075Z","shell.execute_reply":"2025-09-08T11:36:44.434246Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======================\n# SUBSET TRAINING \n# ======================\ndef create_balanced_subset(df, subset_size=5000, min_positive_ratio=0.05):\n \"\"\"Create a subset that maintains better class balance\"\"\"\n \n # Separate positive and negative cases\n positive_cases = df[df['target'] == 1]\n negative_cases = df[df['target'] == 0]\n \n # Calculate how many positives we want (at least 5% of subset)\n min_positives = int(subset_size * min_positive_ratio)\n available_positives = len(positive_cases)\n \n # Use all available positives if we don't have enough\n n_positives = min(min_positives, available_positives)\n n_negatives = subset_size - n_positives\n \n print(f\"Creating balanced subset:\")\n print(f\" Available positives: {available_positives}\")\n print(f\" Using positives: {n_positives}\")\n print(f\" Using negatives: {n_negatives}\")\n \n # Sample\n if n_positives > 0:\n positive_subset = positive_cases.sample(n=n_positives, random_state=42)\n else:\n positive_subset = pd.DataFrame()\n \n negative_subset = negative_cases.sample(n=n_negatives, random_state=42)\n \n # Combine\n balanced_subset = pd.concat([positive_subset, negative_subset])\n \n print(f\"Subset created:\")\n print(f\" Total: {len(balanced_subset)}\")\n print(f\" Positive: {len(positive_subset)} ({len(positive_subset)/len(balanced_subset)*100:.1f}%)\")\n print(f\" Negative: {len(negative_subset)} ({len(negative_subset)/len(balanced_subset)*100:.1f}%)\")\n \n return balanced_subset.sample(frac=1, random_state=42).reset_index(drop=True) # Shuffle\n\n# Create balanced subsets for faster training\nprint(\"Creating balanced subsets for faster training...\")\ntrain_subset = create_balanced_subset(train_df, subset_size=3000, min_positive_ratio=0.05)\nval_subset = create_balanced_subset(val_df, subset_size=600, min_positive_ratio=0.05)\n\nprint(f\"Using subsets: Train {len(train_subset)}, Val {len(val_subset)}\")\n\n\ntransform = transforms.Compose([\n transforms.RandomResizedCrop(224, scale=(0.8, 1.0)),\n transforms.RandomHorizontalFlip(),\n transforms.RandomVerticalFlip(),\n transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),\n transforms.ToTensor(),\n transforms.Normalize(mean=[0.485, 0.456, 0.406],\n std=[0.229, 0.224, 0.225])\n])\n\n# Create subset datasets and loaders\ntrain_subset_dataset = FusionDataset(train_subset, feature_columns, transform)\nval_subset_dataset = FusionDataset(val_subset, feature_columns, transform)\n\ntrain_subset_loader = DataLoader(train_subset_dataset, batch_size=94, shuffle=True, num_workers=2)\nval_subset_loader = DataLoader(val_subset_dataset, batch_size=86, shuffle=False, num_workers=2)\n\nprint(f\"Subset loaders created: {len(train_subset_loader)} train batches, {len(val_subset_loader)} val batches\")\n\ndef train_epoch(model, dataloader, criterion, optimizer, device):\n model.train()\n running_loss = 0.0\n all_preds, all_targets, all_probs = [], [], []\n \n for (images, metadata), targets in tqdm(dataloader, desc='Training'):\n images, metadata, targets = images.to(device), metadata.to(device), targets.to(device)\n \n optimizer.zero_grad()\n outputs = model(images, metadata).squeeze()\n loss = criterion(outputs, targets.squeeze())\n loss.backward()\n optimizer.step()\n \n running_loss += loss.item()\n probs = torch.sigmoid(outputs)\n \n all_preds.extend((probs > 0.5).float().cpu().numpy())\n all_probs.extend(probs.detach().cpu().numpy())\n all_targets.extend(targets.cpu().numpy())\n \n epoch_loss = running_loss / len(dataloader)\n epoch_auc = roc_auc_score(all_targets, all_probs)\n epoch_acc = accuracy_score(all_targets, all_preds)\n return epoch_loss, epoch_auc, epoch_acc\n\ndef validate_epoch(model, dataloader, criterion, device):\n model.eval()\n running_loss = 0.0\n all_preds, all_targets, all_probs = [], [], []\n \n with torch.no_grad():\n for (images, metadata), targets in tqdm(dataloader, desc='Validation'):\n images, metadata, targets = images.to(device), metadata.to(device), targets.to(device)\n \n outputs = model(images, metadata).squeeze()\n loss = criterion(outputs, targets.squeeze())\n \n running_loss += loss.item()\n probs = torch.sigmoid(outputs)\n \n all_preds.extend((probs > 0.5).float().cpu().numpy())\n all_probs.extend(probs.cpu().numpy())\n all_targets.extend(targets.cpu().numpy())\n \n epoch_loss = running_loss / len(dataloader)\n epoch_auc = roc_auc_score(all_targets, all_probs)\n epoch_acc = accuracy_score(all_targets, all_preds)\n return epoch_loss, epoch_auc, epoch_acc\n\n# ======================\n# TRAINING LOOP WITH SUBSETS\n# ======================\n# Compute class weights\npos_weight = torch.tensor([len(train_subset[train_subset['target']==0]) / \n len(train_subset[train_subset['target']==1])]).to(device)\n\ncriterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight)\noptimizer = optim.Adam(fusion_model.parameters(), lr=1e-4)\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', patience=3)\n\n# Create models directory if it doesn't exist\nos.makedirs('/kaggle/working/models', exist_ok=True)\n\n# Generate unique model filename with timestamp\ntimestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\nbest_model_path = f'/kaggle/working/models/best_fusion_model_{timestamp}.pth'\n\nbest_auc = 0.0\nepochs_without_improvement = 0\nearly_stopping_patience = 5 # Stop if no improvement for 5 epochs\nhistory = {'train_loss': [], 'train_auc': [], 'train_acc': [], 'val_loss': [], 'val_auc': [],\n 'val_acc':[]}\n\nprint(\"Starting fusion model training with subsets...\")\nprint(f\"Early stopping enabled with patience: {early_stopping_patience}\")\nprint(f\"Best model will be saved to: {best_model_path}\")\nprint(\"=\" * 50)\n\nfor epoch in range(20):\n print(f\"\\nEpoch {epoch+1}/20\")\n print(\"-\" * 40)\n \n # Train with subset\n train_loss, train_auc, train_acc = train_epoch(fusion_model, train_subset_loader, criterion, optimizer, device)\n \n # Validate with subset\n val_loss, val_auc, val_acc = validate_epoch(fusion_model, val_subset_loader, criterion, device)\n \n # Update history\n history['train_loss'].append(train_loss)\n history['train_auc'].append(train_auc)\n history['train_acc'].append(train_acc)\n history['val_loss'].append(val_loss)\n history['val_auc'].append(val_auc)\n history['val_acc'].append(val_acc)\n \n # Save best model\n if val_auc > best_auc:\n best_auc = val_auc\n epochs_without_improvement = 0\n torch.save(fusion_model.state_dict(), 'best_fusion_model.pth')\n print(f\"New best model saved with AUC: {best_auc:.4f}\")\n print(f\"Model saved to: {best_model_path}\")\n\n # Immediate verification\n if os.path.exists(best_model_path):\n file_size = os.path.getsize(best_model_path) / (1024 * 1024)\n print(f\"File verified: {file_size:.2f} MB\")\n else:\n print(\"Warning: File not found immediately after saving!\")\n else:\n epochs_without_improvement += 1\n print(f\"No improvement for {epochs_without_improvement} epoch(s)\")\n \n # Learning rate scheduling\n scheduler.step(val_auc)\n \n print(f\"Train Loss: {train_loss:.4f}, Train AUC: {train_auc:.4f}, Train Acc: {train_acc:.4f}\")\n print(f\"Val Loss: {val_loss:.4f}, Val AUC: {val_auc:.4f}, Val Acc: {val_acc:.4f}\")\n print(f\"Learning Rate: {optimizer.param_groups[0]['lr']:.2e}\")\n\n # Early stopping check\n if epochs_without_improvement >= early_stopping_patience:\n print(f\"\\nEarly stopping triggered after {epoch+1} epochs!\")\n print(f\"No improvement for {early_stopping_patience} consecutive epochs.\")\n break\n \nprint(f\"\\nTraining completed! Best Validation AUC: {best_auc:.4f}\")\n\n# ======================\n# MODEL VERIFICATION & DOWNLOAD LINKS\n# ======================\nprint(\"\\n\" + \"=\"*60)\nprint(\"MODEL SAVING VERIFICATION\")\nprint(\"=\"*60)\n\n# Check if model file exists\nmodel_file_exists = os.path.exists(best_model_path)\n\nprint(f\"Best model path: {best_model_path}\")\nprint(f\"Model file exists: {'True' if model_file_exists else 'False'}\")\n\nif model_file_exists:\n # Get file size\n file_size = os.path.getsize(best_model_path) / (1024 * 1024)\n print(f\"Model file size: {file_size:.2f} MB\")\n \n # Load the best model\n try:\n fusion_model.load_state_dict(torch.load(best_model_path))\n print(\"Model loaded successfully!\")\n except Exception as e:\n print(f\"Error loading model: {e}\")\n \n # Create download link\n from IPython.display import FileLink, display\n print(\"\\nDOWNLOAD LINK:\")\n display(FileLink(best_model_path, result_html_prefix=\"Click to download: \"))\nelse:\n print(\"Model file not found! Checking directory...\")\n print(\"\\nFILES IN /kaggle/working/:\")\n !ls -la /kaggle/working/\n print(\"\\nFILES IN /kaggle/working/models/:\")\n !ls -la /kaggle/working/models/ 2>/dev/null || echo \"Models directory doesn't exist\"\n\n# Force sync to ensure files are visible\n!sync\nprint(\"\\nFile system synced. Check Kaggle file browser on the right.\")\n\nprint(f\"\\nBest model should be at: {best_model_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-08T11:36:49.101248Z","iopub.execute_input":"2025-09-08T11:36:49.102011Z","iopub.status.idle":"2025-09-08T12:19:54.953443Z","shell.execute_reply.started":"2025-09-08T11:36:49.101985Z","shell.execute_reply":"2025-09-08T12:19:54.952664Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Evaluation, Analysis & Reporting (Wong Kang Yi)","metadata":{}},{"cell_type":"code","source":"# ================================\n# Full Evaluation Code with Metrics, Visualization, and Error Analysis\n# ================================\n\nimport os\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom torchvision.models import efficientnet_b0\nimport torch.nn as nn\nimport pandas as pd\nfrom PIL import Image\nimport numpy as np\nfrom sklearn.metrics import (\n accuracy_score, roc_auc_score, confusion_matrix,\n precision_score, recall_score, f1_score, roc_curve, precision_recall_curve\n)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom torch.serialization import safe_globals\nfrom tqdm import tqdm\n\n# -------------------------------\n# Configuration\n# -------------------------------\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nBATCH_SIZE = 32\nIMAGE_SIZE = 224\nMODEL_PATH = \"/kaggle/input/melanoma-efficientnet-subset-model-pth/melanoma_efficientnet_subset_model.pth\"\n\nVAL_CSV = \"/kaggle/input/validate-dataset/val_processed.csv\"\n\n\n\nIMAGE_DIR = \"/kaggle/input/siim-isic-melanoma-classification/jpeg/train\"\n\n# -------------------------------\n# Dataset Class\n# -------------------------------\nclass MelanomaDataset(Dataset):\n def __init__(self, dataframe, image_dir, transform=None, is_test=False):\n self.df = dataframe.reset_index(drop=True)\n self.image_dir = image_dir\n self.transform = transform\n self.is_test = is_test\n\n def __len__(self):\n return len(self.df)\n\n def __getitem__(self, idx):\n image_name = self.df.loc[idx, 'image_name']\n image_path = os.path.join(self.image_dir, f\"{image_name}.jpg\")\n try:\n image = Image.open(image_path).convert('RGB')\n except Exception as e:\n print(f\"Error loading {image_path}: {e}\")\n image = Image.new('RGB', (224, 224), (0, 0, 0))\n\n if self.transform:\n image = self.transform(image)\n\n if self.is_test:\n return image, image_name\n else:\n target = torch.tensor(self.df.loc[idx, 'target'], dtype=torch.float32)\n return image, target\n\n# -------------------------------\n# Transforms\n# -------------------------------\ntransform = transforms.Compose([\n transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n transforms.ToTensor(),\n transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\n\n# -------------------------------\n# Load Validation Data\n# -------------------------------\nval_df = pd.read_csv(VAL_CSV)\nval_dataset = MelanomaDataset(val_df, IMAGE_DIR, transform=transform)\nval_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False)\n\n# -------------------------------\n# Model Definition\n# -------------------------------\nclass MelanomaEfficientNet(nn.Module):\n def __init__(self, model_name='efficientnet_b0', num_classes=1, dropout_rate=0.3):\n super(MelanomaEfficientNet, self).__init__()\n if model_name == 'efficientnet_b0':\n self.backbone = efficientnet_b0(pretrained=False)\n num_features = self.backbone.classifier[1].in_features\n else:\n raise ValueError(\"Unsupported model_name\")\n\n self.backbone.classifier = nn.Sequential(\n nn.Dropout(p=dropout_rate),\n nn.Linear(num_features, 512),\n nn.ReLU(),\n nn.Dropout(p=dropout_rate),\n nn.Linear(512, num_classes)\n )\n\n def forward(self, x):\n return self.backbone(x)\n\n# -------------------------------\n# Load Pretrained Checkpoint Safely\n# -------------------------------\nwith safe_globals(['scalar']):\n checkpoint = torch.load(MODEL_PATH, map_location=DEVICE, weights_only=False)\n\n# Initialize model\nmodel = MelanomaEfficientNet(model_name='efficientnet_b0', num_classes=1, dropout_rate=0.3).to(DEVICE)\n\n# Load state dict\nmodel.load_state_dict(checkpoint['model_state_dict'])\nmodel.eval()\nprint(\"Pretrained EfficientNet loaded successfully!\")\n\n# -------------------------------\n# Evaluation Loop\n# -------------------------------\nall_preds = []\nall_labels = []\nimage_names = []\n\nwith torch.no_grad():\n for i, (images, labels) in enumerate(tqdm(val_loader, desc=\"Evaluating\")):\n images = images.to(DEVICE)\n labels = labels.to(DEVICE)\n outputs = model(images).squeeze()\n probs = torch.sigmoid(outputs)\n\n all_preds.extend(probs.cpu().numpy())\n all_labels.extend(labels.cpu().numpy())\n image_names.extend(val_df.iloc[i * BATCH_SIZE:(i + 1) * BATCH_SIZE]['image_name'].tolist())\n\n# -------------------------------\n# Metrics\n# -------------------------------\nbinary_preds = [1 if p >= 0.5 else 0 for p in all_preds]\n\naccuracy = accuracy_score(all_labels, binary_preds)\nroc_auc = roc_auc_score(all_labels, all_preds)\nprecision = precision_score(all_labels, binary_preds)\nrecall = recall_score(all_labels, binary_preds)\nf1 = f1_score(all_labels, binary_preds)\ncm = confusion_matrix(all_labels, binary_preds)\n\nprint(f\"\\nValidation Accuracy : {accuracy:.4f}\")\nprint(f\"Validation ROC-AUC : {roc_auc:.4f}\")\nprint(f\"Precision : {precision:.4f}\")\nprint(f\"Recall : {recall:.4f}\")\nprint(f\"F1-Score : {f1:.4f}\")\nprint(\"\\nConfusion Matrix:\")\nprint(cm)\n\n# -------------------------------\n# Confusion Matrix Heatmap\n# -------------------------------\nplt.figure(figsize=(6, 5))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=['Negative', 'Positive'], yticklabels=['Negative', 'Positive'])\nplt.title(\"Confusion Matrix\")\nplt.xlabel(\"Predicted\")\nplt.ylabel(\"Actual\")\nplt.show()\n\n# -------------------------------\n# ROC Curve\n# -------------------------------\nfpr, tpr, _ = roc_curve(all_labels, all_preds)\nplt.figure(figsize=(6, 5))\nplt.plot(fpr, tpr, label=f\"ROC AUC = {roc_auc:.4f}\")\nplt.plot([0, 1], [0, 1], linestyle='--', color='gray')\nplt.xlabel(\"False Positive Rate\")\nplt.ylabel(\"True Positive Rate\")\nplt.title(\"ROC Curve\")\nplt.legend()\nplt.show()\n\n# -------------------------------\n# Precision-Recall Curve\n# -------------------------------\nprecision_vals, recall_vals, _ = precision_recall_curve(all_labels, all_preds)\nplt.figure(figsize=(6, 5))\nplt.plot(recall_vals, precision_vals, label=\"Precision-Recall Curve\")\nplt.xlabel(\"Recall\")\nplt.ylabel(\"Precision\")\nplt.title(\"Precision-Recall Curve\")\nplt.legend()\nplt.show()\n\n# -------------------------------\n# Accuracy & F1 Bar Chart\n# -------------------------------\nplt.figure(figsize=(6, 4))\nplt.bar(['Accuracy', 'F1-Score'], [accuracy, f1], color=['skyblue', 'orange'])\nplt.title(\"Accuracy vs F1-Score\")\nplt.ylim(0, 1)\nplt.show()\n\n# -------------------------------\n# Error Analysis: Misclassified Images\n# -------------------------------\nmisclassified = []\nfor idx, (true_label, pred_label) in enumerate(zip(all_labels, binary_preds)):\n if true_label != pred_label:\n misclassified.append((image_names[idx], true_label, pred_label))\n\nprint(f\"\\nTotal Misclassified: {len(misclassified)}\")\nprint(\"Sample Misclassified Cases:\")\nprint(misclassified[:10])\n\n# -------------------------------\n# Patient-Level Mistakes (if patient_id exists)\n# -------------------------------\nif 'patient_id' in val_df.columns:\n misclassified_df = val_df.iloc[[idx for idx, (t, p) in enumerate(zip(all_labels, binary_preds)) if t != p]]\n patient_counts = misclassified_df['patient_id'].value_counts().head(10)\n print(\"\\nTop Patients with Most Misclassifications:\")\n print(patient_counts)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-10T13:51:23.787836Z","iopub.execute_input":"2025-09-10T13:51:23.788256Z","iopub.status.idle":"2025-09-10T13:58:57.558188Z","shell.execute_reply.started":"2025-09-10T13:51:23.788225Z","shell.execute_reply":"2025-09-10T13:58:57.556956Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================================\n# Full Evaluation Code for Fusion Model with Metrics, Visualization, and Error Analysis\n# =====================================\n\nimport os\nimport torch\nimport torch.nn as nn\nimport pandas as pd\nimport numpy as np\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision.models import efficientnet_b0\nfrom types import MethodType\nfrom torchvision import transforms\nfrom PIL import Image\nfrom sklearn.metrics import (\n accuracy_score, roc_auc_score, confusion_matrix,\n precision_score, recall_score, f1_score,\n roc_curve, precision_recall_curve\n)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\n\n# -------------------------------\n# Configuration (edit if needed)\n# -------------------------------\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nBATCH_SIZE = 32\nIMAGE_SIZE = 224\nMODEL_PATH = \"/kaggle/input/best-fusion-model/best_fusion_model.pth\"\nVAL_CSV = \"/kaggle/input/validate-dataset/val_processed.csv\"\nIMAGE_DIR = \"/kaggle/input/siim-isic-melanoma-classification/jpeg/train\"\n\n# feature columns\nfeature_columns = ['age_approx', 'sex', 'anatom_site_general_challenge']\n\n# encoding maps\nsex_map = {'male': 0, 'female': 1}\nsite_map = {\n 'torso': 0, 'head/neck': 1, 'lower extremity': 2,\n 'upper extremity': 3, 'palms/soles': 4,\n 'oral/genital': 5, 'back': 6\n}\n\n# -------------------------------\n# Utilities: monkey-patch EfficientNet\n# -------------------------------\ndef make_efficientnet_with_extract_features():\n model = efficientnet_b0(weights=None)\n\n if not hasattr(model, \"extract_features\"):\n def extract_features(self, x):\n x = self.features(x)\n x = nn.functional.adaptive_avg_pool2d(x, 1).flatten(1)\n return x\n model.extract_features = MethodType(extract_features, model)\n\n return model\n\n# -------------------------------\n# Load checkpoint and inspect\n# -------------------------------\nraw = torch.load(MODEL_PATH, map_location='cpu')\n\nif isinstance(raw, dict):\n if 'model_state_dict' in raw:\n state_dict_in = raw['model_state_dict']\n print(\"Checkpoint contains 'model_state_dict' key.\")\n else:\n state_dict_in = raw\n print(\"Checkpoint is a raw state_dict.\")\nelse:\n state_dict_in = None\n checkpoint_model_object = raw\n print(\"Checkpoint is a whole model object.\")\n\n# detect expected metadata input dim\nexpected_meta_dim = None\nif state_dict_in is not None:\n for k in state_dict_in.keys():\n if (k.endswith('meta_mlp.0.weight') or k.endswith('meta_mlp.0.bias')\n or '.meta_mlp.0.weight' in k):\n w = state_dict_in[k]\n if isinstance(w, torch.Tensor) and w.dim() == 2:\n expected_meta_dim = w.shape[1]\n break\n\n if expected_meta_dim is None:\n for k, v in state_dict_in.items():\n if 'meta_mlp' in k and k.endswith('.weight'):\n if isinstance(v, torch.Tensor) and v.dim() == 2:\n expected_meta_dim = v.shape[1]\n break\n\nif expected_meta_dim is None:\n expected_meta_dim = len(feature_columns)\n\nprint(f\"Expected metadata input dim (in checkpoint or fallback): {expected_meta_dim}\")\n\n# -------------------------------\n# Dataset\n# -------------------------------\nclass FusionDataset(Dataset):\n def __init__(self, df, image_dir, feature_columns,\n transform=None, expected_meta_dim=None):\n self.df = df.reset_index(drop=True)\n self.image_dir = image_dir\n self.feature_columns = feature_columns\n self.transform = transform\n self.expected_meta_dim = expected_meta_dim or len(feature_columns)\n\n def __len__(self):\n return len(self.df)\n\n def _encode_value(self, col, val):\n if pd.isna(val):\n return 0.0\n if col == 'sex':\n if isinstance(val, str) and val in sex_map:\n return float(sex_map[val])\n elif isinstance(val, str):\n return float(hash(val) % 1000) / 100.0\n return float(val)\n elif col == 'anatom_site_general_challenge':\n if isinstance(val, str) and val in site_map:\n return float(site_map[val])\n elif isinstance(val, str):\n return float(hash(val) % 1000) / 100.0\n return float(val)\n else:\n return float(val)\n\n def __getitem__(self, idx):\n row = self.df.iloc[idx]\n image_path = os.path.join(self.image_dir, f\"{row['image_name']}.jpg\")\n\n try:\n img = Image.open(image_path).convert('RGB')\n except Exception:\n img = Image.new('RGB', (IMAGE_SIZE, IMAGE_SIZE), (0, 0, 0))\n\n if self.transform:\n img = self.transform(img)\n\n meta_vals = [\n self._encode_value(col, row.get(col, 0))\n for col in self.feature_columns\n ]\n\n if len(meta_vals) < self.expected_meta_dim:\n meta_vals += [0.0] * (self.expected_meta_dim - len(meta_vals))\n elif len(meta_vals) > self.expected_meta_dim:\n meta_vals = meta_vals[:self.expected_meta_dim]\n\n metadata = torch.tensor(meta_vals, dtype=torch.float32)\n target = torch.tensor(float(row['target']), dtype=torch.float32)\n\n return (img, metadata), target\n\n# -------------------------------\n# DataLoader\n# -------------------------------\ntransform = transforms.Compose([\n transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n transforms.ToTensor(),\n transforms.Normalize(mean=[0.485,0.456,0.406],\n std=[0.229,0.224,0.225])\n])\n\nval_df = pd.read_csv(VAL_CSV)\nval_dataset = FusionDataset(\n val_df, IMAGE_DIR, feature_columns,\n transform=transform, expected_meta_dim=expected_meta_dim\n)\nval_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE,\n shuffle=False, num_workers=2, pin_memory=True)\n\n# -------------------------------\n# FusionModel\n# -------------------------------\nclass FusionModel(nn.Module):\n def __init__(self, image_model, meta_input_dim,\n meta_hidden=[128, 64], fusion_hidden=[512, 256],\n dropout=0.3):\n super().__init__()\n self.image_model = image_model\n\n with torch.no_grad():\n dummy = torch.randn(1, 3, IMAGE_SIZE, IMAGE_SIZE)\n if not hasattr(self.image_model, \"extract_features\"):\n def extract_features(self_img, x):\n x = self_img.features(x)\n x = nn.functional.adaptive_avg_pool2d(x, 1).flatten(1)\n return x\n self.image_model.extract_features = MethodType(\n extract_features, self.image_model\n )\n\n img_feat = self.image_model.extract_features(dummy)\n if isinstance(img_feat, torch.Tensor):\n img_out_dim = img_feat.shape[1]\n else:\n raise RuntimeError(\"extract_features did not return Tensor\")\n\n layers = []\n prev_dim = meta_input_dim\n for h in meta_hidden:\n layers += [nn.Linear(prev_dim, h),\n nn.BatchNorm1d(h),\n nn.ReLU(),\n nn.Dropout(dropout)]\n prev_dim = h\n self.meta_mlp = nn.Sequential(*layers)\n meta_out_dim = prev_dim\n\n fusion_layers = []\n prev_dim = img_out_dim + meta_out_dim\n for h in fusion_hidden:\n fusion_layers += [nn.Linear(prev_dim, h),\n nn.BatchNorm1d(h),\n nn.ReLU(),\n nn.Dropout(dropout)]\n prev_dim = h\n fusion_layers.append(nn.Linear(prev_dim, 1))\n self.fusion = nn.Sequential(*fusion_layers)\n\n def forward(self, images, metadata):\n img_features = self.image_model.extract_features(images)\n meta_features = self.meta_mlp(metadata)\n combined = torch.cat([img_features, meta_features], dim=1)\n out = self.fusion(combined)\n return out.squeeze(1)\n\n# -------------------------------\n# Build model\n# -------------------------------\nimage_model = make_efficientnet_with_extract_features()\nfusion_model = FusionModel(image_model=image_model,\n meta_input_dim=expected_meta_dim).to(DEVICE)\nprint(f\"Constructed FusionModel. Total params: {sum(p.numel() for p in fusion_model.parameters()):,}\")\n\n# -------------------------------\n# Load checkpoint\n# -------------------------------\nif state_dict_in is None:\n fusion_model = checkpoint_model_object.to(DEVICE)\n fusion_model.eval()\n print(\"Using the whole-model object from checkpoint directly.\")\nelse:\n ckpt = state_dict_in\n remapped = {}\n for k, v in ckpt.items():\n new_k = k\n if k.startswith(\"image_model.backbone.\"):\n new_k = k.replace(\"image_model.backbone.\", \"image_model.\")\n remapped[new_k] = v\n\n model_state = fusion_model.state_dict()\n compatible_state = {}\n skipped = []\n\n for k, v in remapped.items():\n if k in model_state:\n if isinstance(v, torch.Tensor) and model_state[k].shape == v.shape:\n compatible_state[k] = v\n else:\n skipped.append((k, tuple(v.shape), tuple(model_state[k].shape)))\n else:\n skipped.append((k, tuple(v.shape), None))\n\n fusion_model.load_state_dict(compatible_state, strict=False)\n fusion_model.to(DEVICE).eval()\n\n print(f\"Loaded {len(compatible_state)} parameters from checkpoint.\")\n if skipped:\n print(f\"Skipped {len(skipped)} keys. Examples:\")\n for s in skipped[:10]:\n print(\" \", s)\n\n# -------------------------------\n# Evaluation\n# -------------------------------\nfusion_model.eval()\nall_preds, all_labels, image_names = [], [], []\n\nwith torch.no_grad():\n for i, ((images, metadata), labels) in enumerate(tqdm(val_loader, desc=\"Evaluating\")):\n images, metadata, labels = images.to(DEVICE), metadata.to(DEVICE), labels.to(DEVICE)\n outputs = fusion_model(images, metadata)\n probs = torch.sigmoid(outputs) if outputs.dtype.is_floating_point else outputs\n all_preds.extend(probs.cpu().numpy().tolist())\n all_labels.extend(labels.cpu().numpy().tolist())\n start = i * BATCH_SIZE\n end = start + images.size(0)\n image_names.extend(val_df.iloc[start:end]['image_name'].tolist())\n\n# -------------------------------\n# Metrics\n# -------------------------------\nall_preds = np.array(all_preds)\nall_labels = np.array(all_labels, dtype=int)\nbinary_preds = (all_preds >= 0.5).astype(int)\n\naccuracy = accuracy_score(all_labels, binary_preds)\nroc_auc = roc_auc_score(all_labels, all_preds) if len(np.unique(all_labels)) > 1 else float('nan')\nprecision = precision_score(all_labels, binary_preds, zero_division=0)\nrecall = recall_score(all_labels, binary_preds, zero_division=0)\nf1 = f1_score(all_labels, binary_preds, zero_division=0)\ncm = confusion_matrix(all_labels, binary_preds)\n\nprint(\"\\n===== Evaluation Summary =====\")\nprint(f\"Samples evaluated: {len(all_labels)}\")\nprint(f\"Accuracy : {accuracy:.4f}\")\nprint(f\"ROC-AUC : {roc_auc:.4f}\")\nprint(f\"Precision: {precision:.4f}\")\nprint(f\"Recall : {recall:.4f}\")\nprint(f\"F1 Score : {f1:.4f}\")\nprint(\"\\nConfusion Matrix:\")\nprint(cm)\n\n# -------------------------------\n# Visualization\n# -------------------------------\nplt.figure(figsize=(6, 5))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues',\n xticklabels=['Neg', 'Pos'], yticklabels=['Neg', 'Pos'])\nplt.title(\"Confusion Matrix\")\nplt.xlabel(\"Predicted\")\nplt.ylabel(\"Actual\")\nplt.show()\n\nif not np.isnan(roc_auc):\n fpr, tpr, _ = roc_curve(all_labels, all_preds)\n plt.figure(figsize=(6, 5))\n plt.plot(fpr, tpr, label=f\"ROC AUC = {roc_auc:.4f}\")\n plt.plot([0, 1], [0, 1], '--', color='gray')\n plt.xlabel(\"False Positive Rate\")\n plt.ylabel(\"True Positive Rate\")\n plt.title(\"ROC Curve\")\n plt.legend()\n plt.show()\n\nprec_vals, rec_vals, _ = precision_recall_curve(all_labels, all_preds)\nplt.figure(figsize=(6, 5))\nplt.plot(rec_vals, prec_vals, label=\"Precision-Recall\")\nplt.xlabel(\"Recall\")\nplt.ylabel(\"Precision\")\nplt.title(\"Precision-Recall Curve\")\nplt.legend()\nplt.show()\n\nplt.figure(figsize=(6, 4))\nplt.bar(['Accuracy', 'F1-Score'], [accuracy, f1])\nplt.ylim(0, 1)\nplt.title(\"Accuracy vs F1\")\nplt.show()\n\n# -------------------------------\n# Error Analysis\n# -------------------------------\nmisclassified = [\n (image_names[idx], int(true), int(pred))\n for idx, (true, pred) in enumerate(zip(all_labels, binary_preds))\n if true != pred\n]\n\nprint(f\"\\nTotal misclassified: {len(misclassified)}\")\nprint(\"Sample misclassified (up to 20):\")\nfor item in misclassified[:20]:\n print(\" \", item)\n\nif 'patient_id' in val_df.columns:\n wrong_indices = [i for i, (t, p) in enumerate(zip(all_labels, binary_preds)) if t != p]\n mis_df = val_df.iloc[wrong_indices]\n if 'patient_id' in mis_df.columns:\n patient_counts = mis_df['patient_id'].value_counts().head(10)\n print(\"\\nTop patients with most misclassifications:\")\n print(patient_counts)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-11T06:31:58.101582Z","iopub.execute_input":"2025-09-11T06:31:58.101925Z"}},"outputs":[],"execution_count":null}]}