{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:07:44.277143Z","iopub.execute_input":"2025-09-01T00:07:44.277667Z","iopub.status.idle":"2025-09-01T00:07:44.531400Z","shell.execute_reply.started":"2025-09-01T00:07:44.277645Z","shell.execute_reply":"2025-09-01T00:07:44.530877Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport torch\nfrom tqdm import tqdm\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torchvision import transforms\nfrom torchvision.models import efficientnet_b0, efficientnet_b3\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torch.optim as optim\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.metrics import roc_auc_score, accuracy_score, classification_report\nimport cv2\nfrom PIL import Image\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:07:47.227215Z","iopub.execute_input":"2025-09-01T00:07:47.227585Z","iopub.status.idle":"2025-09-01T00:07:53.109811Z","shell.execute_reply.started":"2025-09-01T00:07:47.227562Z","shell.execute_reply":"2025-09-01T00:07:53.108976Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Handling & Preprocessing (Look Wey Shen)","metadata":{}},{"cell_type":"code","source":"# (1) Explore Dataset\n# Dataset path\ndata_path = \"/kaggle/input/siim-isic-melanoma-classification\"\n\n# List files\nprint(os.listdir(data_path))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:07:56.596582Z","iopub.execute_input":"2025-09-01T00:07:56.596993Z","iopub.status.idle":"2025-09-01T00:07:56.601846Z","shell.execute_reply.started":"2025-09-01T00:07:56.596971Z","shell.execute_reply":"2025-09-01T00:07:56.600981Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load Metadata (CSV File)\ntrain_df = pd.read_csv(f\"{data_path}/train.csv\")\ntest_df = pd.read_csv(f\"{data_path}/test.csv\")\n\nprint(f\"Training data shape: {train_df.shape}\")\nprint(f\"Test data shape: {test_df.shape}\")\nprint(\"\\nTraining data columns:\", train_df.columns.tolist())\nprint(\"\\nFirst 5 rows:\")\nprint(train_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:07:58.477749Z","iopub.execute_input":"2025-09-01T00:07:58.478009Z","iopub.status.idle":"2025-09-01T00:07:58.588252Z","shell.execute_reply.started":"2025-09-01T00:07:58.477990Z","shell.execute_reply":"2025-09-01T00:07:58.587645Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# (2) Metadata Cleaning and Preprocessing\ndef clean_metadata(df, is_train=True):\n    \"\"\"Clean and preprocess metadata\"\"\"\n    df_clean = df.copy()\n    \n    # Handle missing values in age\n    if 'age_approx' in df_clean.columns:\n        # Fill missing age with median\n        median_age = df_clean['age_approx'].median()\n        df_clean['age_approx'].fillna(median_age, inplace=True)\n        \n        # Normalize age (0-1 scale)\n        df_clean['age_normalized'] = df_clean['age_approx'] / 100.0\n        print(f\"Age missing values filled with median: {median_age}\")\n    \n    # Handle missing values in sex\n    if 'sex' in df_clean.columns:\n        # Fill missing sex with mode\n        mode_sex = df_clean['sex'].mode()[0] if not df_clean['sex'].mode().empty else 'male'\n        df_clean['sex'].fillna(mode_sex, inplace=True)\n        \n        # One-hot encode sex\n        sex_dummies = pd.get_dummies(df_clean['sex'], prefix='sex')\n        df_clean = pd.concat([df_clean, sex_dummies], axis=1)\n        print(f\"Sex missing values filled with mode  : {mode_sex}\")\n    \n    # Handle missing values in anatomical site\n    if 'anatom_site_general_challenge' in df_clean.columns:\n        # Fill missing site with 'unknown'\n        df_clean['anatom_site_general_challenge'].fillna('unknown', inplace=True)\n        \n        # One-hot encode anatomical site\n        site_dummies = pd.get_dummies(df_clean['anatom_site_general_challenge'], prefix='site')\n        df_clean = pd.concat([df_clean, site_dummies], axis=1)\n        print(\"Anatomical site missing values filled with 'unknown'\")\n    \n    # Create additional features\n    if is_train and 'target' in df_clean.columns:\n        # Calculate class weights for imbalanced dataset\n        target_counts = df_clean['target'].value_counts()\n        print(f\"\\nClass distribution:\")\n        print(f\"Benign (0)   : {target_counts[0]} ({target_counts[0]/len(df_clean)*100:.2f}%)\")\n        print(f\"Malignant (1): {target_counts[1]} ({target_counts[1]/len(df_clean)*100:.2f}%)\")\n    \n    return df_clean","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:01.282290Z","iopub.execute_input":"2025-09-01T00:08:01.282560Z","iopub.status.idle":"2025-09-01T00:08:01.289836Z","shell.execute_reply.started":"2025-09-01T00:08:01.282539Z","shell.execute_reply":"2025-09-01T00:08:01.289035Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Clean training and test metadata\ntrain_clean = clean_metadata(train_df, is_train=True)\ntest_clean = clean_metadata(test_df, is_train=False)\n\nprint(f\"\\nCleaned training data shape: {train_clean.shape}\")\nprint(f\"Cleaned test data shape    : {test_clean.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:04.586744Z","iopub.execute_input":"2025-09-01T00:08:04.587018Z","iopub.status.idle":"2025-09-01T00:08:04.628784Z","shell.execute_reply.started":"2025-09-01T00:08:04.586997Z","shell.execute_reply":"2025-09-01T00:08:04.628153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# (3) Patient-based Train/Validation Split\ndef create_patient_split(df, test_size=0.2, random_state=42):\n    \"\"\"Create train/validation split by patient ID to avoid data leakage\"\"\"\n    \n    # Get unique patients\n    unique_patients = df['patient_id'].unique()\n    print(f\"Total unique patients: {len(unique_patients)}\")\n    \n    # Split patients (not individual images)\n    train_patients, val_patients = train_test_split(\n        unique_patients, \n        test_size=test_size, \n        random_state=random_state,\n        stratify=None  # Can't stratify by patient easily, would need more complex logic\n    )\n    \n    # Create train/validation dataframes\n    train_split = df[df['patient_id'].isin(train_patients)].copy()\n    val_split = df[df['patient_id'].isin(val_patients)].copy()\n    \n    print(f\"Training patients    : {len(train_patients)}\")\n    print(f\"Validation patients  : {len(val_patients)}\")\n    print(f\"Training images      : {len(train_split)}\")\n    print(f\"Validation images    : {len(val_split)}\")\n    \n    # Check target distribution in splits\n    if 'target' in df.columns:\n        print(f\"\\nTarget distribution in training split:\")\n        print(train_split['target'].value_counts(normalize=True))\n        print(f\"\\nTarget distribution in validation split:\")\n        print(val_split['target'].value_counts(normalize=True))\n    \n    return train_split, val_split","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:08.878100Z","iopub.execute_input":"2025-09-01T00:08:08.878355Z","iopub.status.idle":"2025-09-01T00:08:08.884150Z","shell.execute_reply.started":"2025-09-01T00:08:08.878335Z","shell.execute_reply":"2025-09-01T00:08:08.883393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create patient-based split\ntrain_split, val_split = create_patient_split(train_clean)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:12.867199Z","iopub.execute_input":"2025-09-01T00:08:12.868034Z","iopub.status.idle":"2025-09-01T00:08:12.887606Z","shell.execute_reply.started":"2025-09-01T00:08:12.868009Z","shell.execute_reply":"2025-09-01T00:08:12.887044Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# (4) Image Preprocessing Functions\nclass ImagePreprocessor:\n    def __init__(self, target_size=(224, 224), normalize=True):\n        self.target_size = target_size\n        self.normalize = normalize\n        \n    def load_and_preprocess_image(self, image_path, augment=False):\n        \"\"\"Load and preprocess a single image\"\"\"\n        try:\n            # Load image\n            image = cv2.imread(image_path)\n            if image is None:\n                print(f\"Warning: Could not load image {image_path}\")\n                return None\n                \n            # Convert BGR to RGB\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n            \n            # Resize image\n            image = cv2.resize(image, self.target_size)\n            \n            # Normalize pixel values to [0, 1]\n            if self.normalize:\n                image = image.astype(np.float32) / 255.0\n            \n            # Apply augmentation if specified\n            if augment:\n                image = self.apply_augmentation(image)\n                \n            return image\n            \n        except Exception as e:\n            print(f\"Error processing image {image_path}: {str(e)}\")\n            return None\n    \n    def apply_augmentation(self, image):\n        \"\"\"Apply basic data augmentation\"\"\"\n        # Random horizontal flip\n        if np.random.random() > 0.5:\n            image = cv2.flip(image, 1)\n        \n        # Random rotation (small angle)\n        if np.random.random() > 0.5:\n            angle = np.random.uniform(-15, 15)\n            rows, cols = image.shape[:2]\n            M = cv2.getRotationMatrix2D((cols/2, rows/2), angle, 1)\n            image = cv2.warpAffine(image, M, (cols, rows))\n        \n        # Random brightness adjustment\n        if np.random.random() > 0.5:\n            brightness = np.random.uniform(0.8, 1.2)\n            image = np.clip(image * brightness, 0, 1)\n            \n        return image\n    \n    def preprocess_batch(self, image_paths, augment=False, batch_size=32):\n        \"\"\"Preprocess a batch of images\"\"\"\n        images = []\n        valid_paths = []\n        \n        for path in image_paths:\n            img = self.load_and_preprocess_image(path, augment=augment)\n            if img is not None:\n                images.append(img)\n                valid_paths.append(path)\n                \n        return np.array(images), valid_paths","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:15.487170Z","iopub.execute_input":"2025-09-01T00:08:15.487423Z","iopub.status.idle":"2025-09-01T00:08:15.495787Z","shell.execute_reply.started":"2025-09-01T00:08:15.487405Z","shell.execute_reply":"2025-09-01T00:08:15.495132Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Initialize preprocessor\npreprocessor = ImagePreprocessor(target_size=(224, 224), normalize=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:18.395992Z","iopub.execute_input":"2025-09-01T00:08:18.396643Z","iopub.status.idle":"2025-09-01T00:08:18.399954Z","shell.execute_reply.started":"2025-09-01T00:08:18.396621Z","shell.execute_reply":"2025-09-01T00:08:18.399206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# (5) Create Data Loading Functions\ndef create_image_paths(df, image_dir):\n    \"\"\"Create full image paths from dataframe\"\"\"\n    return [os.path.join(image_dir, f\"{img_id}.jpg\") for img_id in df['image_name']]\n\ndef save_processed_data(train_df, val_df, test_df, output_dir='processed_data'):\n    \"\"\"Save processed dataframes\"\"\"\n    os.makedirs(output_dir, exist_ok=True)\n    \n    # Save cleaned metadata\n    train_df.to_csv(os.path.join(output_dir, 'train_processed.csv'), index=False)\n    val_df.to_csv(os.path.join(output_dir, 'val_processed.csv'), index=False)\n    test_df.to_csv(os.path.join(output_dir, 'test_processed.csv'), index=False)\n    \n    print(f\"Processed data saved to {output_dir}/\")\n    \n    # Save preprocessing summary\n    with open(os.path.join(output_dir, 'preprocessing_summary.txt'), 'w') as f:\n        f.write(\"SIIM-ISIC Data Preprocessing Summary\\n\")\n        f.write(\"=\"*40 + \"\\n\\n\")\n        f.write(f\"Training samples: {len(train_df)}\\n\")\n        f.write(f\"Validation samples: {len(val_df)}\\n\")\n        f.write(f\"Test samples: {len(test_df)}\\n\")\n        f.write(f\"Image target size: {preprocessor.target_size}\\n\")\n        f.write(f\"Normalization applied: {preprocessor.normalize}\\n\")\n        \n        # Feature columns\n        feature_cols = [col for col in train_df.columns if col not in ['image_name', 'patient_id', 'target']]\n        f.write(f\"\\nFeature columns ({len(feature_cols)}):\\n\")\n        for col in feature_cols:\n            f.write(f\"  - {col}\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:21.545941Z","iopub.execute_input":"2025-09-01T00:08:21.546196Z","iopub.status.idle":"2025-09-01T00:08:21.552983Z","shell.execute_reply.started":"2025-09-01T00:08:21.546176Z","shell.execute_reply":"2025-09-01T00:08:21.552290Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# (6) Execute Preprocessing Pipeline\n# Create image paths\ntrain_paths = create_image_paths(train_split, f\"{data_path}/jpeg/train/\")\nval_paths = create_image_paths(val_split, f\"{data_path}/jpeg/train/\")\ntest_paths = create_image_paths(test_clean, f\"{data_path}/jpeg/test/\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:27.986521Z","iopub.execute_input":"2025-09-01T00:08:27.986823Z","iopub.status.idle":"2025-09-01T00:08:28.026013Z","shell.execute_reply.started":"2025-09-01T00:08:27.986802Z","shell.execute_reply":"2025-09-01T00:08:28.025428Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Validate that images exist\ndef validate_image_paths(paths, df, split_name):\n    \"\"\"Validate that image files exist\"\"\"\n    existing_paths = []\n    valid_indices = []\n    \n    for i, path in enumerate(paths):\n        if os.path.exists(path):\n            existing_paths.append(path)\n            valid_indices.append(i)\n    \n    print(f\"{split_name}: {len(existing_paths)}/{len(paths)} images found\")\n    return df.iloc[valid_indices].reset_index(drop=True), existing_paths","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:29.510749Z","iopub.execute_input":"2025-09-01T00:08:29.511225Z","iopub.status.idle":"2025-09-01T00:08:29.515487Z","shell.execute_reply.started":"2025-09-01T00:08:29.511205Z","shell.execute_reply":"2025-09-01T00:08:29.514791Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Validate all splits\ntrain_final, train_paths_final = validate_image_paths(train_paths, train_split, \"Training\")\nval_final, val_paths_final = validate_image_paths(val_paths, val_split, \"Validation\")\ntest_final, test_paths_final = validate_image_paths(test_paths, test_clean, \"Test\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:08:32.360854Z","iopub.execute_input":"2025-09-01T00:08:32.361610Z","iopub.status.idle":"2025-09-01T00:10:41.253088Z","shell.execute_reply.started":"2025-09-01T00:08:32.361577Z","shell.execute_reply":"2025-09-01T00:10:41.252404Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save processed data\nsave_processed_data(train_final, val_final, test_final)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:10:41.254349Z","iopub.execute_input":"2025-09-01T00:10:41.254590Z","iopub.status.idle":"2025-09-01T00:10:41.505719Z","shell.execute_reply.started":"2025-09-01T00:10:41.254572Z","shell.execute_reply":"2025-09-01T00:10:41.505087Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Training samples   : {len(train_final)}\")\nprint(f\"Validation samples : {len(val_final)}\")\nprint(f\"Test samples       : {len(test_final)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:10:41.506363Z","iopub.execute_input":"2025-09-01T00:10:41.506598Z","iopub.status.idle":"2025-09-01T00:10:41.510949Z","shell.execute_reply.started":"2025-09-01T00:10:41.506581Z","shell.execute_reply":"2025-09-01T00:10:41.510234Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load and display a sample image\nif len(train_paths_final) > 0:\n    # Load a sample image\n    sample_image = preprocessor.load_and_preprocess_image(train_paths_final[0])\n    if sample_image is not None:\n        print(f\"Sample image shape: {sample_image.shape}\")\n        print(f\"Sample image data type: {sample_image.dtype}\")\n        print(f\"Sample image value range: [{sample_image.min():.3f}, {sample_image.max():.3f}]\")\n        \n        plt.figure(figsize=(8, 6))\n        plt.imshow(sample_image)\n        plt.title(\"Sample Preprocessed Image\")\n        plt.axis('off')\n        plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:10:41.512200Z","iopub.execute_input":"2025-09-01T00:10:41.512432Z","iopub.status.idle":"2025-09-01T00:10:42.050754Z","shell.execute_reply.started":"2025-09-01T00:10:41.512417Z","shell.execute_reply":"2025-09-01T00:10:42.049979Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Image Model (CNN) (Tan Jian Hua)","metadata":{}},{"cell_type":"code","source":"# Check if CUDA is available\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:10:54.625848Z","iopub.execute_input":"2025-09-01T00:10:54.626348Z","iopub.status.idle":"2025-09-01T00:10:54.680981Z","shell.execute_reply.started":"2025-09-01T00:10:54.626325Z","shell.execute_reply":"2025-09-01T00:10:54.680266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# STEP 2: Custom Dataset Class\n# ================================\n\nclass MelanomaDataset(Dataset):\n    def __init__(self, dataframe, image_dir, transform=None, is_test=False):\n        \"\"\"\n        Custom dataset for melanoma classification\n        \n        Args:\n            dataframe: pandas DataFrame with image_name and target columns\n            image_dir: directory containing images\n            transform: torchvision transforms\n            is_test: whether this is test data (no targets)\n        \"\"\"\n        self.df = dataframe.reset_index(drop=True)\n        self.image_dir = image_dir\n        self.transform = transform\n        self.is_test = is_test\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        # Get image name and construct path\n        image_name = self.df.loc[idx, 'image_name']\n        image_path = os.path.join(self.image_dir, f\"{image_name}.jpg\")\n        \n        # Load image\n        try:\n            image = Image.open(image_path).convert('RGB')\n        except Exception as e:\n            print(f\"Error loading image {image_path}: {e}\")\n            # Return a blank image if loading fails\n            image = Image.new('RGB', (224, 224), (0, 0, 0))\n        \n        # Apply transforms\n        if self.transform:\n            image = self.transform(image)\n        \n        if self.is_test:\n            return image, image_name\n        else:\n            target = torch.tensor(self.df.loc[idx, 'target'], dtype=torch.float32)\n            return image, target","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:10:56.756079Z","iopub.execute_input":"2025-09-01T00:10:56.756643Z","iopub.status.idle":"2025-09-01T00:10:56.762440Z","shell.execute_reply.started":"2025-09-01T00:10:56.756623Z","shell.execute_reply":"2025-09-01T00:10:56.761651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# STEP 3: Data Transforms\n# ================================\n\ndef get_transforms(image_size=224, augment=True):\n    \"\"\"Get training and validation transforms\"\"\"\n    \n    if augment:\n        train_transform = transforms.Compose([\n            transforms.Resize((image_size, image_size)),\n            transforms.RandomHorizontalFlip(p=0.5),\n            transforms.RandomVerticalFlip(p=0.5),\n            transforms.RandomRotation(degrees=20),\n            transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),\n            transforms.RandomResizedCrop(image_size, scale=(0.8, 1.0)),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=[0.485, 0.456, 0.406], \n                               std=[0.229, 0.224, 0.225])  # ImageNet normalization\n        ])\n    else:\n        train_transform = transforms.Compose([\n            transforms.Resize((image_size, image_size)),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=[0.485, 0.456, 0.406], \n                               std=[0.229, 0.224, 0.225])\n        ])\n    \n    val_transform = transforms.Compose([\n        transforms.Resize((image_size, image_size)),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], \n                           std=[0.229, 0.224, 0.225])\n    ])\n    \n    return train_transform, val_transform","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:11:00.250884Z","iopub.execute_input":"2025-09-01T00:11:00.251372Z","iopub.status.idle":"2025-09-01T00:11:00.257527Z","shell.execute_reply.started":"2025-09-01T00:11:00.251352Z","shell.execute_reply":"2025-09-01T00:11:00.256663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# STEP 4: EfficientNet Model\n# ================================\n\nclass MelanomaEfficientNet(nn.Module):\n    def __init__(self, model_name='efficientnet_b0', num_classes=1, pretrained=True, dropout_rate=0.3):\n        \"\"\"\n        EfficientNet model for melanoma classification\n        \n        Args:\n            model_name: which EfficientNet variant to use\n            num_classes: number of output classes (1 for binary classification)\n            pretrained: whether to use pretrained weights\n            dropout_rate: dropout rate for regularization\n        \"\"\"\n        super(MelanomaEfficientNet, self).__init__()\n        \n        # Load pretrained EfficientNet\n        if model_name == 'efficientnet_b0':\n            self.backbone = efficientnet_b0(pretrained=pretrained)\n            num_features = self.backbone.classifier[1].in_features\n        elif model_name == 'efficientnet_b3':\n            self.backbone = efficientnet_b3(pretrained=pretrained)\n            num_features = self.backbone.classifier[1].in_features\n        else:\n            raise ValueError(f\"Unsupported model: {model_name}\")\n        \n        # Replace classifier\n        self.backbone.classifier = nn.Sequential(\n            nn.Dropout(p=dropout_rate),\n            nn.Linear(num_features, 512),\n            nn.ReLU(),\n            nn.Dropout(p=dropout_rate),\n            nn.Linear(512, num_classes)\n        )\n        \n    def forward(self, x):\n        return self.backbone(x)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:11:03.846265Z","iopub.execute_input":"2025-09-01T00:11:03.846539Z","iopub.status.idle":"2025-09-01T00:11:03.852476Z","shell.execute_reply.started":"2025-09-01T00:11:03.846520Z","shell.execute_reply":"2025-09-01T00:11:03.851736Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# STEP 5: Training Functions\n# ================================\n\ndef train_one_epoch(model, dataloader, criterion, optimizer, device, epoch):\n    \"\"\"Train the model for one epoch\"\"\"\n    model.train()\n    running_loss = 0.0\n    running_acc = 0.0\n    \n    pbar = tqdm(dataloader, desc=f'Epoch {epoch+1} - Training')\n    for images, targets in pbar:\n        images, targets = images.to(device), targets.to(device)\n        \n        # Zero gradients\n        optimizer.zero_grad()\n        \n        # Forward pass\n        outputs = model(images).squeeze()\n        loss = criterion(outputs, targets)\n        \n        # Backward pass\n        loss.backward()\n        optimizer.step()\n        \n        # Statistics\n        running_loss += loss.item()\n        predictions = torch.sigmoid(outputs) > 0.5\n        running_acc += (predictions == targets).float().mean().item()\n        \n        pbar.set_postfix({'Loss': f'{loss.item():.4f}'})\n    \n    epoch_loss = running_loss / len(dataloader)\n    epoch_acc = running_acc / len(dataloader)\n    \n    return epoch_loss, epoch_acc\n\ndef validate_one_epoch(model, dataloader, criterion, device, epoch):\n    \"\"\"Validate the model for one epoch\"\"\"\n    model.eval()\n    running_loss = 0.0\n    all_predictions = []\n    all_targets = []\n    \n    with torch.no_grad():\n        pbar = tqdm(dataloader, desc=f'Epoch {epoch+1} - Validation')\n        for images, targets in pbar:\n            images, targets = images.to(device), targets.to(device)\n            \n            # Forward pass\n            outputs = model(images).squeeze()\n            loss = criterion(outputs, targets)\n            \n            # Statistics\n            running_loss += loss.item()\n            predictions = torch.sigmoid(outputs)\n            \n            all_predictions.extend(predictions.cpu().numpy())\n            all_targets.extend(targets.cpu().numpy())\n            \n            pbar.set_postfix({'Loss': f'{loss.item():.4f}'})\n    \n    epoch_loss = running_loss / len(dataloader)\n    \n    # Calculate metrics\n    all_predictions = np.array(all_predictions)\n    all_targets = np.array(all_targets)\n    \n    auc_score = roc_auc_score(all_targets, all_predictions)\n    acc_score = accuracy_score(all_targets, all_predictions > 0.5)\n    \n    return epoch_loss, auc_score, acc_score\n\ndef train_model(model, train_loader, val_loader, num_epochs=20, learning_rate=1e-4, weight_decay=1e-5, early_stopping_patience=5, early_stopping=True):\n    \"\"\"Complete training loop\"\"\"\n    \n    # Loss function and optimizer\n    criterion = nn.BCEWithLogitsLoss()\n    optimizer = optim.AdamW(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', patience=3, factor=0.5, verbose=True)\n    \n    # Training history\n    history = {\n        'train_loss': [], 'train_acc': [],\n        'val_loss': [], 'val_auc': [], 'val_acc': []\n    }\n    \n    best_auc = 0.0\n    best_model_state = None\n    epochs_without_improvement = 0\n    \n    print(\"Starting training...\")\n    if early_stopping:\n        print(f\"Early stopping enabled with patience: {early_stopping_patience}\")\n    print(\"=\"*50)\n    \n    for epoch in range(num_epochs):\n        # Train\n        train_loss, train_acc = train_one_epoch(model, train_loader, criterion, optimizer, device, epoch)\n        \n        # Validate\n        val_loss, val_auc, val_acc = validate_one_epoch(model, val_loader, criterion, device, epoch)\n        \n        # Learning rate scheduling\n        scheduler.step(val_loss)\n        \n        # Save best model\n        if val_auc > best_auc:\n            best_auc = val_auc\n            best_model_state = model.state_dict().copy()\n            epochs_without_improvement = 0\n            print(f\"New best AUC: {best_auc:.4f}\")\n        else:\n            epochs_without_improvement += 1\n            print(f\"No improvement for {epochs_without_improvement} epoch(s)\")\n        \n        # Update history\n        history['train_loss'].append(train_loss)\n        history['train_acc'].append(train_acc)\n        history['val_loss'].append(val_loss)\n        history['val_auc'].append(val_auc)\n        history['val_acc'].append(val_acc)\n        \n        # Print epoch results\n        print(f\"Epoch {epoch+1}/{num_epochs}:\")\n        print(f\"  Train Loss: {train_loss:.4f}, Train Acc: {train_acc:.4f}\")\n        print(f\"  Val Loss: {val_loss:.4f}, Val AUC: {val_auc:.4f}, Val Acc: {val_acc:.4f}\")\n        print(f\"  Current LR: {optimizer.param_groups[0]['lr']:.2e}\")\n\n        # Early stopping check\n        if early_stopping and epochs_without_improvement >= early_stopping_patience:\n            print(f\"\\nEarly stopping triggered after {epoch+1} epochs!\")\n            print(f\"No improvement for {early_stopping_patience} consecutive epochs.\")\n            break\n        \n        print(\"-\" * 50)\n    \n    # Load best model\n    model.load_state_dict(best_model_state)\n    print(f\"\\nTraining completed! Best validation AUC: {best_auc:.4f}\")\n    \n    return model, history\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:11:06.845998Z","iopub.execute_input":"2025-09-01T00:11:06.846248Z","iopub.status.idle":"2025-09-01T00:11:06.859259Z","shell.execute_reply.started":"2025-09-01T00:11:06.846230Z","shell.execute_reply":"2025-09-01T00:11:06.858541Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# STEP 6: Visualization Functions\n# ================================\n\ndef plot_training_history(history):\n    \"\"\"Plot training history\"\"\"\n    fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 10))\n    \n    epochs = range(1, len(history['train_loss']) + 1)\n    \n    # Loss\n    ax1.plot(epochs, history['train_loss'], 'b-', label='Training Loss')\n    ax1.plot(epochs, history['val_loss'], 'r-', label='Validation Loss')\n    ax1.set_title('Model Loss')\n    ax1.set_xlabel('Epoch')\n    ax1.set_ylabel('Loss')\n    ax1.legend()\n    ax1.grid(True)\n    \n    # Accuracy\n    ax2.plot(epochs, history['train_acc'], 'b-', label='Training Accuracy')\n    ax2.plot(epochs, history['val_acc'], 'r-', label='Validation Accuracy')\n    ax2.set_title('Model Accuracy')\n    ax2.set_xlabel('Epoch')\n    ax2.set_ylabel('Accuracy')\n    ax2.legend()\n    ax2.grid(True)\n    \n    # AUC\n    ax3.plot(epochs, history['val_auc'], 'g-', label='Validation AUC')\n    ax3.set_title('Validation AUC')\n    ax3.set_xlabel('Epoch')\n    ax3.set_ylabel('AUC')\n    ax3.legend()\n    ax3.grid(True)\n    \n    # Learning rate would go here if tracked\n    ax4.axis('off')\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:11:12.476013Z","iopub.execute_input":"2025-09-01T00:11:12.476499Z","iopub.status.idle":"2025-09-01T00:11:12.482607Z","shell.execute_reply.started":"2025-09-01T00:11:12.476477Z","shell.execute_reply":"2025-09-01T00:11:12.482008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# Create subset (for testing)\n# ================================\ndef create_balanced_subset(df, subset_size=5000, min_positive_ratio=0.05):\n    \"\"\"Create a subset that maintains better class balance\"\"\"\n    \n    # Separate positive and negative cases\n    positive_cases = df[df['target'] == 1]\n    negative_cases = df[df['target'] == 0]\n    \n    # Calculate how many positives we want (at least 5% of subset)\n    min_positives = int(subset_size * min_positive_ratio)\n    available_positives = len(positive_cases)\n    \n    # Use all available positives if we don't have enough\n    n_positives = min(min_positives, available_positives)\n    n_negatives = subset_size - n_positives\n    \n    print(f\"Creating balanced subset:\")\n    print(f\"  Available positives: {available_positives}\")\n    print(f\"  Using positives: {n_positives}\")\n    print(f\"  Using negatives: {n_negatives}\")\n    \n    # Sample\n    if n_positives > 0:\n        positive_subset = positive_cases.sample(n=n_positives, random_state=42)\n    else:\n        positive_subset = pd.DataFrame()\n    \n    negative_subset = negative_cases.sample(n=n_negatives, random_state=42)\n    \n    # Combine\n    balanced_subset = pd.concat([positive_subset, negative_subset])\n    \n    print(f\"Subset created:\")\n    print(f\"  Total: {len(balanced_subset)}\")\n    print(f\"  Positive: {len(positive_subset)} ({len(positive_subset)/len(balanced_subset)*100:.1f}%)\")\n    print(f\"  Negative: {len(negative_subset)} ({len(negative_subset)/len(balanced_subset)*100:.1f}%)\")\n    \n    return balanced_subset.sample(frac=1, random_state=42).reset_index(drop=True)  # Shuffle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:11:16.760585Z","iopub.execute_input":"2025-09-01T00:11:16.761048Z","iopub.status.idle":"2025-09-01T00:11:16.766802Z","shell.execute_reply.started":"2025-09-01T00:11:16.761024Z","shell.execute_reply":"2025-09-01T00:11:16.766188Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# STEP 7: Main Training Script\n# ================================\n\ndef main(use_subset=False, subset_size=5000):\n    \"\"\"Main function to run the training\"\"\"\n    \n    # Configuration\n    CONFIG = {\n        'model_name': 'efficientnet_b0',  # or 'efficientnet_b3'\n        'image_size': 224,                # 224 is good for b0, 300 for b3\n        'batch_size': 96,                 # Adjust based on GPU memory\n        'num_epochs': 20,\n        'learning_rate': 1e-4,\n        'weight_decay': 1e-5,\n        'num_workers': 4,                 # Adjust based on your system\n        'dropout_rate': 0.3,\n        'early_stopping': True,           # Enable/disable early stopping\n        'early_stopping_patience': 5,     # Stop if no improvement for 5 epochs\n        'use_subset': use_subset,\n        'subset_size': subset_size\n    }\n    \n    # Data paths (adjust these to match your setup)\n    DATA_PATH = \"/kaggle/input/siim-isic-melanoma-classification\"\n    PROCESSED_DATA_PATH = \"processed_data\"  # From member 1's preprocessing\n    \n    # Load processed data\n    train_df = pd.read_csv(f\"{PROCESSED_DATA_PATH}/train_processed.csv\")\n    val_df = pd.read_csv(f\"{PROCESSED_DATA_PATH}/val_processed.csv\")\n    test_df = pd.read_csv(f\"{PROCESSED_DATA_PATH}/test_processed.csv\")\n\n    print(\"=\"*50)\n    print(\"ORIGINAL DATASET SIZES:\")\n    print(f\"Training samples: {len(train_df)}\")\n    print(f\"Validation samples: {len(val_df)}\")\n    print(f\"Test samples: {len(test_df)}\")\n\n    # CREATE SUBSET IF REQUESTED\n    if use_subset:\n        print(\"\\n\" + \"=\"*50)\n        print(\"CREATING SUBSET FOR FAST EXPERIMENTATION:\")\n        \n        # Create balanced subsets\n        train_df = create_balanced_subset(train_df, subset_size=subset_size, min_positive_ratio=0.05)\n        val_df = create_balanced_subset(val_df, subset_size=subset_size//5, min_positive_ratio=0.05)  # 20% of train size\n        \n        print(\"\\nFINAL SUBSET SIZES:\")\n        print(f\"Training samples: {len(train_df)}\")\n        print(f\"Validation samples: {len(val_df)}\")\n        print(f\"Test samples: {len(test_df)} (unchanged)\")\n    \n    print(\"=\"*50)\n    \n    # Get transforms\n    train_transform, val_transform = get_transforms(\n        image_size=CONFIG['image_size'], \n        augment=True\n    )\n    \n    # Create datasets\n    train_dataset = MelanomaDataset(\n        train_df, \n        f\"{DATA_PATH}/jpeg/train\",\n        transform=train_transform\n    )\n    \n    val_dataset = MelanomaDataset(\n        val_df, \n        f\"{DATA_PATH}/jpeg/train\",\n        transform=val_transform\n    )\n    \n    # Create data loaders\n    train_loader = DataLoader(\n        train_dataset, \n        batch_size=CONFIG['batch_size'],\n        shuffle=True, \n        num_workers=CONFIG['num_workers'],\n        pin_memory=True if torch.cuda.is_available() else False,\n        persistent_workers=True if CONFIG['num_workers'] > 0 else False,\n        prefetch_factor=2 if CONFIG['num_workers'] > 0 else 2\n    )\n    \n    val_loader = DataLoader(\n        val_dataset, \n        batch_size=CONFIG['batch_size'],\n        shuffle=False, \n        num_workers=CONFIG['num_workers'],\n        pin_memory=True if torch.cuda.is_available() else False,\n        persistent_workers=True if CONFIG['num_workers'] > 0 else False,\n        prefetch_factor=2 if CONFIG['num_workers'] > 0 else 2\n    )\n    \n    print(f\"Train batches: {len(train_loader)}\")\n    print(f\"Validation batches: {len(val_loader)}\")\n    \n    # Create model\n    model = MelanomaEfficientNet(\n        model_name=CONFIG['model_name'],\n        num_classes=1,\n        pretrained=True,\n        dropout_rate=CONFIG['dropout_rate']\n    ).to(device)\n    \n    print(f\"Model created: {CONFIG['model_name']}\")\n    print(f\"Total parameters: {sum(p.numel() for p in model.parameters()):,}\")\n    print(f\"Trainable parameters: {sum(p.numel() for p in model.parameters() if p.requires_grad):,}\")\n    \n    # Train the model\n    trained_model, history = train_model(\n        model, \n        train_loader, \n        val_loader,\n        num_epochs=CONFIG['num_epochs'],\n        learning_rate=CONFIG['learning_rate'],\n        weight_decay=CONFIG['weight_decay']\n    )\n    \n    # Plot training history\n    plot_training_history(history)\n    \n    # Save the trained model with subset info\n    model_filename = f\"melanoma_efficientnet_{'subset_' if use_subset else 'full_'}model.pth\"\n    torch.save({\n        'model_state_dict': trained_model.state_dict(),\n        'config': CONFIG,\n        'history': history\n    }, model_filename)\n    \n    print(f\"Model saved as '{model_filename}'\")\n    \n    return trained_model, history","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:11:19.766080Z","iopub.execute_input":"2025-09-01T00:11:19.766579Z","iopub.status.idle":"2025-09-01T00:11:19.777063Z","shell.execute_reply.started":"2025-09-01T00:11:19.766556Z","shell.execute_reply":"2025-09-01T00:11:19.776466Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# STEP 8: Inference Functions\n# ================================\n\ndef load_trained_model(model_path, model_name='efficientnet_b0', dropout_rate=0.3):\n    \"\"\"Load a trained model\"\"\"\n    checkpoint = torch.load(model_path, map_location=device)\n    \n    model = MelanomaEfficientNet(\n        model_name=model_name,\n        num_classes=1,\n        pretrained=False,  # We're loading our own weights\n        dropout_rate=dropout_rate\n    ).to(device)\n    \n    model.load_state_dict(checkpoint['model_state_dict'])\n    model.eval()\n    \n    return model, checkpoint.get('config', {}), checkpoint.get('history', {})\n\ndef predict_test_set(model, test_df, data_path, transform, batch_size=32):\n    \"\"\"Generate predictions for test set\"\"\"\n    test_dataset = MelanomaDataset(\n        test_df, \n        f\"{data_path}/jpeg/test\",\n        transform=transform,\n        is_test=True\n    )\n    \n    test_loader = DataLoader(\n        test_dataset, \n        batch_size=batch_size,\n        shuffle=False, \n        num_workers=2\n    )\n    \n    model.eval()\n    predictions = []\n    image_names = []\n    \n    with torch.no_grad():\n        for images, names in tqdm(test_loader, desc='Predicting'):\n            images = images.to(device)\n            outputs = model(images).squeeze()\n            probs = torch.sigmoid(outputs)\n            \n            predictions.extend(probs.cpu().numpy())\n            image_names.extend(names)\n    \n    return predictions, image_names\n\n# ================================\n# HOW TO USE IT\n# ================================\n\n# Option 1: Quick experiment with subset (30-60 minutes)\n# trained_model, history = main(use_subset=True, subset_size=3000)\n\n# Option 2: Full training (3-5 hours)  \n# trained_model, history = main(use_subset=False)\n\n# Option 3: Different subset sizes\n# trained_model, history = main(use_subset=True, subset_size=8000)  # Larger subset\n\n# Run the training when this script is executed\nif __name__ == \"__main__\":\n    # Uncomment the line below to start training\n    trained_model, training_history = main(use_subset=True, subset_size=3000)\n    print(\"Setup complete! Uncomment the last line to start training.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T00:11:25.965921Z","iopub.execute_input":"2025-09-01T00:11:25.966381Z","iopub.status.idle":"2025-09-01T01:36:06.635522Z","shell.execute_reply.started":"2025-09-01T00:11:25.966358Z","shell.execute_reply":"2025-09-01T01:36:06.634825Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Metadata + Fusion Model (Seng Zi Jun)","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Evaluation, Analysis & Reporting (Wong Kang Yi)","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}