{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"},{"sourceId":10525161,"sourceType":"datasetVersion","datasetId":6514077},{"sourceId":10526916,"sourceType":"datasetVersion","datasetId":6515018},{"sourceId":10531415,"sourceType":"datasetVersion","datasetId":6517233},{"sourceId":10531533,"sourceType":"datasetVersion","datasetId":6517263}],"dockerImageVersionId":30840,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing Required Libraries","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\nimport pandas as pd\nimport os\nfrom sklearn.metrics import accuracy_score, confusion_matrix, f1_score\nimport timm\nimport random\nimport time\nfrom collections import OrderedDict\nfrom torch.cuda import amp\nimport numpy as np\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.optim.optimizer\nfrom torchvision import transforms as T\nimport matplotlib.pyplot as plt\nfrom torchvision.io import read_image\nimport cv2\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import f1_score\nimport seaborn as sns\nfrom tqdm import tqdm\nimport concurrent.futures\nfrom torch.utils.data import random_split\nfrom sklearn.metrics import classification_report, f1_score\n\nprint(torch.__version__)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seed(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)  # Seed for hash-based operations\n    np.random.seed(seed)                      # Seed for NumPy\n    torch.manual_seed(seed)                   # Seed for PyTorch (CPU)\n    torch.cuda.manual_seed(seed)              # Seed for PyTorch (GPU)\n    torch.backends.cudnn.deterministic = True # Make CuDNN deterministic\n    torch.backends.cudnn.benchmark = True     # Enable benchmark mode for CuDNN","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"seed(19071591)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# EDA (Exploratory Data Analysis)","metadata":{}},{"cell_type":"code","source":"label_title = {\"0\" : \"No DR\",\"1\" : \"Mild\",\"2\" : \"Moderate\",\"3\" :\"Severe\",\"4\" : \"Proliferative DR\"}\nclass_labels=[\"No DR\",\"Mild\",\"Moderate\",\"Severe\",\"Proliferative DR\"]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df= pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ntrain_df[\"filename\"] = train_df[\"id_code\"].map(lambda x:os.path.join(\"train_images\",x+\".png\"))\ntest_df= pd.read_csv('../input/aptos2019-blindness-detection/test.csv')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Number of train samples: ', train_df.shape[0])\nprint('Number of test samples: ', test_df.shape[0])\ndisplay(train_df.head())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df['diagnosis'].value_counts()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.diagnosis.hist()\nplt.title('Distribution of Diagnoses')\nplt.xlabel('Diagnosis')\nplt.ylabel('Frequency')\nplt.xticks(range(5), class_labels)\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dfDDR= pd.read_csv('../input/ddr-dataset/DDR dataset/train.csv')\ntrain_dfDDR[\"filename\"] = train_dfDDR[\"id_code\"].map(lambda x:os.path.join(\"train_images\",x+\".png\"))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Number of train samples: ', train_dfDDR.shape[0])\ndisplay(train_dfDDR.head())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dfDDR['diagnosis'].value_counts()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dfDDR.diagnosis.hist()\nplt.title('Distribution of Diagnoses')\nplt.xlabel('Diagnosis')\nplt.ylabel('Frequency')\nplt.xticks(range(5), class_labels)\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"import shutil\noutput_dir = \"/kaggle/working/\"\ndataset_dir = os.path.join(output_dir, \"train_images\")\nif not os.path.exists(dataset_dir):\n    os.makedirs(dataset_dir)\n\naptos_dir = \"../input/aptos2019-blindness-detection/train_images\"\nddr_dir = \"../input/ddr-dataset/DDR dataset/train_images\"\n\naptos_df = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\naptos_df[\"filename\"] = aptos_df[\"id_code\"].map(lambda x:os.path.join(aptos_dir,x+\".png\"))\n\nddr_df = pd.read_csv('../input/ddr-dataset/DDR dataset/train.csv')\nddr_df[\"filename\"] = ddr_df[\"id_code\"].map(lambda x:os.path.join(ddr_dir,x+\".png\"))\n\nfor index, row in aptos_df.iterrows():\n    src_path = row[\"filename\"]\n    dst_path = os.path.join(dataset_dir, os.path.basename(src_path))\n    if os.path.exists(src_path):\n      shutil.copy(src_path, dst_path)\n\nfor index, row in ddr_df.iterrows():\n    src_path = row[\"filename\"]\n    dst_path = os.path.join(dataset_dir, os.path.basename(src_path))\n    if os.path.exists(src_path):\n      shutil.copy(src_path, dst_path)\n\nmerged_df = pd.concat([aptos_df, ddr_df], ignore_index=True)\nmerged_df.to_csv('/kaggle/working/train.csv', index=False)\n\nprint(\"Sao chép ảnh hoàn tất!\")\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"!cd /kaggle/working/\n!zip -r datasets-dr.zip train_images train.csv\n\nfrom IPython.display import FileLink\nFileLink(r'datasets-dr.zip')\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df= pd.read_csv('../input/datasets-dr/Datasets/train.csv')\ntrain_df[\"filename\"] = train_df[\"id_code\"].map(lambda x:os.path.join(\"train_images\",x+\".png\"))\ntest_df= pd.read_csv('../input/aptos2019-blindness-detection/test.csv')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.diagnosis.hist()\nplt.title('Distribution of Diagnoses')\nplt.xlabel('Diagnosis')\nplt.ylabel('Frequency')\nplt.xticks(range(5), class_labels)\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Number of train samples: ', train_df.shape[0])\ntrain_df['diagnosis'].value_counts()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.set_style(\"white\")\nfigure, ax = plt.subplots(2,5, figsize=(20, 12))\nax = ax.flatten()\nfor i,row in train_df.iloc[0:10,:].iterrows():\n    image = cv2.imread(os.path.join(\"../input/datasets-dr/Datasets/train_images\",row[\"id_code\"]+\".png\"))\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    ax[i].imshow(image)\n    ax[i].set_title(label_title[str(row[\"diagnosis\"])])\n    plt.tight_layout()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Preprocessing","metadata":{}},{"cell_type":"code","source":"def crop_image(image, tol = 7):\n    if image.ndim == 2:\n        mask = image > tol\n        return image[np.ix_(mask.any(1), mask.any(0))]\n    elif image.ndim == 3:\n        image_gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)\n        mask = image_gray > tol\n        check_shape = image[:,:,0][np.ix_(mask.any(1), mask.any(0))].shape[0]\n        if(check_shape == 0):\n            return image\n        else:\n            image1 = image [:,:,0][np.ix_(mask.any(1), mask.any(0))]\n            image2 = image [:,:,1][np.ix_(mask.any(1), mask.any(0))]\n            image3 = image [:,:,2][np.ix_(mask.any(1), mask.any(0))]\n            image = np.stack([image1, image2, image3], axis = -1)\n        return image\n\ndef add_black_padding_and_resize(image, img_size):\n    h, w = image.shape[:2]\n    new_h, new_w = img_size, img_size\n    if h > w:\n        scale_factor = img_size / h\n    else:\n        scale_factor = img_size / w\n\n    new_h = int(h * scale_factor)\n    new_w = int(w * scale_factor)\n    resized_image = cv2.resize(image, (new_w, new_h))\n    top = (img_size - new_h) // 2\n    bottom = img_size - new_h - top\n    left = (img_size - new_w) // 2\n    right = img_size - new_w - left\n\n    padded_resized_image = cv2.copyMakeBorder(resized_image, top, bottom, left, right, cv2.BORDER_CONSTANT, value= 0)\n    return padded_resized_image","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Set input and output directories\ninput_dir = '/kaggle/input/datasets-dr/Datasets/train_images/'\noutput_dir = '/kaggle/working/processed_images/'\nos.makedirs(output_dir, exist_ok=True) ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_image(image, img_size = 256):\n    image_id = image['id_code'] \n    image_file = image_id + '.png' \n    image_path = os.path.join(input_dir, image_file)\n\n    if os.path.exists(image_path):\n        image = cv2.imread(image_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = crop_image(image)\n        image = add_black_padding_and_resize(image, img_size)\n        \n        # Split the image into its channels (BGR format)\n        red, green, blue = cv2.split(image)\n        clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\n        red = clahe.apply(red)\n        green = clahe.apply(green)\n        blue = clahe.apply(blue)\n        \n        #Kernel\n        kernel_horizontal = np.array([[1, 1, 1, 1, 1]], dtype=np.uint8)\n        kernel_vertical = np.array([[1],\n                                [1],\n                                [1],\n                                [1],\n                                [1]], dtype=np.uint8)\n        kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (9, 9))\n    \n        #Top/Black hat for Channel Red\n        blackhat_h = cv2.morphologyEx(red, cv2.MORPH_BLACKHAT, kernel_horizontal)\n        blackhat_v = cv2.morphologyEx(red, cv2.MORPH_BLACKHAT, kernel_vertical)\n        tophat = cv2.morphologyEx(red, cv2.MORPH_TOPHAT, kernel)\n        blackhat = cv2.morphologyEx(red, cv2.MORPH_BLACKHAT, kernel)\n        image_tophat = cv2.add(red, tophat)\n        image_blackhat = np.maximum(np.maximum(blackhat_h, blackhat_v), blackhat)\n        image_red = cv2.subtract(image_tophat, image_blackhat)\n        \n        #Top/Black hat for Channel Green\n        blackhat_h = cv2.morphologyEx(green, cv2.MORPH_BLACKHAT, kernel_horizontal)\n        blackhat_v = cv2.morphologyEx(green, cv2.MORPH_BLACKHAT, kernel_vertical)\n        tophat = cv2.morphologyEx(green, cv2.MORPH_TOPHAT, kernel)\n        blackhat = cv2.morphologyEx(green, cv2.MORPH_BLACKHAT, kernel)\n        image_tophat = cv2.add(green, tophat)\n        image_blackhat = np.maximum(np.maximum(blackhat_h, blackhat_v), blackhat)\n        image_green = cv2.subtract(image_tophat, image_blackhat)\n    \n        image = cv2.merge([image_red, image_green, blue])\n        \n        output_path = os.path.join(output_dir, image_file)\n        cv2.imwrite(output_path, image)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:\n    list(tqdm(executor.map(process_image, \n                           [image for _, image in train_df.iterrows()]), \n                           total=train_df.shape[0], \n                           desc=\"Processing images\", \n                           unit=\"image\"))\nprint(\"Processing complete for all images.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.set_style(\"white\")\nfigure, ax = plt.subplots(2,5, figsize=(20, 12))\nax = ax.flatten()\nfor i,row in train_df.iloc[0:10,:].iterrows():\n    image = cv2.imread(os.path.join(\"/kaggle/working/processed_images\",row[\"id_code\"]+\".png\"))\n    red, green, blue = cv2.split(image)\n    clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\n    ima = clahe.apply(red)\n    #image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    ax[i].imshow(image)\n    ax[i].set_title(label_title[str(row[\"diagnosis\"])])\n    plt.tight_layout()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Splitting and Transformation","metadata":{}},{"cell_type":"code","source":"DATA_DIR = \"/kaggle/working/processed_images\"\nTRAIN_DIR = \"/kaggle/working/processed_images\"\nCSV_PATH = \"/kaggle/input/datasets-dr/Datasets/train.csv\"\nMODEL_PATH = \"./kaggle/working/\"\nLEARNING_RATE = 1e-4\nTRAIN_BATCH_SIZE = 32\nVALID_BATCH_SIZE = 32\nTRAIN_SPLIT = 0.8\nNUM_WORKERS = 2\nUSE_AMP = True\nEPOCHS=30","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class RetinopathyDataset(Dataset):\n    def __init__(self, image_dir, csv_file, transforms=None):\n        self.data = pd.read_csv(csv_file)\n        self.transforms = transforms\n        self.image_dir = image_dir\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join(self.image_dir, self.data.loc[idx, 'id_code'] + '.png')\n\n        tensor_image = read_image(img_name)\n        label = torch.tensor(self.data.loc[idx, 'diagnosis'], dtype=torch.long)\n\n        if self.transforms is not None:\n            tensor_image = self.transforms(tensor_image)\n\n        return (tensor_image, label)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_trasforms= T.Compose([\n    T.RandomHorizontalFlip(),\n    T.RandomVerticalFlip(),\n    T.RandomRotation(degrees=60),\n    T.ColorJitter(brightness=0.10),\n    T.ConvertImageDtype(torch.float32),\n    T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\nfull_dataset = RetinopathyDataset(TRAIN_DIR, CSV_PATH, transforms=train_trasforms)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_size = int(TRAIN_SPLIT * len(full_dataset))\ntest_size = len(full_dataset) - train_size\ntrain_dataset, val_dataset = torch.utils.data.random_split(full_dataset, [train_size, test_size])\n\ntrain_loader = DataLoader(full_dataset, batch_size=TRAIN_BATCH_SIZE, shuffle=True, \n                          num_workers=NUM_WORKERS, drop_last=True, pin_memory=False)\nval_loader = DataLoader(val_dataset, batch_size=VALID_BATCH_SIZE, shuffle=True, \n                        num_workers=NUM_WORKERS, drop_last=True, pin_memory=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Helper Functions and Utilities for Training and Evaluation","metadata":{}},{"cell_type":"code","source":"@torch.no_grad()\ndef accuracy(output, target):\n    with torch.no_grad():\n        _, predicted = output.max(1)  # Get the class index with the highest score\n        correct = predicted.eq(target).sum().item()  # Count correct predictions\n        total = target.size(0)  # Total number of samples\n        accuracy = 100.0 * correct / total  # Compute accuracy percentage\n    return accuracy\n\ndef set_debug_apis(state: bool = False):\n    torch.autograd.profiler.profile(enabled=state)\n    torch.autograd.profiler.emit_nvtx(enabled=state)\n    torch.autograd.set_detect_anomaly(mode=state)\n\ndef print_size_of_model(model):\n    torch.save(model.state_dict(), \"temp.p\")  # Save model state\n    print(\"Size (MB):\", os.path.getsize(\"temp.p\") / 1e6)  # Convert bytes to MB\n    os.remove(\"temp.p\")  # Clean up temporary file","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(torch.cuda.is_available())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Fine Tuning The Models","metadata":{}},{"cell_type":"code","source":"def train_step(model: torch.nn.Module, train_loader, criterion, device: str, optimizer, scheduler=None, num_batches: int = None, log_interval: int = 100, scaler=None):\n    model = model.to(device)\n    model.train()\n\n    metrics = OrderedDict()\n\n    total_loss = 0\n    correct_predictions = 0\n    total_samples = 0\n\n    for batch_idx, (inputs, target) in enumerate(train_loader):\n        inputs = inputs.to(device)\n        target = target.to(device)\n\n        # Zero the parameter gradients\n        optimizer.zero_grad()\n\n        # Mixed precision training if scaler is provided\n        if scaler is not None:\n            with torch.cuda.amp.autocast():\n                output = model(inputs)\n                loss = criterion(output, target)\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n        else:\n            output = model(inputs)\n            loss = criterion(output, target)\n            loss.backward()\n            optimizer.step()\n\n        # Update the scheduler if it's provided\n        if scheduler is not None:\n            scheduler.step()\n\n        # Update metrics\n        total_loss += loss.item() * inputs.size(0)\n        _, predicted = output.max(1)\n        correct_predictions += predicted.eq(target).sum().item()\n        total_samples += inputs.size(0)\n        \n        # Calculate and update the metrics directly in the main loop\n        metrics[\"loss\"] = total_loss / total_samples\n        metrics[\"accuracy\"] = 100.0 * correct_predictions / total_samples\n\n        # Break if num_batches is set and we've reached the limit\n        if num_batches is not None and batch_idx + 1 >= num_batches:\n            break\n\n    return metrics","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"@torch.no_grad()  # No gradient calculation needed during validation\ndef val_step(model: torch.nn.Module, val_loader, criterion, device: str, num_batches=None, log_interval: int = 100):\n    model = model.to(device)\n    model.eval()  # Set the model to evaluation mode\n\n    metrics = OrderedDict()\n\n    total_loss = 0\n    correct_predictions = 0\n    total_samples = 0\n\n    for batch_idx, (inputs, target) in enumerate(val_loader):\n        inputs = inputs.to(device)\n        target = target.to(device)\n\n        # Forward pass (no gradient computation)\n        output = model(inputs)\n        loss = criterion(output, target)\n\n        # Update metrics\n        total_loss += loss.item() * inputs.size(0)\n        _, predicted = output.max(1)  # Get predictions\n        correct_predictions += predicted.eq(target).sum().item()  # Compare predictions with targets\n        total_samples += inputs.size(0)\n\n        # Calculate and update the metrics directly in the main loop\n        metrics[\"loss\"] = total_loss / total_samples\n        metrics[\"accuracy\"] = 100.0 * correct_predictions / total_samples\n\n        # Break if num_batches is specified and we've reached the limit\n        if num_batches is not None and batch_idx + 1 >= num_batches:\n            break\n\n    return metrics","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MODEL_SWINV2_SMALL = \"swinv2_small_window16_256\"\nMODEL_SWINV2_SMALL_SAVE=  \"/kaggle/working/swinv2.Csv\"\nCHECKPOINT_MODEL_SWINV2_SMALL_DIR = \"/kaggle/working/model_swinv2/\"\nos.makedirs(CHECKPOINT_MODEL_SWINV2_SMALL_DIR, exist_ok=True)\nset_debug_apis(False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MODEL_FastVit = \"fastvit_s12\"\nMODEL_FastVit_SAVE=  \"/kaggle/working/fastvit_s12.Csv\"\nCHECKPOINT_MODEL_FastVit_DIR = \"/kaggle/working/model_fastvit/\"\nos.makedirs(CHECKPOINT_MODEL_FastVit_DIR, exist_ok=True)\nset_debug_apis(False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.utils.class_weight import compute_class_weight\n# Extract the class labels (diagnosis column)\nclass_labels = train_df['diagnosis'].values\n\n# Compute class weights using sklearn\nunique_classes = np.unique(class_labels)\nclass_weights = compute_class_weight('balanced', classes=unique_classes, y=class_labels)\n\n# Create a dictionary that maps class labels to their corresponding weights\nclass_weight_dict = {class_label: weight for class_label, weight in zip(unique_classes, class_weights)}\n\nprint(\"Class Weights:\", class_weight_dict)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch.nn.functional as F\nclass FocalLoss(nn.Module):\n    def __init__(self, class_weights, gamma=1.5, reduction='mean'):\n        super(FocalLoss, self).__init__()\n        self.alpha = torch.tensor(class_weights).float().to(device) \n        self.gamma = gamma\n        self.reduction = reduction\n\n    def forward(self, inputs, targets):\n        targets = targets.long() # Đảm bảo targets là kiểu Long\n        ce_loss = F.cross_entropy(inputs, targets, reduction='none')\n        pt = torch.exp(-ce_loss)\n        focal_loss = self.alpha[targets] * (1 - pt)**self.gamma * ce_loss\n        if self.reduction == 'mean':\n            return focal_loss.mean()\n        elif self.reduction == 'sum':\n            return focal_loss.sum()\n        else:\n            return focal_loss","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# MODEL_SWINV2_SMALL","metadata":{}},{"cell_type":"code","source":"model= timm.create_model(MODEL_SWINV2_SMALL, pretrained=True, num_classes=5)\ncriterion = FocalLoss(class_weights=list(class_weight_dict.values()), gamma=2, reduction='mean').to(device)\noptimizer= optim.AdamW(model.parameters(), lr=LEARNING_RATE)\n\nif USE_AMP:\n    from torch.cuda import amp\n    scaler = torch.amp.GradScaler()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Initialize lists to store metrics\ntrain_loss = []\ntrain_accuracy = []\nval_loss = []\nval_accuracy = []\n\n# Record start time\nstart_time = time.time()\n\n# Total number of batches for training and validation across all epochs\ntotal_train_batches = len(train_loader) * EPOCHS\ntotal_val_batches = len(val_loader) * EPOCHS\ntotal_batches = total_train_batches + total_val_batches\n\n# Start epoch loop with a single tqdm progress bar for the entire training process\nwith tqdm(total=total_batches, desc=\"Training Progress\", unit=\"batch\") as pbar:\n    for epoch in range(EPOCHS):\n        \n        # Perform training step\n        train_metrics = train_step(model, train_loader, criterion, device, optimizer, scaler=scaler)\n        train_loss.append(train_metrics[\"loss\"])\n        train_accuracy.append(train_metrics[\"accuracy\"])\n\n        # Update progress bar after training for the current epoch\n        pbar.set_postfix({\n            \"Train Loss\": f\"{train_metrics['loss']:.4f}\",\n            \"Train Acc\": f\"{train_metrics['accuracy']:.2f}%\",\n            \"Stage\": \"Train\"\n        })\n        pbar.update(len(train_loader))  # Update progress bar by the number of batches in the training loader\n\n        # Perform validation step\n        val_metrics = val_step(model, val_loader, criterion, device)\n        val_loss.append(val_metrics[\"loss\"])\n        val_accuracy.append(val_metrics[\"accuracy\"])\n\n        # Update progress bar after validation for the current epoch\n        pbar.set_postfix({\n            \"Val Loss\": f\"{val_metrics['loss']:.4f}\",\n            \"Val Acc\": f\"{val_metrics['accuracy']:.2f}%\",\n            \"Stage\": \"Val\"\n        })\n        pbar.update(len(val_loader))  # Update progress bar by the number of batches in the validation loader\n\n        # Print epoch summary (not per-batch)\n        print(f\"Epoch {epoch+1} Summary:\")\n        print(f\"Training loss = {train_metrics['loss']:.4f}, Training accuracy = {train_metrics['accuracy']:.2f}%\")\n        print(f\"Validation loss = {val_metrics['loss']:.4f}, Validation accuracy = {val_metrics['accuracy']:.2f}%\")\n\n        # Save model checkpoint\n        checkpoint_path = os.path.join(CHECKPOINT_MODEL_SWINV2_SMALL_DIR, f\"{MODEL_SWINV2_SMALL}_epoch_{epoch+1}.pt\")\n        torch.save(model.state_dict(), checkpoint_path)\n\n# Record end time\nend_time = time.time()\n\n# Calculate total training time\ntotal_time = end_time - start_time\nprint(f\"\\nTotal training time: {total_time:.2f} seconds\")\n\n# Create a DataFrame to store the metrics\nmetrics_df = pd.DataFrame({\n    \"epoch\": range(1, EPOCHS + 1),\n    \"train_loss\": train_loss,\n    \"train_accuracy\": train_accuracy,\n    \"val_loss\": val_loss,\n    \"val_accuracy\": val_accuracy,\n})\n\n# Save the DataFrame to CSV\nmetrics_df.to_csv(MODEL_SWINV2_SMALL_SAVE, index=False)\nprint(\"Training metrics saved to CSV.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#metrics_path = f\"{MODEL_SWINV2_SMALL}_metrics.csv\"\n#metrics_df = pd.read_csv(MODEL_SWINV2_SMALL_SAVE)\n\nmetrics_path = f\"{MODEL_SWINV2_SMALL}_metrics.csv\"\nmetrics_df = pd.read_csv('../input/model-swinv2/model_swinv2/swinv2.Csv')\n\n# Set Seaborn style for better aesthetics\nsns.set_theme(style=\"whitegrid\")\n\n# Create a figure for visualizations\nplt.figure(figsize=(12, 8))\n\n# Plot training and validation loss\nplt.subplot(2, 1, 1)\nsns.lineplot(x='epoch', y='train_loss', data=metrics_df, label='Train Loss', color='blue')\nsns.lineplot(x='epoch', y='val_loss', data=metrics_df, label='Validation Loss', color='orange')\nplt.title(\"Training and Validation Loss per Epoch\", fontsize=14, fontweight='bold')\nplt.xlabel(\"Epoch\", fontsize=12)\nplt.ylabel(\"Loss\", fontsize=12)\nplt.legend()\nplt.grid(alpha=0.3)\n\n# Plot training and validation accuracy\nplt.subplot(2, 1, 2)\nsns.lineplot(x='epoch', y='train_accuracy', data=metrics_df, label='Train Accuracy', color='green')\nsns.lineplot(x='epoch', y='val_accuracy', data=metrics_df, label='Validation Accuracy', color='red')\nplt.title(\"Training and Validation Accuracy per Epoch\", fontsize=14, fontweight='bold')\nplt.xlabel(\"Epoch\", fontsize=12)\nplt.ylabel(\"Accuracy\", fontsize=12)\nplt.legend()\nplt.grid(alpha=0.3)\n\n# Adjust spacing between plots\nplt.tight_layout()\n\n# Save the visualization as a file with transparent background\n#output_plot_path = f\"{MODEL_SWINV2_SMALL}_training_visualization.png\"\n#plt.savefig(output_plot_path, dpi=300, transparent=True)\n#print(f\"Visualization saved to {output_plot_path}\")\n\n# Show the plots\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_model_from_checkpoint(checkpoint_path, model_class, device):\n    model.load_state_dict(torch.load(checkpoint_path, map_location=device))\n    model.to(device)\n    model.eval()  \n    return model","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Predict on validation data\ndef predict(model, dataloader, device):\n    model.eval()\n    y_true = []\n    y_pred = []\n\n    with torch.no_grad():  \n        for inputs, labels in dataloader:  \n            inputs = inputs.to(device)\n            labels = labels.to(device)\n            outputs = model(inputs)\n            _, preds = torch.max(outputs, 1)\n\n            # Collect true and predicted labels\n            y_true.extend(labels.cpu().numpy())\n            y_pred.extend(preds.cpu().numpy())\n\n    return np.array(y_true), np.array(y_pred)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define paths and device\n#checkpoint_path = \"/kaggle/working/model_swinv/swinv2_small_window16_256_epoch_24.pt\"\ncheckpoint_path = \"../input/model-swinv2/model_swinv2/model_swinv2/swinv2_small_window16_256_epoch_24.pt\"\nmodel_class=5\nmodel = load_model_from_checkpoint(checkpoint_path, model_class, device)\n\ny_true_train, y_pred_train = predict(model, train_loader, device)\ny_true_val, y_pred_val = predict(model, val_loader, device)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_accuracy= accuracy_score(y_true_train, y_pred_train)\nValidation_accuracy = accuracy_score(y_true_val, y_pred_val)\nprint(f\"Train Accuracy: {round(train_accuracy * 100, 2)}%\")\nprint(f\"Validation Accuracy: {round(Validation_accuracy * 100, 2)}%\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Generate Classification Report\nreport = classification_report(y_true_val, y_pred_val, digits=2)\nprint(\"\\nClassification Report:\\n\", report)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cm = confusion_matrix(y_true_val, y_pred_val)\n\n# Plot confusion matrix using seaborn\nplt.figure(figsize=(8, 6))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues', \n            xticklabels=[label_title[str(i)] for i in range(cm.shape[0])], \n            yticklabels=[label_title[str(i)] for i in range(cm.shape[0])])\nplt.xlabel('Predicted')\nplt.ylabel('True')\nplt.title('Confusion Matrix')\nplt.xticks(rotation=45)\nplt.yticks(rotation=45)\nplt.savefig('confusion_matrix_swin', dpi=300, transparent=True)\n\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate errors per class (misclassifications)\nclass_errors = {}\nnum_classes = cm.shape[0]\n\nfor i in range(num_classes):\n    total = np.sum(cm[i, :])  # Total number of instances of class i\n    incorrect = total - cm[i, i]  # Misclassified instances of class i\n    error_rate = incorrect / total if total != 0 else 0\n    class_errors[label_title[str(i)]] = round(error_rate, 4)  # Round error_rate and map to class name\n\n# Create a DataFrame for the error rates\nerror_data = pd.DataFrame(list(class_errors.items()), columns=['Class', 'Error Rate'])\n\nfig, ax = plt.subplots(figsize=(8, 4))  # Adjust size as needed\nax.axis('off')  # Turn off axis for the table\n\n# Create and display the table, making it fill the figure\ntable = ax.table(cellText=error_data.values, colLabels=error_data.columns, cellLoc='center', loc='center')\ntable.auto_set_font_size(False)\ntable.set_fontsize(12)\ntable.auto_set_column_width(col=list(range(len(error_data.columns))))\ntable.scale(20, 5)  # Scale table size (adjust the values as needed)\nplt.savefig('table_error_rate_swin', dpi=300, transparent=True)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!cd /kaggle/working/\n!zip -r model_swinv2.zip model_swinv2 swinv2_small_window16_256_training_visualization.png swinv2.Csv","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from IPython.display import FileLink\nFileLink(r'model_swinv2.zip')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# MODEL_FastVit","metadata":{}},{"cell_type":"code","source":"model= timm.create_model(MODEL_FastVit, pretrained=True, num_classes=5)\nweights_tensor = torch.tensor(list(class_weight_dict.values())).float()\ncriterion= nn.CrossEntropyLoss(weight=weights_tensor.to(device))\noptimizer= optim.AdamW(model.parameters(), lr=LEARNING_RATE)\n\nif USE_AMP:\n    from torch.cuda import amp\n    scaler = torch.amp.GradScaler()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Initialize lists to store metrics\ntrain_loss = []\ntrain_accuracy = []\nval_loss = []\nval_accuracy = []\n\n# Record start time\nstart_time = time.time()\n\n# Total number of batches for training and validation across all epochs\ntotal_train_batches = len(train_loader) * EPOCHS\ntotal_val_batches = len(val_loader) * EPOCHS\ntotal_batches = total_train_batches + total_val_batches\n\n# Start epoch loop with a single tqdm progress bar for the entire training process\nwith tqdm(total=total_batches, desc=\"Training Progress\", unit=\"batch\") as pbar:\n    for epoch in range(EPOCHS):\n        \n        # Perform training step\n        train_metrics = train_step(model, train_loader, criterion, device, optimizer, scaler=scaler)\n        train_loss.append(train_metrics[\"loss\"])\n        train_accuracy.append(train_metrics[\"accuracy\"])\n\n        # Update progress bar after training for the current epoch\n        pbar.set_postfix({\n            \"Train Loss\": f\"{train_metrics['loss']:.4f}\",\n            \"Train Acc\": f\"{train_metrics['accuracy']:.2f}%\",\n            \"Stage\": \"Train\"\n        })\n        pbar.update(len(train_loader))  # Update progress bar by the number of batches in the training loader\n\n        # Perform validation step\n        val_metrics = val_step(model, val_loader, criterion, device)\n        val_loss.append(val_metrics[\"loss\"])\n        val_accuracy.append(val_metrics[\"accuracy\"])\n\n        # Update progress bar after validation for the current epoch\n        pbar.set_postfix({\n            \"Val Loss\": f\"{val_metrics['loss']:.4f}\",\n            \"Val Acc\": f\"{val_metrics['accuracy']:.2f}%\",\n            \"Stage\": \"Val\"\n        })\n        pbar.update(len(val_loader))  # Update progress bar by the number of batches in the validation loader\n\n        # Print epoch summary (not per-batch)\n        print(f\"Epoch {epoch+1} Summary:\")\n        print(f\"Training loss = {train_metrics['loss']:.4f}, Training accuracy = {train_metrics['accuracy']:.2f}%\")\n        print(f\"Validation loss = {val_metrics['loss']:.4f}, Validation accuracy = {val_metrics['accuracy']:.2f}%\")\n\n        # Save model checkpoint\n        checkpoint_path = os.path.join(CHECKPOINT_MODEL_FastVit_DIR, f\"{MODEL_FastVit}_epoch_{epoch+1}.pt\")\n        torch.save(model.state_dict(), checkpoint_path)\n\n# Record end time\nend_time = time.time()\n\n# Calculate total training time\ntotal_time = end_time - start_time\nprint(f\"\\nTotal training time: {total_time:.2f} seconds\")\n\n# Create a DataFrame to store the metrics\nmetrics_df = pd.DataFrame({\n    \"epoch\": range(1, EPOCHS + 1),\n    \"train_loss\": train_loss,\n    \"train_accuracy\": train_accuracy,\n    \"val_loss\": val_loss,\n    \"val_accuracy\": val_accuracy,\n})\n\n# Save the DataFrame to CSV\nmetrics_df.to_csv(MODEL_FastVit_SAVE, index=False)\nprint(\"Training metrics saved to CSV.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#metrics_path = f\"{MODEL_FastVit}_metrics.csv\" \n#metrics_df = pd.read_csv(MODEL_FastVit_SAVE)\n\nmetrics_path = f\"{MODEL_FastVit}_metrics.csv\"\nmetrics_df = pd.read_csv('../input/model-fastvit/model_fastvit/fastvit_s12.Csv')\n\n# Set Seaborn style for better aesthetics\nsns.set_theme(style=\"whitegrid\")\n\n# Create a figure for visualizations\nplt.figure(figsize=(12, 8))\n\n# Plot training and validation loss\nplt.subplot(2, 1, 1)\nsns.lineplot(x='epoch', y='train_loss', data=metrics_df, label='Train Loss', color='blue')\nsns.lineplot(x='epoch', y='val_loss', data=metrics_df, label='Validation Loss', color='orange')\nplt.title(\"Training and Validation Loss per Epoch\", fontsize=14, fontweight='bold')\nplt.xlabel(\"Epoch\", fontsize=12)\nplt.ylabel(\"Loss\", fontsize=12)\nplt.legend()\nplt.grid(alpha=0.3)\n\n# Plot training and validation accuracy\nplt.subplot(2, 1, 2)\nsns.lineplot(x='epoch', y='train_accuracy', data=metrics_df, label='Train Accuracy', color='green')\nsns.lineplot(x='epoch', y='val_accuracy', data=metrics_df, label='Validation Accuracy', color='red')\nplt.title(\"Training and Validation Accuracy per Epoch\", fontsize=14, fontweight='bold')\nplt.xlabel(\"Epoch\", fontsize=12)\nplt.ylabel(\"Accuracy\", fontsize=12)\nplt.legend()\nplt.grid(alpha=0.3)\n\n# Adjust spacing between plots\nplt.tight_layout()\n\n# Save the visualization as a file with transparent background\n#output_plot_path = f\"{MODEL_FastVit}_training_visualization.png\"\n#plt.savefig(output_plot_path, dpi=300, transparent=True)\n#print(f\"Visualization saved to {output_plot_path}\")\n\n# Show the plots\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_model_from_checkpoint(checkpoint_path, model_class, device):\n    model.load_state_dict(torch.load(checkpoint_path, map_location=device))\n    model.to(device)\n    model.eval()  \n    return model","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Predict on validation data\ndef predict(model, dataloader, device):\n    model.eval()\n    y_true = []\n    y_pred = []\n\n    with torch.no_grad():  \n        for inputs, labels in dataloader:  \n            inputs = inputs.to(device)\n            labels = labels.to(device)\n            outputs = model(inputs)\n            _, preds = torch.max(outputs, 1)\n\n            # Collect true and predicted labels\n            y_true.extend(labels.cpu().numpy())\n            y_pred.extend(preds.cpu().numpy())\n\n    return np.array(y_true), np.array(y_pred)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define paths and device\ncheckpoint_path = \"../input/model-fastvit/model_fastvit/model_fastvit/fastvit_s12_epoch_29.pt\"\nmodel_class=5\nmodel = load_model_from_checkpoint(checkpoint_path, model_class, device)\n\ny_true_train, y_pred_train = predict(model, train_loader, device)\ny_true_val, y_pred_val = predict(model, val_loader, device)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_accuracy= accuracy_score(y_true_train, y_pred_train)\nValidation_accuracy = accuracy_score(y_true_val, y_pred_val)\nprint(f\"Train Accuracy: {round(train_accuracy * 100, 2)}%\")\nprint(f\"Validation Accuracy: {round(Validation_accuracy * 100, 2)}%\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Generate Classification Report\nreport = classification_report(y_true_val, y_pred_val, digits=2)\nprint(\"\\nClassification Report:\\n\", report)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cm = confusion_matrix(y_true_val, y_pred_val)\n\n# Plot confusion matrix using seaborn\nplt.figure(figsize=(8, 6))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues', \n            xticklabels=[label_title[str(i)] for i in range(cm.shape[0])], \n            yticklabels=[label_title[str(i)] for i in range(cm.shape[0])])\nplt.xlabel('Predicted')\nplt.ylabel('True')\nplt.title('Confusion Matrix')\nplt.xticks(rotation=45)\nplt.yticks(rotation=45)\nplt.savefig('confusion_matrix_fastvit', dpi=300, transparent=True)\n\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate errors per class (misclassifications)\nclass_errors = {}\nnum_classes = cm.shape[0]\n\nfor i in range(num_classes):\n    total = np.sum(cm[i, :])  # Total number of instances of class i\n    incorrect = total - cm[i, i]  # Misclassified instances of class i\n    error_rate = incorrect / total if total != 0 else 0\n    class_errors[label_title[str(i)]] = round(error_rate, 4)  # Round error_rate and map to class name\n\n# Create a DataFrame for the error rates\nerror_data = pd.DataFrame(list(class_errors.items()), columns=['Class', 'Error Rate'])\n\nfig, ax = plt.subplots(figsize=(8, 4))  # Adjust size as needed\nax.axis('off')  # Turn off axis for the table\n\n# Create and display the table, making it fill the figure\ntable = ax.table(cellText=error_data.values, colLabels=error_data.columns, cellLoc='center', loc='center')\ntable.auto_set_font_size(False)\ntable.set_fontsize(12)\ntable.auto_set_column_width(col=list(range(len(error_data.columns))))\ntable.scale(20, 5)  # Scale table size (adjust the values as needed)\nplt.savefig('table_error_rate_fastvit', dpi=150, transparent=True)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!cd /kaggle/working/\n!zip -r model_fastvit.zip model_fastvit fastvit_s12_training_visualization.png confusion_matrix_fastvit.png fastvit_s12.Csv table_error_rate_fastvit.png","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from IPython.display import FileLink\nFileLink(r'model_fastvit.zip')","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}