{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"}],"dockerImageVersionId":30805,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!nvidia-smi","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !pip install -q torch==1.8.1+cu111 torchvision==0.9.1+cu111 -f https://download.pytorch.org/whl/torch_stable.html\n# !conda install pytorch torchvision torchaudio cudatoolkit=11.1 -c pytorch -c nvidia\n!pip install timm","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !pip uninstall torchtext -y\n# !pip uninstall torchaudio -y","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\nimport pandas as pd\nimport os\nimport timm\nimport random\nimport time\nfrom collections import OrderedDict\nfrom torch.cuda import amp\nimport numpy as np\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torchvision import transforms as T\nfrom torchvision.io import read_image\nfrom tqdm import tqdm\nprint(torch.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T22:34:47.288626Z","iopub.execute_input":"2024-12-07T22:34:47.289316Z","iopub.status.idle":"2024-12-07T22:34:53.160256Z","shell.execute_reply.started":"2024-12-07T22:34:47.289283Z","shell.execute_reply":"2024-12-07T22:34:53.159360Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pwd","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n@torch.no_grad()\ndef accuracy(output, target, topk=(1,)):\n    \"\"\"Computes the accuracy over the k top predictions for the specified values of k\"\"\"\n    maxk = max(topk)\n    batch_size = target.size(0)\n    _, pred = output.topk(maxk, 1, True, True)\n    pred = pred.t()\n    correct = pred.eq(target.reshape(1, -1).expand_as(pred))\n    return [correct[:k].reshape(-1).float().sum(0) * 100. / batch_size for k in topk]\n\n\ndef set_debug_apis(state: bool = False):\n    torch.autograd.profiler.profile(enabled=state)\n    torch.autograd.profiler.emit_nvtx(enabled=state)\n    torch.autograd.set_detect_anomaly(mode=state)\n\n\ndef seed_everything(seed):\n    \"\"\"\n    Makes code deterministic using a given seed.\n    Internally sets all seeds of torch, numpy and random.\n    \"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\n\ndef print_size_of_model(model):\n    torch.save(model.state_dict(), \"temp.p\")\n    print(\"Size (MB):\", os.path.getsize(\"temp.p\") / 1e6)\n    os.remove(\"temp.p\")\n\n\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T22:35:03.543055Z","iopub.execute_input":"2024-12-07T22:35:03.543764Z","iopub.status.idle":"2024-12-07T22:35:03.553088Z","shell.execute_reply.started":"2024-12-07T22:35:03.543725Z","shell.execute_reply":"2024-12-07T22:35:03.552164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class RetinopathyDataset(Dataset):\n    def __init__(self, image_dir, csv_file, transforms=None):\n        self.data = pd.read_csv(csv_file)\n        self.transforms = transforms\n        self.image_dir = image_dir\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        # img_name = os.path.join('../input/aptos2019-blindness-detection/train_images',\n        #                         self.data.loc[idx, 'id_code'] + '.png')\n\n        img_name = os.path.join(self.image_dir, self.data.loc[idx, 'id_code'] + '.png')\n\n        tensor_image = read_image(img_name)\n        label = torch.tensor(self.data.loc[idx, 'diagnosis'], dtype=torch.long)\n\n        if self.transforms is not None:\n            tensor_image = self.transforms(tensor_image)\n\n        return (tensor_image, label)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T22:35:05.169612Z","iopub.execute_input":"2024-12-07T22:35:05.170182Z","iopub.status.idle":"2024-12-07T22:35:05.175962Z","shell.execute_reply.started":"2024-12-07T22:35:05.170148Z","shell.execute_reply":"2024-12-07T22:35:05.174978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(torch.cuda.is_available())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T22:35:07.652650Z","iopub.execute_input":"2024-12-07T22:35:07.653400Z","iopub.status.idle":"2024-12-07T22:35:07.710482Z","shell.execute_reply.started":"2024-12-07T22:35:07.653337Z","shell.execute_reply":"2024-12-07T22:35:07.709488Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_step(model: nn.Module, train_loader, criterion,\n               device: str, optimizer,\n               scheduler=None, num_batches: int = None,\n               log_interval: int = 100,\n               scaler=None,):\n    \"\"\"\n    Performs one step of training. Calculates loss, forward pass, computes gradient and returns metrics.\n    Args:\n        model : A pytorch CNN Model.\n        train_loader : Train loader.\n        criterion : Loss function to be optimized.\n        device : \"cuda\" or \"cpu\"\n        optimizer : Torch optimizer to train.\n        scheduler : Learning rate scheduler.\n        num_batches : (optional) Integer To limit training to certain number of batches.\n        log_interval : (optional) Defualt 100. Integer to Log after specified batch ids in every batch.\n        scaler: (optional)  Pass torch.cuda.amp.GradScaler() for fp16 precision Training.\n    \"\"\"\n\n    model = model.to(device)\n    start_train_step = time.time()\n    metrics = OrderedDict()\n    model.train()\n    last_idx = len(train_loader) - 1\n    batch_time_m = AverageMeter()\n    # data_time_m = AverageMeter()\n    losses_m = AverageMeter()\n    top1_m = AverageMeter()\n    top5_m = AverageMeter()\n    cnt = 0\n    batch_start = time.time()\n    # num_updates = epoch * len(loader)\n\n    for batch_idx, (inputs, target) in enumerate(train_loader):\n        last_batch = batch_idx == last_idx\n        # data_time_m.update(time.time() - batch_start)\n        inputs = inputs.to(device)\n        target = target.to(device)\n\n        # zero the parameter gradients\n        optimizer.zero_grad()\n\n        if scaler is not None:\n            with amp.autocast():\n                output = model(inputs)\n                loss = criterion(output, target)\n                # Scale the loss using Grad Scaler\n            scaler.scale(loss).backward()\n            # Step using scaler.step()\n            scaler.step(optimizer)\n            # Update for next iteration\n            scaler.update()\n\n        else:\n            output = model(inputs)\n            loss = criterion(output, target)\n            loss.backward()\n            optimizer.step()\n\n        if scheduler is not None:\n            scheduler.step()\n\n        cnt += 1\n        acc1, acc5 = accuracy(output, target, topk=(1, 5))\n\n        top1_m.update(acc1.item(), output.size(0))\n        top5_m.update(acc5.item(), output.size(0))\n        losses_m.update(loss.item(), inputs.size(0))\n\n        batch_time_m.update(time.time() - batch_start)\n        batch_start = time.time()\n        if last_batch or batch_idx % log_interval == 0:  # If we reach the log intervel\n            print(\n                \"Batch Train Time: {batch_time.val:.3f} ({batch_time.avg:.3f})  \"\n                \"Loss: {loss.val:>7.4f} ({loss.avg:>6.4f})  \"\n                \"Top 1 Accuracy: {top1.val:>7.4f} ({top1.avg:>7.4f})  \"\n                \"Top 5 Accuracy: {top5.val:>7.4f} ({top5.avg:>7.4f})\".format(\n                    batch_time=batch_time_m, loss=losses_m, top1=top1_m, top5=top5_m))\n\n        if num_batches is not None:\n            if cnt >= num_batches:\n                end_train_step = time.time()\n                metrics[\"loss\"] = losses_m.avg\n                metrics[\"top1\"] = top1_m.avg\n                metrics[\"top5\"] = top5_m.avg\n                print(f\"Done till {num_batches} train batches\")\n                print(f\"Time taken for train step = {end_train_step - start_train_step} sec\")\n                return metrics\n\n    metrics[\"loss\"] = losses_m.avg\n    metrics[\"top1\"] = top1_m.avg\n    metrics[\"top5\"] = top5_m.avg\n    end_train_step = time.time()\n    print(f\"Time taken for train step = {end_train_step - start_train_step} sec\")\n    return metrics","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T22:35:09.349657Z","iopub.execute_input":"2024-12-07T22:35:09.350332Z","iopub.status.idle":"2024-12-07T22:35:09.361726Z","shell.execute_reply.started":"2024-12-07T22:35:09.350298Z","shell.execute_reply":"2024-12-07T22:35:09.360875Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"@torch.no_grad()\ndef val_step(model: nn.Module, val_loader, criterion,\n             device: str, num_batches=None,\n             log_interval: int = 100):\n\n    \"\"\"\n    Performs one step of validation. Calculates loss, forward pass and returns metrics.\n    Args:\n        model : A pytorch CNN Model.\n        val_loader : Validation loader.\n        criterion : Loss function to be optimized.\n        device : \"cuda\" or \"cpu\"\n        num_batches : (optional) Integer To limit validation to certain number of batches.\n        log_interval : (optional) Defualt 100. Integer to Log after specified batch ids in every batch.\n    \"\"\"\n\n    model = model.to(device)\n    start_val_step = time.time()\n    last_idx = len(val_loader) - 1\n    batch_time_m = AverageMeter()\n    # data_time_m = AverageMeter()\n    losses_m = AverageMeter()\n    top1_m = AverageMeter()\n    top5_m = AverageMeter()\n    cnt = 0\n    model.eval()\n    batch_start = time.time()\n    metrics = OrderedDict()\n\n    for batch_idx, (inputs, target) in enumerate(val_loader):\n        last_batch = batch_idx == last_idx\n        inputs = inputs.to(device)\n        target = target.to(device)\n\n        output = model(inputs)\n        loss = criterion(output, target)\n        cnt += 1\n        acc1, acc5 = accuracy(output, target, topk=(1, 5))\n        reduced_loss = loss.data\n\n        losses_m.update(reduced_loss.item(), inputs.size(0))\n        top1_m.update(acc1.item(), output.size(0))\n        top5_m.update(acc5.item(), output.size(0))\n        batch_time_m.update(time.time() - batch_start)\n\n        batch_start = time.time()\n\n        if (last_batch or batch_idx % log_interval == 0):  # If we reach the log intervel\n            print(\n                \"Batch Inference Time: {batch_time.val:.3f} ({batch_time.avg:.3f})  \"\n                \"Loss: {loss.val:>7.4f} ({loss.avg:>6.4f})  \"\n                \"Top 1 Accuracy: {top1.val:>7.4f} ({top1.avg:>7.4f})  \"\n                \"Top 5 Accuracy: {top5.val:>7.4f} ({top5.avg:>7.4f})\".format(\n                    batch_time=batch_time_m, loss=losses_m, top1=top1_m, top5=top5_m))\n\n        if num_batches is not None:\n            if cnt >= num_batches:\n                end_val_step = time.time()\n                metrics[\"loss\"] = losses_m.avg\n                metrics[\"top1\"] = top1_m.avg\n                metrics[\"top5\"] = top5_m.avg\n                print(f\"Done till {num_batches} validation batches\")\n                print(f\"Time taken for validation step = {end_val_step - start_val_step} sec\")\n                return metrics\n\n    metrics[\"loss\"] = losses_m.avg\n    metrics[\"top1\"] = top1_m.avg\n    metrics[\"top5\"] = top5_m.avg\n    print(\"Finished the validation epoch\")\n\n    end_val_step = time.time()\n    print(f\"Time taken for validation step = {end_val_step - start_val_step} sec\")\n    return metrics\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T22:35:10.272040Z","iopub.execute_input":"2024-12-07T22:35:10.272393Z","iopub.status.idle":"2024-12-07T22:35:10.281704Z","shell.execute_reply.started":"2024-12-07T22:35:10.272359Z","shell.execute_reply":"2024-12-07T22:35:10.280778Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DATA_DIR = \"../input/aptos2019-blindness-detection/\"\nTRAIN_DIR = \"../input/aptos2019-blindness-detection/train_images\"\nCSV_PATH = \"../input/aptos2019-blindness-detection/train.csv\"\nMODEL_PATH = \".\"\n\n\nTRAIN_SPLIT = 0.8\n# Automaticllay\n# VAL_SPLIT = 0.2\n\nLEARNING_RATE = 1e-4\nTRAIN_BATCH_SIZE = 32\nVALID_BATCH_SIZE = 32\n\nNUM_WORKERS = 2\nEPOCHS = 20\n\nIMG_WIDTH = 224\nIMG_HEIGHT = 224\n\nMODEL_NAME = \"pit_b_224\"\n\nMODEL_SAVE = MODEL_PATH + MODEL_NAME\nUSE_AMP = True\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-07T22:35:46.688012Z","iopub.execute_input":"2024-12-07T22:35:46.688577Z","iopub.status.idle":"2024-12-07T22:35:46.693434Z","shell.execute_reply.started":"2024-12-07T22:35:46.688542Z","shell.execute_reply":"2024-12-07T22:35:46.692499Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"seed_everything(42)\nset_debug_apis(False)\n\ntrain_trasforms = T.Compose([\n    T.ConvertImageDtype(torch.float32),\n    T.Resize((IMG_WIDTH, IMG_HEIGHT)),\n    T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\nfull_dataset = RetinopathyDataset(TRAIN_DIR, CSV_PATH, transforms=train_trasforms)\n\ntrain_size = int(TRAIN_SPLIT * len(full_dataset))\ntest_size = len(full_dataset) - train_size\n\ntrain_dataset, val_dataset = torch.utils.data.random_split(full_dataset, [train_size, test_size])\n\ntrain_loader = DataLoader(full_dataset, batch_size=TRAIN_BATCH_SIZE,\n                          shuffle=False, num_workers=NUM_WORKERS, drop_last=True, pin_memory=False)\n\nval_loader = DataLoader(val_dataset, batch_size=VALID_BATCH_SIZE, shuffle=False,\n                        num_workers=NUM_WORKERS, drop_last=True, pin_memory=False)\n\n# for batch_idx, (inputs, target) in enumerate(train_loader):\n#     print(batch_idx)\n#     # print(inputs)\n#     print(target)\n#     break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T22:35:52.913604Z","iopub.execute_input":"2024-12-07T22:35:52.914003Z","iopub.status.idle":"2024-12-07T22:35:52.964172Z","shell.execute_reply.started":"2024-12-07T22:35:52.913961Z","shell.execute_reply":"2024-12-07T22:35:52.963278Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = timm.create_model(MODEL_NAME, pretrained=True, num_classes=5)\n\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.AdamW(model.parameters(), lr=LEARNING_RATE)\n\nif torch.cuda.is_available():\n    device = \"cuda\"\nelse:\n    device = \"cpu\"\n\nif USE_AMP:\n    from torch.cuda import amp\n    scaler = amp.GradScaler()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T22:35:56.248123Z","iopub.execute_input":"2024-12-07T22:35:56.248501Z","iopub.status.idle":"2024-12-07T22:35:58.520153Z","shell.execute_reply.started":"2024-12-07T22:35:56.248465Z","shell.execute_reply":"2024-12-07T22:35:58.519168Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_loss = []\ntrain_top1_acc = []\nval_loss = []\nval_top1_acc = []\n\nfor epoch in tqdm(range(EPOCHS)):\n    train_metrics = train_step(model, train_loader, criterion, device, optimizer, scaler=scaler)\n    train_loss.append(train_metrics[\"loss\"])\n    print(f\"Training loss = {train_metrics['loss']}\")\n    train_top1_acc.append(train_metrics[\"top1\"])\n\n    val_metrics = val_step(model, val_loader, criterion, device)\n    val_loss.append(val_metrics[\"loss\"])\n    print(f\"Validation loss = {val_metrics['loss']}\")\n    val_top1_acc.append(val_metrics[\"top1\"])\n    \n    torch.save(model.state_dict(), f\"{MODEL_NAME}_{epoch}\" + \".pt\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T22:36:02.564843Z","iopub.execute_input":"2024-12-07T22:36:02.565171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.plot(train_loss)\nplt.title(f\"Training Results for {MODEL_NAME}\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Train Loss\")\nplt.show()\n\n\nplt.plot(train_top1_acc)\nplt.title(f\"Training Results for {MODEL_NAME}\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Training Top1 Accuracy\")\nplt.show()\n\n\nplt.plot(val_loss)\nplt.title(f\"Validation Results for {MODEL_NAME}\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Validation Loss\")\nplt.show()\n\n\nplt.plot(val_top1_acc)\nplt.title(f\"Validation Results for {MODEL_NAME}\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Validation Top1 Accuracy\")\nplt.show()\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}