{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"scrolled":true,"execution":{"iopub.status.busy":"2023-03-03T17:28:20.159554Z","iopub.execute_input":"2023-03-03T17:28:20.160447Z","iopub.status.idle":"2023-03-03T17:28:49.740805Z","shell.execute_reply.started":"2023-03-03T17:28:20.160402Z","shell.execute_reply":"2023-03-03T17:28:49.739372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install timm\n!pip install torch torchvision","metadata":{"execution":{"iopub.status.busy":"2023-03-03T17:28:49.742813Z","iopub.execute_input":"2023-03-03T17:28:49.743150Z","iopub.status.idle":"2023-03-03T17:29:10.507190Z","shell.execute_reply.started":"2023-03-03T17:28:49.743118Z","shell.execute_reply":"2023-03-03T17:29:10.505787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader\nfrom torchvision.transforms import transforms\nfrom datetime import datetime\nimport timm\n# Some Native Python Libraries\nimport gc\nimport os\nimport time\nimport random\nfrom datetime import datetime\n\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\nfrom sklearn import model_selection, metrics","metadata":{"execution":{"iopub.status.busy":"2023-03-03T17:30:22.630581Z","iopub.execute_input":"2023-03-03T17:30:22.631297Z","iopub.status.idle":"2023-03-03T17:30:23.578506Z","shell.execute_reply.started":"2023-03-03T17:30:22.631258Z","shell.execute_reply":"2023-03-03T17:30:23.577392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# general global variables\nDATA_PATH = \"../input/cassava-leaf-disease-classification\"\nTRAIN_PATH = \"../input/cassava-leaf-disease-classification/train_images/\"\nTEST_PATH = \"../input/cassava-leaf-disease-classification/test_images/\"\nMODEL_PATH = (\n    \"../input/vit-base-models-pretrained-pytorch/jx_vit_base_p16_224-80ecf9dd.pth\"\n)\n\n# model specific global variables\n\n# THIS IS THE IMAGE SIZE IN PIXELS:\nIMG_SIZE = 224\n\nBATCH_SIZE = 16\n\nLR = 2e-05\n\n    N_EPOCHS = 10","metadata":{"execution":{"iopub.status.busy":"2023-03-03T17:30:26.200845Z","iopub.execute_input":"2023-03-03T17:30:26.202934Z","iopub.status.idle":"2023-03-03T17:30:26.209275Z","shell.execute_reply.started":"2023-03-03T17:30:26.202880Z","shell.execute_reply":"2023-03-03T17:30:26.207967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lets take a look at the data:\nfrom sklearn.model_selection import train_test_split\n\ndf = pd.read_csv(os.path.join(DATA_PATH, \"train.csv\"))\n\n# Splitting the dataset into training and testsets:\ntrain_df, valid_df = train_test_split(\n    df, test_size=0.1, random_state=42, stratify=df.label.values\n)\n\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-03T17:30:27.859203Z","iopub.execute_input":"2023-03-03T17:30:27.859858Z","iopub.status.idle":"2023-03-03T17:30:27.896746Z","shell.execute_reply.started":"2023-03-03T17:30:27.859818Z","shell.execute_reply":"2023-03-03T17:30:27.895638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.label.value_counts().plot(kind=\"bar\")","metadata":{"execution":{"iopub.status.busy":"2023-03-03T17:30:29.014078Z","iopub.execute_input":"2023-03-03T17:30:29.014554Z","iopub.status.idle":"2023-03-03T17:30:29.230334Z","shell.execute_reply.started":"2023-03-03T17:30:29.014518Z","shell.execute_reply":"2023-03-03T17:30:29.229244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CassavaDataset(torch.utils.data.Dataset):\n    \"\"\"\n    Helper Class to create the pytorch dataset\n    \"\"\"\n\n    def __init__(self, df, data_path=DATA_PATH, mode=\"train\", transforms=None):\n        super().__init__()\n        self.df_data = df.values\n        self.data_path = data_path\n        self.transforms = transforms\n        self.mode = mode\n        self.data_dir = \"train_images\" if mode == \"train\" else \"test_images\"\n\n    def __len__(self):\n        return len(self.df_data)\n\n    def __getitem__(self, index):\n        img_name, label = self.df_data[index]\n        img_path = os.path.join(self.data_path, self.data_dir, img_name)\n        img = Image.open(img_path).convert(\"RGB\")\n\n        if self.transforms is not None:\n            image = self.transforms(img)\n\n        return image, label","metadata":{"execution":{"iopub.status.busy":"2023-03-03T17:30:30.432503Z","iopub.execute_input":"2023-03-03T17:30:30.433380Z","iopub.status.idle":"2023-03-03T17:30:30.443123Z","shell.execute_reply.started":"2023-03-03T17:30:30.433336Z","shell.execute_reply":"2023-03-03T17:30:30.442024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transforms_train = transforms.Compose(\n    [\n        # Resizing the image to the previously stated size\n        transforms.Resize((IMG_SIZE, IMG_SIZE)),\n        # Randomly flipping the images horizontally with the probability of 30%\n        transforms.RandomHorizontalFlip(p=0.3),\n        # Randomly flipping the images vertically with the probability of 30%\n        transforms.RandomVerticalFlip(p=0.3),\n        # Randomly Rotating the images by 10 degrees\n        transforms.RandomRotation(10),\n        # Randomly 10 degrees worth of Affine\n        transforms.RandomAffine(10),\n        # Croppping the images to the stated\n        transforms.RandomResizedCrop(IMG_SIZE),\n        # Converting the image to tensor\n        transforms.ToTensor(),\n        # Normalizing\n        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n    ]\n)\n\ntransforms_valid = transforms.Compose(\n    [\n        transforms.Resize((IMG_SIZE, IMG_SIZE)),\n        transforms.ToTensor(),\n        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n    ]\n)","metadata":{"execution":{"iopub.status.busy":"2023-03-03T17:30:32.259148Z","iopub.execute_input":"2023-03-03T17:30:32.259844Z","iopub.status.idle":"2023-03-03T17:30:32.267643Z","shell.execute_reply.started":"2023-03-03T17:30:32.259804Z","shell.execute_reply":"2023-03-03T17:30:32.266511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ViTBase16(nn.Module):\n    def __init__(self, n_classes, pretrained=False):\n\n        super(ViTBase16, self).__init__()\n\n        self.model = timm.create_model(\"vit_base_patch16_224\", pretrained=False)\n\n        # I mean it is pretrained?!\n        if pretrained:\n            self.model.load_state_dict(torch.load(MODEL_PATH))\n\n        self.model.head = nn.Linear(self.model.head.in_features, n_classes)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n\n    def train_one_epoch(self, train_loader, criterion, optimizer, device):\n\n        # The Accuracy starts from Zero:\n        epoch_loss = 0.0\n        epoch_accuracy = 0.0\n\n        # This bad boy trains the model\n        self.model.train()\n        for i, (data, target) in enumerate(train_loader):\n\n            # Clear the gradients of all optimized variables\n            optimizer.zero_grad()\n            # forward pass: compute predicted outputs by passing inputs to the model\n            output = self.forward(data)\n            # calculate the batch loss\n            loss = criterion(output, target)\n            # backward pass: compute gradient of the loss with respect to model parameters\n            loss.backward()\n            # Calculate Accuracy\n            accuracy = (output.argmax(dim=1) == target).float().mean()\n            # update training loss and accuracy\n            epoch_loss += loss\n            epoch_accuracy += accuracy\n\n            # perform a single optimization step (parameter update)\n            if device.type == \"xla\":\n                xm.optimizer_step(optimizer)\n\n                if i % 20 == 0:\n                    xm.master_print(f\"\\tBATCH {i+1}/{len(train_loader)} - LOSS: {loss}\")\n\n            else:\n                optimizer.step()\n\n        return epoch_loss / len(train_loader), epoch_accuracy / len(train_loader)\n\n    def validate_one_epoch(self, valid_loader, criterion, device):\n        # keep track of validation loss\n        valid_loss = 0.0\n        valid_accuracy = 0.0\n\n        ######################\n        # validate the model #\n        ######################\n        self.model.eval()\n        for data, target in valid_loader:\n            # move tensors to GPU if CUDA is available\n            if device.type == \"cuda\":\n                data, target = data.cuda(), target.cuda()\n            elif device.type == \"xla\":\n                data = data.to(device, dtype=torch.float32)\n                target = target.to(device, dtype=torch.int64)\n\n            with torch.no_grad():\n                # forward pass: compute predicted outputs by passing inputs to the model\n                output = self.model(data)\n                # calculate the batch loss\n                loss = criterion(output, target)\n                # Calculate Accuracy\n                accuracy = (output.argmax(dim=1) == target).float().mean()\n                # update average validation loss and accuracy\n                valid_loss += loss\n                valid_accuracy += accuracy\n\n        return valid_loss / len(valid_loader), valid_accuracy / len(valid_loader)","metadata":{"execution":{"iopub.status.busy":"2023-03-03T17:30:33.337654Z","iopub.execute_input":"2023-03-03T17:30:33.338665Z","iopub.status.idle":"2023-03-03T17:30:33.352655Z","shell.execute_reply.started":"2023-03-03T17:30:33.338602Z","shell.execute_reply":"2023-03-03T17:30:33.351516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_gpu(model, epochs, device, criterion, optimizer, train_loader, valid_loader=None):\n    valid_loss_min = np.Inf  # track change in validation loss\n\n    # keeping track of losses as they happen\n    train_losses = []\n    valid_losses = []\n    train_accs = []\n    valid_accs = []\n\n    for epoch in range(1, epochs + 1):\n        gc.collect()\n        model.train()\n        train_loss = 0.0\n        correct = 0\n        total = 0\n        for data, target in train_loader:\n            data, target = data.to(device), target.to(device)\n            optimizer.zero_grad()\n            output = model(data)\n            loss = criterion(output, target)\n            loss.backward()\n            optimizer.step()\n\n            train_loss += loss.item() * data.size(0)\n            _, predicted = torch.max(output.data, 1)\n            total += target.size(0)\n            correct += (predicted == target).sum().item()\n\n        train_loss = train_loss / len(train_loader.dataset)\n        train_acc = 100 * correct / total\n        train_losses.append(train_loss)\n        train_accs.append(train_acc)\n\n        if valid_loader is not None:\n            gc.collect()\n            model.eval()\n            valid_loss = 0.0\n            correct = 0\n            total = 0\n            with torch.no_grad():\n                for data, target in valid_loader:\n                    data, target = data.to(device), target.to(device)\n                    output = model(data)\n                    loss = criterion(output, target)\n\n                    valid_loss += loss.item() * data.size(0)\n                    _, predicted = torch.max(output.data, 1)\n                    total += target.size(0)\n                    correct += (predicted == target).sum().item()\n\n            valid_loss = valid_loss / len(valid_loader.dataset)\n            valid_acc = 100 * correct / total\n            valid_losses.append(valid_loss)\n            valid_accs.append(valid_acc)\n\n            # print training/validation statistics\n            print(f\"\\n\\t[TRAIN] EPOCH {epoch} - LOSS: {train_loss}, ACCURACY: {train_acc}\")\n            print(f\"\\t[VALID] EPOCH {epoch} - LOSS: {valid_loss}, ACCURACY: {valid_acc}\\n\")\n\n            # save model if validation loss has decreased\n            if valid_loss <= valid_loss_min and epoch != 1:\n                print(\"Validation loss decreased ({:.4f} --> {:.4f}).  Saving model ...\".format(valid_loss_min, valid_loss))\n                torch.save(model.state_dict(), 'best_model.pth')\n                valid_loss_min = valid_loss\n\n    return {'train_loss': train_losses, 'valid_losses': valid_losses, 'train_acc': train_accs, 'valid_acc': valid_accs}\n","metadata":{"execution":{"iopub.status.busy":"2023-03-03T17:30:34.950743Z","iopub.execute_input":"2023-03-03T17:30:34.951431Z","iopub.status.idle":"2023-03-03T17:30:34.967219Z","shell.execute_reply.started":"2023-03-03T17:30:34.951393Z","shell.execute_reply":"2023-03-03T17:30:34.966109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ViTBase16(n_classes=5, pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-03T17:30:36.129727Z","iopub.execute_input":"2023-03-03T17:30:36.130301Z","iopub.status.idle":"2023-03-03T17:30:42.828953Z","shell.execute_reply.started":"2023-03-03T17:30:36.130264Z","shell.execute_reply":"2023-03-03T17:30:42.827915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _run():\n    train_dataset = CassavaDataset(train_df, transforms=transforms_train)\n    valid_dataset = CassavaDataset(valid_df, transforms=transforms_valid)\n    \n    train_loader = torch.utils.data.DataLoader(\n        dataset=train_dataset,\n        batch_size=BATCH_SIZE,\n        shuffle=True,\n        num_workers=8,\n    )\n\n    valid_loader = torch.utils.data.DataLoader(\n        dataset=valid_dataset,\n        batch_size=BATCH_SIZE,\n        shuffle=False,\n        num_workers=8,\n    )\n\n    criterion = nn.CrossEntropyLoss()\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    model.to(device)\n\n    optimizer = torch.optim.Adam(model.parameters(), lr=LR)\n\n    print(f\"INITIALIZING TRAINING ON {device}...\")\n    start_time = datetime.now()\n    print(f\"Start Time: {start_time}\")\n\n    logs = fit_gpu(\n        model=model,\n        epochs=N_EPOCHS,\n        device=device,\n        criterion=criterion,\n        optimizer=optimizer,\n        train_loader=train_loader,\n        valid_loader=valid_loader,\n    )\n\n    print(f\"Execution time: {datetime.now() - start_time}\")\n\n    print(\"Saving Model\")\n    torch.save(model.state_dict(), f'model_5e_{datetime.now().strftime(\"%Y%m%d-%H%M\")}.pth')\n","metadata":{"execution":{"iopub.status.busy":"2023-03-03T17:30:45.778731Z","iopub.execute_input":"2023-03-03T17:30:45.779573Z","iopub.status.idle":"2023-03-03T17:30:45.788380Z","shell.execute_reply.started":"2023-03-03T17:30:45.779535Z","shell.execute_reply":"2023-03-03T17:30:45.787159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Start training processes\ndef _mp_fn():\n    torch.set_default_tensor_type(\"torch.FloatTensor\")\n    a = _run()\n","metadata":{"execution":{"iopub.status.busy":"2023-03-03T17:30:49.209121Z","iopub.execute_input":"2023-03-03T17:30:49.210065Z","iopub.status.idle":"2023-03-03T17:30:49.215901Z","shell.execute_reply.started":"2023-03-03T17:30:49.210008Z","shell.execute_reply":"2023-03-03T17:30:49.214286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.multiprocessing as mp\n\n\n_mp_fn = _mp_fn()\nmp.set_start_method('spawn', force=True)\nn_gpus = torch.cuda.device_count()\n","metadata":{"execution":{"iopub.status.busy":"2023-03-03T17:30:50.969909Z","iopub.execute_input":"2023-03-03T17:30:50.970273Z","iopub.status.idle":"2023-03-03T18:38:14.578673Z","shell.execute_reply.started":"2023-03-03T17:30:50.970242Z","shell.execute_reply":"2023-03-03T18:38:14.576924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FLAGS = {}\nmp.spawn(_mp_fn, nprocs=n_gpus, args=(FLAGS,))\n","metadata":{"execution":{"iopub.status.busy":"2023-03-03T18:42:45.670768Z","iopub.execute_input":"2023-03-03T18:42:45.671876Z","iopub.status.idle":"2023-03-03T18:42:47.341099Z","shell.execute_reply.started":"2023-03-03T18:42:45.671830Z","shell.execute_reply":"2023-03-03T18:42:47.339634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}