{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport shutil\nimport tensorflow as tf\nfrom tensorflow.keras import models, layers\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\n\nfrom tensorflow.keras.optimizers import Adam\n\nimport warnings\nwarnings.simplefilter(\"ignore\")\n\nimport os, cv2, json\nfrom PIL import Image","metadata":{"execution":{"iopub.status.busy":"2022-09-25T05:33:09.084010Z","iopub.execute_input":"2022-09-25T05:33:09.084476Z","iopub.status.idle":"2022-09-25T05:33:14.671458Z","shell.execute_reply.started":"2022-09-25T05:33:09.084421Z","shell.execute_reply":"2022-09-25T05:33:14.670460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_dir = \"/kaggle/input/cassava-leaf-disease-classification/\"\ntrain_dir = dataset_dir + \"train_images/\"\ntrain_df_loc = dataset_dir + \"/train.csv\"\nworking_train = \"/kaggle/working/train/\"\nworking_test = \"/kaggle/working/test/\"\nworking_valid = \"/kaggle/working/valid/\"","metadata":{"execution":{"iopub.status.busy":"2022-09-25T05:33:14.675938Z","iopub.execute_input":"2022-09-25T05:33:14.676240Z","iopub.status.idle":"2022-09-25T05:33:14.685522Z","shell.execute_reply.started":"2022-09-25T05:33:14.676213Z","shell.execute_reply":"2022-09-25T05:33:14.683769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(train_df_loc)\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2022-09-25T05:33:14.687057Z","iopub.execute_input":"2022-09-25T05:33:14.687445Z","iopub.status.idle":"2022-09-25T05:33:14.746508Z","shell.execute_reply.started":"2022-09-25T05:33:14.687404Z","shell.execute_reply":"2022-09-25T05:33:14.745672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir train \n!mkdir test\n!mkdir valid\n!mkdir './train/CBB'\n!mkdir './train/CBSD'\n!mkdir './train/CGM'\n!mkdir './train/CMD'\n!mkdir './train/H'\n\n!mkdir './valid/CBB'\n!mkdir './valid/CBSD'\n!mkdir './valid/CGM'\n!mkdir './valid/CMD'\n!mkdir './valid/H'\n\n!mkdir './test/CBB'\n!mkdir './test/CBSD'\n!mkdir './test/CGM'\n!mkdir './test/CMD'\n!mkdir './test/H'","metadata":{"execution":{"iopub.status.busy":"2022-09-25T05:33:14.749794Z","iopub.execute_input":"2022-09-25T05:33:14.750636Z","iopub.status.idle":"2022-09-25T05:33:32.374402Z","shell.execute_reply.started":"2022-09-25T05:33:14.750600Z","shell.execute_reply":"2022-09-25T05:33:32.373031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subdir = {0 : 'CBB', 1 : 'CBSD', 2 : 'CGM', 3 : 'CMD', 4 : 'H'}\ntrain_cnt = {0 : 0, 1 : 0, 2 : 0, 3 : 0, 4 : 0}\nall_cnt = {0 : 1087, 1 : 2189, 2 : 2386, 3 : 13158, 4 : 2577 }\nfor index, row in train_df.iterrows():\n    label = row['label']\n    train_cnt[label] += 1\n    image_name = row['image_id']\n    image = cv2.imread(train_dir + image_name)\n#     cv2.imshow(image, (2,2))\n    if train_cnt[label] <= 0.8 * all_cnt[label]:\n        cv2.imwrite(working_train + subdir[label] + '/' + image_name, image)\n    elif train_cnt[label] <= 0.9 * all_cnt[label]:\n        cv2.imwrite(working_test + subdir[label] + '/' + image_name, image)\n    else :\n        cv2.imwrite(working_valid + subdir[label] + '/' + image_name, image)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-25T05:33:32.376334Z","iopub.execute_input":"2022-09-25T05:33:32.377391Z","iopub.status.idle":"2022-09-25T05:41:21.337071Z","shell.execute_reply.started":"2022-09-25T05:33:32.377346Z","shell.execute_reply":"2022-09-25T05:41:21.335537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"ViT part begin","metadata":{}},{"cell_type":"code","source":"print(\"hello\")","metadata":{"execution":{"iopub.status.busy":"2022-09-25T05:41:21.338425Z","iopub.execute_input":"2022-09-25T05:41:21.338795Z","iopub.status.idle":"2022-09-25T05:41:21.344692Z","shell.execute_reply.started":"2022-09-25T05:41:21.338758Z","shell.execute_reply":"2022-09-25T05:41:21.343592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q git+https://github.com/huggingface/transformers","metadata":{"execution":{"iopub.status.busy":"2022-09-25T05:41:21.346254Z","iopub.execute_input":"2022-09-25T05:41:21.346868Z","iopub.status.idle":"2022-09-25T05:42:03.065234Z","shell.execute_reply.started":"2022-09-25T05:41:21.346831Z","shell.execute_reply":"2022-09-25T05:42:03.064027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision\nfrom torchvision.transforms import ToTensor\n\ntrain_ds = torchvision.datasets.ImageFolder('./train/', transform=ToTensor())\nvalid_ds = torchvision.datasets.ImageFolder('./valid/', transform=ToTensor())\ntest_ds = torchvision.datasets.ImageFolder('./test/', transform=ToTensor())","metadata":{"execution":{"iopub.status.busy":"2022-09-25T05:42:03.067931Z","iopub.execute_input":"2022-09-25T05:42:03.068394Z","iopub.status.idle":"2022-09-25T05:42:03.384582Z","shell.execute_reply.started":"2022-09-25T05:42:03.068348Z","shell.execute_reply":"2022-09-25T05:42:03.383574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import ViTModel\nfrom transformers.modeling_outputs import SequenceClassifierOutput\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass ViTForImageClassification(nn.Module):\n    def __init__(self, num_labels=5):\n        super(ViTForImageClassification, self).__init__()\n        self.vit = ViTModel.from_pretrained('google/vit-base-patch16-224-in21k')\n        self.dropout = nn.Dropout(0.1)\n        self.classifier = nn.Linear(self.vit.config.hidden_size, num_labels)\n        self.num_labels = num_labels\n\n    def forward(self, pixel_values, labels):\n        outputs = self.vit(pixel_values=pixel_values)\n        output = self.dropout(outputs.last_hidden_state[:,0])\n        logits = self.classifier(output)\n\n        loss = None\n        if labels is not None:\n          loss_fct = nn.CrossEntropyLoss()\n          loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))\n        if loss is not None:\n          return logits, loss.item()\n        else:\n          return logits, None","metadata":{"execution":{"iopub.status.busy":"2022-09-25T05:42:03.385810Z","iopub.execute_input":"2022-09-25T05:42:03.386254Z","iopub.status.idle":"2022-09-25T05:42:04.361918Z","shell.execute_reply.started":"2022-09-25T05:42:03.386204Z","shell.execute_reply":"2022-09-25T05:42:04.360773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 3\nBATCH_SIZE = 16\nLEARNING_RATE = 2e-5","metadata":{"execution":{"iopub.status.busy":"2022-09-25T05:42:04.366197Z","iopub.execute_input":"2022-09-25T05:42:04.366565Z","iopub.status.idle":"2022-09-25T05:42:04.371882Z","shell.execute_reply.started":"2022-09-25T05:42:04.366525Z","shell.execute_reply":"2022-09-25T05:42:04.370972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import ViTFeatureExtractor\nimport torch.nn as nn\nimport torch\n# Define Model\nmodel = ViTForImageClassification(len(train_ds.classes))    \n# Feature Extractor\nfeature_extractor = ViTFeatureExtractor.from_pretrained('google/vit-base-patch16-224-in21k')\n# Adam Optimizer\noptimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)\n# Cross Entropy Loss\nloss_func = nn.CrossEntropyLoss()\n# Use GPU if available  \ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu') \nif torch.cuda.is_available():\n    model.cuda() ","metadata":{"execution":{"iopub.status.busy":"2022-09-25T05:42:04.374641Z","iopub.execute_input":"2022-09-25T05:42:04.375314Z","iopub.status.idle":"2022-09-25T05:42:20.502787Z","shell.execute_reply.started":"2022-09-25T05:42:04.375276Z","shell.execute_reply":"2022-09-25T05:42:20.501760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.utils.data as data\nfrom torch.autograd import Variable\nimport numpy as np\n\nprint(\"Number of train samples: \", len(train_ds))\nprint(\"Number of test samples: \", len(test_ds))\nprint(\"Detected Classes are: \", train_ds.class_to_idx) \n\ntrain_loader = data.DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True,  num_workers=4)\ntest_loader  = data.DataLoader(test_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=4) \n\n# Train the model\nfor epoch in range(EPOCHS):        \n  for step, (x, y) in enumerate(train_loader):\n    # Change input array into list with each batch being one element\n    x = np.split(np.squeeze(np.array(x)), BATCH_SIZE)\n    # Remove unecessary dimension\n    for index, array in enumerate(x):\n      x[index] = np.squeeze(array)\n    # Apply feature extractor, stack back into 1 tensor and then convert to tensor\n    x = torch.tensor(np.stack(feature_extractor(x)['pixel_values'], axis=0))\n    # Send to GPU if available\n    x, y  = x.to(device), y.to(device)\n    b_x = Variable(x)   # batch x (image)\n    b_y = Variable(y)   # batch y (target)\n    # Feed through model\n    output, loss = model(b_x, None)\n    # Calculate loss\n    if loss is None: \n      loss = loss_func(output, b_y)   \n      optimizer.zero_grad()     \n      loss.backward()\n      optimizer.step()\n\n    if step % 50 == 0:\n      # Get the next batch for testing purposes\n      test = next(iter(test_loader))\n      test_x = test[0]\n      # Reshape and get feature matrices as needed\n      test_x = np.split(np.squeeze(np.array(test_x)), BATCH_SIZE)\n      for index, array in enumerate(test_x):\n        test_x[index] = np.squeeze(array)\n      test_x = torch.tensor(np.stack(feature_extractor(test_x)['pixel_values'], axis=0))\n      # Send to appropirate computing device\n      test_x = test_x.to(device)\n      test_y = test[1].to(device)\n      # Get output (+ respective class) and compare to target\n      test_output, loss = model(test_x, test_y)\n      test_output = test_output.argmax(1)\n      # Calculate Accuracy\n      accuracy = (test_output == test_y).sum().item() / BATCH_SIZE\n      print('Epoch: ', epoch, '| train loss: %.4f' % loss, '| test accuracy: %.2f' % accuracy)","metadata":{"execution":{"iopub.status.busy":"2022-09-25T05:42:20.504320Z","iopub.execute_input":"2022-09-25T05:42:20.504902Z","iopub.status.idle":"2022-09-25T05:56:16.445776Z","shell.execute_reply.started":"2022-09-25T05:42:20.504863Z","shell.execute_reply":"2022-09-25T05:56:16.444172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# torch.save(model, './ViT-partial-model.pt')","metadata":{"execution":{"iopub.status.busy":"2022-09-25T05:56:16.447532Z","iopub.status.idle":"2022-09-25T05:56:16.448056Z","shell.execute_reply.started":"2022-09-25T05:56:16.447789Z","shell.execute_reply":"2022-09-25T05:56:16.447816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from IPython.display import FileLink\n# FileLink(r'ViT-partial-model.pt')\n# os.path.getsize('./ViT-partial-model.pt')","metadata":{"execution":{"iopub.status.busy":"2022-09-25T05:56:16.449922Z","iopub.status.idle":"2022-09-25T05:56:16.450435Z","shell.execute_reply.started":"2022-09-25T05:56:16.450172Z","shell.execute_reply":"2022-09-25T05:56:16.450196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\n\nimport torch_xla\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.xla_multiprocessing as xmp\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.core.xla_model as xm\nmodel.save(model.state_dict(), f'model_partial_{datetime.now().strftime(\"%Y%m%d-%H%M\")}.pth')","metadata":{"execution":{"iopub.status.busy":"2022-09-25T06:25:06.471844Z","iopub.execute_input":"2022-09-25T06:25:06.472890Z","iopub.status.idle":"2022-09-25T06:25:06.501301Z","shell.execute_reply.started":"2022-09-25T06:25:06.472851Z","shell.execute_reply":"2022-09-25T06:25:06.499785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"submit - part","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(\"../input/cassava-leaf-disease-classification/sample_submission.csv\")\ntest_df","metadata":{"execution":{"iopub.status.busy":"2022-09-25T05:59:48.597804Z","iopub.execute_input":"2022-09-25T05:59:48.598396Z","iopub.status.idle":"2022-09-25T05:59:48.621965Z","shell.execute_reply.started":"2022-09-25T05:59:48.598348Z","shell.execute_reply":"2022-09-25T05:59:48.621053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\n\nfor image_id in test_df.image_id:\n    image = Image.open(\"../input/cassava-leaf-disease-classification/test_images/\"+image_id)\n    image = np.expand_dims(image, axis = 0)\n    preds.append(np.argmax(model.predict(image)))\n\ntest_df['label'] = preds\ntest_df.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-09-25T06:01:56.254127Z","iopub.execute_input":"2022-09-25T06:01:56.254832Z","iopub.status.idle":"2022-09-25T06:01:56.307568Z","shell.execute_reply.started":"2022-09-25T06:01:56.254795Z","shell.execute_reply":"2022-09-25T06:01:56.306236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\n\nEVAL_BATCH = 1\neval_loader  = data.DataLoader(valid_ds, batch_size=EVAL_BATCH, shuffle=True, num_workers=4) \n# Disable grad\nwith torch.no_grad():\n    \n  inputs, target = next(iter(eval_loader))\n  # Reshape and get feature matrices as needed\n  print(inputs.shape)\n  inputs = inputs[0].permute(1, 2, 0)\n  # Save original Input\n  originalInput = inputs\n  for index, array in enumerate(inputs):\n    inputs[index] = np.squeeze(array)\n  inputs = torch.tensor(np.stack(feature_extractor(inputs)['pixel_values'], axis=0))\n\n  # Send to appropriate computing device\n  inputs = inputs.to(device)\n  target = target.to(device)\n \n  # Generate prediction\n  prediction, loss = model(inputs, target)\n    \n  # Predicted class value using argmax\n  predicted_class = np.argmax(prediction.cpu())\n  value_predicted = list(valid_ds.class_to_idx.keys())[list(valid_ds.class_to_idx.values()).index(predicted_class)]\n  value_target = list(valid_ds.class_to_idx.keys())[list(valid_ds.class_to_idx.values()).index(target)]\n        \n  # Show result\n  plt.imshow(originalInput)\n  plt.xlim(224,0)\n  plt.ylim(224,0)\n  plt.title(f'Prediction: {value_predicted} - Actual target: {value_target}')\n  plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pytorch-xla-env-setup.py --no-index --find-links=file:///kaggle/input/vision-transformer-vit-tutorial-baseline/pytorch-xla-env-setup.py\n!pip install torch-1.7-cp37-cp37m-linux_x86_64.whl --no-index --find-links=file:///kaggle/input/vision-transformer-vit-tutorial-baseline/torch-1.7-cp37-cp37m-linux_x86_64.whl\n!pip install torch_xla-1.7-cp37-cp37m-linux_x86_64.whl --no-index --find-links=file:///kaggle/input/vision-transformer-vit-tutorial-baseline/torch_xla-1.7-cp37-cp37m-linux_x86_64.whl\n!pip install torchvision-1.7-cp37-cp37m-linux_x86_64.whl --no-index --find-links=file:///kaggle/input/vision-transformer-vit-tutorial-baseline/torchvision-1.7-cp37-cp37m-linux_x86_64.whl\n\n# ../input/vision-transformer-vit-tutorial-baseline/torchvision-1.7-cp37-cp37m-linux_x86_64.whl\n# ../input/vision-transformer-vit-tutorial-baseline/torch_xla-1.7-cp37-cp37m-linux_x86_64.whl\n# ../input/vision-transformer-vit-tutorial-baseline/pytorch-xla-env-setup.py\n# ../input/vision-transformer-vit-tutorial-baseline/torch-1.7-cp37-cp37m-linux_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2022-09-25T07:14:35.258341Z","iopub.execute_input":"2022-09-25T07:14:35.258723Z","iopub.status.idle":"2022-09-25T07:14:43.549022Z","shell.execute_reply.started":"2022-09-25T07:14:35.258693Z","shell.execute_reply":"2022-09-25T07:14:43.547847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(model.state_dict(), 'model_weights-vit-partial.pth')","metadata":{"execution":{"iopub.status.busy":"2022-09-25T07:17:38.676463Z","iopub.execute_input":"2022-09-25T07:17:38.676874Z","iopub.status.idle":"2022-09-25T07:17:39.211100Z","shell.execute_reply.started":"2022-09-25T07:17:38.676837Z","shell.execute_reply":"2022-09-25T07:17:39.209985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import FileLink\nFileLink(r'model_weights-vit-partial.pth')\n","metadata":{"execution":{"iopub.status.busy":"2022-09-25T07:20:20.612731Z","iopub.execute_input":"2022-09-25T07:20:20.613152Z","iopub.status.idle":"2022-09-25T07:20:20.620053Z","shell.execute_reply.started":"2022-09-25T07:20:20.613118Z","shell.execute_reply":"2022-09-25T07:20:20.619112Z"},"trusted":true},"execution_count":null,"outputs":[]}]}