{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-01T00:21:27.793793Z","iopub.execute_input":"2021-09-01T00:21:27.794352Z","iopub.status.idle":"2021-09-01T00:21:27.798294Z","shell.execute_reply.started":"2021-09-01T00:21:27.794258Z","shell.execute_reply":"2021-09-01T00:21:27.797651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **0. Importing Libraries**","metadata":{}},{"cell_type":"code","source":"!pip install torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:21:34.155907Z","iopub.execute_input":"2021-09-01T00:21:34.156373Z","iopub.status.idle":"2021-09-01T00:24:26.803449Z","shell.execute_reply.started":"2021-09-01T00:21:34.156345Z","shell.execute_reply":"2021-09-01T00:24:26.802258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:24:26.806367Z","iopub.execute_input":"2021-09-01T00:24:26.806817Z","iopub.status.idle":"2021-09-01T00:25:15.913024Z","shell.execute_reply.started":"2021-09-01T00:24:26.806768Z","shell.execute_reply":"2021-09-01T00:25:15.912075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Importing libraries\nimport os\nimport pandas as pd\nfrom scipy import stats\nimport numpy as np\nimport glob\nimport tensorflow as tf\n#import timm\nimport random\nimport time\nimport copy\nfrom operator import itemgetter\n\nfrom collections import OrderedDict, namedtuple\nimport joblib\n\nimport logging\nimport sys\n\nfrom PIL import Image\nimport cv2\nimport albumentations\nimport io\nimport IPython.display as display\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\nimport torch.optim as optim\nimport torch_xla\nimport torch_xla.core.xla_model as xm\nimport torch_xla.debug.metrics as met\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.distributed.xla_multiprocessing as xmp\nimport torch_xla.utils.utils as xu\nimport torchvision\nfrom torchvision import datasets, transforms\nfrom torch.utils.data import Dataset, DataLoader, ConcatDataset\nimport torchvision.transforms as transforms\nfrom torchvision.utils import make_grid\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics, model_selection\n\nimport warnings\nwarnings.filterwarnings(\"ignore\");","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:25:55.138561Z","iopub.execute_input":"2021-09-01T00:25:55.138972Z","iopub.status.idle":"2021-09-01T00:26:02.431336Z","shell.execute_reply.started":"2021-09-01T00:25:55.138932Z","shell.execute_reply":"2021-09-01T00:26:02.430443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **1. Data Loading**","metadata":{}},{"cell_type":"code","source":"# Loading the paths of all the files\ntrain_files = glob.glob('../input/tpu-getting-started/*/train/*.tfrec')\nval_files = glob.glob('../input/tpu-getting-started/*/val/*.tfrec')\ntest_files = glob.glob('../input/tpu-getting-started/*/test/*.tfrec')","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:26:11.429110Z","iopub.execute_input":"2021-09-01T00:26:11.429470Z","iopub.status.idle":"2021-09-01T00:26:11.589651Z","shell.execute_reply.started":"2021-09-01T00:26:11.429440Z","shell.execute_reply":"2021-09-01T00:26:11.588844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Printing samples\nprint(train_files[:5]), len(train_files)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:26:13.618663Z","iopub.execute_input":"2021-09-01T00:26:13.619269Z","iopub.status.idle":"2021-09-01T00:26:13.627734Z","shell.execute_reply.started":"2021-09-01T00:26:13.619236Z","shell.execute_reply":"2021-09-01T00:26:13.627040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loading the training data\ntrain_feature_description = {\n    'class': tf.io.FixedLenFeature([], tf.int64),\n    'id': tf.io.FixedLenFeature([], tf.string),\n    'image': tf.io.FixedLenFeature([], tf.string),\n}\n\ndef _parse_image_function(example_proto):\n  # Parse the input tf.Example proto using the dictionary above.\n  return tf.io.parse_single_example(example_proto, train_feature_description)\n\ntrain_ids = []\ntrain_class = []\ntrain_images = []\n\nfor i in train_files:\n  train_image_dataset = tf.data.TFRecordDataset(i)\n\n  train_image_dataset = train_image_dataset.map(_parse_image_function)\n\n  ids = [str(id_features['id'].numpy())[2:-1] for id_features in train_image_dataset] # [2:-1] is done to remove b' from 1st and 'from last in train id names\n  train_ids = train_ids + ids\n\n  classes = [int(class_features['class'].numpy()) for class_features in train_image_dataset]\n  train_class = train_class + classes\n\n  images = [image_features['image'].numpy() for image_features in train_image_dataset]\n  train_images = train_images + images\n","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:26:17.076996Z","iopub.execute_input":"2021-09-01T00:26:17.077439Z","iopub.status.idle":"2021-09-01T00:27:40.736252Z","shell.execute_reply.started":"2021-09-01T00:26:17.077413Z","shell.execute_reply":"2021-09-01T00:27:40.735018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loading the validation data\nval_feature_description = {\n    'class': tf.io.FixedLenFeature([], tf.int64),\n    'id': tf.io.FixedLenFeature([], tf.string),\n    'image': tf.io.FixedLenFeature([], tf.string),\n}\n\ndef _parse_image_function(example_proto):\n  # Parse the input tf.Example proto using the dictionary above.\n  return tf.io.parse_single_example(example_proto, val_feature_description)\n\nval_ids = []\nval_class = []\nval_images = []\n\nfor i in val_files:\n    val_image_dataset = tf.data.TFRecordDataset(i)\n\n    val_image_dataset = val_image_dataset.map(_parse_image_function)\n\n    ids = [str(image_features['id'].numpy())[2:-1] for image_features in val_image_dataset]\n    val_ids += ids\n\n    classes = [int(image_features['class'].numpy()) for image_features in val_image_dataset]\n    val_class += classes \n\n    images = [image_features['image'].numpy() for image_features in val_image_dataset]\n    val_images += images","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:27:40.738266Z","iopub.execute_input":"2021-09-01T00:27:40.738678Z","iopub.status.idle":"2021-09-01T00:28:08.115803Z","shell.execute_reply.started":"2021-09-01T00:27:40.738638Z","shell.execute_reply":"2021-09-01T00:28:08.115070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loading the testing data\ntest_feature_description = {\n    'id': tf.io.FixedLenFeature([], tf.string),\n    'image': tf.io.FixedLenFeature([], tf.string),\n}\n\ndef _parse_image_function_test(example_proto):\n    return tf.io.parse_single_example(example_proto, test_feature_description)\n\ntest_ids = []\ntest_images = []\nfor i in test_files:\n    test_image_dataset = tf.data.TFRecordDataset(i)\n    \n    test_image_dataset = test_image_dataset.map(_parse_image_function_test)\n\n    ids = [str(id_features['id'].numpy())[2:-1] for id_features in test_image_dataset]\n    test_ids = test_ids + ids\n\n    images = [image_features['image'].numpy() for image_features in test_image_dataset]\n    test_images = test_images + images","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:28:08.117200Z","iopub.execute_input":"2021-09-01T00:28:08.117694Z","iopub.status.idle":"2021-09-01T00:28:49.689509Z","shell.execute_reply.started":"2021-09-01T00:28:08.117650Z","shell.execute_reply":"2021-09-01T00:28:49.688811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display images\nimport IPython.display as display\nprint(val_ids[1], ' | ', val_class[1])\ndisplay.display(display.Image(data=val_images[1]))","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:28:49.690619Z","iopub.execute_input":"2021-09-01T00:28:49.691036Z","iopub.status.idle":"2021-09-01T00:28:49.699639Z","shell.execute_reply.started":"2021-09-01T00:28:49.690988Z","shell.execute_reply":"2021-09-01T00:28:49.698667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **2. Data preparation**","metadata":{}},{"cell_type":"code","source":"# Creating the dataset class\nclass FlowersDataset():\n    def __init__(self, ids, cls, imgs, transforms, is_test=False):\n        super()\n        self.ids = ids\n        if not is_test:\n            self.cls = cls\n        self.imgs = imgs\n        self.transforms = transforms\n        self.is_test = is_test\n    \n    def __len__(self):\n        return len(self.ids)\n\n    def __getitem__(self, idx):\n        img = self.imgs[idx]\n        img = Image.open(io.BytesIO(img))\n        img = self.transforms(img)\n        if self.is_test:\n            return img, -1, self.ids[idx]\n        return img, int(self.cls[idx]), self.ids[idx]","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:39:54.530012Z","iopub.execute_input":"2021-09-01T00:39:54.530622Z","iopub.status.idle":"2021-09-01T00:39:54.540385Z","shell.execute_reply.started":"2021-09-01T00:39:54.530578Z","shell.execute_reply":"2021-09-01T00:39:54.539334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating the transformers\ntrain_transforms = transforms.Compose([\n                        transforms.RandomResizedCrop(224),\n                        transforms.RandomHorizontalFlip(),\n                        transforms.RandomVerticalFlip(),\n                        transforms.ToTensor(),\n                        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n                        transforms.RandomErasing()\n                    ])\n\ntest_transforms = valid_transforms = transforms.Compose([\n                        transforms.CenterCrop(224),\n                        transforms.Resize(224),\n                        transforms.ToTensor(),\n                        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n                    ])","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:40:32.248090Z","iopub.execute_input":"2021-09-01T00:40:32.248568Z","iopub.status.idle":"2021-09-01T00:40:32.254429Z","shell.execute_reply.started":"2021-09-01T00:40:32.248539Z","shell.execute_reply":"2021-09-01T00:40:32.253645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating the training and validation datasets\ntrain_ds = FlowersDataset(train_ids, train_class, train_images, train_transforms)\nvalid_ds = FlowersDataset(val_ids, val_class, val_images, valid_transforms)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:40:41.344410Z","iopub.execute_input":"2021-09-01T00:40:41.344728Z","iopub.status.idle":"2021-09-01T00:40:41.349500Z","shell.execute_reply.started":"2021-09-01T00:40:41.344701Z","shell.execute_reply":"2021-09-01T00:40:41.348461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating the TPU device\ndevice = xm.xla_device()","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:40:44.481687Z","iopub.execute_input":"2021-09-01T00:40:44.482014Z","iopub.status.idle":"2021-09-01T00:40:50.354416Z","shell.execute_reply.started":"2021-09-01T00:40:44.481988Z","shell.execute_reply":"2021-09-01T00:40:50.353197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Building the Distributed samplers \ntrain_sampler = torch.utils.data.distributed.DistributedSampler(\n        train_ds,\n        num_replicas=xm.xrt_world_size(),\n        rank=xm.get_ordinal(),\n        shuffle=True\n    )\n\nvalid_sampler = torch.utils.data.distributed.DistributedSampler(\n        valid_ds,\n        num_replicas=xm.xrt_world_size(),\n        rank=xm.get_ordinal(),\n        shuffle=True\n    )\n    \ntrain_loader = DataLoader(train_ds, 128, sampler=train_sampler, num_workers=4, pin_memory=True)\nval_loader = DataLoader(valid_ds, 128, sampler=valid_sampler, num_workers=4, pin_memory=True)\n\ndatasets_sizes = {\n    'train': len(train_ds),\n    'val': len(valid_ds),\n}\ndatasets_sizes","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:41:02.965706Z","iopub.execute_input":"2021-09-01T00:41:02.966039Z","iopub.status.idle":"2021-09-01T00:41:02.975621Z","shell.execute_reply.started":"2021-09-01T00:41:02.966012Z","shell.execute_reply":"2021-09-01T00:41:02.974770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualising the channels of a sample\ntransforms_example = transforms.Compose([\n                        transforms.CenterCrop(224),\n                        transforms.ToTensor(),])\n\nexampleset = FlowersDataset(train_ids, train_class, train_images, transforms_example)\n\nx, y, _ = next(iter(DataLoader(exampleset)))\n\nchannels = ['Red', 'Green', 'Blue']\ncmaps = [plt.cm.Reds_r, plt.cm.Greens_r, plt.cm.Blues_r]\n\nfig, ax = plt.subplots(1, 4, figsize=(15, 10))\n\nfor i, axs in enumerate(fig.axes[:3]):\n    axs.imshow(x[0][i,:,:], cmap=cmaps[i])\n    axs.set_title(f'{channels[i]} Channel')\n    axs.set_xticks([])\n    axs.set_yticks([])\n    \nax[3].imshow(x[0].permute(1,2,0))\nax[3].set_title('Three Channels')\nax[3].set_xticks([])\nax[3].set_yticks([]);","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:41:11.999837Z","iopub.execute_input":"2021-09-01T00:41:12.000152Z","iopub.status.idle":"2021-09-01T00:41:12.437080Z","shell.execute_reply.started":"2021-09-01T00:41:12.000128Z","shell.execute_reply":"2021-09-01T00:41:12.436166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# inspecting the normalisation of each channel\nchannels = 3\nloaders = {\n    'train':train_loader,\n    'val':val_loader,\n}\nfor channel in range(channels):\n    for x in ['train', 'val']:\n        #number of pixels in the dataset = number of all pixels in one object * number of all objects in the dataset\n        num_pxl = datasets_sizes[x]*224*224\n    \n        #we go through the butches and sum up the pixels of the objects, \n        #which then divide the sum by the number of all pixels to calculate the average\n        total_sum = 0\n        for batch in loaders[x]:\n            layer = list(map(itemgetter(channel), batch[0]))\n            layer = torch.stack(layer, dim=0)\n            total_sum += layer.sum()\n        mean = total_sum / num_pxl\n\n        #we calculate the standard deviation using the formula that I indicated above\n        sum_sqrt = 0\n        for batch in loaders[x]: \n            layer = list(map(itemgetter(channel), batch[0]))\n            sum_sqrt += ((torch.stack(layer, dim=0) - mean).pow(2)).sum()\n        std = torch.sqrt(sum_sqrt / num_pxl)\n        \n        print(f'|channel:{channel+1}| {x} - mean: {mean}, std: {std}')","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:41:22.918167Z","iopub.execute_input":"2021-09-01T00:41:22.918499Z","iopub.status.idle":"2021-09-01T00:54:21.515791Z","shell.execute_reply.started":"2021-09-01T00:41:22.918473Z","shell.execute_reply":"2021-09-01T00:54:21.514021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot pixel distribution histogram\nx, y, _ = next(iter(exampleset))\n\ndef plotHist(img):\n  plt.figure(figsize=(10,5))\n  plt.subplot(1,2,1)\n  plt.imshow(x.permute(1,2,0))\n  plt.axis('off')\n  histo = plt.subplot(1,2,2)\n  histo.set_ylabel('Count')\n  histo.set_xlabel('Pixel Intensity')\n  plt.hist(img.numpy().flatten(), bins=10, lw=0, alpha=0.5, color='r')\n\nplotHist(x)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:54:21.520136Z","iopub.execute_input":"2021-09-01T00:54:21.520466Z","iopub.status.idle":"2021-09-01T00:54:21.859722Z","shell.execute_reply.started":"2021-09-01T00:54:21.520435Z","shell.execute_reply":"2021-09-01T00:54:21.858856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Normalizing the data\ndef norm_out(img):\n    \n    img = img.permute(1,2,0)\n    mean = torch.FloatTensor([0.485, 0.456, 0.406])\n    std = torch.FloatTensor([0.229, 0.224, 0.225])\n    \n    img = img*std + mean\n        \n    return np.clip(img,0,1)\n\ndef show_batch(dl):\n    \n    for images, labels, _ in dl:\n        fig, ax = plt.subplots(figsize=(25, 25))\n        ax.set_xticks([]); ax.set_yticks([])\n        #images = norm_out(images[:60])\n        ax.imshow(norm_out(make_grid(images[:60], nrow=10)))#.permute(1, 2, 0))\n        ax.set_title('Images without augmentation', fontsize=40)\n        break\n        \nshow_batch(loaders['val'])","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:55:03.952426Z","iopub.execute_input":"2021-09-01T00:55:03.952738Z","iopub.status.idle":"2021-09-01T00:55:07.698975Z","shell.execute_reply.started":"2021-09-01T00:55:03.952712Z","shell.execute_reply":"2021-09-01T00:55:07.697945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_batch(dl):\n    for images, labels, _ in dl:\n        fig, ax = plt.subplots(figsize=(25, 25))\n        ax.set_xticks([]); ax.set_yticks([])\n        ax.imshow(make_grid(images[:60], nrow=10).permute(1, 2, 0))\n        ax.set_title('Images with augmentation', fontsize=40)\n        break\n        \nshow_batch(loaders['train'])","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:55:09.868964Z","iopub.execute_input":"2021-09-01T00:55:09.869380Z","iopub.status.idle":"2021-09-01T00:55:13.799011Z","shell.execute_reply.started":"2021-09-01T00:55:09.869340Z","shell.execute_reply":"2021-09-01T00:55:13.798055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Printing the shape of a batch\nfor img, _, _ in loaders['train']:\n    print(img.shape)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:56:15.582526Z","iopub.execute_input":"2021-09-01T00:56:15.582876Z","iopub.status.idle":"2021-09-01T00:56:18.524593Z","shell.execute_reply.started":"2021-09-01T00:56:15.582837Z","shell.execute_reply":"2021-09-01T00:56:18.522054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **3. Training and Test**","metadata":{}},{"cell_type":"code","source":"# One epoch of training\ndef train(loader, epoch, model, optimizer, criterion):\n    #tracker = xm.RateTracker()\n    model.train()\n    running_loss = 0.\n    running_corrects = 0.\n    tot = 0\n    for idx, (ip, tgt, _) in enumerate(loader):\n        ip, tgt = ip.to(device), tgt.to(device)                            \n        output = model(ip)\n        loss = criterion(output, tgt)\n        tot += ip.shape[0]\n        # Append outputs\n        _, pred = output.max(dim=1)\n        running_corrects += torch.sum(pred == tgt.data)\n        # compute gradient and do SGD step\n        optimizer.zero_grad()\n        loss.backward()\n        #torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n        #optimizer.step()\n        xm.optimizer_step(optimizer)\n        running_loss += loss.item()*ip.size(0)\n    return running_corrects, running_loss","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:56:25.673940Z","iopub.execute_input":"2021-09-01T00:56:25.674272Z","iopub.status.idle":"2021-09-01T00:56:25.682457Z","shell.execute_reply.started":"2021-09-01T00:56:25.674242Z","shell.execute_reply":"2021-09-01T00:56:25.681585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# One validation epoch\ndef test(loader, model, criterion):\n    with torch.no_grad():\n        model.eval()\n        running_loss = 0.\n        running_corrects = 0.\n        tot = 0\n        for i, (ip, tgt, _) in enumerate(loader):\n            ip, tgt = ip.to(device), tgt.to(device)\n            output = model(ip)\n            loss = criterion(output, tgt)\n            tot += ip.shape[0]\n            _, pred = output.max(dim=1)\n            running_corrects += torch.sum(pred == tgt.data)\n            running_loss += loss.item()*ip.size(0)\n        return running_corrects, running_loss","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:56:49.867752Z","iopub.execute_input":"2021-09-01T00:56:49.868082Z","iopub.status.idle":"2021-09-01T00:56:49.874283Z","shell.execute_reply.started":"2021-09-01T00:56:49.868053Z","shell.execute_reply":"2021-09-01T00:56:49.873665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predicting on test dataset\ndef predict(model, loader, device):\n    with torch.no_grad():\n        torch.cuda.empty_cache()\n        model.eval()\n        preds = dict()\n        for i, (ip, _, ids) in enumerate(loader):\n            ip = ip.to(device)\n            output = model(ip)\n            _, pred = output.max(dim=1)\n            for i, j in zip(ids, pred.cpu().detach()):\n                preds[i] = j\n        return preds","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:56:54.957873Z","iopub.execute_input":"2021-09-01T00:56:54.958226Z","iopub.status.idle":"2021-09-01T00:56:54.964524Z","shell.execute_reply.started":"2021-09-01T00:56:54.958193Z","shell.execute_reply":"2021-09-01T00:56:54.963231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loss and Acc variables\nlosses = {'train':[], 'val':[]}\naccuracies = {'train':[], 'val':[]}","metadata":{"execution":{"iopub.status.busy":"2021-09-01T00:56:58.896904Z","iopub.execute_input":"2021-09-01T00:56:58.897242Z","iopub.status.idle":"2021-09-01T00:56:58.901627Z","shell.execute_reply.started":"2021-09-01T00:56:58.897216Z","shell.execute_reply":"2021-09-01T00:56:58.900662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"hi\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ftting function for XLA TPU\n\ndef fit(seed, epochs, model):\n    # Train and valid dataloaders\n    xm.master_print('Creating a model {}...'.format(seed))\n    device = xm.xla_device()\n    WRAPPED_MODEL = xmp.MpModelWrapper(model)\n    model = WRAPPED_MODEL.to(device)\n    model.to(device)  \n    criterion = nn.CrossEntropyLoss()\n\n    if seed==1:\n        optimizer = torch.optim.Adam(model.head.parameters(), lr=0.001* xm.xrt_world_size(), betas=(0.9, 0.999), eps=1e-08, weight_decay=0)\n    if seed==2 or seed==3:\n        optimizer = torch.optim.Adam(model.fc.parameters(), lr=0.001* xm.xrt_world_size(), betas=(0.9, 0.999), eps=1e-08, weight_decay=0)\n    if seed==4 or seed==0:\n        optimizer = torch.optim.Adam(model.classifier.parameters(), lr=0.001* xm.xrt_world_size(), betas=(0.9, 0.999), eps=1e-08, weight_decay=0)\n    # scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.7, patience=3, verbose=True)\n\n    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 4, gamma=0.1)\n    since = time.time()\n    best_model = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n    for epoch in range(epochs):\n    \n        #train\n        xm.master_print('Epoch: {}/{}'.format(epoch+1, epochs))\n        para_loader = pl.ParallelLoader(train_loader, [device])\n        running_corrects, running_loss = train(para_loader.per_device_loader(device), epoch, model, optimizer, criterion)\n        epoch_loss = running_loss / datasets_sizes['train']\n        epoch_acc = running_corrects/datasets_sizes['train']\n        losses['train'].append(epoch_loss)\n        accuracies['train'].append(epoch_acc)\n        xm.master_print('{} - loss:{}, accuracy{}'.format('train', epoch_loss, epoch_acc))\n\n        #val\n        para_loader = pl.ParallelLoader(val_loader, [device])\n        running_corrects, running_loss = test(para_loader.per_device_loader(device), model, criterion)\n        epoch_loss = running_loss / datasets_sizes['val']\n        epoch_acc = running_corrects/datasets_sizes['val']\n        losses['val'].append(epoch_loss)\n        accuracies['val'].append(epoch_acc)\n        xm.master_print('{} - loss:{}, accuracy{}'.format('val', epoch_loss, epoch_acc))\n\n        #epoch end\n        xm.master_print('Time: {}m {}s'.format((time.time()- since)//60, (time.time()- since)%60))\n        xm.master_print('=='*31)\n        if epoch_acc > best_acc:\n            best_acc = epoch_acc\n            best_model = copy.deepcopy(model.state_dict())\n        scheduler.step()\n      \n    time_elapsed = time.time() - since\n    xm.master_print('CLASSIFIER TRAINING TIME {}m {}s'.format(time_elapsed//60, time_elapsed%60))\n    xm.master_print('=='*31)\n    model.load_state_dict(best_model)\n\n    for param in model.parameters():\n        param.requires_grad=True\n\n    optimizer = torch.optim.Adam(model.parameters(), lr=0.0001* xm.xrt_world_size(), betas=(0.9, 0.999), eps=1e-08, weight_decay=0)  \n#   scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.7, patience=3, verbose=True)\n    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 4, gamma=0.1)\n    \n    for epoch in range(epochs):\n\n        #train\n        xm.master_print('Epoch: {}/{}'.format(epoch+1, epochs))\n        para_loader = pl.ParallelLoader(train_loader, [device])\n        running_corrects, running_loss = train(para_loader.per_device_loader(device), epoch, model, optimizer, criterion)\n        epoch_loss = running_loss / datasets_sizes['train']\n        epoch_acc = running_corrects/datasets_sizes['train']\n        losses['train'].append(epoch_loss)\n        accuracies['train'].append(epoch_acc)\n        xm.master_print('{} - loss:{}, accuracy{}'.format('train', epoch_loss, epoch_acc))\n\n        #val\n        para_loader = pl.ParallelLoader(val_loader, [device])\n        running_corrects, running_loss = test(para_loader.per_device_loader(device), model, criterion)\n        epoch_loss = running_loss / datasets_sizes['val']\n        epoch_acc = running_corrects/datasets_sizes['val']\n        losses['val'].append(epoch_loss)\n        accuracies['val'].append(epoch_acc)\n        xm.master_print('{} - loss:{}, accuracy{}'.format('val', epoch_loss, epoch_acc))\n\n        #epoch end\n        xm.master_print('Time: {}m {}s'.format((time.time()- since)//60, (time.time()- since)%60))\n        xm.master_print('=='*31)\n        if epoch_acc > best_acc:\n            best_acc = epoch_acc\n            best_model = copy.deepcopy(model.state_dict())\n        scheduler.step()\n\n    time_elapsed = time.time() - since\n    xm.master_print('ALL NET TRAINING TIME {}m {}s'.format(time_elapsed//60, time_elapsed%60))\n    xm.master_print('=='*31)\n\n    model.load_state_dict(best_model)\n    \n    predictions = predict(model, testloader, device)\n    for key in predictions.keys():\n        ensemble_df.loc[ensemble_df['id'] == key, 'model_' + str(seed + 1)] = int((predictions[key]).item())\n  \n    xm.master_print('Prediction Saved! \\n')","metadata":{"execution":{"iopub.status.busy":"2021-09-01T01:21:55.443058Z","iopub.execute_input":"2021-09-01T01:21:55.443458Z","iopub.status.idle":"2021-09-01T01:21:55.468050Z","shell.execute_reply.started":"2021-09-01T01:21:55.443426Z","shell.execute_reply":"2021-09-01T01:21:55.467293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"densenet121 = torchvision.models.densenet121(pretrained=True)\nfor param in densenet121.parameters():\n    param.requires_grad=False\n\ndensenet121.classifier = nn.Linear(in_features=densenet121.classifier.in_features, out_features=104, bias=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T01:04:45.099910Z","iopub.execute_input":"2021-09-01T01:04:45.100307Z","iopub.status.idle":"2021-09-01T01:04:46.376375Z","shell.execute_reply.started":"2021-09-01T01:04:45.100267Z","shell.execute_reply":"2021-09-01T01:04:46.375581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install timm","metadata":{"execution":{"iopub.status.busy":"2021-09-01T01:06:09.225828Z","iopub.execute_input":"2021-09-01T01:06:09.226161Z","iopub.status.idle":"2021-09-01T01:06:18.253496Z","shell.execute_reply.started":"2021-09-01T01:06:09.226134Z","shell.execute_reply":"2021-09-01T01:06:18.252530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import timm\nViT  = timm.create_model(\"vit_base_patch16_224\", pretrained=True)\nfor param in ViT.parameters():\n    param.requires_grad=False\n\nViT.head = nn.Linear(ViT.head.in_features, 104)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T01:06:28.175392Z","iopub.execute_input":"2021-09-01T01:06:28.175716Z","iopub.status.idle":"2021-09-01T01:06:39.060075Z","shell.execute_reply.started":"2021-09-01T01:06:28.175686Z","shell.execute_reply":"2021-09-01T01:06:39.058682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"googlenet = torchvision.models.googlenet(pretrained=True)\nfor param in googlenet.parameters():\n    param.grad_requires = False\n\ngooglenet.fc = nn.Linear(in_features=googlenet.fc.in_features, out_features=104, bias=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T01:07:06.943739Z","iopub.execute_input":"2021-09-01T01:07:06.944106Z","iopub.status.idle":"2021-09-01T01:07:10.289666Z","shell.execute_reply.started":"2021-09-01T01:07:06.944074Z","shell.execute_reply":"2021-09-01T01:07:10.288942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resnet101 = torchvision.models.resnet101(pretrained=True)\nfor param in resnet101.parameters():\n    param.grad_requires = False\n\nresnet101.fc = nn.Linear(in_features=resnet101.fc.in_features, out_features=104, bias=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T01:07:27.695428Z","iopub.execute_input":"2021-09-01T01:07:27.695743Z","iopub.status.idle":"2021-09-01T01:07:33.434944Z","shell.execute_reply.started":"2021-09-01T01:07:27.695715Z","shell.execute_reply":"2021-09-01T01:07:33.434138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg19_bn = torchvision.models.vgg19_bn(pretrained=True)\nfor param in vgg19_bn.parameters():\n    param.grad_requires = False\n\nvgg19_bn.classifier[6] = nn.Linear(4096, 104, bias=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T01:07:59.210103Z","iopub.execute_input":"2021-09-01T01:07:59.210467Z","iopub.status.idle":"2021-09-01T01:08:37.792373Z","shell.execute_reply.started":"2021-09-01T01:07:59.210438Z","shell.execute_reply":"2021-09-01T01:08:37.791406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_transforms = transforms.Compose([\n                        transforms.CenterCrop(224),\n                        transforms.Resize(224),\n                        transforms.ToTensor(),\n                        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n                    ])\n\ntest_ds = FlowersDataset(test_ids, [], test_images, test_transforms, True)\ntestloader = DataLoader(test_ds, 128, num_workers=4, pin_memory=True, shuffle=False)\n\nsubmit_df = pd.read_csv('../input/tpu-getting-started/sample_submission.csv')\nensemble_df = submit_df.copy()\n\n\nmodels = [densenet121, ViT, googlenet, resnet101, vgg19_bn]\n\nnum_models = len(models)\nnum_epochs = 10\n\nfor seed in range(num_models):\n    preds = fit(seed=seed, epochs=num_epochs, model=models[seed])","metadata":{"execution":{"iopub.status.busy":"2021-09-01T01:22:03.802494Z","iopub.execute_input":"2021-09-01T01:22:03.803056Z","iopub.status.idle":"2021-09-01T09:09:36.035876Z","shell.execute_reply.started":"2021-09-01T01:22:03.803008Z","shell.execute_reply":"2021-09-01T09:09:36.034262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **4 Submit Preparing**","metadata":{}},{"cell_type":"code","source":"ensemble_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T09:09:36.039570Z","iopub.execute_input":"2021-09-01T09:09:36.039902Z","iopub.status.idle":"2021-09-01T09:09:36.080805Z","shell.execute_reply.started":"2021-09-01T09:09:36.039867Z","shell.execute_reply":"2021-09-01T09:09:36.080153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Final prediction\nfinal_pred = ensemble_df.iloc[:,2:].mode(axis=1).iloc[:,0]\nsubmit_df.label = final_pred.astype(int)\nsubmit_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T09:09:36.084115Z","iopub.execute_input":"2021-09-01T09:09:36.084508Z","iopub.status.idle":"2021-09-01T09:09:39.559937Z","shell.execute_reply.started":"2021-09-01T09:09:36.084469Z","shell.execute_reply":"2021-09-01T09:09:39.559025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a submission file\nsubmit_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T09:18:01.283909Z","iopub.execute_input":"2021-09-01T09:18:01.284411Z","iopub.status.idle":"2021-09-01T09:18:01.308740Z","shell.execute_reply.started":"2021-09-01T09:18:01.284370Z","shell.execute_reply":"2021-09-01T09:18:01.307660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(5, 2, figsize=(15, 15))\nmodelname = ['DenseNet', 'ViT', 'GoogLeNet', 'ResNet101', 'VGG16 with BN']\n\nepochs=10\n\ni=0\n\nfor row in range(5):\n\n    epoch_list = list(range(1,epochs*2+1))\n\n    ax[row][0].plot(epoch_list, accuracies['train'][i:20+i], '-o', label='Train Accuracy')\n    ax[row][0].plot(epoch_list, accuracies['val'][i:20+i], '-o', label='Validation Accuracy')\n    ax[row][0].plot([epochs for x in range(20)],  np.linspace(min(accuracies['train'][i:20+i]).cpu(), max(accuracies['train'][i:20+i]).cpu(), 20), color='r', label='Unfreeze net')\n    ax[row][0].set_xticks(np.arange(0, epochs*2+1, 5))\n    ax[row][0].set_ylabel('Accuracy Value')\n    ax[row][0].set_xlabel('Epoch')\n    ax[row][0].set_title('Accuracy {}'.format(modelname[row]))\n    ax[row][0].legend(loc=\"best\")\n\n    ax[row][1].plot(epoch_list, losses['train'][i:20+i], '-o', label='Train Loss')\n    ax[row][1].plot(epoch_list, losses['val'][i:20+i], '-o',label='Validation Loss')\n    ax[row][1].plot([epochs for x in range(20)], np.linspace(min(losses['train'][i:20+i]), max(losses['train'][i:20+i]), 20), color='r', label='Unfreeze net')\n    ax[row][1].set_xticks(np.arange(0, epochs*2+1, 5))\n    ax[row][1].set_ylabel('Loss Value')\n    ax[row][1].set_xlabel('Epoch')\n    ax[row][1].set_title('Loss {}'.format(modelname[row]))\n    ax[row][1].legend(loc=\"best\")\n    fig.tight_layout()\n    fig.subplots_adjust(top=1.5, wspace=0.3)\n\n    i+=20","metadata":{"execution":{"iopub.status.busy":"2021-09-01T09:09:39.588717Z","iopub.execute_input":"2021-09-01T09:09:39.589008Z","iopub.status.idle":"2021-09-01T09:09:42.835712Z","shell.execute_reply.started":"2021-09-01T09:09:39.588981Z","shell.execute_reply":"2021-09-01T09:09:42.834889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#60sec*30min*2 = 1h\nimport time\nfor i in range(60*30*2):\n    time.sleep(1)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T09:09:42.837002Z","iopub.execute_input":"2021-09-01T09:09:42.837306Z","iopub.status.idle":"2021-09-01T09:15:17.727926Z","shell.execute_reply.started":"2021-09-01T09:09:42.837279Z","shell.execute_reply":"2021-09-01T09:15:17.726274Z"},"trusted":true},"execution_count":null,"outputs":[]}]}