{"cells":[{"metadata":{},"cell_type":"markdown","source":"Use the JPEG image with pytorch instead of the original TFRecord format: \n\nhttps://www.kaggle.com/c/flower-classification-with-tpus/data\n\nA simple tutorial of XLA for starter:\n\nhttps://www.kaggle.com/blaxkdolphin/tpustarter-xla\n\nOfficial XLA tutorial:\n\nhttps://github.com/pytorch/xla/\n\n\nOther good pytorch implement for this competition:\n\nhttps://www.kaggle.com/yasufuminakama/flower-pytorch-xla-se-resnext50\n\nhttps://www.kaggle.com/dhananjay3/fast-pytorch-xla-for-tpu-with-multiprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np \nimport pandas as pd\nfrom glob import glob\nfrom collections import deque\nimport matplotlib.pyplot as plt\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.utils.data.distributed import DistributedSampler\n\nimport torchvision\nimport torchvision.transforms as T \nimport torchvision.models as models\n\n# imports the torch_xla package\nimport torch_xla\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.distributed.xla_multiprocessing as xmp","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build custom Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"root = '../input/104-flowers-garden-of-eden'\n\ntrain_df = pd.DataFrame()\nfolder = os.listdir(root)\nfor f in folder:\n    classes = os.listdir(os.path.join(root,f,'train'))\n    for c in classes:\n        images = os.listdir(os.path.join(root,f,'train',c))\n        tmp_df = pd.DataFrame(images,columns=['image_name'])\n        tmp_df['class'] = c\n        tmp_df['folder'] = f\n        tmp_df['type'] = 'train'\n        train_df = train_df.append(tmp_df, ignore_index=True)\nprint('train:',train_df.shape)   \n\nval_df = pd.DataFrame()\nfolder = os.listdir(root)\nfor f in folder:\n    classes = os.listdir(os.path.join(root,f,'val'))\n    for c in classes:\n        images = os.listdir(os.path.join(root,f,'val',c))\n        tmp_df = pd.DataFrame(images,columns=['image_name'])\n        tmp_df['class'] = c\n        tmp_df['folder'] = f\n        tmp_df['type'] = 'val'\n        val_df = val_df.append(tmp_df, ignore_index=True)\nprint('val:',val_df.shape)     ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('num class:', len(CLASSES))\ntrain_df['label'] = train_df['class'].apply(lambda x: CLASSES.index(x))\nval_df['label'] = val_df['class'].apply(lambda x: CLASSES.index(x))  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.DataFrame()\nf = 'jpeg-224x224'\n\nimages = os.listdir(os.path.join(root,f,'test'))\ntmp_df = pd.DataFrame(images,columns=['image_name'])\ntmp_df['class'] = 'unknown'\ntmp_df['folder'] = f\ntmp_df['type'] = 'test'\ntest_df = test_df.append(tmp_df, ignore_index=True)\nprint('test:',test_df.shape)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class flowerDataset(Dataset):\n    def __init__(self, df, root = '../input/104-flowers-garden-of-eden'):\n        self.df = df\n        self.root = root\n        self.transforms = T.Compose([T.Resize((224,224)), T.ToTensor()])\n        \n    def __getitem__(self, idx):\n\n        img_path = os.path.join(self.root, \n                                self.df.iloc[idx]['folder'], \n                                self.df.iloc[idx]['type'],\n                                self.df.iloc[idx]['class'],\n                                self.df.iloc[idx]['image_name'])\n        img = Image.open(img_path)\n        img_tensor = self.transforms(img)\n        target_tensor = torch.tensor(self.df.iloc[idx]['label'], dtype=torch.long)\n        return img_tensor, target_tensor\n    \n    def __len__(self):\n        return len(self.df)\n    \n    \nclass testDataset(Dataset):\n    def __init__(self, df, root = '../input/104-flowers-garden-of-eden'):\n        self.df = df\n        self.root = root\n        self.transforms = T.Compose([T.ToTensor()])\n        \n    def __getitem__(self, idx):\n        img_path = os.path.join(self.root, self.df.iloc[idx]['folder'], \n                                self.df.iloc[idx]['type'],\n                                self.df.iloc[idx]['image_name'])\n        img = Image.open(img_path)\n        img_tensor = self.transforms(img)\n        return img_tensor,  self.df.iloc[idx]['image_name'][:-5]\n    \n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = flowerDataset(train_df)\nprint(train_dataset.__len__())\n\ntrain_loader = DataLoader(train_dataset, batch_size = 32, shuffle = True, drop_last = True)\ntrain_iter = iter(train_loader)\n\nimages, labels = next(train_iter)\nprint(images.size())\nprint(labels.size())\n\nplot_size = 32\n\nfig = plt.figure(figsize=(25, 10))\nfor idx in np.arange(plot_size):\n    ax = fig.add_subplot(4, plot_size/4, idx+1, xticks=[], yticks=[])\n    ax.imshow(np.transpose(images[idx], (1, 2, 0)))\n    ax.set_title(classes[labels[idx].item()])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Useing Multiple Cloud TPU cores"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_net():\n    torch.manual_seed(FLAGS['seed'])\n    \n    device = xm.xla_device()   \n    world_size = xm.xrt_world_size()\n    \n    xm.master_print(f'device: {device}')\n    xm.master_print(f'world_size: {world_size}')\n    \n    ### train loader\n    train_dataset = flowerDataset(train_df)\n    train_sampler = DistributedSampler(train_dataset,\n                                       num_replicas = world_size,\n                                       rank = xm.get_ordinal(),\n                                       shuffle = True)\n    train_loader = DataLoader(train_dataset,\n                              batch_size = FLAGS['batch_size'],\n                              sampler = train_sampler,\n                              num_workers = FLAGS['num_workers'],\n                              drop_last = True)\n    \n    ### val loader\n    val_dataset = flowerDataset(val_df)\n    val_sampler = DistributedSampler(val_dataset,\n                                     num_replicas = world_size,\n                                     rank = xm.get_ordinal(),\n                                     shuffle = True)\n    \n    val_loader = DataLoader(val_dataset,\n                            batch_size = FLAGS['batch_size'],\n                            sampler = val_sampler,\n                            num_workers = FLAGS['num_workers'],\n                            drop_last = True)\n    \n    #### model\n    model = models.resnet18()\n    model.load_state_dict(torch.load('/kaggle/input/resnet18/resnet18.pth'))\n    model.fc = nn.Linear(512, 104)\n    model.to(device)\n    \n    ### Scale learning rate to num cores\n    optimizer = optim.SGD(model.parameters(), \n                          lr = FLAGS['learning_rate'] * world_size,\n                          momentum = FLAGS['momentum'], \n                          weight_decay=5e-4)\n    \n    loss_fn = torch.nn.CrossEntropyLoss()\n    \n    def train_loop_fn(loader):\n        tracker = xm.RateTracker()\n        model.train()\n        loss_window = deque(maxlen = FLAGS['log_steps'])\n        for x, (data, target) in enumerate(loader):\n            optimizer.zero_grad()\n            output = model(data)\n            loss = loss_fn(output, target)\n            loss_window.append(loss.item())\n            loss.backward()\n            xm.optimizer_step(optimizer)\n            tracker.add(FLAGS['batch_size'])\n            if (x+1) % FLAGS['log_steps'] == 0:\n                print('[xla:{}]({}) Loss={:.5f} '.format(xm.get_ordinal(), x+1, np.mean(loss_window)), flush=True)\n                \n    def val_loop_fn(loader):\n        total_samples, correct = 0, 0\n        model.eval()\n        for data, target in loader:\n            with torch.no_grad():\n                output = model(data)\n            pred = output.max(1, keepdim=True)[1]\n            correct += pred.eq(target.view_as(pred)).sum().item()\n            total_samples += data.size()[0]\n\n        accuracy = 100.0 * correct / total_samples\n        print('[xla:{}] Accuracy={:.2f}%'.format(xm.get_ordinal(), accuracy), flush=True)\n        return accuracy\n\n\n    for epoch in range(1,FLAGS['num_epochs'] + 1):\n        para_loader = pl.ParallelLoader(train_loader, [device])\n        train_loop_fn(para_loader.per_device_loader(device))\n        xm.master_print(\"Finished training epoch {}\".format(epoch))\n        \n        para_loader = pl.ParallelLoader(val_loader, [device])\n        accuracy = val_loop_fn(para_loader.per_device_loader(device))\n        \n        best_accuracy = 0.0\n        if accuracy > best_accuracy:\n            xm.save(model.state_dict(), 'trained_resnet18_model.pth')\n            best_accuracy = accuracy\n        \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _mp_fn(rank, flags):\n    global FLAGS\n    FLAGS = flags\n    torch.set_default_tensor_type('torch.FloatTensor')\n    train_start = time.time()\n    train_net()\n    elapsed_train_time = time.time() - train_start\n    print(\"Process\", rank, \"finished training. Train time was:\", elapsed_train_time)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define Parameters\nFLAGS = {}\nFLAGS['seed'] = 1\nFLAGS['num_workers'] = 4\nFLAGS['num_cores'] = 8\nFLAGS['num_epochs'] = 10\nFLAGS['log_steps'] = 50\nFLAGS['batch_size'] = 16\nFLAGS['learning_rate'] = 0.0001\nFLAGS['momentum'] = 0.9\n\nxmp.spawn(_mp_fn, args = (FLAGS,), nprocs = FLAGS['num_cores'],start_method='fork')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = models.resnet18()\nmodel.fc = nn.Linear(512, 104)\nmodel.load_state_dict(torch.load('trained_resnet18_model.pth'))\n\ndevice = xm.xla_device()\nmodel.to(device)\nmodel.eval()\n\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 128\ntest_dataset = testDataset(test_df)\ntest_loader = DataLoader(test_dataset, batch_size = batch_size)\nn = test_dataset.__len__()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label = []\nid = []\nfor x, (images, names) in enumerate(test_loader):\n    images = images.to(device)\n    with torch.no_grad():\n        output = model(images)\n    preds = list(output.max(1)[1].cpu().numpy())\n    label.extend(preds)\n    id.extend(names)\n    print('\\rProcess {} %'.format(round(100*x*batch_size/n)),end=\"\")\n    \nprint('\\rProcess 100 %')    \n\npredictions = pd.DataFrame(data={'id':id,'label':label})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}