{"cells":[{"metadata":{},"cell_type":"markdown","source":"##  Install Dependence"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-24T09:01:25.859974Z","start_time":"2020-09-24T09:01:25.854922Z"}},"cell_type":"markdown","source":"### Install XLA"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:40.078935Z","start_time":"2020-09-28T03:07:40.074176Z"},"_kg_hide-output":true,"trusted":false},"cell_type":"code","source":"!pip uninstall -y torch torchvision ","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:40.095975Z","start_time":"2020-09-28T03:07:40.083602Z"},"_kg_hide-output":true,"trusted":false},"cell_type":"code","source":"file_url = 'https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py'\n!curl {file_url} -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --version 20200529 --apt-packages libomp5 libopenblas-dev","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-27T06:53:39.154952Z","start_time":"2020-09-27T06:53:39.150041Z"},"scrolled":true},"cell_type":"markdown","source":"[torch-fix: 20200528](https://github.com/pytorch/pytorch/commit/3d2fce6bc39c7cfa2872a43773da74e0da79757a#diff-724910f4a1ffa1852a02e149b181ac22)"},{"metadata":{},"cell_type":"markdown","source":"### Pytorch Lightning"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:40.108098Z","start_time":"2020-09-28T03:07:40.099983Z"},"hide_input":false,"_kg_hide-output":true,"trusted":false},"cell_type":"code","source":"!pip install pytorch_lightning","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:40.122694Z","start_time":"2020-09-28T03:07:40.111112Z"},"_kg_hide-output":true,"trusted":false},"cell_type":"code","source":"!pip install tfrecord","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Import Library"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:42.105388Z","start_time":"2020-09-28T03:07:40.125634Z"},"trusted":false},"cell_type":"code","source":"import os\nimport io\nimport warnings\nimport random\nimport glob\nimport psutil\nimport pytorch_lightning as pl\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport torch\nimport torchvision\nimport matplotlib.pyplot as plt\n\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import (Dataset, DataLoader)\nfrom torchvision.transforms import (\n        Resize,\n        Compose,\n        ToTensor,\n        Normalize,\n        RandomOrder,\n        ColorJitter,\n        RandomRotation,\n        RandomGrayscale,\n        RandomResizedCrop,\n        RandomVerticalFlip,\n        RandomHorizontalFlip)\n\nfrom PIL import Image, ImageDraw, ImageFont\n\n# from torchsummary import summary\nfrom pytorch_lightning.callbacks import EarlyStopping\nfrom pytorch_lightning.callbacks import ModelCheckpoint\n\nfrom tfrecord import reader\nfrom tfrecord.tools.tfrecord2idx import create_index\n\n%matplotlib inline\n\nsns.set(style='white', font_scale=1.2)\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Show Version"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:42.114268Z","start_time":"2020-09-28T03:07:42.107735Z"},"trusted":false},"cell_type":"code","source":"np.__version__, pd.__version__, sns.__version__","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:42.128601Z","start_time":"2020-09-28T03:07:42.118707Z"},"trusted":false},"cell_type":"code","source":"torch.__version__, torchvision.__version__, pl.__version__","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Global Constants"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T04:05:47.127013Z","start_time":"2020-09-28T04:05:47.017441Z"},"trusted":false},"cell_type":"code","source":"RNG_SEED = 9527\n\nKGGL_NAME = 'tpu-getting-started'\nKGGL_ROOT = '/kaggle/working'\nDATA_ROOT = f'/kaggle/input/{KGGL_NAME}'\nWORK_ROOT = f'{KGGL_ROOT}/{KGGL_NAME}'\nCKPT_PATH = f'{WORK_ROOT}/checkpoints/best.ckpt'\nSUBMITCSV = f'{KGGL_ROOT}/submission.csv'\nFONT_PATH = '/usr/share/fonts/truetype/dejavu/DejaVuSerif-Bold.ttf'\n\nINPUT_SIZE = 192 # 192, 224, 331, 512\nBATCH_SIZE = 64\nNUM_WORKERS = 1 # psutil.cpu_count()\n\nMAX_EPOCHS = 50\n\nDATASET_MEAN = (0.5, 0.5, 0.5)\nDATASET_STD = (0.5, 0.5, 0.5)\n\nCLASS_NAMES = [\n    'pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', \n    'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', \n    'globe thistle', 'snapdragon', \"colt's foot\", 'king protea', 'spear thistle', \n    'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', \n    'balloon flower', 'giant white arum lily', 'fire lily', 'pincushion flower', \n    'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', \n    'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william', \n    'carnation', 'garden phlox', 'love in the mist', 'cosmos',  'alpine sea holly', \n    'ruby-lipped cattleya', 'cape flower', 'great masterwort',  'siam tulip', \n    'lenten rose', 'barberton daisy', 'daffodil',  'sword lily', 'poinsettia', \n    'bolero deep blue',  'wallflower', 'marigold', 'buttercup', 'daisy', \n    'common dandelion', 'petunia', 'wild pansy', 'primula',  'sunflower', \n    'lilac hibiscus', 'bishop of llandaff', 'gaura',  'geranium', 'orange dahlia', \n    'pink-yellow dahlia', 'cautleya spicata',  'japanese anemone', 'black-eyed susan', \n    'silverbush', 'californian poppy',  'osteospermum', 'spring crocus', 'iris', \n    'windflower',  'tree poppy', 'gazania', 'azalea', 'water lily',  'rose', \n    'thorn apple', 'morning glory', 'passion flower',  'lotus', 'toad lily', \n    'anthurium', 'frangipani',  'clematis', 'hibiscus', 'columbine', 'desert-rose', \n    'tree mallow', 'magnolia', 'cyclamen ', 'watercress',  'canna lily', \n    'hippeastrum ', 'bee balm', 'pink quill',  'foxglove', 'bougainvillea', \n    'camellia', 'mallow',  'mexican petunia',  'bromelia', 'blanket flower', \n    'trumpet creeper',  'blackberry lily', 'common tulip', 'wild rose']\n\nNUM_CLASSES = len(CLASS_NAMES)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:43.301480Z","start_time":"2020-09-28T03:07:42.153824Z"},"trusted":false},"cell_type":"code","source":"!ls -l $DATA_ROOT\n!mkdir -p $WORK_ROOT","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Random Seed"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:43.315694Z","start_time":"2020-09-28T03:07:43.305785Z"},"trusted":false},"cell_type":"code","source":"torch.manual_seed(RNG_SEED)\nnp.random.seed(RNG_SEED)\nrandom.seed(RNG_SEED)\n\n# torch.backends.cudnn.deterministic = True\n# torch.backends.cudnn.benchmark = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepare Data"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:43.331577Z","start_time":"2020-09-28T03:07:43.318871Z"},"trusted":false},"cell_type":"code","source":"IMG_SCALE = 'tfrecords-jpeg-{}x{}'.format(INPUT_SIZE, INPUT_SIZE)\ntrain_files = glob.glob(f'{DATA_ROOT}/{IMG_SCALE}/train/*.tfrec')\nvalid_files = glob.glob(f'{DATA_ROOT}/{IMG_SCALE}/val/*.tfrec')\ntest_files = glob.glob(f'{DATA_ROOT}/{IMG_SCALE}/test/*.tfrec')\nprint('Files:', \\\n      '\\n\\tTrain tfrec Count:', len(train_files), \\\n      '\\n\\tValid tfrec Count:', len(valid_files), \\\n      '\\n\\tTest  tfrec Count:', len(test_files))","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-24T10:03:46.069555Z","start_time":"2020-09-24T10:03:46.060425Z"}},"cell_type":"markdown","source":"### Create Records Index"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:43.765636Z","start_time":"2020-09-28T03:07:43.334518Z"},"trusted":false},"cell_type":"code","source":"def create_indexes(phase):\n    tfrec_files = glob.glob(f'{DATA_ROOT}/{IMG_SCALE}/{phase}/*.tfrec')\n    dirpath = os.path.dirname(tfrec_files[0].replace(DATA_ROOT, WORK_ROOT))\n    if not os.path.exists(dirpath):\n        os.makedirs(dirpath)\n    patterns = []\n    for tfrec_path in tfrec_files:  \n        index_path = tfrec_path.replace(DATA_ROOT, WORK_ROOT).replace('.tfrec', '.index')\n        create_index(tfrec_path, index_path)\n        patterns.append(tfrec_path[len(DATA_ROOT)+1:-6])\n    return patterns\n        \ntrain_patterns = create_indexes('train')\nvalid_patterns = create_indexes('val')\ntest_patterns  = create_indexes('test')","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:43.773666Z","start_time":"2020-09-28T03:07:43.768232Z"},"trusted":false},"cell_type":"code","source":"train_patterns[:5]","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:43.794508Z","start_time":"2020-09-28T03:07:43.776119Z"},"trusted":false},"cell_type":"code","source":"class TFRecordFlowersDataset(torch.utils.data.IterableDataset):\n    def __init__(self, patterns, labeled=True, augtrans=None, imgtrans=None):\n        super().__init__()\n        self.labeled = labeled\n        self.augtrans = augtrans\n        self.imgtrans = imgtrans \n        self.imagecnt = 0\n        self.imagepat = []\n        for pattern in patterns:\n            tfrec_path = DATA_ROOT + f'/{pattern}.tfrec'\n            index_path = WORK_ROOT + f'/{pattern}.index'\n            self.imagecnt += len(np.loadtxt(index_path, dtype=np.int64)[:, 0])\n            self.imagepat.append((tfrec_path, index_path))\n        print('Count:', self.imagecnt)\n\n    def __iter__(self):\n        worker_info = torch.utils.data.get_worker_info()\n        if worker_info is not None:\n            shard = worker_info.id, worker_info.num_workers\n            np.random.seed(worker_info.seed % np.iinfo(np.uint32).max)\n        else:\n            shard = None\n            \n        if self.labeled:\n            description = {'image': 'byte', 'class': 'int'}\n        else:\n            description = {'image': 'byte', 'id': 'byte'}\n            \n        for tfrec_path, index_path in self.imagepat:\n            it = reader.tfrecord_loader(\n                tfrec_path,\n                index_path,\n                description,\n                shard\n            )\n            for elem in it:\n                img = Image.open(io.BytesIO(elem['image']), mode='r').convert('RGB')\n                if self.augtrans:\n                    img = self.augtrans(img)\n                if self.imgtrans:\n                    img = self.imgtrans(img)\n                tag = elem['class'].item() if self.labeled else str(elem['id'], encoding='utf-8')\n                yield img, tag\n                \n    def __len__(self):\n        return self.imagecnt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train and Valid Data"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:43.811168Z","start_time":"2020-09-28T03:07:43.797137Z"},"trusted":false},"cell_type":"code","source":"# train_dataset = TFRecordFlowersDataset(train_patterns)\n# valid_dataset = TFRecordFlowersDataset(valid_patterns)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Test Data"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:43.822491Z","start_time":"2020-09-28T03:07:43.813931Z"},"trusted":false},"cell_type":"code","source":"# test_dataset = TFRecordFlowersDataset(test_patterns, labeled=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Sample Data Exploration"},{"metadata":{},"cell_type":"markdown","source":"### Demo  Image"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:44.038574Z","start_time":"2020-09-28T03:07:43.825172Z"},"trusted":false},"cell_type":"code","source":"sample_dataset = TFRecordFlowersDataset(train_patterns)\nsample_iter = iter(sample_dataset)\nsample_data = [next(sample_iter) for _ in range(10)]\nsample_dataloader = DataLoader(sample_dataset, batch_size=BATCH_SIZE, shuffle=False)\nsample_data[0], len(sample_dataloader)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:44.655945Z","start_time":"2020-09-28T03:07:44.041316Z"},"trusted":false},"cell_type":"code","source":"fig, axes = plt.subplots(nrows=2, ncols=5, sharey=True, figsize=(12,4))\nfor r in range(2):\n    for c in range(5):\n        axes[r][c].set_xticks([])\n        axes[r][c].set_yticks([])\n        axes[r][c].imshow(np.array(sample_data[r*2 + c][0]).astype('uint8')) # 'gray_r'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define Display Image Method"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:44.667263Z","start_time":"2020-09-28T03:07:44.658535Z"},"trusted":false},"cell_type":"code","source":"def draw_image(imgdata, labelname, augtrans=None):\n    img = imgdata.copy()\n    if augtrans is not None:\n        img = augtrans(img)\n        \n    font_obj = ImageFont.truetype(FONT_PATH, 16)\n    draw_img = ImageDraw.Draw(img)\n    draw_img.text((0, 0), labelname, font=font_obj, fill=(255, 255, 255))\n    return np.array(img)\n\ndef grid_image(imgs_list, cols=5):\n    images = torch.as_tensor(imgs_list) # [(W, H, C)...] to (B, H, W, C)\n    images = images.permute(0, 3, 1, 2) # (B, H, W, C) to (B, C, H, W)\n    images = torchvision.utils.make_grid(images, nrow=cols) # (C, 2*H, 4*W)\n    images = images.permute(1, 2, 0) # (H, W, C)\n    return images","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Display Raw Sample Images"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:45.845187Z","start_time":"2020-09-28T03:07:44.669937Z"},"trusted":false},"cell_type":"code","source":"plt.figure(figsize=(16, 8))\n\nimages_2x5 = [\n    draw_image(\n        imgdata=img,\n        labelname=CLASS_NAMES[labelid],\n    ) for img, labelid in sample_data\n]\n\nplt.xticks([])\nplt.yticks([])\nplt.imshow(grid_image(images_2x5, cols=5));","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Augment Transform"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:45.853836Z","start_time":"2020-09-28T03:07:45.847661Z"},"trusted":false},"cell_type":"code","source":"aug_trans = RandomOrder([\n    RandomRotation(degrees=30),\n    RandomVerticalFlip(p=0.3),\n    RandomHorizontalFlip(p=0.3),\n    ColorJitter(brightness=0.55, contrast=0.3, saturation=0.25, hue=0),\n])\n\nimg_trans = Compose([\n    RandomResizedCrop((INPUT_SIZE, INPUT_SIZE)),\n    ToTensor(),\n    Normalize(mean=DATASET_MEAN, std=DATASET_STD),\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Display Augment Images"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:47.342233Z","start_time":"2020-09-28T03:07:45.858182Z"},"trusted":false},"cell_type":"code","source":"plt.figure(figsize=(24, 12))\n\naugment_images_2x5 = [\n    draw_image(\n        imgdata=img,\n        labelname=CLASS_NAMES[labelid],\n        augtrans = aug_trans\n    ) for img, labelid in sample_data\n]\n\nplt.xticks([])\nplt.yticks([])\nplt.imshow(grid_image(augment_images_2x5, cols=5));","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:47.352017Z","start_time":"2020-09-28T03:07:47.346637Z"},"trusted":false},"cell_type":"code","source":"del sample_dataloader\ndel sample_dataset\ndel sample_data","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build Network"},{"metadata":{},"cell_type":"markdown","source":"### Load Pretrained Model"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:49.193476Z","start_time":"2020-09-28T03:07:47.355358Z"},"trusted":false},"cell_type":"code","source":"backbone = torchvision.models.vgg16(pretrained=True)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:49.199031Z","start_time":"2020-09-28T03:07:49.195827Z"},"trusted":false},"cell_type":"code","source":"# backbone","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:49.216187Z","start_time":"2020-09-28T03:07:49.201356Z"},"trusted":false},"cell_type":"code","source":"# summary(backbone, (3, INPUT_SIZE, INPUT_SIZE), device='cpu')","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-21T09:47:38.878375Z","start_time":"2020-09-21T09:47:36.189307Z"}},"cell_type":"markdown","source":"### Freezing Partial Layers"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:49.228469Z","start_time":"2020-09-28T03:07:49.218596Z"},"trusted":false},"cell_type":"code","source":"# layer_index = 0\nfor param in backbone.features.parameters():\n    # if layer_index > 18:\n    #     break\n    # layer_index += 1\n    param.requires_grad = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model/ Loss/Optimizer "},{"metadata":{"trusted":true},"cell_type":"code","source":"METRICS = {\n    'epoch':[0],\n    'train_loss':[0],\n    'train_acc':[0],\n    'val_acc':[0],\n    'val_loss':[0],\n    'lr': [0],\n}\n\ndef log_last_metric():\n    print('{}: train_loss[{}], train_acc[{}], val_loss[{}], val_acc[{}], lr{}'.format(\n        METRICS['epoch'][-1] + 1,\n        round(METRICS['train_loss'][-1], 3),\n        round(METRICS['train_acc'][-1], 3),\n        round(METRICS['val_loss'][-1], 3),\n        round(METRICS['val_acc'][-1], 3), METRICS['lr']\n    ))","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:49.278868Z","start_time":"2020-09-28T03:07:49.230918Z"},"_kg_hide-output":true,"trusted":false},"cell_type":"code","source":"class ClassifierNet(pl.LightningModule):\n    def __init__(self, extractor=None, num_classes=NUM_CLASSES):\n        super().__init__()\n        if extractor is not None:\n            self.features = extractor\n            self.classifier = nn.Sequential(\n                nn.AdaptiveAvgPool2d(output_size=(7, 7)),\n                nn.Flatten(start_dim=1, end_dim=-1),\n                nn.Linear(in_features=25088, out_features=2048, bias=True),\n                nn.ReLU(inplace=True),\n                nn.Dropout(p=0.5, inplace=False),\n                nn.Linear(in_features=2048, out_features=1024, bias=True),\n                nn.ReLU(inplace=True),\n                nn.Dropout(p=0.5, inplace=False),\n                nn.Linear(in_features=1024, out_features=256, bias=True),\n                nn.ReLU(inplace=True),\n                # nn.Dropout(p=0.5, inplace=False),\n                nn.Linear(in_features=256, out_features=num_classes, bias=True)\n            )\n        else:    \n            self.features = nn.Sequential(\n                nn.BatchNorm2d(num_features=3, momentum=0.1),\n                nn.Conv2d(in_channels=3, out_channels=64, kernel_size=7, stride=2, padding=1, bias=True),\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(num_features=64, momentum=0.1, affine=True, track_running_stats=True),\n                nn.MaxPool2d(kernel_size=7, stride=1, padding=3, ceil_mode=False),\n                nn.Dropout(inplace=True, p=0.5),\n                nn.Conv2d(in_channels=64, out_channels=64, kernel_size=5, stride=2, padding=2, bias=True),\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(num_features=64, momentum=0.1, affine=True, track_running_stats=True),\n                nn.MaxPool2d(kernel_size=5, stride=1, padding=2, ceil_mode=False),\n                nn.Dropout(inplace=True, p=0.5)\n            )\n            self.classifier = nn.Sequential(\n               nn.AdaptiveAvgPool2d(output_size=(5, 5)),\n               nn.Flatten(start_dim=1, end_dim=-1),\n               nn.Linear(in_features=1600, out_features=128, bias=True),\n               nn.Dropout(inplace=True, p=0.5),\n               nn.Linear(in_features=128, out_features=num_classes, bias=True),\n            )\n  \n    def forward(self, x, *args, **kwargs):\n        x = self.features(x)\n        x = self.classifier(x)\n        return x\n            \n    @property\n    def metrics(self):\n        return self.metrics\n        \n    def configure_optimizers(self):\n        optimizer = torch.optim.Adam(\n            model.parameters(),\n            lr=0.001,\n        )\n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n            optimizer,\n            mode='min',\n            factor=0.1,\n            patience=3,\n            min_lr=1e-6)\n        return [optimizer], [scheduler]\n    \n    def prepare_data(self):\n        self.result_df = None\n        self.train_dataset = TFRecordFlowersDataset(train_patterns, True, aug_trans, img_trans) \n        self.valid_dataset = TFRecordFlowersDataset(valid_patterns, True, None, img_trans) \n        self.test_dataset  = TFRecordFlowersDataset(test_patterns, False, None, img_trans)\n\n    def train_dataloader(self):\n        return DataLoader(\n                self.train_dataset,\n                batch_size=BATCH_SIZE,\n                num_workers=NUM_WORKERS,\n                drop_last=True,\n                shuffle=False)\n    \n    def training_step(self, batch, batch_idx):\n        x, y_true = batch\n        y_pred = self(x)\n        loss = F.cross_entropy(y_pred, y_true, reduction='mean')\n        acc = (torch.argmax(y_pred, dim=1) == y_true).float().mean()\n        return {'loss': loss, 'acc': acc}\n\n    def training_epoch_end(self, outputs):\n        loss = torch.stack([x['loss'] for x in outputs]).mean()\n        acc = torch.stack([x['acc'] for x in outputs]).mean()\n        METRICS['epoch'].append(self.current_epoch)\n        METRICS['train_loss'].append(loss.cpu().item())\n        METRICS['train_acc'].append(acc.cpu().item())\n        return {'progress_bar': {'train_loss': loss, 'train_acc': acc}}\n\n    def val_dataloader(self):\n        return DataLoader(\n            self.valid_dataset,\n            batch_size=BATCH_SIZE,\n            num_workers=NUM_WORKERS,\n            drop_last=False,\n            shuffle=False)\n    \n    def validation_step(self, batch, batch_idx):\n        x, y_true = batch\n        y_pred = self(x)\n        loss = F.cross_entropy(y_pred, y_true, reduction='mean')\n        acc = (torch.argmax(y_pred, dim=1) == y_true).float().mean()\n        return {'val_loss': loss, 'val_acc': acc}\n\n    def validation_epoch_end(self, outputs):\n        loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n        acc = torch.stack([x['val_acc'] for x in outputs]).mean()\n        METRICS['val_loss'].append(loss.cpu().item())\n        METRICS['val_acc'].append(acc.cpu().item())\n        log_last_metric() # kaggle debug\n        return {'progress_bar': {'val_loss': loss, 'val_acc': acc}}\n    \n    def test_dataloader(self):\n        return DataLoader(\n            self.test_dataset,\n            batch_size=BATCH_SIZE,\n            num_workers=NUM_WORKERS,\n            drop_last=False,\n            shuffle=False)\n    \n    def test_step(self, batch, batch_idx):\n        x, imgid = batch\n        y_pred = torch.argmax(self(x), dim=1).cpu().numpy()\n        log = {'imgid': imgid, 'label': y_pred}\n        return log\n\n    def test_epoch_end(self, outputs):\n        imgid = np.concatenate([x['imgid'] for x in outputs])\n        label = np.concatenate([x['label'] for x in outputs])\n        result = {'id': imgid, 'label': label}\n        self.result_df = pd.DataFrame(data=result)  # TODO submission\n        return result\n    \nclass ClassifierTrainer(pl.Trainer):\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n\n    def on_validation_start(self):\n        lrs = []\n        for scheduler in self.lr_schedulers:\n            ss = scheduler['scheduler']\n            if isinstance(ss, torch.optim.lr_scheduler.ReduceLROnPlateau):\n                for i, param_group in enumerate(ss.optimizer.param_groups):\n                    lrs.append(np.float32(param_group['lr']))\n            else:\n                lrs.extend([np.float32(x) for x in ss.get_last_lr()])\n        self.add_progress_bar_metrics({'lr': lrs})\n        METRICS['lr'] = lrs\n        return super().on_validation_start()    \n    \n    # Workaround TPU on pytorch_lightning Fail (distrib_data_parallel.py)\n    def transfer_distrib_spawn_state_on_fit_end(self, model, mp_queue, results):\n        if self.global_rank == 0 and mp_queue is not None: \n            mp_queue.put(CKPT_PATH) # best_path\n            mp_queue.put({})        # results   # Test Phase: cannot pass real results (will block)\n            mp_queue.put(None)      # last_path\n\n    def save_spawn_weights(self, model):\n        if self.is_global_zero:\n            super().save_checkpoint(CKPT_PATH, weights_only=True)\n        \n    def load_spawn_weights(self, original_model):\n        return original_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"if torch.cuda.is_available():\n    args = {'gpus':[0]}\nelse:\n    args = {'tpu_cores':[1], 'precision':16}\n\ntrainer = ClassifierTrainer(\n    max_epochs=MAX_EPOCHS,\n    logger=False,\n    log_gpu_memory='min_max',\n    weights_summary='top',\n    num_sanity_val_steps=0,\n    progress_bar_refresh_rate=1,\n    check_val_every_n_epoch=1,\n    default_root_dir=WORK_ROOT,\n    resume_from_checkpoint=None,\n    early_stop_callback=EarlyStopping(monitor='val_loss', patience=7, mode='min'),\n    checkpoint_callback=ModelCheckpoint(monitor='val_loss', period=5, mode='min'),\n    **args\n)\n\nmodel = ClassifierNet(backbone.features, num_classes=NUM_CLASSES)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:16:24.048231Z","start_time":"2020-09-28T03:09:39.060605Z"},"scrolled":false,"trusted":false},"cell_type":"code","source":"trainer.fit(model);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predict"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:49.303406Z","start_time":"2020-09-28T03:07:40.182Z"},"trusted":false},"cell_type":"code","source":"trainer.test(model);\n\nresult_df = model.result_df\nplt.figure(figsize=(22, 10))\nsns.countplot(x='label',data=result_df).set_title(\"Predict Data Distribution\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot Metrics"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:49.307759Z","start_time":"2020-09-28T03:07:40.188Z"},"trusted":false},"cell_type":"code","source":"num_epoch = len(METRICS['epoch'])\nfig, axs = plt.subplots(1, 2, figsize=(16, 8))\naxs[0].plot(METRICS['epoch'], METRICS['train_acc'])\naxs[0].plot(METRICS['epoch'], METRICS['val_acc'])\naxs[0].set_title('Accuracy')\naxs[0].set_ylabel('Accuracy')\naxs[0].set_xlabel('Epoch')\naxs[0].legend(['train', 'val'], loc='best')\n\naxs[1].plot(METRICS['epoch'], METRICS['train_loss'])\naxs[1].plot(METRICS['epoch'], METRICS['val_loss'])\naxs[1].set_title('Loss')\naxs[1].set_ylabel('Loss')\naxs[1].set_xlabel('Epoch')\naxs[1].legend(['train', 'val'], loc='best');","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T08:26:58.770807Z","start_time":"2020-09-22T08:19:54.498Z"}},"cell_type":"markdown","source":"## Submission "},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:49.310090Z","start_time":"2020-09-28T03:07:40.192Z"},"trusted":false},"cell_type":"code","source":"!rm -rf $KGGL_ROOT/*\n\nresult_df.to_csv(SUBMITCSV, index=False)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-28T03:07:49.312206Z","start_time":"2020-09-28T03:07:40.196Z"},"trusted":false},"cell_type":"code","source":"!ls -l ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}