{"cells":[{"metadata":{},"cell_type":"markdown","source":"Credit to https://www.kaggle.com/debarshichanda/tpu-training.\nI modified some codes and made it work.<br>\nMy training code: https://www.kaggle.com/qinhui1999/pytorch-tpu-training-v1"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"\npackage_path = '../input/pytorch-image-models/pytorch-image-models-master' #'../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\nimport sys; sys.path.append(package_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport copy\nimport time\nimport random\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\n#import torch_optimizer as optim\nfrom torch.optim import lr_scheduler\nimport torchvision\nfrom torchvision import models\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.cuda import amp\n\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.utils import class_weight\n\nfrom tqdm.notebook import tqdm\nfrom collections import defaultdict\nfrom datetime import datetime\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport timm\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG:\n    model_name = 'resnet18d'#'tf_efficientnet_b4_ns'\n    img_size = 380\n    scheduler = 'CosineAnnealingWarmRestarts'\n    T_max = 10\n    T_0 = 10\n    lr = 1e-4\n    min_lr = 1e-6\n    batch_size = 64#16*4\n    weight_decay = 1e-6\n    seed = 42\n    num_classes = 5\n    num_epochs = 10#10\n    n_fold = 5\n    smoothing = 0.2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_seed(seed = 42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed(CFG.seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT_DIR = \"../input/cassava-leaf-disease-classification\"\nTRAIN_DIR = \"../input/cassava-leaf-disease-classification/train_images\"\nTEST_DIR = \"../input/cassava-leaf-disease-classification/test_images\"\n\nMODEL_PATH='../input/timmpretrainedresnet/resnet18d_ra2-48a79e06.pth'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv(os.path.join(ROOT_DIR, 'sample_submission.csv'))\nsubmission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaLeafDataset(nn.Module):\n    def __init__(self, root_dir, df, transforms=None):\n        self.root_dir = root_dir\n        self.df = df\n        self.labels=df['label'].values\n        self.image_ids=df['image_id'].values\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        img_path = os.path.join(self.root_dir, self.image_ids[index])\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n        label = self.labels[index]\n        \n        if self.transforms:\n            img = self.transforms(image=img)[\"image\"]\n            \n        return img, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_transforms = {\n    \"train\": A.Compose([\n        A.RandomResizedCrop(CFG.img_size, CFG.img_size),\n#         A.Transpose(p=0.5),\n        A.HorizontalFlip(p=0.5),\n#         A.VerticalFlip(p=0.5),\n#         A.ShiftScaleRotate(p=0.5),\n#         A.HueSaturationValue(\n#                 hue_shift_limit=0.2, \n#                 sat_shift_limit=0.2, \n#                 val_shift_limit=0.2, \n#                 p=0.5\n#             ),\n#         A.RandomBrightnessContrast(\n#                 brightness_limit=(-0.1,0.1), \n#                 contrast_limit=(-0.1, 0.1), \n#                 p=0.5\n#             ),\n        A.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0\n            ),\n#         A.CoarseDropout(p=0.5),\n#         A.Cutout(p=0.5),\n        ToTensorV2()], p=1.),\n    \n    \"valid\": A.Compose([\n        A.CenterCrop(CFG.img_size, CFG.img_size, p=1.),\n        A.Resize(CFG.img_size, CFG.img_size),\n        A.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0\n            ),\n        ToTensorV2()], p=1.)\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tqdm\nfrom tqdm.notebook import tqdm as tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class EffNet(nn.Module):\n#     def __init__(self, n_classes, pretrained=True):\n#         super(EffNet, self).__init__()\n#         self.model = timm.create_model(CFG.model_name, pretrained=True)\n#         num_features = self.model.classifier.in_features\n#         self.model.classifier = nn.Linear(num_features, CFG.num_classes)\n\n#     def forward(self, x):\n#         x = self.model(x)\n#         return x\n\nclass EffNet(nn.Module):\n    def __init__(self, n_classes, pretrained=True):\n        super(EffNet, self).__init__()\n        self.model = timm.create_model(CFG.model_name, pretrained=False)\n        self.model.load_state_dict(torch.load(MODEL_PATH))\n        self.logit = nn.Linear(512, CFG.num_classes)\n\n    def forward(self, x):\n        batch_size, C, H, W = x.shape\n        logit = self.model.forward_features(x)\n        logit = F.adaptive_avg_pool2d(logit,1).reshape(batch_size,-1)\n        \n        logit=self.logit(logit)\n        return logit  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef predict(model,valid_loader,  device):\n    # keep track of validation loss\n    \n    \n\n    ######################\n    # validate the model #\n    ######################\n    model.eval()\n    preds=[]\n    props=[]\n    for data, target in tqdm(valid_loader):\n        # move tensors to GPU if CUDA is available\n\n        data = data.to(device, dtype=torch.float32)\n        target = target.to(device, dtype=torch.int64)\n\n        with torch.no_grad():\n            # forward pass: compute predicted outputs by passing inputs to the model\n            output = model(data)\n            output=F.softmax(output).cpu().numpy()\n            props.extend(output)\n            preds+=output.argmax(1).tolist()\n    preds=np.array(preds)\n    props=np.array(props)\n    return preds,props","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_dataset = TestDataset(df=submission_df, transforms=test_transforms)\nvalid_dataset = CassavaLeafDataset(TEST_DIR, submission_df, transforms=data_transforms[\"valid\"])\nvalid_loader = torch.utils.data.DataLoader(\n        dataset=valid_dataset,\n        batch_size=CFG.batch_size*4,\n\n        drop_last=False,\n        num_workers=4,\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint='../input/pytorch-tpu-training-v1/Fold-4_0.859375_epoch-7.pth'\ndevice = 'cuda'\nmodel = EffNet(n_classes=CFG.num_classes)\nmodel.to(device)\nmodel.load_state_dict(torch.load(checkpoint, map_location=device))\n\npreds,props=predict(model,valid_loader,device)\nprint(preds.shape,props.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# np.savez_compressed('props',a=props)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['label'] = preds\nsubmission_df.to_csv('submission.csv', index=False)\nsubmission_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}