{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-14T19:15:19.876989Z","iopub.execute_input":"2021-06-14T19:15:19.877538Z","iopub.status.idle":"2021-06-14T19:15:19.88953Z","shell.execute_reply.started":"2021-06-14T19:15:19.877423Z","shell.execute_reply":"2021-06-14T19:15:19.888189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport shutil\nimport math\nimport torch\nimport torch.optim as optim\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision\nfrom torchvision import transforms\n\n# !pip install -U tensorboardcolab\n# from tensorboardcolab import TensorBoardColab\n\ntorch.manual_seed(470)\ntorch.cuda.manual_seed(470)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:15:19.891597Z","iopub.execute_input":"2021-06-14T19:15:19.892321Z","iopub.status.idle":"2021-06-14T19:15:21.307624Z","shell.execute_reply.started":"2021-06-14T19:15:19.892156Z","shell.execute_reply":"2021-06-14T19:15:21.306676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.datasets import load_files\nimport cv2\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:15:21.309401Z","iopub.execute_input":"2021-06-14T19:15:21.309752Z","iopub.status.idle":"2021-06-14T19:15:22.57706Z","shell.execute_reply.started":"2021-06-14T19:15:21.309723Z","shell.execute_reply":"2021-06-14T19:15:22.576014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# training & optimization hyper-parameters\nmax_epoch = 100\nlearning_rate = 0.001\nbatch_size = 64\ndevice = 'cuda'\n\n# model hyper-parameters\noutput_dim = 10\n\n# Boolean value to select training process\ntraining_process = True\n\nMEAN = (0.4452, 0.4457, 0.4464)\nSTD = (0.2592, 0.2596, 0.2600)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:25:15.539625Z","iopub.execute_input":"2021-06-14T19:25:15.539986Z","iopub.status.idle":"2021-06-14T19:25:15.546795Z","shell.execute_reply.started":"2021-06-14T19:25:15.539954Z","shell.execute_reply":"2021-06-14T19:25:15.545744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = '../input/state-farm-distracted-driver-detection/imgs/train'\ntest_dir = '../input/state-farm-distracted-driver-detection/imgs/test'\n\n\nclasses = ['c0','c1','c2','c3','c4','c5','c6','c7','c8','c9']","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:15:22.587496Z","iopub.execute_input":"2021-06-14T19:15:22.588033Z","iopub.status.idle":"2021-06-14T19:15:22.597784Z","shell.execute_reply.started":"2021-06-14T19:15:22.58799Z","shell.execute_reply":"2021-06-14T19:15:22.596877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\ntransform = transforms.Compose([\n   transforms.Resize((256, 256)),\n   transforms.RandomHorizontalFlip(),\n   transforms.RandomChoice([\n       transforms.ColorJitter(0.2, 0.2, 0.2, 0.2),\n       transforms.RandomResizedCrop(256),\n       transforms.RandomAffine(\n           degrees=10, translate=(0.2, 0.2),\n           scale=(0.8, 1.2), shear=15, resample=Image.BILINEAR)\n   ]),\n   transforms.ToTensor(),\n   transforms.Normalize(MEAN, STD),\n])\n# transform = transforms.Compose(\n#    [transforms.Resize((224, 224)),\n#     transforms.ToTensor(),\n#     transforms.Normalize((0.3143, 0.3804, 0.3732), (0.2872, 0.3306, 0.3319))])\n","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:25:29.243144Z","iopub.execute_input":"2021-06-14T19:25:29.243519Z","iopub.status.idle":"2021-06-14T19:25:29.252797Z","shell.execute_reply.started":"2021-06-14T19:25:29.243481Z","shell.execute_reply":"2021-06-14T19:25:29.251788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_EXTENSIONS = [ '.jpg', '.JPG', '.jpeg', '.JPEG', '.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP', ]\n\ndef is_image_file(filename):\n    return any(filename.endswith(extension) for extension in IMG_EXTENSIONS)\n\ndef find_classes(dir):\n    classes = [d for d in os.listdir(dir) if os.path.isdir(os.path.join(dir, d))]\n    classes.sort()\n    class_to_idx = {classes[i]: i for i in range(len(classes))}\n    return classes, class_to_idx\n\ndef make_dataset(dir, class_to_idx):\n    images = []\n    dir = os.path.expanduser(dir)\n    for target in sorted(os.listdir(dir)):\n        d = os.path.join(dir, target)\n        if not os.path.isdir(d):\n            continue\n        for root, _, fnames in sorted(os.walk(d)):\n            for fname in sorted(fnames):\n                if is_image_file(fname):\n                    path = os.path.join(root, fname)\n                    item = (path, class_to_idx[target])\n                    images.append(item)\n    return images\n\ndef make_testdataset(dir):\n    images = []\n    dir = os.path.expanduser(dir)\n    for root, _, fnames in sorted(os.walk(dir)):\n        for fname in sorted(fnames):\n            if is_image_file(fname):\n                path = os.path.join(root, fname)\n                item = (path, fname)\n                images.append(item)\n    return images\n\ndef pil_loader(path):\n    # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835)\n    with open(path, 'rb') as f:\n        with Image.open(f) as img:\n            return img.convert('RGB')\n        \ndef accimage_loader(path):\n    import accimage\n    try:\n        return accimage.Image(path)\n    except IOError:\n        # Potentially a decoding problem, fall back to PIL.Image\n        return pil_loader(path)\n    \ndef default_loader(path):\n    from torchvision import get_image_backend\n    if get_image_backend() == 'accimage':\n        return accimage_loader(path)\n    else:\n        return pil_loader(path)\n    \nclass trainImageFolder(Dataset):\n    \"\"\"A generic data loader where the images are arranged in this way: ::\n        root/dog/xxx.png\n        root/dog/xxy.png\n        root/dog/xxz.png\n\n        root/cat/123.png\n        root/cat/nsdf3.png\n        root/cat/asd932_.png\n    Args:\n        root (string): Root directory path.\n        transform (callable, optional): A function/transform that takes in an PIL image and returns a transformed version.\\\n            E.g, ``transforms.RandomCrop``\n        target_transform (callable, optional): A function/transform that takes in the target and transforms it.\n        loader (callable, optional): A function to load an image given its path.\n        Attributes: classes (list): List of the class names.\n        class_to_idx (dict): Dict with items (class_name, class_index).\n        imgs (list): List of (image path, class_index) tuples \n    \"\"\" \n    def __init__(self, root, transform=None, target_transform=None, loader=default_loader): \n        classes, class_to_idx = find_classes(root)\n        imgs = make_dataset(root, class_to_idx) \n        if len(imgs) == 0: \n            raise(RuntimeError(\"Found 0 images in subfolders of: \" + root + \"\\n\" \"Supported image extensions are: \" + \",\".join(IMG_EXTENSIONS))) \n        self.root = root \n        self.imgs = imgs \n        self.classes = classes \n        self.class_to_idx = class_to_idx \n        self.transform = transform \n        self.target_transform = target_transform \n        self.loader = loader \n        \n    def __getitem__(self, index): \n        \"\"\" Args: \n                index (int): Index \n            Returns: \n                tuple: (image, target) where target is class_index of the target class. \n        \"\"\" \n        path, target = self.imgs[index] \n        img = self.loader(path) \n        if self.transform is not None: \n            img = self.transform(img) \n        if self.target_transform is not None: \n            target = self.target_transform(target)\n        return img, target \n        \n    def __len__(self): \n        return len(self.imgs)\n\nclass testImageFolder(Dataset):\n    \"\"\"\n    Args:\n        root (string): Root directory path.\n        transform (callable, optional): A function/transform that takes in an PIL image and returns a transformed version.\\\n            E.g, ``transforms.RandomCrop``\n        target_transform (callable, optional): A function/transform that takes in the target and transforms it.\n        loader (callable, optional): A function to load an image given its path.\n        Attributes: classes (list): List of the class names.\n        class_to_idx (dict): Dict with items (class_name, class_index).\n        imgs (list): List of (image path, class_index) tuples \n    \"\"\" \n    def __init__(self, root, transform=None, loader=default_loader): \n        #classes, class_to_idx = find_classes(root)\n        imgs = make_testdataset(root) \n        if len(imgs) == 0: \n            raise(RuntimeError(\"Found 0 images in subfolders of: \" + root + \"\\n\" \"Supported image extensions are: \" + \",\".join(IMG_EXTENSIONS))) \n        self.root = root \n        self.imgs = imgs \n        #self.classes = classes \n        #self.class_to_idx = class_to_idx \n        self.transform = transform \n        #self.target_transform = target_transform \n        self.loader = loader \n        \n    def __getitem__(self, index): \n        \"\"\" Args: \n                    index (int): Index \n                Returns: \n                    tuple: (image, fname) where fname is image name.\n        \"\"\" \n        path, fname = self.imgs[index] \n        img = self.loader(path) \n        if self.transform is not None: \n            img = self.transform(img) \n        #if self.target_transform is not None: \n            #target = self.target_transform(target)\n        return img, fname\n        \n    def __len__(self): \n        return len(self.imgs)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:15:22.611259Z","iopub.execute_input":"2021-06-14T19:15:22.611743Z","iopub.status.idle":"2021-06-14T19:15:22.640501Z","shell.execute_reply.started":"2021-06-14T19:15:22.611687Z","shell.execute_reply":"2021-06-14T19:15:22.639173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = trainImageFolder(root=train_dir, transform=transform)\n#test_data = torchvision.datasets.ImageFolder(root=test_dir, transform=transform)\n\ntrain_size = int(0.8*len(train_data))\nval_size = len(train_data) - train_size\nprint(train_size, val_size)\n\ntrain_dataset, val_dataset = torch.utils.data.random_split(train_data, [train_size, val_size])\n\n\ntrain_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=2)\nval_dataloader= torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=2)\n#test_dataloader = torch.utils.data.DataLoader(test_data, batch_size=batch_size, shuffle=False, num_workers=2)\n\nprint(train_data.class_to_idx)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:15:22.644281Z","iopub.execute_input":"2021-06-14T19:15:22.644801Z","iopub.status.idle":"2021-06-14T19:15:27.814838Z","shell.execute_reply.started":"2021-06-14T19:15:22.644752Z","shell.execute_reply":"2021-06-14T19:15:27.81378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntest_dataset = testImageFolder(test_dir, transform=transform)\nprint(len(test_dataset))\ntest_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, \n                               num_workers=2)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:15:27.816712Z","iopub.execute_input":"2021-06-14T19:15:27.817363Z","iopub.status.idle":"2021-06-14T19:17:41.099335Z","shell.execute_reply.started":"2021-06-14T19:15:27.817316Z","shell.execute_reply":"2021-06-14T19:17:41.098066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def adaptive_pool_feat_mult(pool_type='avg'):\n    if pool_type == 'catavgmax':\n        return 2\n    else:\n        return 1\n\n\nclass SelectAdaptivePool2d(nn.Module):\n    \"\"\"Selectable global pooling layer with dynamic input kernel size\n    \"\"\"\n    def __init__(self, output_size=1, pool_type='fast', flatten=False):\n        super(SelectAdaptivePool2d, self).__init__()\n        self.pool_type = pool_type or ''  # convert other falsy values to empty string for consistent TS typing\n        self.flatten = flatten\n        if pool_type == '':\n            self.pool = nn.Identity()  # pass through\n        elif pool_type == 'fast':\n            assert output_size == 1\n            self.pool = FastAdaptiveAvgPool2d(self.flatten)\n            self.flatten = False\n        elif pool_type == 'avg':\n            self.pool = nn.AdaptiveAvgPool2d(output_size)\n        elif pool_type == 'avgmax':\n            self.pool = AdaptiveAvgMaxPool2d(output_size)\n        elif pool_type == 'catavgmax':\n            self.pool = AdaptiveCatAvgMaxPool2d(output_size)\n        elif pool_type == 'max':\n            self.pool = nn.AdaptiveMaxPool2d(output_size)\n        else:\n            assert False, 'Invalid pool type: %s' % pool_type\n\n    def is_identity(self):\n        return self.pool_type == ''\n\n    def forward(self, x):\n        x = self.pool(x)\n        if self.flatten:\n            x = x.flatten(1)\n        return x\n\n    def feat_mult(self):\n        return adaptive_pool_feat_mult(self.pool_type)\n\n    def __repr__(self):\n        return self.__class__.__name__ + ' (' \\\n               + 'pool_type=' + self.pool_type \\\n               + ', flatten=' + str(self.flatten) + ')'\n\n\n\ndef _create_pool(num_features, num_classes, pool_type='avg', use_conv=False):\n    flatten_in_pool = not use_conv  # flatten when we use a Linear layer after pooling\n    if not pool_type:\n        assert num_classes == 0 or use_conv,\\\n            'Pooling can only be disabled if classifier is also removed or conv classifier is used'\n        flatten_in_pool = False  # disable flattening if pooling is pass-through (no pooling)\n    global_pool = SelectAdaptivePool2d(pool_type=pool_type, flatten=flatten_in_pool)\n    num_pooled_features = num_features * global_pool.feat_mult()\n    return global_pool, num_pooled_features\n\n\ndef _create_fc(num_features, num_classes, pool_type='avg', use_conv=False):\n    if num_classes <= 0:\n        fc = nn.Identity()  # pass-through (no classifier)\n    elif use_conv:\n        fc = nn.Conv2d(num_features, num_classes, 1, bias=True)\n    else:\n        # NOTE: using my Linear wrapper that fixes AMP + torchscript casting issue\n        fc = Linear(num_features, num_classes, bias=True)\n    return fc\n\n\ndef create_classifier(num_features, num_classes, pool_type='avg', use_conv=False):\n    global_pool, num_pooled_features = _create_pool(num_features, num_classes, pool_type, use_conv=use_conv)\n    fc = _create_fc(num_pooled_features, num_classes, use_conv=use_conv)\n    return global_pool, fc\n\n\nclass DlaBasic(nn.Module):\n    \"\"\"DLA Basic\"\"\"\n\n    def __init__(self, inplanes, planes, stride=1, dilation=1, **_):\n        super(DlaBasic, self).__init__()\n        self.conv1 = nn.Conv2d(\n            inplanes, planes, kernel_size=3, stride=stride, padding=dilation, bias=False, dilation=dilation)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = nn.Conv2d(\n            planes, planes, kernel_size=3, stride=1, padding=dilation, bias=False, dilation=dilation)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.stride = stride\n\n    def forward(self, x, shortcut=None):\n        if shortcut is None:\n            shortcut = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n\n        out += shortcut\n        out = self.relu(out)\n\n        return out\n\n\nclass DlaBottleneck(nn.Module):\n    \"\"\"DLA/DLA-X Bottleneck\"\"\"\n    expansion = 2\n\n    def __init__(self, inplanes, outplanes, stride=1, dilation=1, cardinality=1, base_width=64):\n        super(DlaBottleneck, self).__init__()\n        self.stride = stride\n        mid_planes = int(math.floor(outplanes * (base_width / 64)) * cardinality)\n        mid_planes = mid_planes // self.expansion\n\n        self.conv1 = nn.Conv2d(inplanes, mid_planes, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(mid_planes)\n        self.conv2 = nn.Conv2d(\n            mid_planes, mid_planes, kernel_size=3, stride=stride, padding=dilation,\n            bias=False, dilation=dilation, groups=cardinality)\n        self.bn2 = nn.BatchNorm2d(mid_planes)\n        self.conv3 = nn.Conv2d(mid_planes, outplanes, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(outplanes)\n        self.relu = nn.ReLU(inplace=True)\n\n    def forward(self, x, shortcut=None):\n        if shortcut is None:\n            shortcut = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        out += shortcut\n        out = self.relu(out)\n\n        return out\n\n\nclass DlaBottle2neck(nn.Module):\n    \"\"\" Res2Net/Res2NeXT DLA Bottleneck\n    Adapted from https://github.com/gasvn/Res2Net/blob/master/dla.py\n    \"\"\"\n    expansion = 2\n\n    def __init__(self, inplanes, outplanes, stride=1, dilation=1, scale=4, cardinality=8, base_width=4):\n        super(DlaBottle2neck, self).__init__()\n        self.is_first = stride > 1\n        self.scale = scale\n        mid_planes = int(math.floor(outplanes * (base_width / 64)) * cardinality)\n        mid_planes = mid_planes // self.expansion\n        self.width = mid_planes\n\n        self.conv1 = nn.Conv2d(inplanes, mid_planes * scale, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(mid_planes * scale)\n\n        num_scale_convs = max(1, scale - 1)\n        convs = []\n        bns = []\n        for _ in range(num_scale_convs):\n            convs.append(nn.Conv2d(\n                mid_planes, mid_planes, kernel_size=3, stride=stride,\n                padding=dilation, dilation=dilation, groups=cardinality, bias=False))\n            bns.append(nn.BatchNorm2d(mid_planes))\n        self.convs = nn.ModuleList(convs)\n        self.bns = nn.ModuleList(bns)\n        if self.is_first:\n            self.pool = nn.AvgPool2d(kernel_size=3, stride=stride, padding=1)\n\n        self.conv3 = nn.Conv2d(mid_planes * scale, outplanes, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(outplanes)\n        self.relu = nn.ReLU(inplace=True)\n\n    def forward(self, x, shortcut=None):\n        if shortcut is None:\n            shortcut = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        spx = torch.split(out, self.width, 1)\n        spo = []\n        for i, (conv, bn) in enumerate(zip(self.convs, self.bns)):\n            sp = spx[i] if i == 0 or self.is_first else sp + spx[i]\n            sp = conv(sp)\n            sp = bn(sp)\n            sp = self.relu(sp)\n            spo.append(sp)\n        if self.scale > 1:\n            spo.append(self.pool(spx[-1]) if self.is_first else spx[-1])\n        out = torch.cat(spo, 1)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        out += shortcut\n        out = self.relu(out)\n\n        return out\n\n\nclass DlaRoot(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size, shortcut):\n        super(DlaRoot, self).__init__()\n        self.conv = nn.Conv2d(\n            in_channels, out_channels, 1, stride=1, bias=False, padding=(kernel_size - 1) // 2)\n        self.bn = nn.BatchNorm2d(out_channels)\n        self.relu = nn.ReLU(inplace=True)\n        self.shortcut = shortcut\n\n    def forward(self, *x):\n        children = x\n        x = self.conv(torch.cat(x, 1))\n        x = self.bn(x)\n        if self.shortcut:\n            x += children[0]\n        x = self.relu(x)\n\n        return x\n\n\nclass DlaTree(nn.Module):\n    def __init__(self, levels, block, in_channels, out_channels, stride=1,\n                 dilation=1, cardinality=1, base_width=64,\n                 level_root=False, root_dim=0, root_kernel_size=1, root_shortcut=False):\n        super(DlaTree, self).__init__()\n        if root_dim == 0:\n            root_dim = 2 * out_channels\n        if level_root:\n            root_dim += in_channels\n        self.downsample = nn.MaxPool2d(stride, stride=stride) if stride > 1 else nn.Identity()\n        self.project = nn.Identity()\n        cargs = dict(dilation=dilation, cardinality=cardinality, base_width=base_width)\n        if levels == 1:\n            self.tree1 = block(in_channels, out_channels, stride, **cargs)\n            self.tree2 = block(out_channels, out_channels, 1, **cargs)\n            if in_channels != out_channels:\n                # NOTE the official impl/weights have  project layers in levels > 1 case that are never\n                # used, I've moved the project layer here to avoid wasted params but old checkpoints will\n                # need strict=False while loading.\n                self.project = nn.Sequential(\n                    nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, bias=False),\n                    nn.BatchNorm2d(out_channels))\n        else:\n            cargs.update(dict(root_kernel_size=root_kernel_size, root_shortcut=root_shortcut))\n            self.tree1 = DlaTree(\n                levels - 1, block, in_channels, out_channels, stride, root_dim=0, **cargs)\n            self.tree2 = DlaTree(\n                levels - 1, block, out_channels, out_channels, root_dim=root_dim + out_channels, **cargs)\n        if levels == 1:\n            self.root = DlaRoot(root_dim, out_channels, root_kernel_size, root_shortcut)\n        self.level_root = level_root\n        self.root_dim = root_dim\n        self.levels = levels\n\n    def forward(self, x, shortcut=None, children=None):\n        children = [] if children is None else children\n        bottom = self.downsample(x)\n        shortcut = self.project(bottom)\n        if self.level_root:\n            children.append(bottom)\n        x1 = self.tree1(x, shortcut)\n        if self.levels == 1:\n            x2 = self.tree2(x1)\n            x = self.root(x2, x1, *children)\n        else:\n            children.append(x1)\n            x = self.tree2(x1, children=children)\n        return x\n\n\nclass DLA(nn.Module):\n    def __init__(self, levels, channels, output_stride=32, num_classes=10, in_chans=3,\n                 cardinality=1, base_width=64, block=DlaBottle2neck, shortcut_root=False,\n                 drop_rate=0.0, global_pool='avg'):\n        super(DLA, self).__init__()\n        self.channels = channels\n        self.num_classes = num_classes\n        self.cardinality = cardinality\n        self.base_width = base_width\n        self.drop_rate = drop_rate\n        assert output_stride == 32  # FIXME support dilation\n\n        self.base_layer = nn.Sequential(\n            nn.Conv2d(in_chans, channels[0], kernel_size=7, stride=1, padding=3, bias=False),\n            nn.BatchNorm2d(channels[0]),\n            nn.ReLU(inplace=True))\n        self.level0 = self._make_conv_level(channels[0], channels[0], levels[0])\n        self.level1 = self._make_conv_level(channels[0], channels[1], levels[1], stride=2)\n        cargs = dict(cardinality=cardinality, base_width=base_width, root_shortcut=shortcut_root)\n        self.level2 = DlaTree(levels[2], block, channels[1], channels[2], 2, level_root=False, **cargs)\n        self.level3 = DlaTree(levels[3], block, channels[2], channels[3], 2, level_root=True, **cargs)\n        self.level4 = DlaTree(levels[4], block, channels[3], channels[4], 2, level_root=True, **cargs)\n        self.level5 = DlaTree(levels[5], block, channels[4], channels[5], 2, level_root=True, **cargs)\n        self.feature_info = [\n            dict(num_chs=channels[0], reduction=1, module='level0'),  # rare to have a meaningful stride 1 level\n            dict(num_chs=channels[1], reduction=2, module='level1'),\n            dict(num_chs=channels[2], reduction=4, module='level2'),\n            dict(num_chs=channels[3], reduction=8, module='level3'),\n            dict(num_chs=channels[4], reduction=16, module='level4'),\n            dict(num_chs=channels[5], reduction=32, module='level5'),\n        ]\n\n        self.num_features = channels[-1]\n        self.global_pool, self.fc = create_classifier(\n            self.num_features, self.num_classes, pool_type=global_pool, use_conv=True)\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n                m.weight.data.normal_(0, math.sqrt(2. / n))\n            elif isinstance(m, nn.BatchNorm2d):\n                m.weight.data.fill_(1)\n                m.bias.data.zero_()\n\n    def _make_conv_level(self, inplanes, planes, convs, stride=1, dilation=1):\n        modules = []\n        for i in range(convs):\n            modules.extend([\n                nn.Conv2d(inplanes, planes, kernel_size=3, stride=stride if i == 0 else 1,\n                          padding=dilation, bias=False, dilation=dilation),\n                nn.BatchNorm2d(planes),\n                nn.ReLU(inplace=True)])\n            inplanes = planes\n        return nn.Sequential(*modules)\n\n    def get_classifier(self):\n        return self.fc\n\n    def reset_classifier(self, num_classes, global_pool='avg'):\n        self.num_classes = num_classes\n        self.global_pool, self.fc = create_classifier(\n            self.num_features, self.num_classes, pool_type=global_pool, use_conv=True)\n\n    def forward_features(self, x):\n        x = self.base_layer(x)\n        x = self.level0(x)\n        x = self.level1(x)\n        x = self.level2(x)\n        x = self.level3(x)\n        x = self.level4(x)\n        x = self.level5(x)\n        return x\n\n    def forward(self, x):\n        x = self.forward_features(x)\n        x = self.global_pool(x)\n        if self.drop_rate > 0.:\n            x = F.dropout(x, p=self.drop_rate, training=self.training)\n        x = self.fc(x)\n        if not self.global_pool.is_identity():\n            x = x.flatten(1)  # conv classifier, flatten if pooling isn't pass-through (disabled)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:17:41.101119Z","iopub.execute_input":"2021-06-14T19:17:41.101522Z","iopub.status.idle":"2021-06-14T19:17:41.172359Z","shell.execute_reply.started":"2021-06-14T19:17:41.101482Z","shell.execute_reply":"2021-06-14T19:17:41.171434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nfrom typing import Any, Callable, Optional, Tuple\n\ndef build_model_with_cfg(\n        model_cls: Callable,\n        variant: str,\n        pretrained: bool,\n        default_cfg: dict,\n        model_cfg: Optional[Any] = None,\n        feature_cfg: Optional[dict] = None,\n        pretrained_strict: bool = True,\n        pretrained_filter_fn: Optional[Callable] = None,\n        pretrained_custom_load: bool = False,\n        kwargs_filter: Optional[Tuple[str]] = None,\n        **kwargs):\n    \"\"\" Build model with specified default_cfg and optional model_cfg\n    This helper fn aids in the construction of a model including:\n      * handling default_cfg and associated pretained weight loading\n      * passing through optional model_cfg for models with config based arch spec\n      * features_only model adaptation\n      * pruning config / model adaptation\n    Args:\n        model_cls (nn.Module): model class\n        variant (str): model variant name\n        pretrained (bool): load pretrained weights\n        default_cfg (dict): model's default pretrained/task config\n        model_cfg (Optional[Dict]): model's architecture config\n        feature_cfg (Optional[Dict]: feature extraction adapter config\n        pretrained_strict (bool): load pretrained weights strictly\n        pretrained_filter_fn (Optional[Callable]): filter callable for pretrained weights\n        pretrained_custom_load (bool): use custom load fn, to load numpy or other non PyTorch weights\n        kwargs_filter (Optional[Tuple]): kwargs to filter before passing to model\n        **kwargs: model args passed through to model __init__\n    \"\"\"\n    pruned = kwargs.pop('pruned', False)\n    features = False\n    feature_cfg = feature_cfg or {}\n    default_cfg = deepcopy(default_cfg) if default_cfg else {}\n    update_default_cfg_and_kwargs(default_cfg, kwargs, kwargs_filter)\n    default_cfg.setdefault('architecture', variant)\n\n    # Setup for feature extraction wrapper done at end of this fn\n    if kwargs.pop('features_only', False):\n        features = True\n        feature_cfg.setdefault('out_indices', (0, 1, 2, 3, 4))\n        if 'out_indices' in kwargs:\n            feature_cfg['out_indices'] = kwargs.pop('out_indices')\n\n    # Build the model\n    model = model_cls(**kwargs) if model_cfg is None else model_cls(cfg=model_cfg, **kwargs)\n    model.default_cfg = default_cfg\n    \n    if pruned:\n        model = adapt_model_from_file(model, variant)\n\n    # For classification models, check class attr, then kwargs, then default to 1k, otherwise 0 for feats\n    num_classes_pretrained = 0 if features else getattr(model, 'num_classes', kwargs.get('num_classes', 1000))\n    if pretrained:\n        if pretrained_custom_load:\n            load_custom_pretrained(model)\n        else:\n            load_pretrained(\n                model,\n                num_classes=num_classes_pretrained,\n                in_chans=kwargs.get('in_chans', 3),\n                filter_fn=pretrained_filter_fn,\n                strict=pretrained_strict)\n\n    # Wrap the model in a feature extraction module if enabled\n    if features:\n        feature_cls = FeatureListNet\n        if 'feature_cls' in feature_cfg:\n            feature_cls = feature_cfg.pop('feature_cls')\n            if isinstance(feature_cls, str):\n                feature_cls = feature_cls.lower()\n                if 'hook' in feature_cls:\n                    feature_cls = FeatureHookNet\n                else:\n                    assert False, f'Unknown feature class {feature_cls}'\n        model = feature_cls(model, **feature_cfg)\n        model.default_cfg = default_cfg_for_features(default_cfg)  # add back default_cfg\n    \n    return model\n\ndef register_model(fn):\n    # lookup containing module\n    mod = sys.modules[fn.__module__]\n    module_name_split = fn.__module__.split('.')\n    module_name = module_name_split[-1] if len(module_name_split) else ''\n\n    # add model to __all__ in module\n    model_name = fn.__name__\n    if hasattr(mod, '__all__'):\n        mod.__all__.append(model_name)\n    else:\n        mod.__all__ = [model_name]\n\n    # add entries to registry dict/sets\n    _model_entrypoints[model_name] = fn\n    _model_to_module[model_name] = module_name\n    _module_to_models[module_name].add(model_name)\n    has_pretrained = False  # check if model has a pretrained url to allow filtering on this\n    if hasattr(mod, 'default_cfgs') and model_name in mod.default_cfgs:\n        # this will catch all models that have entrypoint matching cfg key, but miss any aliasing\n        # entrypoints or non-matching combos\n        has_pretrained = 'url' in mod.default_cfgs[model_name] and 'http' in mod.default_cfgs[model_name]['url']\n        _model_default_cfgs[model_name] = deepcopy(mod.default_cfgs[model_name])\n    if has_pretrained:\n        _model_has_pretrained.add(model_name)\n    return fn\n    '''","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:23:00.492578Z","iopub.execute_input":"2021-06-14T19:23:00.492923Z","iopub.status.idle":"2021-06-14T19:23:00.512288Z","shell.execute_reply.started":"2021-06-14T19:23:00.49289Z","shell.execute_reply":"2021-06-14T19:23:00.511326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n__all__ = ['DLA']\n#from timm.data import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD\n\ndef _cfg(url='', **kwargs):\n    return {\n        'url': url,\n        'num_classes': 1000, 'input_size': (3, 224, 224), 'pool_size': (7, 7),\n        'crop_pct': 0.875, 'interpolation': 'bilinear',\n        'mean': MEAN, 'std': STD,\n        'first_conv': 'base_layer.0', 'classifier': 'fc',\n        **kwargs\n    }\n\n\ndefault_cfgs = {\n    'dla34': _cfg(url='http://dl.yf.io/dla/models/imagenet/dla34-ba72cf86.pth'),\n    'dla46_c': _cfg(url='http://dl.yf.io/dla/models/imagenet/dla46_c-2bfd52c3.pth'),\n    'dla46x_c': _cfg(url='http://dl.yf.io/dla/models/imagenet/dla46x_c-d761bae7.pth'),\n    'dla60x_c': _cfg(url='http://dl.yf.io/dla/models/imagenet/dla60x_c-b870c45c.pth'),\n    'dla60': _cfg(url='http://dl.yf.io/dla/models/imagenet/dla60-24839fc4.pth'),\n    'dla60x': _cfg(url='http://dl.yf.io/dla/models/imagenet/dla60x-d15cacda.pth'),\n    'dla102': _cfg(url='http://dl.yf.io/dla/models/imagenet/dla102-d94d9790.pth'),\n    'dla102x': _cfg(url='http://dl.yf.io/dla/models/imagenet/dla102x-ad62be81.pth'),\n    'dla102x2': _cfg(url='http://dl.yf.io/dla/models/imagenet/dla102x2-262837b6.pth'),\n    'dla169': _cfg(url='http://dl.yf.io/dla/models/imagenet/dla169-0914e092.pth'),\n    'dla60_res2net': _cfg(\n        url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-res2net/res2net_dla60_4s-d88db7f9.pth'),\n    'dla60_res2next': _cfg(\n        url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-res2net/res2next_dla60_4s-d327927b.pth'),\n}\n'''","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:25:42.224075Z","iopub.execute_input":"2021-06-14T19:25:42.224463Z","iopub.status.idle":"2021-06-14T19:25:42.234867Z","shell.execute_reply.started":"2021-06-14T19:25:42.224421Z","shell.execute_reply":"2021-06-14T19:25:42.233746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nimport sys\nfrom collections import defaultdict\nfrom copy import deepcopy\n#__all__ = ['list_models', 'is_model', 'model_entrypoint', 'list_modules', 'is_model_in_modules',\n#           'is_model_default_key', 'has_model_default_key', 'get_model_default_value', 'is_model_pretrained']\n\n_module_to_models = defaultdict(set)  # dict of sets to check membership of model in module\n_model_to_module = {}  # mapping of model names to module names\n_model_entrypoints = {}  # mapping of model names to entrypoint fns\n_model_has_pretrained = set()  # set of model names that have pretrained weight url present\n_model_default_cfgs = dict()  # central repo for model default_cfgs\n\ndef _create_dla(variant, pretrained=False, **kwargs):\n    return build_model_with_cfg(\n        DLA, variant, pretrained,\n        default_cfg=default_cfgs[variant],\n        pretrained_strict=False,\n        feature_cfg=dict(out_indices=(1, 2, 3, 4, 5)),\n        **kwargs)\n\n\n@register_model\ndef dla60_res2net(pretrained=False, **kwargs):\n    model_kwargs = dict(\n        levels=(1, 1, 1, 2, 3, 1), channels=(16, 32, 128, 256, 512, 1024),\n        block=DlaBottle2neck, cardinality=1, base_width=28, **kwargs)\n    return _create_dla('dla60_res2net', pretrained, **model_kwargs)\n\n\n@register_model\ndef dla60_res2next(pretrained=False,**kwargs):\n    model_kwargs = dict(\n        levels=(1, 1, 1, 2, 3, 1), channels=(16, 32, 128, 256, 512, 1024),\n        block=DlaBottle2neck, cardinality=8, base_width=4, **kwargs)\n    return _create_dla('dla60_res2next', pretrained, **model_kwargs)\n\n\n@register_model\ndef dla34(pretrained=False, **kwargs):  # DLA-34\n    model_kwargs = dict(\n        levels=[1, 1, 1, 2, 2, 1], channels=[16, 32, 64, 128, 256, 512],\n        block=DlaBasic, **kwargs)\n    return _create_dla('dla34', pretrained, **model_kwargs)\n\n\n@register_model\ndef dla46_c(pretrained=False, **kwargs):  # DLA-46-C\n    model_kwargs = dict(\n        levels=[1, 1, 1, 2, 2, 1], channels=[16, 32, 64, 64, 128, 256],\n        block=DlaBottleneck, **kwargs)\n    return _create_dla('dla46_c', pretrained, **model_kwargs)\n\n\n@register_model\ndef dla46x_c(pretrained=False, **kwargs):  # DLA-X-46-C\n    model_kwargs = dict(\n        levels=[1, 1, 1, 2, 2, 1], channels=[16, 32, 64, 64, 128, 256],\n        block=DlaBottleneck, cardinality=32, base_width=4, **kwargs)\n    return _create_dla('dla46x_c', pretrained, **model_kwargs)\n\n\n@register_model\ndef dla60x_c(pretrained=False, **kwargs):  # DLA-X-60-C\n    model_kwargs = dict(\n        levels=[1, 1, 1, 2, 3, 1], channels=[16, 32, 64, 64, 128, 256],\n        block=DlaBottleneck, cardinality=32, base_width=4, **kwargs)\n    return _create_dla('dla60x_c', pretrained, **model_kwargs)\n\n\n@register_model\ndef dla60(pretrained=False, **kwargs):  # DLA-60\n    model_kwargs = dict(\n        levels=[1, 1, 1, 2, 3, 1], channels=[16, 32, 128, 256, 512, 1024],\n        block=DlaBottleneck, **kwargs)\n    return _create_dla('dla60', pretrained, **model_kwargs)\n\n\n@register_model\ndef dla60x(pretrained=False, **kwargs):  # DLA-X-60\n    model_kwargs = dict(\n        levels=[1, 1, 1, 2, 3, 1], channels=[16, 32, 128, 256, 512, 1024],\n        block=DlaBottleneck, cardinality=32, base_width=4, **kwargs)\n    return _create_dla('dla60x', pretrained, **model_kwargs)\n\n\n@register_model\ndef dla102(pretrained=False, **kwargs):  # DLA-102\n    model_kwargs = dict(\n        levels=[1, 1, 1, 3, 4, 1], channels=[16, 32, 128, 256, 512, 1024],\n        block=DlaBottleneck, shortcut_root=True, **kwargs)\n    return _create_dla('dla102', pretrained, **model_kwargs)\n\n\n@register_model\ndef dla102x(pretrained=False, **kwargs):  # DLA-X-102\n    model_kwargs = dict(\n        levels=[1, 1, 1, 3, 4, 1], channels=[16, 32, 128, 256, 512, 1024],\n        block=DlaBottleneck, cardinality=32, base_width=4, shortcut_root=True, **kwargs)\n    return _create_dla('dla102x', pretrained, **model_kwargs)\n\n\n@register_model\ndef dla102x2(pretrained=False, **kwargs):  # DLA-X-102 64\n    model_kwargs = dict(\n        levels=[1, 1, 1, 3, 4, 1], channels=[16, 32, 128, 256, 512, 1024],\n        block=DlaBottleneck, cardinality=64, base_width=4, shortcut_root=True, **kwargs)\n    return _create_dla('dla102x2', pretrained, **model_kwargs)\n\n\n@register_model\ndef dla169(pretrained=False, **kwargs):  # DLA-169\n    model_kwargs = dict(\n        levels=[1, 1, 2, 3, 5, 1], channels=[16, 32, 128, 256, 512, 1024],\n        block=DlaBottleneck, shortcut_root=True, **kwargs)\n    return _create_dla('dla169', pretrained, **model_kwargs)\n'''","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:41:14.435659Z","iopub.execute_input":"2021-06-14T19:41:14.436057Z","iopub.status.idle":"2021-06-14T19:41:14.4678Z","shell.execute_reply.started":"2021-06-14T19:41:14.436023Z","shell.execute_reply":"2021-06-14T19:41:14.466579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nimport logging\n_logger = logging.getLogger(__name__)\ndef overlay_external_default_cfg(default_cfg, kwargs):\n    \"\"\" Overlay 'external_default_cfg' in kwargs on top of default_cfg arg.\n    \"\"\"\n    external_default_cfg = kwargs.pop('external_default_cfg', None)\n    if external_default_cfg:\n        default_cfg.pop('url', None)  # url should come from external cfg\n        default_cfg.pop('hf_hub', None)  # hf hub id should come from external cfg\n        default_cfg.update(external_default_cfg)\n        \ndef update_default_cfg_and_kwargs(default_cfg, kwargs, kwargs_filter):\n    \"\"\" Update the default_cfg and kwargs before passing to model\n    FIXME this sequence of overlay default_cfg, set default kwargs, filter kwargs\n    could/should be replaced by an improved configuration mechanism\n    Args:\n        default_cfg: input default_cfg (updated in-place)\n        kwargs: keyword args passed to model build fn (updated in-place)\n        kwargs_filter: keyword arg keys that must be removed before model __init__\n    \"\"\"\n    # Overlay default cfg values from `external_default_cfg` if it exists in kwargs\n    overlay_external_default_cfg(default_cfg, kwargs)\n    # Set model __init__ args that can be determined by default_cfg (if not already passed as kwargs)\n    default_kwarg_names = ('num_classes', 'global_pool', 'in_chans')\n    if default_cfg.get('fixed_input_size', False):\n        # if fixed_input_size exists and is True, model takes an img_size arg that fixes its input size\n        default_kwarg_names += ('img_size',)\n    set_default_kwargs(kwargs, names=default_kwarg_names, default_cfg=default_cfg)\n    # Filter keyword args for task specific model variants (some 'features only' models, etc.)\n    filter_kwargs(kwargs, names=kwargs_filter)\n    \ndef set_default_kwargs(kwargs, names, default_cfg):\n    for n in names:\n        # for legacy reasons, model __init__args uses img_size + in_chans as separate args while\n        # default_cfg has one input_size=(C, H ,W) entry\n        if n == 'img_size':\n            input_size = default_cfg.get('input_size', None)\n            if input_size is not None:\n                assert len(input_size) == 3\n                kwargs.setdefault(n, input_size[-2:])\n        elif n == 'in_chans':\n            input_size = default_cfg.get('input_size', None)\n            if input_size is not None:\n                assert len(input_size) == 3\n                kwargs.setdefault(n, input_size[0])\n        else:\n            default_val = default_cfg.get(n, None)\n            if default_val is not None:\n                kwargs.setdefault(n, default_cfg[n])\n\ndef filter_kwargs(kwargs, names):\n    if not kwargs or not names:\n        return\n    for n in names:\n        kwargs.pop(n, None)\n\ndef load_pretrained(model, default_cfg=None, num_classes=1000, in_chans=3, filter_fn=None, strict=True, progress=False):\n    \"\"\" Load pretrained checkpoint\n    Args:\n        model (nn.Module) : PyTorch model module\n        default_cfg (Optional[Dict]): default configuration for pretrained weights / target dataset\n        num_classes (int): num_classes for model\n        in_chans (int): in_chans for model\n        filter_fn (Optional[Callable]): state_dict filter fn for load (takes state_dict, model as args)\n        strict (bool): strict load of checkpoint\n        progress (bool): enable progress bar for weight download\n    \"\"\"\n    default_cfg = default_cfg or getattr(model, 'default_cfg', None) or {}\n    pretrained_url = default_cfg.get('url', None)\n    hf_hub_id = default_cfg.get('hf_hub', None)\n    if not pretrained_url and not hf_hub_id:\n        _logger.warning(\"No pretrained weights exist for this model. Using random initialization.\")\n        return\n    if hf_hub_id and has_hf_hub(necessary=not pretrained_url):\n        _logger.info(f'Loading pretrained weights from Hugging Face hub ({hf_hub_id})')\n        state_dict = load_state_dict_from_hf(hf_hub_id)\n    else:\n        _logger.info(f'Loading pretrained weights from url ({pretrained_url})')\n        state_dict = load_state_dict_from_url(pretrained_url, progress=progress, map_location='cpu')\n    if filter_fn is not None:\n        # for backwards compat with filter fn that take one arg, try one first, the two\n        try:\n            state_dict = filter_fn(state_dict)\n        except TypeError:\n            state_dict = filter_fn(state_dict, model)\n\n    input_convs = default_cfg.get('first_conv', None)\n    if input_convs is not None and in_chans != 3:\n        if isinstance(input_convs, str):\n            input_convs = (input_convs,)\n        for input_conv_name in input_convs:\n            weight_name = input_conv_name + '.weight'\n            try:\n                state_dict[weight_name] = adapt_input_conv(in_chans, state_dict[weight_name])\n                _logger.info(\n                    f'Converted input conv {input_conv_name} pretrained weights from 3 to {in_chans} channel(s)')\n            except NotImplementedError as e:\n                del state_dict[weight_name]\n                strict = False\n                _logger.warning(\n                    f'Unable to convert pretrained {input_conv_name} weights, using random init for this layer.')\n\n    classifiers = default_cfg.get('classifier', None)\n    label_offset = default_cfg.get('label_offset', 0)\n    if classifiers is not None:\n        if isinstance(classifiers, str):\n            classifiers = (classifiers,)\n        if num_classes != default_cfg['num_classes']:\n            for classifier_name in classifiers:\n                # completely discard fully connected if model num_classes doesn't match pretrained weights\n                del state_dict[classifier_name + '.weight']\n                del state_dict[classifier_name + '.bias']\n            strict = False\n        elif label_offset > 0:\n            for classifier_name in classifiers:\n                # special case for pretrained weights with an extra background class in pretrained weights\n                classifier_weight = state_dict[classifier_name + '.weight']\n                state_dict[classifier_name + '.weight'] = classifier_weight[label_offset:]\n                classifier_bias = state_dict[classifier_name + '.bias']\n                state_dict[classifier_name + '.bias'] = classifier_bias[label_offset:]\n\n    model.load_state_dict(state_dict, strict=strict)\n    '''","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:41:20.101472Z","iopub.execute_input":"2021-06-14T19:41:20.101851Z","iopub.status.idle":"2021-06-14T19:41:20.12534Z","shell.execute_reply.started":"2021-06-14T19:41:20.101818Z","shell.execute_reply":"2021-06-14T19:41:20.124306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from torch.hub import load_state_dict_from_url, download_url_to_file, urlparse, HASH_REGEX","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:41:23.533457Z","iopub.execute_input":"2021-06-14T19:41:23.533835Z","iopub.status.idle":"2021-06-14T19:41:23.538432Z","shell.execute_reply.started":"2021-06-14T19:41:23.533801Z","shell.execute_reply":"2021-06-14T19:41:23.537054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_classifier = DLA(levels=(1, 1, 1, 2, 3, 1), channels=(16, 32, 64, 128, 256, 512), block=DlaBottleneck, cardinality=1, base_width=28, drop_rate=0.15)\n#my_classifier = dla34(pretrained=True)\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nmy_classifier = my_classifier.to(device)\n\n# Print your neural network structure\n#print(my_classifier)\n\noptimizer = optim.Adam(my_classifier.parameters(), lr=learning_rate, weight_decay=0.0001)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:41:24.78816Z","iopub.execute_input":"2021-06-14T19:41:24.788552Z","iopub.status.idle":"2021-06-14T19:41:45.208931Z","shell.execute_reply.started":"2021-06-14T19:41:24.788516Z","shell.execute_reply":"2021-06-14T19:41:45.206336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"ckpt_dir = os.path.join('../output/kaggle/working', 'checkpoints')\nif not os.path.exists(ckpt_dir):\n  os.makedirs(ckpt_dir)\n  \nbest_acc = 0.\nckpt_path = os.path.join(ckpt_dir, 'lastest.pt')\nif os.path.exists(ckpt_path):\n  ckpt = torch.load(ckpt_path)\n  try:\n    my_classifier.load_state_dict(ckpt['my_classifier'])\n    optimizer.load_state_dict(ckpt['optimizer'])\n    best_acc = ckpt['best_acc']\n  except RuntimeError as e:\n      print('wrong checkpoint')\n  else:    \n    print('checkpoint is loaded !')\n    print('current best accuracy : %.2f' % best_acc)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:17:41.314842Z","iopub.status.idle":"2021-06-14T19:17:41.315698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\n# Basic settings\nname='main'\nckpt_dir='ckpts'\nckpt_reload='10'\ngpu=True\nlog_dir='logs'\nlog_iter = 100\n\nresult_dir = Path('../output/kaggle/working') / 'Rdla60_res2next_lr0.001_bs64' / name\nckpt_dir = result_dir / ckpt_dir\nckpt_dir.mkdir(parents=True, exist_ok=True)\nlog_dir = result_dir / log_dir\nlog_dir.mkdir(parents=True, exist_ok=True)\n\ndevice = 'cuda' if torch.cuda.is_available() and gpu else 'cpu'\n","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:17:41.316891Z","iopub.status.idle":"2021-06-14T19:17:41.317571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.tensorboard import SummaryWriter \nfrom PIL import Image\nwriter = SummaryWriter(log_dir)\n\nif training_process:\n  it = 0\n  train_losses = []\n  val_losses = []\n  train_accs = []\n  val_accs = []\n  for epoch in range(max_epoch):\n    # train phase\n    my_classifier.train()\n    for inputs, labels in train_dataloader:\n      it += 1\n\n      # load data to the GPU.\n      #P8.1. Send 'inputs' and 'labels' to either cpu or gpu using 'device' variable\n      inputs = inputs.to(device)\n      labels = labels.to(device)\n\n\n      # feed data into the network and get outputs.\n      # P8.2. Feed `inputs` into the network, get an output, and keep it in a variable called `logit`. \n      logits = my_classifier(inputs)\n\n\n      # calculate loss\n      # Note: `F.cross_entropy` function receives logits, or pre-softmax outputs, rather than final probability scores.\n      # P8.3. Compute loss using `logit` and `labels`, and keep it in a variable called `loss` \n      loss =  F.cross_entropy(logits, labels)\n\n\n      # Note: You should flush out gradients computed at the previous step before computing gradients at the current step. \n      #       Otherwise, gradients will accumulate.\n      # P8.4. flush out the previously computed gradient \n      optimizer.zero_grad()\n\n\n      # backprogate loss.\n      # P8.5. backward the computed loss. \n      loss.backward()\n\n\n      # P8.6. update the network weights. \n      optimizer.step()\n\n\n      # calculate accuracy.\n      acc = (logits.argmax(dim=1) == labels).float().mean()\n\n      if it % 2000 == 0 and writer is not None:\n          # P8.7. Log `loss` with a tag name 'train_loss' using `writer`. Use `global_step` as a timestamp for the log. \n          # writer.writer_your_code_here (one-liner).\n          writer.add_scalar('train_loss', loss, global_step=epoch)\n\n          # P8.8. Log `accuracy` with a tag name 'train_accuracy' using `writer`. Use `global_step` as a timestamp for the log. \n          # writer.writer_your_code_here (one-liner).\n          writer.add_scalar('train_accuracy', acc, global_step=epoch)\n\n          print('[epoch:{}, iteration:{}] train loss : {:.4f} train accuracy : {:.4f}'.format(epoch+1, it, loss.item(), acc.item()))\n\n    # save losses in a list so that we can visualize them later.\n    train_losses.append(loss)  \n    train_accs.append(acc)\n\n    \n    # test phase\n    n = 0.\n    val_loss = 0.\n    val_acc = 0.\n\n    my_classifier.eval()\n    for val_inputs, val_labels in val_dataloader:\n      #P8.9. Send 'inputs' and 'labels' to either cpu or gpu using 'device' variable\n      #test_inputs = write your code here (one-liner).\n      #test_labels = write your code here (one-liner).\n      val_inputs = val_inputs.to(device)\n      val_labels = val_labels.to(device)\n\n\n      # P8.10. Feed `inputs` into the network, get an output, and keep it in a variable called `logit`. \n      # logits = write your code here (one-liner).\n      logits = my_classifier(val_inputs)\n\n\n      # Yes, for your convenience.\n      val_loss += F.cross_entropy(logits, val_labels, reduction='sum').item()\n      val_acc += (logits.argmax(dim=1) == val_labels).float().sum().item()\n      n += val_inputs.size(0)\n        \n    val_loss /= n\n    val_acc /= n\n    val_losses.append(val_loss)\n    val_accs.append(val_acc)\n\n\n    # P8.11. Log `test_loss` with a tag name 'test_loss' using `writer`. Use `global_step` as a timestamp for the log.\n    # writer.write_your_code_here (one-liner).\n    writer.add_scalar('val_loss', val_loss, global_step=epoch)\n\n\n    # P8.12. Log `test_accuracy` with a tag name 'test_accuracy' using `writer`. Use `global_step` as a timestamp for the log.\n    # writer.write_your_code_here (one-liner).\n    writer.add_scalar('val_accuracy', val_acc, global_step=epoch)\n\n\n    print('[epoch:{}, iteration:{}] val_loss : {:.4f} val accuracy : {:.4f}'.format(epoch+1, it, val_loss, val_acc)) \n\n    writer.flush()\n    # save checkpoint whenever there is some improvement in performance\n    if val_acc > best_acc:\n      best_acc = val_acc\n      # Save records.\n      ckpt = {'my_classifier':my_classifier.state_dict(),\n              'optimizer':optimizer.state_dict(),\n              'best_acc':best_acc}\n      torch.save(ckpt, ckpt_path)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:17:41.319065Z","iopub.status.idle":"2021-06-14T19:17:41.319792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.plot(train_losses, label='train loss')\nplt.plot(val_losses, label='val loss')\nplt.legend()\n\nif not training_process:\n  # Re-load trained model\n  my_classifier.load_state_dict(ckpt['my_classifier'])\n  optimizer.load_state_dict(ckpt['optimizer'])\n\n  # Testing\n  n = 0.\n  test_loss = 0.\n  test_acc = 0.\n  my_classifier.eval()\n  for test_inputs, test_labels in val_dataloader:\n    test_inputs = test_inputs.to(device)\n    test_labels = test_labels.to(device)\n\n    logits = my_classifier(test_inputs)\n    test_loss += F.cross_entropy(logits, test_labels, reduction='sum').item()\n    test_acc += (logits.argmax(dim=1) == test_labels).float().sum().item()\n    n += test_inputs.size(0)\n\n  test_loss /= n\n  test_acc /= n\n  print('Test_loss : {:.4f}, Test accuracy : {:.4f}'.format(test_loss, test_acc))","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:17:41.32095Z","iopub.status.idle":"2021-06-14T19:17:41.321711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(train_accs, label='train acc')\nplt.plot(val_accs, label='val acc')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:17:41.322909Z","iopub.status.idle":"2021-06-14T19:17:41.323649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi = open('result.csv','w')     #result.csv 파일을 write모드로 open\nfi.write(\"img,c0,c1,c2,c3,c4,c5,c6,c7,c8,c9\\n\")\nmy_classifier.load_state_dict(ckpt['my_classifier'])\nmy_classifier.eval()\nfor test_inputs, image_names in test_dataloader:\n    test_inputs = test_inputs.to(device)\n    logits = my_classifier(test_inputs)\n    probs = F.softmax(logits, dim=1)\n    probs = probs.tolist()\n    for i in range(len(probs)):\n        fi.write(image_names[i])\n        for j in range(10):\n            fi.write(','+str(probs[i][j]))\n        fi.write('\\n')\nfi.close()\nprint('output file created!')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:17:41.324911Z","iopub.status.idle":"2021-06-14T19:17:41.325752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\npd.read_csv('result.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:17:41.326891Z","iopub.status.idle":"2021-06-14T19:17:41.327593Z"},"trusted":true},"execution_count":null,"outputs":[]}]}