{"cells":[{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"from __future__ import print_function, division, absolute_import\n\n%matplotlib inline\n# %matplotlib notebook\n\nimport os, sys, gc\nimport numpy as np\nimport random\nimport glob\n\nfrom tqdm import tqdm_notebook as tqdm\n\nimport torch\nimport ignite\nimport torch.nn as nn\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom rcic_pytorch_utils import *  ## our utility script https://www.kaggle.com/hmendonca/rcic-pytorch-utils\n# !cat rcic-pytorch-utils.py","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fold = 1; head_num = 4; head_run = 3\ntarget = 'sirna_ct'\nresolution = 256\ndropout_rate = 0.4 # regularisation prior to ArcNet features\nweight_decay = 2e-4\nchannel_size = 200 # ArcNet feature size\nmargin = 0.45      # ArcNet margin\nlearning_rate = 0.035\nwarmup_learning_rate = learning_rate/100\nfocal_gamma = 3.5\n\nremove_head = False # reset ArcNet head (features and fc layers)\ntrain_to = -18  ## only train top half of the model\nfreeze_at_1st_n_last = False  # freeze all but the last layers, at the first and last epoches\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\n\nuse_amp = True if device == 'cuda' else False\ntraining = True if device == 'cuda' else False\ntrain_batch_size = 96 if device == 'cuda' else 16\neval_batch_size = 16 if device == 'cuda' else 8\n\n# debug = is_interactive()\ndebug = False\ncreate_sub = (device == 'cuda' or debug)\n\nprint(f'Interactive:{is_interactive()} Debug:{debug} Device:{device} Training:{training}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path_data = '/kaggle/input/recursion-cellular-image-classification'\nmodel_path = glob.glob(f'/kaggle/input/*/fold{fold}*.pth')[0]\nbest_sub = glob.glob(f'/kaggle/input/*/submission.csv')[0]\n\nprint(f'Training from model  {model_path}')\nprint(f'with pseudo labels from  {best_sub}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_everything(42000 + head_num*100 + head_run*10 + fold)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(torch.__version__, ignite.__version__, os.cpu_count())\nif device == 'cuda': print(torch.cuda.get_device_name())","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# install NVIDIA Apex if needed to support mixed precision training\nif use_amp and training:\n    try:\n        from apex import amp\n    except ImportError:\n#         !git clone https://github.com/NVIDIA/apex\n#         !pip install --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" apex/\n        !pip install  -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ../input/*/*/NVIDIA-apex*\n        from apex import amp","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model\n\n\nLet's define some helpful modules:\n- Flatten \n- Swish \n\nThe reason why Swish is not implemented in `torch.nn` can be found [here](https://github.com/pytorch/pytorch/pull/3182).\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Swish(nn.Module):\n    def forward(self, x):\n        return x * torch.sigmoid(x)\n\n\nclass Flatten(nn.Module):\n    def forward(self, x):\n        return x.reshape(x.shape[0], -1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's visualize Swish transform vs ReLU:"},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pylab as plt\n%matplotlib inline\n\nd = torch.linspace(-10.0, 10.0)\ns = Swish()\nres = s(d)\nres2 = torch.relu(d)\n\nplt.title(\"Swish transformation\")\nplt.plot(d.numpy(), res.numpy(), label='Swish')\nplt.plot(d.numpy(), res2.numpy(), label='ReLU')\n_ = plt.legend()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now let's define `SqueezeExcitation` module"},{"metadata":{"trusted":true},"cell_type":"code","source":"class SqueezeExcitation(nn.Module):\n    \n    def __init__(self, inplanes, se_planes):\n        super(SqueezeExcitation, self).__init__()\n        self.reduce_expand = nn.Sequential(\n            nn.Conv2d(inplanes, se_planes, \n                      kernel_size=1, stride=1, padding=0, bias=True),\n            Swish(),\n            nn.Conv2d(se_planes, inplanes, \n                      kernel_size=1, stride=1, padding=0, bias=True),\n            nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        x_se = torch.mean(x, dim=(-2, -1), keepdim=True)\n        x_se = self.reduce_expand(x_se)\n        return x_se * x\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Next, we can define `MBConv`.\n\n**Note on implementation**: in Tensorflow (and PyTorch ports) convolutions use `SAME` padding option which in PyTorch requires\na specific padding computation and additional operation to apply. We will use built-in padding argument of the convolution."},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.nn import functional as F\n\nclass MBConv(nn.Module):\n    def __init__(self, inplanes, planes, kernel_size, stride, \n                 expand_rate=1.0, se_rate=0.25, \n                 drop_connect_rate=0.2):\n        super(MBConv, self).__init__()\n\n        expand_planes = int(inplanes * expand_rate)\n        se_planes = max(1, int(inplanes * se_rate))\n\n        self.expansion_conv = None        \n        if expand_rate > 1.0:\n            self.expansion_conv = nn.Sequential(\n                nn.Conv2d(inplanes, expand_planes, \n                          kernel_size=1, stride=1, padding=0, bias=False),\n                nn.BatchNorm2d(expand_planes, momentum=0.01, eps=1e-3),\n                Swish()\n            )\n            inplanes = expand_planes\n\n        self.depthwise_conv = nn.Sequential(\n            nn.Conv2d(inplanes, expand_planes,\n                      kernel_size=kernel_size, stride=stride, \n                      padding=kernel_size // 2, groups=expand_planes,\n                      bias=False),\n            nn.BatchNorm2d(expand_planes, momentum=0.01, eps=1e-3),\n            Swish()\n        )\n\n        self.squeeze_excitation = SqueezeExcitation(expand_planes, se_planes)\n        \n        self.project_conv = nn.Sequential(\n            nn.Conv2d(expand_planes, planes, \n                      kernel_size=1, stride=1, padding=0, bias=False),\n            nn.BatchNorm2d(planes, momentum=0.01, eps=1e-3),\n        )\n\n        self.with_skip = stride == 1\n        self.drop_connect_rate = torch.tensor(drop_connect_rate, requires_grad=False)\n    \n    def _drop_connect(self, x):        \n        keep_prob = 1.0 - self.drop_connect_rate\n        drop_mask = torch.rand(x.shape[0], 1, 1, 1) + keep_prob\n        drop_mask = drop_mask.type_as(x)\n        drop_mask.floor_()\n        return drop_mask * x / keep_prob\n        \n    def forward(self, x):\n        z = x\n        if self.expansion_conv is not None:\n            x = self.expansion_conv(x)\n\n        x = self.depthwise_conv(x)\n        x = self.squeeze_excitation(x)\n        x = self.project_conv(x)\n        \n        # Add identity skip\n        if x.shape == z.shape and self.with_skip:            \n            if self.training and self.drop_connect_rate is not None:\n                self._drop_connect(x)\n            x += z\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"And finally, we can implement generic `EfficientNet':"},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import OrderedDict\nimport math\n\n\ndef init_weights(module):    \n    if isinstance(module, nn.Conv2d):    \n        nn.init.kaiming_normal_(module.weight, a=0, mode='fan_out')\n    elif isinstance(module, nn.Linear):\n        init_range = 1.0 / math.sqrt(module.weight.shape[1])\n        nn.init.uniform_(module.weight, a=-init_range, b=init_range)\n        \n        \nclass EfficientNet(nn.Module):\n        \n    def _setup_repeats(self, num_repeats):\n        return int(math.ceil(self.depth_coefficient * num_repeats))\n    \n    def _setup_channels(self, num_channels):\n        num_channels *= self.width_coefficient\n        new_num_channels = math.floor(num_channels / self.divisor + 0.5) * self.divisor\n        new_num_channels = max(self.divisor, new_num_channels)\n        if new_num_channels < 0.9 * num_channels:\n            new_num_channels += self.divisor\n        return new_num_channels\n\n    def __init__(self, num_classes, \n                 width_coefficient=1.0,\n                 depth_coefficient=1.0,\n                 se_rate=0.25,\n                 dropout_rate=0.2,\n                 drop_connect_rate=0.2):\n        super(EfficientNet, self).__init__()\n        \n        self.width_coefficient = width_coefficient\n        self.depth_coefficient = depth_coefficient\n        self.divisor = 8\n                \n        list_channels = [32, 16, 24, 40, 80, 112, 192, 320, 1280]\n        list_channels = [self._setup_channels(c) for c in list_channels]\n                \n        list_num_repeats = [1, 2, 2, 3, 3, 4, 1]\n        list_num_repeats = [self._setup_repeats(r) for r in list_num_repeats]        \n        \n        expand_rates = [1, 6, 6, 6, 6, 6, 6]\n        strides = [1, 2, 2, 2, 1, 2, 1]\n        kernel_sizes = [3, 3, 5, 3, 5, 5, 3]\n\n        # Define stem:\n        self.stem = nn.Sequential(\n            nn.Conv2d(3, list_channels[0], kernel_size=3, stride=2, padding=1, bias=False),\n            nn.BatchNorm2d(list_channels[0], momentum=0.01, eps=1e-3),\n            Swish()\n        )\n        \n        # Define MBConv blocks\n        blocks = []\n        counter = 0\n        num_blocks = sum(list_num_repeats)\n        for idx in range(7):\n            \n            num_channels = list_channels[idx]\n            next_num_channels = list_channels[idx + 1]\n            num_repeats = list_num_repeats[idx]\n            expand_rate = expand_rates[idx]\n            kernel_size = kernel_sizes[idx]\n            stride = strides[idx]\n            drop_rate = drop_connect_rate * counter / num_blocks\n            \n            name = \"MBConv{}_{}\".format(expand_rate, counter)\n            blocks.append((\n                name,\n                MBConv(num_channels, next_num_channels, \n                       kernel_size=kernel_size, stride=stride, expand_rate=expand_rate, \n                       se_rate=se_rate, drop_connect_rate=drop_rate)\n            ))\n            counter += 1\n            for i in range(1, num_repeats):                \n                name = \"MBConv{}_{}\".format(expand_rate, counter)\n                drop_rate = drop_connect_rate * counter / num_blocks                \n                blocks.append((\n                    name,\n                    MBConv(next_num_channels, next_num_channels, \n                           kernel_size=kernel_size, stride=1, expand_rate=expand_rate, \n                           se_rate=se_rate, drop_connect_rate=drop_rate)                                    \n                ))\n                counter += 1\n        \n        self.blocks = nn.Sequential(OrderedDict(blocks))\n        \n        # Define head\n        self.head = nn.Sequential(\n            nn.Conv2d(list_channels[-2], list_channels[-1], \n                      kernel_size=1, bias=False),\n            nn.BatchNorm2d(list_channels[-1], momentum=0.01, eps=1e-3),\n            Swish(),\n            nn.AdaptiveAvgPool2d(1),\n            Flatten(),\n            nn.Dropout(p=dropout_rate),\n            nn.Linear(list_channels[-1], num_classes)\n        )\n\n        self.apply(init_weights)\n        \n    def forward(self, x):\n        f = self.stem(x)\n        f = self.blocks(f)\n        y = self.head(f)\n        return y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**All EfficientNet models can be defined using the following parametrization:\n```\n# (width_coefficient, depth_coefficient, resolution, dropout_rate)\n'efficientnet-b0': (1.0, 1.0, 224, 0.2),\n'efficientnet-b1': (1.0, 1.1, 240, 0.2),\n'efficientnet-b2': (1.1, 1.2, 260, 0.3),\n'efficientnet-b3': (1.2, 1.4, 300, 0.3),\n'efficientnet-b4': (1.4, 1.8, 380, 0.4),\n'efficientnet-b5': (1.6, 2.2, 456, 0.4),\n'efficientnet-b6': (1.8, 2.6, 528, 0.5),\n'efficientnet-b7': (2.0, 3.1, 600, 0.5),\n```    \nLet's define and train the third last one: `EfficientNet-B4"},{"metadata":{},"cell_type":"markdown","source":"## Dataflow\n\nLet's setup the dataflow:\n- load train and test datasets\n- setup train/test image transforms\n- setup train/test data loaders\n\nAccording to the EfficientNet paper, authors borrowed training settings from other publications and the dataflow for CIFAR100 is the following:\n\n- input images to the network during training are resized to the model resolution\n- horizontally flipped randomly and augmented using cutout.\n- each mini-batch contained 256 examples\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torchvision.utils as vutils\n\nimport pandas as pd\nfrom sklearn import preprocessing\nfrom sklearn.utils import shuffle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"le = preprocessing.LabelEncoder()\nle.fit_transform(['HEPG2', 'HUVEC', 'RPE', 'U2OS'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(path_data+'/train.csv')\ndf['set'] = 'train'\ndf_test = pd.read_csv(path_data+'/test.csv')\ndf_test['set'] = 'test'\nbest = pd.read_csv(best_sub, index_col=0)\ndf_test_pseudo = df_test.set_index('id_code').join(best, how='inner').reset_index()\n#df_test['sirna'] = pd.read_csv(best_sub).sirna ## add pseudo-labels from best known submission\n\ndf_ctrl = pd.read_csv(path_data+'/train_controls.csv')\ndf_ctrl['set'] = 'train'\ndf_ctrl_test = pd.read_csv(path_data+'/test_controls.csv')\ndf_ctrl_test['set'] = 'test'\ndf_ctrl = pd.concat([df_ctrl, df_ctrl_test], ignore_index=False, sort=False)\ndel df_ctrl_test\n\nn_targets = df.sirna.nunique() + df_ctrl.sirna.nunique()\n\nfor d in [df, df_test, df_test_pseudo, df_ctrl]:\n    d['cell_type'] = d.experiment.apply(lambda s: s[:-3])\n    d['cell_type_n'] = le.transform(d.cell_type)\n    if d is not df_test:\n        d['sirna_ct'] = d.sirna + (d.cell_type_n * n_targets)\n\nn_classes = df.sirna_ct.nunique() + df_ctrl.sirna_ct.nunique()\nprint(n_targets, n_classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test_pseudo.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('cell types:', df.cell_type.unique().tolist())\nprint('train experiments:\\n', df.experiment.unique().tolist())\nprint('test experiments:\\n', df_test.experiment.unique().tolist())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_experiments = get_fold(df, fold)\nprint('validation experiments:\\n', valid_experiments)\n\n# add 3/4 of validation to the training set to mimic effects of pseudo-labelling in tests set?\n#df_train = df[~df.experiment.isin(valid_experiments) | (df.plate != fold)]\ndf_train = df[~df.experiment.isin(valid_experiments)]\ndf_valid = df[df.experiment.isin(valid_experiments)]\n\n## add pseudo-labels to the oversampled training data (2/3)\ndf_train = pd.concat([df_train, df_train, df_test_pseudo], ignore_index=False, sort=False)\n\n_ = df_train.groupby('experiment').id_code.count().plot(kind='bar')\nplt.tight_layout()\n\ndf_train.sample(9)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Oversample minority classes in training"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.yscale('log')\ntarget_dist = df_train.groupby(target)[target].count()\nmax_n = target_dist.max()\nprint(max_n)\n\n_ = target_dist.hist(bins=np.arange(max_n+1))\ntarget_dist.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if training or not debug:\n    for label,n in target_dist.items():\n        copies = min(int(round(max_n/n) - 1), 4)\n        if copies != 0:\n    #         print(f'label:{label} n:{n} copies:{copies}')\n            addon = [df_train[df_train[target] == label]] * copies\n            df_train = pd.concat([df_train] + addon, ignore_index=False, sort=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# show some copies\nprint(df_train.shape)\ndf_train[df_train.id_code == f'U2OS-0{5-fold}_1_D12']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.yscale('log')\ntarget_dist = df_train.groupby(target)[target].count()\n_ = target_dist.hist(bins=np.arange(target_dist.max()+1))\ntarget_dist.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_ = df_train.groupby('cell_type').id_code.count().plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Add controls"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_ctrl = df_ctrl[df_train.columns]\n_ = df_ctrl.groupby('cell_type').id_code.count().plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## train on some controls too. A different number per cell type is chosen to improve the type balance in the training set\n\n# U2OS: add all controls\ndf_train = pd.concat([df_train, df_ctrl[(df_ctrl['cell_type_n'] == 4)]],\n                     ignore_index=False, sort=False)\n# RPE & HEPG2 controls from only one plate (each plate repeats the same controls again)\ndf_train = pd.concat([df_train, df_ctrl[(\n    df_ctrl['cell_type_n'].isin([0,2])) & ((\n    df_ctrl['plate'] != fold) | (\n    df_ctrl.experiment.isin(df_test.experiment.unique()) | df_ctrl.experiment.isin(valid_experiments)))]],\n                     ignore_index=False, sort=False)\n# HUVEC only controls from test and validation\ndf_train = pd.concat([df_train, df_ctrl[(\n    df_ctrl['cell_type_n'] == 1) & (\n    df_ctrl['plate'] == fold) & (\n    df_ctrl.experiment.isin(df_test.experiment.unique()) | df_ctrl.experiment.isin(valid_experiments))]],\n                     ignore_index=False, sort=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.yscale('log')\ntarget_dist = df_train.groupby(target)[target].count()\n_ = target_dist.hist(bins=np.arange(target_dist.max()+1))\ntarget_dist.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_ = df_train.groupby('cell_type').id_code.count().plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if debug:\n    df_train = df_train[df_train[target].isin(df_train[target].sample(50))] # some random classes\n    df_valid = df_valid[df_valid[target].isin(df_train[target].unique())]   # same classes\n\nprint(df_train.shape, df_valid.shape, df_test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Calculate class weights to balance loss function"},{"metadata":{"trusted":true},"cell_type":"code","source":"class_weights = 1. / target_dist\nclass_weights *= n_classes / class_weights.sum()\n# class_weights = class_weights**0.5  # smoothing non-linearity\nclass_weights.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_weights = class_weights.values.ravel()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Get pixels statistics by experiment"},{"metadata":{"trusted":true},"cell_type":"code","source":"exp_stats = get_rcic_exp_stats(path_data)\nexp_stats.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not is_interactive():\n    del df; del df_ctrl; del target_dist\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ExpNormTwinDataset(Dataset):\n    ''' Multi channel datatset normalised by experiment image stats\n        During training, for every image a 'twin' image is also selected from the same class from another random experiment\n    '''\n    def __init__(self, df, img_dir, target, mode='train', sites=[1,2], channels=[1,2,3,4,5,6],\n                 img_stats=None, transform=None):\n        self.df = df\n        self.channels = channels\n        self.sites = sites\n        self.target = target\n        self.mode = mode\n        self.img_dir = img_dir\n        self.stats = img_stats\n        self.transform = transform\n        \n    @staticmethod\n    def _load_channel(file_name):\n        img = cv2.imread(file_name, cv2.IMREAD_UNCHANGED)\n        return np.float32(img)\n\n    def _get_img(self, rec, site):\n        dset, experiment, well, plate = rec.set, rec.experiment, rec.well, rec.plate\n        paths = [os.path.join(self.img_dir, dset, experiment, f'Plate{plate}', f'{well}_s{site}_w{channel}.png')\n                 for channel in self.channels]\n        img = [self._load_channel(img_path) for img_path in paths]\n\n        ## norm\n        if self.stats is not None:\n            stats = self.stats.loc[experiment, ['mean', 'std']]\n            # mean subtract\n            img = [i-m for i,m in zip(img, stats['mean'].values)]\n            # norm to 1 std\n            img = [i/s for i,s in zip(img, stats['std'].values)]\n        \n        img = np.stack(img, axis=-1)\n#         print(stats)\n#         print(img.shape, img.mean(axis=(1,0)).tolist(), img.std(axis=(1,0)).tolist(), float(img.min()), float(img.max()))\n        if self.transform:\n            img = self.transform(image=img)['image']\n        return img\n        \n    def _get_twin(self, rec, site):\n        experiment, target = rec.experiment, rec[self.target]\n        twin = self.df[(self.df[self.target] == target) & (self.df.experiment != experiment)].sample(1).iloc[0]\n        return self._get_img(twin, site)\n        \n    def __getitem__(self, index):\n        rec = self.df.iloc[index]\n        if self.mode == 'train':\n            # returns a random site and a random twin\n            img = self._get_img(rec, random.choice(self.sites))\n            twin = self._get_twin(rec, random.choice(self.sites))\n            return img, twin, rec[self.target]\n        elif self.mode == 'eval':\n            # returns a random site\n            img = self._get_img(rec, random.choice(self.sites))\n            return img, rec[self.target]\n        elif self.mode == 'test':\n            # returns raw images of all available sites\n            return [self._get_img(rec, site) for site in self.sites]\n\n    def __len__(self):\n        \"\"\" Total number of samples in the dataset \"\"\"\n        return len(self.df)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import *\nfrom albumentations.pytorch import ToTensor","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transform = Compose([\n    Rotate(45, p=0.666),\n#     GaussianBlur(blur_limit=3, p=0.5),\n#     RandomBrightness(limit=0.01, p=0.5),\n#     GaussNoise(var_limit=(0.001, 0.01), p=0.5),\n#     OneOrOther(\n#         RandomSizedCrop(min_max_height=np.int32([resolution*0.99, resolution*1.02]),\n#                         height=resolution, width=resolution),\n#     p=0.5),\n    AverageCrop(resolution, resolution),\n    HorizontalFlip(p=0.5),\n    VerticalFlip(p=0.5),\n    Transpose(p=0.5),\n    ToTensor(),\n])\n\nvalid_transform = Compose([\n    CenterCrop(resolution, resolution),\n    ToTensor(),\n])\n\n\ntrain_dataset = ExpNormTwinDataset(df_train, path_data, target=target, img_stats=exp_stats,\n                                   mode='train', transform=train_transform)\ntrain_eval_dataset = ExpNormTwinDataset(df_valid, path_data, target=target, img_stats=exp_stats,\n                                   mode='eval', transform=valid_transform)\n\n## TTA loaders\nvalid_dataset = ExpNormTwinDataset(df_valid, path_data, target=target, img_stats=exp_stats,\n                                   mode='test', transform=ToTensor())\ntest_dataset = ExpNormTwinDataset(df_test, path_data, target=target, img_stats=exp_stats,\n                                  mode='test', transform=ToTensor())\n\nprint(len(train_dataset), len(train_eval_dataset), len(valid_dataset), len(test_dataset))\nprint(test_dataset[0][0].shape, train_eval_dataset[0][0].shape, train_dataset[0][0].shape)\n\nraw_image_size = test_dataset[0][1].shape[-1]\nraw_image_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import DataLoader\n\nnum_workers = os.cpu_count()\nprint('num_workers:', num_workers)\n\ntrain_loader = DataLoader(train_dataset, batch_size=train_batch_size, num_workers=num_workers, \n                          shuffle=True, drop_last=True, pin_memory=True)\n\neval_train_loader = DataLoader(train_eval_dataset, batch_size=eval_batch_size, num_workers=num_workers, \n                               shuffle=False, drop_last=False, pin_memory=True)\n\n## TTA loaders\nvalid_loader = DataLoader(valid_dataset, batch_size=eval_batch_size, num_workers=num_workers, \n                               shuffle=False, drop_last=False, pin_memory=True)\n\ntest_loader = DataLoader(test_dataset, batch_size=eval_batch_size, num_workers=num_workers, \n                         shuffle=False, drop_last=False, pin_memory=True)\n\nprint(len(train_loader), len(eval_train_loader), len(valid_loader), len(test_loader))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot some training images\nbatch, twins, targets = next(iter(train_loader))\n\nplt.figure(figsize=(16, 8))\nplt.axis(\"off\")\nplt.title(\"Training Images\")\n_ = plt.imshow( # show every second channel\n    normalize_channels(\n        vutils.make_grid(batch[:16,::2], padding=2, normalize=False).cpu().numpy().transpose((1, 2, 0)),\n    )\n)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(16, 8))\nplt.axis(\"off\")\nplt.title(\"Training Twins\")\n_ = plt.imshow( # show every second channel\n    normalize_channels(\n        vutils.make_grid(twins[:16,::2], padding=2, normalize=False).cpu().numpy().transpose((1, 2, 0)),\n    )\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"targets = targets[:16]\ntargets.reshape([len(targets)//8,8])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## mean should be around 0 and std around 1\n{'mean':batch.mean(dim=(0,2,3)), 'std':batch.std(dim=(0,2,3))}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del batch\ntorch.cuda.empty_cache()\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load pretrained weights"},{"metadata":{"trusted":true},"cell_type":"code","source":"n_channels = len(test_dataset.channels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_n_remap(model, model_path, device):\n    model_state = torch.load(model_path, map_location=device)\n\n    # A basic remapping is required\n    mapping = { k:v for k,v in zip(model_state.keys(), model.state_dict().keys()) }\n#         print(mapping)\n    mapped_model_state = OrderedDict([\n        (mapping[k], v) for k,v in model_state.items() if k in mapping.keys()\n    ])\n\n    model.load_state_dict(mapped_model_state, strict=False)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import OrderedDict\n\n## from: https://github.com/filipradenovic/cnnimageretrieval-pytorch\ndef gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super().__init__()\n        self.p = nn.Parameter(torch.ones(1)*p)\n        self.eps = eps\n\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)\n        \n    def __repr__(self):\n        return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + ', ' + 'eps=' + str(self.eps) + ')'\n\ndef get_model(model_path=None, n_classes=n_classes):\n    model = EfficientNet(num_classes=n_classes, \n                         width_coefficient=1.4, depth_coefficient=1.8,\n                         dropout_rate=dropout_rate)\n\n    # fix n_channels\n    orig_stem = model.stem[0]\n    model.stem[0] = nn.Conv2d(\n                      in_channels=n_channels,\n                      out_channels=model.stem[0].out_channels,\n                      kernel_size=model.stem[0].kernel_size,\n                      stride=model.stem[0].stride,\n                      padding=model.stem[0].padding,\n                      bias=model.stem[0].bias)\n#     model.stem[0].weight.data = F.interpolate(orig_stem.weight.data.unsqueeze(0),\n#                                           size=[n_channels]+list(orig_stem.kernel_size),\n#                                           mode='trilinear', align_corners=False)[0]\n\n#     model.head[3] = GeM(p=1.25)\n    # remove head.fc\n    del model.head[6]\n\n    if model_path is not None:\n        model = load_n_remap(model, model_path, device)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### ArcNet from: https://github.com/pudae/kaggle-humpback/blob/master/tasks/identifier.py  \nimport types\nimport math\n\nclass ArcModule(nn.Linear):\n    def __init__(self, in_features, out_features, s, m):\n        super().__init__(in_features=in_features, out_features=out_features, bias=False)\n        self.s = s\n        self.m = m\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n\n    def forward(self, inputs, labels=None):\n        cos_th = F.linear(inputs, F.normalize(self.weight))\n        if labels is None:\n            assert not self.training\n            return cos_th * self.s\n        cos_th = cos_th.clamp(-1, 1)\n    \n        ## add margin\n        sin_th = torch.sqrt(1.0 - torch.pow(cos_th, 2))\n        cos_th_m = cos_th * self.cos_m - sin_th * self.sin_m\n        cos_th_m = torch.where(cos_th > self.th, cos_th_m, cos_th - self.mm)\n\n        ## truncate margin?\n        cond_v = cos_th - self.th\n        cond = cond_v <= 0\n        cos_th_m[cond] = (cos_th - self.mm)[cond]\n\n        ## add margin only to correct class\n        if labels.dim() == 1:\n            labels = labels.unsqueeze(-1)\n        onehot = torch.zeros_like(cos_th)\n        onehot.scatter_(1, labels, 1)\n        outputs = onehot * cos_th_m + (1.0 - onehot) * cos_th\n        outputs = outputs * self.s\n        return outputs\n\n\nclass ArcNet(nn.Module):\n    def __init__(self, criterion, backbone, in_features, num_classes, channel_size, s=65, m=margin):\n        super().__init__()\n        assert num_classes % 2 == 0\n        self.num_classes = num_classes\n        self.criterion = criterion\n        self.backbone = backbone\n        self.feat = nn.Sequential(\n            nn.Linear(in_features, channel_size, bias=False),\n            nn.BatchNorm1d(channel_size, momentum=0.01),\n        )\n        self.arc = ArcModule(channel_size, num_classes, s=s, m=m)\n\n    def features(self, images, labels=None):\n        features = self.backbone(images)\n        features = self.feat(features)\n        features = F.normalize(features, dim=-1)\n        return features\n\n    def forward(self, images, twins=None, labels=None):\n        if labels is None:\n            features = self.features(images)\n            return self.arc(features)\n        else:\n            return self.loss(images, twins, labels)\n\n    # add additional cost to separate features according to their siRNA (not part of the original ArcNet)\n    def loss(self, images, twins, labels, cos_w=3):\n        f0 = self.features(images)\n        f1 = self.features(twins)\n        targets = (labels % n_targets)\n        same_target = targets.expand(size=(len(targets), len(targets)))\n        same_target = 2*(same_target == same_target.t()).type_as(f0) - 1  ## 1 for equal targets and -1 for different ones\n        cos_dist = 1 - (torch.mm(f0, f1.t()) * same_target)\n        cos_dist = cos_dist**2  ## squared dist to focus on large errors\n        loss = cos_w * (torch.mean(cos_dist)) # + torch.diag(cos_dist).mean())  ## additional cost for twins (diagonal)\n        pred0 = self.arc(f0, labels)\n        pred1 = self.arc(f1, labels)\n        return loss + 0.5 * (self.criterion(pred0, labels) + self.criterion(pred1, labels))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"backbone = get_model(model_path if remove_head else None, n_classes)\nmodel = ArcNet(criterion=FocalLoss(gamma=focal_gamma, alpha=class_weights),\n               backbone=backbone,\n               in_features=backbone.head[0].out_channels,\n               num_classes=n_classes,\n               channel_size=channel_size)\nif not remove_head:\n    model = load_n_remap(model, model_path, device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# batch = next(iter(train_loader))\n# with torch.no_grad():\n#     print(model.to(device).loss(batch[0].to(device), batch[1].to(device), batch[2].to(device)))\n# del batch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_first_kernels(model.backbone.stem[0].weight)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check norms\nplot_norms([(n,p) for n,p in model.named_parameters() if 'blocks' not in n])","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We will finetune the model on GPU with AMP fp32/fp16 using nvidia/apex package."},{"metadata":{"trusted":true},"cell_type":"code","source":"if device == \"cuda\":\n    assert torch.cuda.is_available()\n    assert torch.backends.cudnn.enabled, \"NVIDIA/Apex:Amp requires cudnn backend to be enabled.\"\n    torch.backends.cudnn.benchmark = True\n\nmodel = model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check cosine distance between features\nif training and not debug: cosine_distance_heatmap(model, next(iter(train_loader)))\n\ntorch.cuda.empty_cache()\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training\nLet's setup focal loss as criterion and SGD as optimizer.\n\nWe will split model parameters into 3 groups: \n\n    1) feature extractor (pretrained weights)\n    2) ArcNet normalised features\n    3) classifier\n\nand define different learning rates for these groups (via learning rate scheduler)."},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"from itertools import chain\n\nimport torch.optim as optim\nimport torch.nn.functional as F\n\nlr = warmup_learning_rate\n\n## Only train layers after the last dimension reduction to same memory + time\nif train_to:\n    training_blocks = model.backbone.blocks[train_to:].parameters()\nelse:\n    training_blocks = chain(model.backbone.stem.parameters(), model.backbone.blocks.parameters())\n\n## freeze first layers prior to the warm-up run\nparams2freeze = None\nif freeze_at_1st_n_last:\n    params2freeze = chain(model.backbone.stem.named_parameters(), model.backbone.blocks.named_parameters())\nelif train_to:\n    params2freeze = chain(model.backbone.stem.named_parameters(), model.backbone.blocks[:train_to].named_parameters())\n\nif params2freeze is not None:\n    for name, param in params2freeze:\n        print(f'\"{name}\" is frozen')\n        param.requires_grad = False\n\n\noptimizer = optim.SGD([\n    {   \"params\": training_blocks,\n        \"lr\": lr * 0.01,\n    },\n    {   \"params\": chain(model.feat[0].parameters(), model.backbone.head.parameters()),\n        \"lr\": lr * 0.2,\n    },\n    {   \"params\": chain(model.feat[1].parameters(), model.arc.parameters()),\n        \"lr\": lr\n    }], \n    momentum=0.9, weight_decay=1e-6, nesterov=True)\n\n# if not use_amp and head_run > 1: optimizer = Lookahead(optimizer)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"if use_amp and training:\n    # Initialize Amp\n    model, optimizer = amp.initialize(model, optimizer, opt_level=\"O2\", num_losses=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Next, let's define a single iteration function `update_fn`. This function is then used by `ignite.engine.Engine` to update model while running over the input data."},{"metadata":{"trusted":true},"cell_type":"code","source":"from ignite.utils import convert_tensor\n\n\ndef update_fn(engine, batch):\n    x0 = convert_tensor(batch[0], device=device, non_blocking=True)\n    x1 = convert_tensor(batch[1], device=device, non_blocking=True)\n    y  = convert_tensor(batch[2], device=device, non_blocking=True)\n\n    model.train()\n\n    # Compute loss \n    loss = model(x0, x1, y)\n\n    optimizer.zero_grad()\n    if use_amp:\n        with amp.scale_loss(loss, optimizer, loss_id=0) as scaled_loss:\n            scaled_loss.backward()\n    else:\n        loss.backward()\n    optimizer.step()\n    \n    return {\n        \"batchloss\": loss.item(),\n    }    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's check `update_fn` and warmup the optimizer momentum"},{"metadata":{"trusted":true},"cell_type":"code","source":"warmup_iters = 1 + (3.5 * n_targets) // train_batch_size\n# if hasattr(optimizer, 'k'): warmup_iters -= (warmup_iters % optimizer.k)  ## complete look ahead iteration\n\nprint(int(warmup_iters))\ntorch.cuda.empty_cache()\ngc.collect()\n\nif training:\n    loss = []\n    loader = iter(train_loader)\n    pbar = tqdm(range(2 if debug else int(warmup_iters)))\n    for i in pbar:\n        res = update_fn(engine=None, batch=next(loader))\n        loss.append(res['batchloss'])\n        pbar.set_description(f\"loss:{loss[-1]:.4f}\"); pbar.refresh()\n\n    if device == 'cuda': print('max_memory_allocated:', torch.cuda.max_memory_allocated())\n    torch.cuda.empty_cache()\n    gc.collect()\n    print('mean loss:', np.mean(loss))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now let's define a trainer and add some practical handlers:\n- log to tensorboard: losses, metrics, lr\n- progress bar\n- models/optimizers checkpointing"},{"metadata":{"trusted":true},"cell_type":"code","source":"from ignite.engine import Engine, Events, create_supervised_evaluator\nfrom ignite.metrics import RunningAverage, Accuracy, Precision, Recall, Loss, TopKCategoricalAccuracy\n\nfrom ignite.contrib.handlers import TensorboardLogger\nfrom ignite.contrib.handlers.tensorboard_logger import OutputHandler, OptimizerParamsHandler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer = Engine(update_fn)\n\ndef siRNAaccuracyMax1108(pred, y):\n    \" Check predictions only on classes present in the validation and test sets \"\n    pred_ = pred.reshape([pred.shape[0], n_classes // n_targets, n_targets]).max(dim=1).values\n    pred_ = torch.argmax(pred_[...,:1108], dim=-1)\n    y_    = (y % n_targets)\n    assert pred_.shape == y_.shape\n    return (pred_ == y_).float().mean().cpu().numpy()\n\nmetrics = {\n#     'Loss': Loss(criterion),\n    'Score': Loss(siRNAaccuracyMax1108),\n    'Accuracy': Accuracy(),\n#     'Precision': Precision(average=True),\n#     'Recall': Recall(average=True),\n    'Top5Accuracy': TopKCategoricalAccuracy(k=5),\n}\n\nevaluator = create_supervised_evaluator(model, metrics=metrics, device=device, non_blocking=True)\n\nhistory = Metrics(evaluator, eval_train_loader, output_transform=lambda out: out['batchloss'], interactive=is_interactive())\nhistory.attach(trainer, \"batchloss\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from datetime import datetime\n\nlog_path = f\"./log\"\ntb_logger = TensorboardLogger(log_dir=log_path)\n\ntb_logger.attach(trainer, \n                 log_handler=OutputHandler('training', ['batchloss', ]), \n                 event_name=Events.ITERATION_COMPLETED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's setup learning rate scheduling:"},{"metadata":{"trusted":true},"cell_type":"code","source":"from ignite.contrib.handlers import CosineAnnealingScheduler, LinearCyclicalScheduler, ParamGroupScheduler\n\nsim_epochs = 2\nepoch_size = len(train_loader)\ncycle_mult = 1\nlr_sched_params = {'param_name':'lr', 'cycle_size':epoch_size*2, 'cycle_mult':cycle_mult,\n                   'start_value_mult':0.7, 'end_value_mult':0.5}\nmom_sched_params = {'param_name':'momentum', 'cycle_size':epoch_size*2, 'cycle_mult':cycle_mult,\n                    'start_value':0.82, 'end_value':0.92}\nwd_sched_params = {'param_name':'weight_decay', 'cycle_size':epoch_size*sim_epochs, 'cycle_mult':cycle_mult,\n                   'start_value':weight_decay/10, 'end_value':weight_decay}\n\nlr = learning_rate\nbody_sched_params = {**lr_sched_params, 'start_value':lr/10, 'end_value':lr/(100*2**head_run)}\nbody_sched = CosineAnnealingScheduler(optimizer.param_groups[0], **body_sched_params)\nfeat_sched_params = {**lr_sched_params, 'start_value':lr/5, 'end_value':lr/(50*2**head_run)}\nfeat_sched = CosineAnnealingScheduler(optimizer.param_groups[1], **feat_sched_params)\nhead_sched_params = {**lr_sched_params, 'start_value':lr, 'end_value':lr/(20*2**head_run)}\nhead_sched = CosineAnnealingScheduler(optimizer.param_groups[2], **feat_sched_params)\n\nmom_sched = CosineAnnealingScheduler(optimizer, **mom_sched_params)\nwd_sched  = CosineAnnealingScheduler(optimizer, **wd_sched_params)\nschedulers = [body_sched, feat_sched, head_sched, mom_sched, wd_sched]\nnames = [\"lr (body)\", \"lr (feat)\", \"lr (head)\", \"momentum\", \"wd\"]\n\nlr_values0 = np.array(body_sched.simulate_values(num_events=epoch_size*sim_epochs, **body_sched_params))\nlr_values1 = np.array(feat_sched.simulate_values(num_events=epoch_size*sim_epochs, **feat_sched_params))\nlr_values2 = np.array(head_sched.simulate_values(num_events=epoch_size*sim_epochs, **head_sched_params))\nwd_values  = np.array(  wd_sched.simulate_values(num_events=epoch_size*sim_epochs, **wd_sched_params))\nmom_values = np.array(mom_sched.simulate_values(num_events=epoch_size*sim_epochs, **mom_sched_params))\n\nfig = plt.figure(figsize=(16, 4))\nax = plt.subplot()\nplt.title(f\"Cosine annealing with start_value_mult={lr_sched_params['start_value_mult']}\")\nax.plot(lr_values0[:, 0], lr_values0[:, 1], label=names[0])\nax.plot(lr_values1[:, 0], lr_values1[:, 1], label=names[1])\nax.plot(lr_values2[:, 0], lr_values2[:, 1], label=names[2])\nax.plot( wd_values[:, 0],  wd_values[:, 1], label=names[4])\nax.set_yscale('log')\nax.set_xlabel('Batches processed')\nax.set_ylabel(\"learning rate\")\nax.legend(frameon=False, loc='upper right')\nax2 = ax.twinx()\nax2.plot(mom_values[:, 0], mom_values[:, 1], label=names[3])\nax2.set_ylabel(\"momentum\")\nax2.legend(frameon=False, loc='lower right')\n# fig.tight_layout()\n_ = ax.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scheduler = ParamGroupScheduler(schedulers=schedulers, names=names)\n\n# Attach single scheduler to the trainer\ntrainer.add_event_handler(Events.ITERATION_STARTED, scheduler)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Log optimizer parameters\ntb_logger.attach(trainer,\n                 log_handler=OptimizerParamsHandler(optimizer, \"lr\"), \n                 event_name=Events.EPOCH_STARTED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from ignite.contrib.handlers import ProgressBar\n\n# Iteration-wise progress bar\npbar = ProgressBar(bar_format=\"\")\npbar.attach(trainer, metric_names=['batchloss',])\n\n# Epoch-wise progress bar with display of training losses\nProgressBar(persist=True, bar_format=\"\").attach(trainer,\n                                                event_name=Events.EPOCH_STARTED,\n                                                closing_event_name=Events.COMPLETED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Log validation metrics:\ntb_logger.attach(evaluator,\n                 log_handler=OutputHandler(tag=\"test\",\n                                           metric_names=list(metrics.keys()),\n                                           another_engine=trainer),\n                 event_name=Events.EPOCH_COMPLETED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now let's setup logging and the best model checkpointing:"},{"metadata":{"trusted":true},"cell_type":"code","source":"import logging\n\n# Setup engine &  logger\ndef setup_logger(logger):\n    handler = logging.StreamHandler()\n    formatter = logging.Formatter(\"%(asctime)s %(name)-12s %(levelname)-8s %(message)s\")\n    handler.setFormatter(formatter)\n    logger.addHandler(handler)\n    logger.setLevel(logging.INFO)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from ignite.handlers import ModelCheckpoint, EarlyStopping, TerminateOnNan\n\ntrainer.add_event_handler(Events.ITERATION_COMPLETED, TerminateOnNan())\n\n# Store the best model\ndef default_score_fn(engine):\n    score = engine.state.metrics['Score']\n    return score\n\nbest_model_handler = ModelCheckpoint(dirname=log_path,\n                                     filename_prefix=\"best\",\n                                     n_saved=10,\n                                     score_name=\"Score\",\n                                     score_function=default_score_fn,\n                                     require_empty=False)\nevaluator.add_event_handler(Events.COMPLETED, best_model_handler, {'model': model, })\n\n# Clear cuda cache between training/testing\n@trainer.on(Events.EPOCH_COMPLETED)\n@evaluator.on(Events.COMPLETED)\ndef empty_cuda_cache(engine):\n    if debug and device == 'cuda': print('max_memory_allocated:', torch.cuda.max_memory_allocated())\n    torch.cuda.empty_cache()\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"trusted":true},"cell_type":"code","source":"# @trainer.on(Events.ITERATION_STARTED)\n# def apply_weight_decay(engine):\n#     with torch.no_grad():\n#         for mod in model.modules():\n#             if not isinstance(mod, nn.modules.batchnorm._BatchNorm):\n#                 for p in mod.parameters():\n#                     p.data -= p.data * weight_decay * lr","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"NUM_EPOCHS = (4 if debug else sim_epochs)\n\nif freeze_at_1st_n_last:\n    @trainer.on(Events.EPOCH_STARTED)\n    def turn_on_layers(engine):\n        epoch = engine.state.epoch\n        ## first and last epoch\n        if (epoch == 1) or (epoch == NUM_EPOCHS):\n            for name, param in model.named_parameters():\n                if ('backbone' not in name) or ('head' in name):\n                    pbar.log_message(f'training \"{name}\"')\n                    param.requires_grad = True\n                else:\n    #                 pbar.log_message(f'\"{name}\" is frozen')\n                    param.requires_grad = False\n        elif (epoch == 2):\n            count = 0\n            for param in training_blocks:  # only re-enable blocks, as head is already training\n                if param.requires_grad == False:\n                    param.requires_grad = True\n                    count += 1\n            pbar.log_message(f\"Epoch {epoch}: training all layers ({count})\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"if training:\n    state = trainer.run(train_loader, max_epochs=NUM_EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if training: plot_first_kernels(model.backbone.stem[0].weight)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check norms\nif training: plot_norms([(n,p) for n,p in model.named_parameters() if 'blocks' not in n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check gradients (last batch)\nif training: plot_grad_flow([(n,p) for n,p in model.named_parameters() if 'blocks' not in n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Results on the validation set:"},{"metadata":{"trusted":true},"cell_type":"code","source":"if training: \n    history.plot(len(train_loader))\n    plt.savefig(f'loss_history{evaluator.state.epoch}_fold{fold}.png')\n    evaluator.state.metrics","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference\n\nLet's load the best model and recompute evaluation metrics on test dataset with a very basic Test-Time-Augmentation to boost the performance"},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model = model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Find the last checkpoint\n!ls {log_path}\ncheckpoints = next(os.walk(log_path))[2]\ncheckpoints = sorted(filter(lambda f: f.endswith(\".pth\"), checkpoints))\nscores = [c.split('=')[-1][:-4] for c in checkpoints]\nbest_epoch = 0\nif len(scores) > 0:\n    best_epoch = np.argmax(scores)\n    print(best_epoch, scores)\n    if not checkpoints:\n        print('No weight files in {}'.format(log_path))\n    else:\n        model_path = f'fold{fold}h{head_num}r{head_run}_{best_epoch}_{scores[best_epoch]}.pth'\n        !cp {os.path.join(log_path, checkpoints[best_epoch])} {model_path}\n        best_model.load_state_dict(torch.load(model_path))\n\nprint(model_path)\n\nif not is_interactive():\n    !rm {log_path}/*.pth","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model = best_model.to(device).eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cosine_distance_heatmap(best_model, next(iter(train_loader)))\n_ = plt.savefig(f'cosine_distance_heatmap{best_epoch}_fold{fold}.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot some test images\nbatch = next(iter(test_loader))\n\nplt.figure(figsize=(16, 8))\nplt.axis(\"off\")\nplt.title(\"Test Images\")\n_ = plt.imshow( normalize_channels(\n    vutils.make_grid(batch[0][:16,::2], padding=2, normalize=False).cpu().numpy().transpose((1, 2, 0))\n) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Classify\nwith torch.no_grad():\n    y_pred = best_model(batch[0][:1].to(device))\n\n# Print predictions\nfor idx in torch.topk(y_pred, k=10)[1].squeeze(0).tolist():\n    prob = torch.softmax(y_pred, dim=-1)[0, idx].item()\n    print('{label:<75} ({p:.8f}%)'.format(label=str(idx), p=prob*100))\n\ndel batch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Create submission and OOF predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"TTA = 9\ndef tta(sites):\n    pred = []\n    for site in sites:\n        site = site.to(device)\n        for i in range(TTA):\n            x = tta9crop(site, i, resolution)\n            pred.append(best_model(x).unsqueeze(0))\n    # concat and calc mean softmax for submission\n    pred = torch.cat(pred)\n    pred = pred.reshape([pred.shape[0], pred.shape[1], n_classes // n_targets, n_targets]).max(dim=-2).values\n    return pred[...,:1108].softmax(dim=-1).mean(dim=0).cpu()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from ignite.contrib.handlers import ProgressBar\n\ndef inference_update_with_tta(engine, batch):\n    global preds, allpreds\n    with torch.no_grad():\n        sites = batch\n        pred = tta(sites)\n        allpreds = np.concatenate([allpreds, (-np.log(pred.numpy())).astype(np.float16)])\n        preds += (torch.argmax(pred, dim=-1) % n_targets).tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if create_sub:\n    allpreds = np.empty([0,1108])\n    preds = []\n\n    inferencer = Engine(inference_update_with_tta)\n    ProgressBar(desc=\"Inference\").attach(inferencer)\n\n    result_state = inferencer.run(valid_loader, max_epochs=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if create_sub:\n    df_valid['pred'] = preds\n    df_valid['acc']  = (df_valid.pred == df_valid.sirna).astype(int)\n\n    print('validation accuracy:', df_valid.acc.mean())\n    _ = df_valid.groupby('cell_type').acc.mean().plot(kind='bar'); plt.show()\n    _ = df_valid.groupby('experiment').acc.mean().plot(kind='bar'); plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if create_sub:\n    pd.DataFrame(allpreds, index=df_valid.id_code).to_pickle(f'df_valid_fold{fold}_log.pkl.gz')\n    print(glob.glob('*.gz'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"if create_sub:\n    allpreds = np.empty([0,1108])\n    preds = []\n\n    inferencer = Engine(inference_update_with_tta)\n    ProgressBar(desc=\"Inference\").attach(inferencer)\n\n    result_state = inferencer.run(test_loader, max_epochs=1)\n\n    pd.DataFrame(allpreds, index=df_test.id_code).to_pickle('df_test_log.pkl.gz')\n    print(glob.glob('*.gz'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if create_sub:\n    test_y = np.exp(-pd.read_pickle('df_test_log.pkl.gz').astype(np.float64)).fillna(0)\n    valid_y = np.exp(-pd.read_pickle(f'df_valid_fold{fold}_log.pkl.gz').astype(np.float64)).fillna(0)\nelse: # recover last model\n    test_y = np.exp(-pd.read_pickle(glob.glob(f'../input/*/df_test_log.pkl.gz')[0]).astype(np.float64)).fillna(0)\n    valid_y = np.exp(-pd.read_pickle(glob.glob(f'../input/*/df_valid_fold{fold}_log.pkl.gz')[0]).astype(np.float64)).fillna(0)\n    \n# normalise across siRNA's (each siRNA is equally likely to appear)\ntest_y = test_y / test_y.sum(axis=0)\nvalid_y = valid_y / valid_y.sum(axis=0)\n\ndf_valid['pred'] = valid_y.values.argmax(axis=1)\ndf_valid['acc']  = (df_valid.pred == df_valid.sirna).astype(int)\n\nprint('validation accuracy (norm):', df_valid.acc.mean())\n# _ = df_valid.groupby('cell_type').acc.mean().plot(kind='bar'); plt.show()\n# _ = df_valid.groupby('experiment').acc.mean().plot(kind='bar'); plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## remove siRNA in the wrong plate and normalise again (both horizontally and vertically)\npreds = remove_leaked_sirna(df_valid.set_index('id_code'), valid_y.copy())\n\nbest_score = -1\nbestpreds = None\nscores = []\nfor k in tqdm(range(10)):\n    preds, newscore, sirna_preds = normalize_both_ways(preds, df_valid.sirna.values)\n    print(f'Newscore:{newscore} Iter:{k}')\n    scores.append(newscore)\n    if newscore > best_score:\n        best_score = newscore\n        best_iter = k\n        bestpreds = sirna_preds\n#     elif device == 'cuda': break  ## stop to save GPU $$$","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(range(len(scores)), scores)\nplt.ylabel(\"score\")\nplt.xlabel(\"iteration\")\n\nbest_score, best_iter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_valid['pred'] = bestpreds\ndf_valid['acc'] = (df_valid.pred == df_valid.sirna).astype(int)\n\nprint('validation accuracy (norm):', df_valid.acc.mean())\n_ = df_valid.groupby('cell_type').acc.mean().plot(kind='bar'); plt.show()\n_ = df_valid.groupby('experiment').acc.mean().plot(kind='bar'); plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = remove_leaked_sirna(df_test.set_index('id_code'), test_y.copy())\n\nfor i in tqdm(range(best_iter+1)):\n    test_preds, _, _ = normalize_both_ways(test_preds)\n\n## assign the most likely unique treatment to each well\npreds = assign_sirna(test_preds)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Finally, the submission csv:"},{"metadata":{"trusted":true},"cell_type":"code","source":"if create_sub:\n    submission = pd.DataFrame({'id_code': pd.read_csv(path_data+'/sample_submission.csv').id_code.values,\n                               'sirna':   np.squeeze(preds).astype(int)})\n    submission.to_csv('submission.csv', index=False)\n    submission.hist(bins=111)\n    submission.head()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"if not is_interactive():\n    # clean up folders\n    !rm -rf apex /tmp/*\n    !ls -lh *","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"04c1cdeb62e644129ecd27b6aa56d1c1":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0511dc30f91145e8a2af427b87f8aa34":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_7250e3d3754a4aba8185c27699df9ca2","max":624,"min":0,"orientation":"horizontal","style":"IPY_MODEL_dbf309a69f9f4645be86af7b2921699f","value":624}},"0a6862e3099c4e70b2dc01e8d9704fcc":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_aa5ad3cff1044b35b0dbb10e9151e10b","placeholder":"​","style":"IPY_MODEL_ab8c8b983aab47c1a5862a81e40d6e5c","value":"100% 1150/1150 [33:56&lt;00:00,  1.51s/it, batchloss=1.24e+01]"}},"0b7069f3392d46cb88d7153997ab6e66":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"106fcbf5278c4371917ad5f9ae6becd0":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"11523ad4ddaa4026bc83c0cb2ed05402":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_0511dc30f91145e8a2af427b87f8aa34","IPY_MODEL_494e919c0d224f5da078b10d970c6fad"],"layout":"IPY_MODEL_54ff05d183054cf38c48d1733f6d04b8"}},"121a5c489c2f41229d69a78a9721b87c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"125fdf61b1cf41dd95168d8dd75a18a3":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"Epoch [3/12]","description_tooltip":null,"layout":"IPY_MODEL_8338561d68034d4b8e7576576bc84a42","max":1150,"min":0,"orientation":"horizontal","style":"IPY_MODEL_5b6b1ff816cb449eaa6f3c62eeedf1a3","value":1150}},"15f8c6f843614663ae9f58480b1ce1d6":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"17ff5af5bee24987b6908f0e70b4ce04":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"189dd0ac5b614f53a26e6fb004b15350":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"Epoch [4/12]","description_tooltip":null,"layout":"IPY_MODEL_1bc39ed9818d461a8bbb0779d3b20912","max":1150,"min":0,"orientation":"horizontal","style":"IPY_MODEL_18ee7f93c2f44f088a0e7f2439baadba","value":81}},"18ee7f93c2f44f088a0e7f2439baadba":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"1bc398347c9b45f8b1bf7c5acbd97d79":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_8b319e83fe914b2f9f66f8471165144c","IPY_MODEL_aab0211f9659418b955ed71276f9ed83"],"layout":"IPY_MODEL_d40a0464bced4bf7b1d77806e81b6f24"}},"1bc39ed9818d461a8bbb0779d3b20912":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1e747dbee5b348dfac78c2ee82cedc90":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_189dd0ac5b614f53a26e6fb004b15350","IPY_MODEL_4c2ae636247b459ba217b6562f7dafb2"],"layout":"IPY_MODEL_106fcbf5278c4371917ad5f9ae6becd0"}},"259481d7bcab4950a90d97c0ad098820":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"Inference","description_tooltip":null,"layout":"IPY_MODEL_f38b66e8f99a468ca6e0af9b18657e98","max":1244,"min":0,"orientation":"horizontal","style":"IPY_MODEL_0b7069f3392d46cb88d7153997ab6e66","value":0}},"29bf3400b6f74738a312c8bd4a347375":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"2f84347b82ad4a17b8d4da4cfc70096c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_629a85902f16498888a45f9138d2ba21","placeholder":"​","style":"IPY_MODEL_17ff5af5bee24987b6908f0e70b4ce04","value":"100% 1150/1150 [35:05&lt;00:00,  1.54s/it, batchloss=1.54e+01]"}},"348f550281a946f686a8e79f49052a24":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"3ace96915df94e1ea30e83c8da0140e6":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"454747d3fb764cdda19b1f49e5754935":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"48b81135cba84e4ab288077d7a8addb8":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"494e919c0d224f5da078b10d970c6fad":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_3ace96915df94e1ea30e83c8da0140e6","placeholder":"​","style":"IPY_MODEL_29bf3400b6f74738a312c8bd4a347375","value":"100% 624/624 [03:56&lt;00:00,  2.64it/s]"}},"4c2ae636247b459ba217b6562f7dafb2":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_cf62d5ff782d4491a426c2f3db52bd23","placeholder":"​","style":"IPY_MODEL_9b1c7d4196a54788b3f378c1849ffacf","value":"  7% 80/1150 [02:27&lt;29:48,  1.67s/it, batchloss=1.25e+01]"}},"54ff05d183054cf38c48d1733f6d04b8":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5b6b1ff816cb449eaa6f3c62eeedf1a3":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"620c756aa9f14c32849763dec894fd7e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_9853a7ee6ee24249b3e845a4f42492aa","max":45,"min":0,"orientation":"horizontal","style":"IPY_MODEL_fa4bf6e7cc3a48fb98acd1c75fbd9dfa","value":45}},"629a85902f16498888a45f9138d2ba21":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6881f39e876645f28063d45a16abcc56":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_620c756aa9f14c32849763dec894fd7e","IPY_MODEL_be5da80cffe548339f3446eb391b3bcd"],"layout":"IPY_MODEL_9085bbf91883437a850548d9eb76d3a4"}},"6f754dbe056f46919f039b6dd9693181":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"7250e3d3754a4aba8185c27699df9ca2":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"72d71620f430459d81a8349ef137e8f5":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"749142d16bcc45eb808afe9f0019732a":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"74cdd861bc5446e9b00dd49695dc2438":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_259481d7bcab4950a90d97c0ad098820","IPY_MODEL_b83c3d4e2544400791ddd3a495e11dff"],"layout":"IPY_MODEL_7bdc54160b10436f83339e3a4849984f"}},"75c358cfe2c74b6bbdeba9c636648d24":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_feb5b0d13cec45519f2f3792d3863f7a","IPY_MODEL_2f84347b82ad4a17b8d4da4cfc70096c"],"layout":"IPY_MODEL_749142d16bcc45eb808afe9f0019732a"}},"7b79957d0d7a40c8a2e4375e369d1251":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7bdc54160b10436f83339e3a4849984f":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7f0a8b4c704e4e9aa5e26c99eb17c1a8":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_9e66782be1f4472e8a15b1b008c2561c","placeholder":"​","style":"IPY_MODEL_348f550281a946f686a8e79f49052a24","value":"100% 1150/1150 [35:12&lt;00:00,  1.52s/it, batchloss=1.88e+01]"}},"80fc473da7494e4985889d29df03015e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"8338561d68034d4b8e7576576bc84a42":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8b319e83fe914b2f9f66f8471165144c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"Epoch","description_tooltip":null,"layout":"IPY_MODEL_db2aa91c157a467383a8e8fb24191b10","max":12,"min":0,"orientation":"horizontal","style":"IPY_MODEL_c1ca5e9744544048967d6b03e54d8086","value":4}},"9085bbf91883437a850548d9eb76d3a4":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9231f22f79bd4b6cba30d52545d1df76":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"Epoch [1/12]","description_tooltip":null,"layout":"IPY_MODEL_72d71620f430459d81a8349ef137e8f5","max":1150,"min":0,"orientation":"horizontal","style":"IPY_MODEL_9bbafc06c9a74370b2ddba545a7002b7","value":1150}},"9853a7ee6ee24249b3e845a4f42492aa":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9b1c7d4196a54788b3f378c1849ffacf":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"9bbafc06c9a74370b2ddba545a7002b7":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"9e66782be1f4472e8a15b1b008c2561c":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a9510d567df84b69a0b534a59e04ebe0":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"aa5ad3cff1044b35b0dbb10e9151e10b":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"aab0211f9659418b955ed71276f9ed83":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_7b79957d0d7a40c8a2e4375e369d1251","placeholder":"​","style":"IPY_MODEL_48b81135cba84e4ab288077d7a8addb8","value":" 33% 4/12 [1:56:24&lt;3:53:52, 1754.01s/it]"}},"ab8c8b983aab47c1a5862a81e40d6e5c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"b83c3d4e2544400791ddd3a495e11dff":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_a9510d567df84b69a0b534a59e04ebe0","placeholder":"​","style":"IPY_MODEL_80fc473da7494e4985889d29df03015e","value":"[1244/1244] 100%|██████████ [24:32&lt;00:00]"}},"bcb9e38a3a684952a072f9ccc01c6d6b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_125fdf61b1cf41dd95168d8dd75a18a3","IPY_MODEL_0a6862e3099c4e70b2dc01e8d9704fcc"],"layout":"IPY_MODEL_15f8c6f843614663ae9f58480b1ce1d6"}},"be5da80cffe548339f3446eb391b3bcd":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_04c1cdeb62e644129ecd27b6aa56d1c1","placeholder":"​","style":"IPY_MODEL_121a5c489c2f41229d69a78a9721b87c","value":"100% 45/45 [05:19&lt;00:00,  6.83s/it]"}},"c1ca5e9744544048967d6b03e54d8086":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"c72d26beb2da45b999c2536180a4bd02":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"cf62d5ff782d4491a426c2f3db52bd23":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d3305dc6f9d44899bad0594a43c75c7f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_9231f22f79bd4b6cba30d52545d1df76","IPY_MODEL_7f0a8b4c704e4e9aa5e26c99eb17c1a8"],"layout":"IPY_MODEL_c72d26beb2da45b999c2536180a4bd02"}},"d40a0464bced4bf7b1d77806e81b6f24":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"db2aa91c157a467383a8e8fb24191b10":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"dbf309a69f9f4645be86af7b2921699f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"f38b66e8f99a468ca6e0af9b18657e98":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"fa4bf6e7cc3a48fb98acd1c75fbd9dfa":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"feb5b0d13cec45519f2f3792d3863f7a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"Epoch [2/12]","description_tooltip":null,"layout":"IPY_MODEL_454747d3fb764cdda19b1f49e5754935","max":1150,"min":0,"orientation":"horizontal","style":"IPY_MODEL_6f754dbe056f46919f039b6dd9693181","value":1150}}},"version_major":2,"version_minor":0}}},"nbformat":4,"nbformat_minor":1}