{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Simple step by step approach with efficientnet for Bengali AI competition . If its useful for you please upvote ."},{"metadata":{},"cell_type":"markdown","source":"Change Log : \nV11 : Coming back with experimenting the basics again with respect to scheduler , augmentations , etc . Will go back to running for long cycle once the code works .\nThis version implemented Over9000 optimizer , OneCycle learning rate and few augmentations are applied and for few placeholder has been kept . \nv10 : Efficentnet b4 gave .9546 in LB . so probably it has good potential . \nv7: Nothing changed . Running for 20 Epochs ,earlier it was 10.\nv6 : Added Competition Metric Calculation Locally . Added Saving best checkpoint based on val recall score . Added some logging for recall .\nv5: Added pretrained weights for greyscale \n"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","id":"GA_kAZP19vM2","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"## Load Libraries \nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport cv2\nimport gc\nimport matplotlib.pyplot as plt\nimport torch.nn.functional as F\nimport os\n# Any results you write to the current directory are saved as output.\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torchvision import transforms,models\nfrom tqdm import tqdm_notebook as tqdm\n\n## This library is for augmentations .\nfrom albumentations import (\n    PadIfNeeded,\n    HorizontalFlip,\n    VerticalFlip,    \n    CenterCrop,    \n    Crop,\n    Compose,\n    Transpose,\n    RandomRotate90,\n    ElasticTransform,\n    GridDistortion, \n    OpticalDistortion,\n    RandomSizedCrop,\n    OneOf,\n    CLAHE,\n    RandomBrightnessContrast,    \n    \n    RandomGamma,\n    ShiftScaleRotate ,\n    GaussNoise,\n    Blur,\n    MotionBlur,   \n    GaussianBlur,\n)\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##Path for data \nPATH = '../input/bengaliai-cv19/'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Data setup part "},{"metadata":{"trusted":true},"cell_type":"code","source":"## Create Data from Parquet file mixing the methods of @hanjoonzhoe and @Iafoss\n\n## Create Crop Function @Iafoss\n\nHEIGHT = 137\nWIDTH = 236\nSIZE = 128\n\ndef bbox(img):\n    rows = np.any(img, axis=1)\n    cols = np.any(img, axis=0)\n    rmin, rmax = np.where(rows)[0][[0, -1]]\n    cmin, cmax = np.where(cols)[0][[0, -1]]\n    return rmin, rmax, cmin, cmax\n\ndef crop_resize(img0, size=SIZE, pad=16):\n    #crop a box around pixels large than the threshold \n    #some images contain line at the sides\n    ymin,ymax,xmin,xmax = bbox(img0[5:-5,5:-5] > 80)\n    #cropping may cut too much, so we need to add it back\n    xmin = xmin - 13 if (xmin > 13) else 0\n    ymin = ymin - 10 if (ymin > 10) else 0\n    xmax = xmax + 13 if (xmax < WIDTH - 13) else WIDTH\n    ymax = ymax + 10 if (ymax < HEIGHT - 10) else HEIGHT\n    img = img0[ymin:ymax,xmin:xmax]\n    #remove lo intensity pixels as noise\n    img[img < 28] = 0\n    lx, ly = xmax-xmin,ymax-ymin\n    l = max(lx,ly) + pad\n    #make sure that the aspect ratio is kept in rescaling\n    img = np.pad(img, [((l-ly)//2,), ((l-lx)//2,)], mode='constant')\n    return cv2.resize(img,(size,size))\n\ndef Resize(df,size=128):\n    resized = {} \n    df = df.set_index('image_id')\n    for i in tqdm(range(df.shape[0])):\n       # image = cv2.resize(df.loc[df.index[i]].values.reshape(137,236),(size,size))\n        image0 = 255 - df.loc[df.index[i]].values.reshape(137,236).astype(np.uint8)\n    #normalize each image by its max val\n        img = (image0*(255.0/image0.max())).astype(np.uint8)\n        image = crop_resize(img)\n        resized[df.index[i]] = image.reshape(-1)\n    resized = pd.DataFrame(resized).T.reset_index()\n    resized.columns = resized.columns.astype(str)\n    resized.rename(columns={'index':'image_id'},inplace=True)\n    return resized\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"%%time\n##Feather data generation for all train_data\nfor i in range(4):\n    data = pd.read_parquet(PATH+f'train_image_data_{i}.parquet')\n    data =Resize(data)\n    data.to_feather(f'train_data_{i}{i}_l.feather')\n    del data\n    gc.collect()\"\"\" \n##TO save RAM I have run this command in another kernel and kept the output for the Kernel as dataset for this Kernel .","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_PATH = \"../input/pytorch-efficientnet-starter-kernel/\"","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","id":"JsfYCnnn9vM5","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"## Load Feather Data \ntrain_all = pd.read_csv(PATH + \"train.csv\")\ntrain = train_all[train_all.grapheme_root.isin([59,60,61,62,63,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95])]\ndata0 = pd.read_feather(DATA_PATH+\"train_data_00_l.feather\")\ndata1 = pd.read_feather(DATA_PATH+'train_data_11_l.feather')\ndata2 = pd.read_feather(DATA_PATH+'train_data_22_l.feather')\ndata3 = pd.read_feather(DATA_PATH+'train_data_33_l.feather')\ndata_full1 = pd.concat([data0,data1,data2,data3],ignore_index=True)\ndata_full = data_full1.loc[data_full1.image_id.isin(train.image_id.values)]\ndel data0,data1,data2,data3,data_full1\ngc.collect()\ndata_full.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import preprocessing\nle = preprocessing.LabelEncoder()\nlabel_grapheme = le.fit_transform(train.grapheme_root.values)\nlabel_conso = le.fit_transform(train.consonant_diacritic.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['grapheme_root'] = label_grapheme\ntrain['consonant_diacritic'] = label_conso","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## A bunch of code copied from internet . Half of them I dont understand yet . However , CutOut is used in this notebook\n##https://github.com/hysts/pytorch_image_classification\nimport numpy as np\nimport torch\nimport torch.nn as nn\n\nclass Cutout:\n    def __init__(self, mask_size, p, cutout_inside, mask_color=1):\n        self.p = p\n        self.mask_size = mask_size\n        self.cutout_inside = cutout_inside\n        self.mask_color = mask_color\n\n        self.mask_size_half = mask_size // 2\n        self.offset = 1 if mask_size % 2 == 0 else 0\n\n    def __call__(self, image):\n        image = np.asarray(image).copy()\n\n        if np.random.random() > self.p:\n            return image\n\n        h, w = image.shape[:2]\n\n        if self.cutout_inside:\n            cxmin, cxmax = self.mask_size_half, w + self.offset - self.mask_size_half\n            cymin, cymax = self.mask_size_half, h + self.offset - self.mask_size_half\n        else:\n            cxmin, cxmax = 0, w + self.offset\n            cymin, cymax = 0, h + self.offset\n\n        cx = np.random.randint(cxmin, cxmax)\n        cy = np.random.randint(cymin, cymax)\n        xmin = cx - self.mask_size_half\n        ymin = cy - self.mask_size_half\n        xmax = xmin + self.mask_size\n        ymax = ymin + self.mask_size\n        xmin = max(0, xmin)\n        ymin = max(0, ymin)\n        xmax = min(w, xmax)\n        ymax = min(h, ymax)\n        image[ymin:ymax, xmin:xmax] = self.mask_color\n        return image\n\n\nclass DualCutout:\n    def __init__(self, mask_size, p, cutout_inside, mask_color=1):\n        self.cutout = Cutout(mask_size, p, cutout_inside, mask_color)\n\n    def __call__(self, image):\n        return np.hstack([self.cutout(image), self.cutout(image)])\n\n\nclass DualCutoutCriterion:\n    def __init__(self, alpha):\n        self.alpha = alpha\n        self.criterion = nn.CrossEntropyLoss(reduction='mean')\n\n    def __call__(self, preds, targets):\n        preds1, preds2 = preds\n        return (self.criterion(preds1, targets) + self.criterion(\n            preds2, targets)) * 0.5 + self.alpha * F.mse_loss(preds1, preds2)\n\n\ndef mixup(data, targets, alpha, n_classes):\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_targets = targets[indices]\n\n    lam = np.random.beta(alpha, alpha)\n    data = data * lam + shuffled_data * (1 - lam)\n    targets = (targets, shuffled_targets, lam)\n\n    return data, targets\n\n\ndef mixup_criterion(preds, targets):\n    targets1, targets2, lam = targets\n    criterion = nn.CrossEntropyLoss(reduction='mean')\n    return lam * criterion(preds, targets1) + (1 - lam) * criterion(\n        preds, targets2)\n    \n\n\nclass RandomErasing:\n    def __init__(self, p, area_ratio_range, min_aspect_ratio, max_attempt):\n        self.p = p\n        self.max_attempt = max_attempt\n        self.sl, self.sh = area_ratio_range\n        self.rl, self.rh = min_aspect_ratio, 1. / min_aspect_ratio\n\n    def __call__(self, image):\n        image = np.asarray(image).copy()\n\n        if np.random.random() > self.p:\n            return image\n\n        h, w = image.shape[:2]\n        image_area = h * w\n\n        for _ in range(self.max_attempt):\n            mask_area = np.random.uniform(self.sl, self.sh) * image_area\n            aspect_ratio = np.random.uniform(self.rl, self.rh)\n            mask_h = int(np.sqrt(mask_area * aspect_ratio))\n            mask_w = int(np.sqrt(mask_area / aspect_ratio))\n\n            if mask_w < w and mask_h < h:\n                x0 = np.random.randint(0, w - mask_w)\n                y0 = np.random.randint(0, h - mask_h)\n                x1 = x0 + mask_w\n                y1 = y0 + mask_h\n                image[y0:y1, x0:x1] = np.random.uniform(0, 1)\n                break\n\n        return image  ","execution_count":null,"outputs":[]},{"metadata":{"id":"mtCv6YoGeSLs","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"## Add Augmentations as suited from Albumentations library\ntrain_aug = Compose([ \n    ShiftScaleRotate(p=1,border_mode=cv2.BORDER_CONSTANT,value =1),\n    OneOf([\n        ElasticTransform(p=0.1, alpha=1, sigma=50, alpha_affine=50,border_mode=cv2.BORDER_CONSTANT,value =1),\n        GridDistortion(distort_limit =0.05 ,border_mode=cv2.BORDER_CONSTANT,value =1, p=0.1),\n        OpticalDistortion(p=0.1, distort_limit= 0.05, shift_limit=0.2,border_mode=cv2.BORDER_CONSTANT,value =1)                  \n        ], p=0.3),\n    OneOf([\n        GaussNoise(var_limit=1.0),\n        Blur(),\n        GaussianBlur(blur_limit=3)\n        ], p=0.4),    \n    RandomGamma(p=0.8)])\n\n## A lot of heavy augmentations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Someone asked for normalization of images . values collected from Iafoss\n\n\nclass ToTensor:\n    def __call__(self, data):\n        if isinstance(data, tuple):\n            return tuple([self._to_tensor(image) for image in data])\n        else:\n            return self._to_tensor(data)\n\n    def _to_tensor(self, data):\n        if len(data.shape) == 3:\n            return torch.from_numpy(data.transpose(2, 0, 1).astype(np.float32))\n        else:\n            return torch.from_numpy(data[None, :, :].astype(np.float32))\n\n\nclass Normalize:\n    def __init__(self, mean, std):\n        self.mean = np.array(mean)\n        self.std = np.array(std)\n\n    def __call__(self, image):\n        image = np.asarray(image).astype(np.float32) / 255.\n        image = (image - self.mean) / self.std\n        return image","execution_count":null,"outputs":[]},{"metadata":{"id":"e1tMjI8m9vM_","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"## Create dataset function\nclass GraphemeDataset(Dataset):\n    def __init__(self,df,label,_type='train',transform =True,aug=train_aug):\n        self.df = df\n        self.label = label\n        self.aug = aug\n        self.transform = transform\n        self.data = df.iloc[:, 1:].values\n    def __len__(self):\n        return len(self.df)\n    def __getitem__(self,idx):\n        label1 = self.label.vowel_diacritic.values[idx]\n        label2 = self.label.grapheme_root.values[idx]\n        label3 = self.label.consonant_diacritic.values[idx]\n        #image = self.df.iloc[idx][1:].values.reshape(128,128).astype(np.float)\n        image = self.data[idx, :].reshape(128,128).astype(np.float)\n        if self.transform:\n            augment = self.aug(image =image)\n            image = augment['image']\n            cutout = Cutout(32,0.5,True,1)\n            image = cutout(image)\n        norm = Normalize([0.0692],[0.2051])\n        image = norm(image)\n\n        return image,label1,label2,label3","execution_count":null,"outputs":[]},{"metadata":{"id":"nkAjCGZuwexT","colab_type":"code","outputId":"1db8b64b-53cb-43ae-8eb6-792857d97798","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"## Do a train-valid split of the data to create dataset and dataloader . Specify random seed to get reproducibility \nfrom sklearn.model_selection import train_test_split\ntrain_df , valid_df = train_test_split(train,test_size=0.20, random_state=42,shuffle=True) ## Split Labels\ndata_train_df, data_valid_df = train_test_split(data_full,test_size=0.20, random_state=42,shuffle =True) ## split data\ndel data_full \ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"id":"jC1hLt6CxXNx","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"##Creating the train and valid dataset for training . Training data has the transform flag ON\ntrain_dataset = GraphemeDataset(data_train_df ,train_df,transform = True) \nvalid_dataset = GraphemeDataset(data_valid_df ,valid_df,transform = False) \ntorch.cuda.empty_cache()\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"id":"EC4YoNn1V6M7","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"##Visulization function for checking Original and augmented image\ndef visualize(original_image,aug_image):\n    fontsize = 18\n    \n    f, ax = plt.subplots(1, 2, figsize=(8, 8))\n\n    ax[0].imshow(original_image, cmap='gray')\n    ax[0].set_title('Original image', fontsize=fontsize)\n    ax[1].imshow(aug_image,cmap='gray')\n    ax[1].set_title('Augmented image', fontsize=fontsize)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## One image taken from raw dataframe another from dataset \norig_image = data_train_df.iloc[0, 1:].values.reshape(128,128).astype(np.float)\naug_image = train_dataset[0][0]","execution_count":null,"outputs":[]},{"metadata":{"id":"glXch-HFU3_u","colab_type":"code","outputId":"ec60c5e1-fed0-4782-a8d9-d6eed1928712","colab":{"base_uri":"https://localhost:8080/","height":271},"trusted":true},"cell_type":"code","source":"## Check the augmentations \nfor i in range (20):\n    aug_image = train_dataset[0][0]\n    visualize (orig_image,aug_image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_df,valid_df,data_train_df,data_valid_df \ntorch.cuda.empty_cache()\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"id":"pj-CALEgxrzF","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"## Create data loader and get ready for training .\nbatch_size = 32 \ntrain_loader = torch.utils.data.DataLoader(train_dataset,batch_size=batch_size,shuffle=True)\nvalid_loader = torch.utils.data.DataLoader(valid_dataset,batch_size=batch_size,shuffle=True)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Model creation part "},{"metadata":{"id":"B2g6gbGU9vNB","colab_type":"code","colab":{},"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"## Mish Activation Function Not yet Used . May be later \nclass Mish(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, x): \n        \n        x = x *( torch.tanh(F.softplus(x)))\n\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Over9000 Optimizer . Inspired by Iafoss . Over and Out !\n##https://github.com/mgrankin/over9000/blob/master/ralamb.py\nimport torch, math\nfrom torch.optim.optimizer import Optimizer\n\n# RAdam + LARS\nclass Ralamb(Optimizer):\n\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)\n        self.buffer = [[None, None, None] for ind in range(10)]\n        super(Ralamb, self).__init__(params, defaults)\n\n    def __setstate__(self, state):\n        super(Ralamb, self).__setstate__(state)\n\n    def step(self, closure=None):\n\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n                if grad.is_sparse:\n                    raise RuntimeError('Ralamb does not support sparse gradients')\n\n                p_data_fp32 = p.data.float()\n\n                state = self.state[p]\n\n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n                else:\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)\n\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                # Decay the first and second moment running average coefficient\n                # m_t\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n                # v_t\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n\n                state['step'] += 1\n                buffered = self.buffer[int(state['step'] % 10)]\n\n                if state['step'] == buffered[0]:\n                    N_sma, radam_step_size = buffered[1], buffered[2]\n                else:\n                    buffered[0] = state['step']\n                    beta2_t = beta2 ** state['step']\n                    N_sma_max = 2 / (1 - beta2) - 1\n                    N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t)\n                    buffered[1] = N_sma\n\n                    # more conservative since it's an approximated value\n                    if N_sma >= 5:\n                        radam_step_size = math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step'])\n                    else:\n                        radam_step_size = 1.0 / (1 - beta1 ** state['step'])\n                    buffered[2] = radam_step_size\n\n                if group['weight_decay'] != 0:\n                    p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)\n\n                # more conservative since it's an approximated value\n                radam_step = p_data_fp32.clone()\n                if N_sma >= 5:\n                    denom = exp_avg_sq.sqrt().add_(group['eps'])\n                    radam_step.addcdiv_(-radam_step_size * group['lr'], exp_avg, denom)\n                else:\n                    radam_step.add_(-radam_step_size * group['lr'], exp_avg)\n\n                radam_norm = radam_step.pow(2).sum().sqrt()\n                weight_norm = p.data.pow(2).sum().sqrt().clamp(0, 10)\n                if weight_norm == 0 or radam_norm == 0:\n                    trust_ratio = 1\n                else:\n                    trust_ratio = weight_norm / radam_norm\n\n                state['weight_norm'] = weight_norm\n                state['adam_norm'] = radam_norm\n                state['trust_ratio'] = trust_ratio\n\n                if N_sma >= 5:\n                    p_data_fp32.addcdiv_(-radam_step_size * group['lr'] * trust_ratio, exp_avg, denom)\n                else:\n                    p_data_fp32.add_(-radam_step_size * group['lr'] * trust_ratio, exp_avg)\n\n                p.data.copy_(p_data_fp32)\n\n        return loss\n\n# Lookahead implementation from https://github.com/rwightman/pytorch-image-models/blob/master/timm/optim/lookahead.py\n\n\"\"\" Lookahead Optimizer Wrapper.\nImplementation modified from: https://github.com/alphadl/lookahead.pytorch\nPaper: `Lookahead Optimizer: k steps forward, 1 step back` - https://arxiv.org/abs/1907.08610\n\"\"\"\nimport torch\nfrom torch.optim.optimizer import Optimizer\nfrom collections import defaultdict\n\nclass Lookahead(Optimizer):\n    def __init__(self, base_optimizer, alpha=0.5, k=6):\n        if not 0.0 <= alpha <= 1.0:\n            raise ValueError(f'Invalid slow update rate: {alpha}')\n        if not 1 <= k:\n            raise ValueError(f'Invalid lookahead steps: {k}')\n        defaults = dict(lookahead_alpha=alpha, lookahead_k=k, lookahead_step=0)\n        self.base_optimizer = base_optimizer\n        self.param_groups = self.base_optimizer.param_groups\n        self.defaults = base_optimizer.defaults\n        self.defaults.update(defaults)\n        self.state = defaultdict(dict)\n        # manually add our defaults to the param groups\n        for name, default in defaults.items():\n            for group in self.param_groups:\n                group.setdefault(name, default)\n\n    def update_slow(self, group):\n        for fast_p in group[\"params\"]:\n            if fast_p.grad is None:\n                continue\n            param_state = self.state[fast_p]\n            if 'slow_buffer' not in param_state:\n                param_state['slow_buffer'] = torch.empty_like(fast_p.data)\n                param_state['slow_buffer'].copy_(fast_p.data)\n            slow = param_state['slow_buffer']\n            slow.add_(group['lookahead_alpha'], fast_p.data - slow)\n            fast_p.data.copy_(slow)\n\n    def sync_lookahead(self):\n        for group in self.param_groups:\n            self.update_slow(group)\n\n    def step(self, closure=None):\n        # print(self.k)\n        #assert id(self.param_groups) == id(self.base_optimizer.param_groups)\n        loss = self.base_optimizer.step(closure)\n        for group in self.param_groups:\n            group['lookahead_step'] += 1\n            if group['lookahead_step'] % group['lookahead_k'] == 0:\n                self.update_slow(group)\n        return loss\n\n    def state_dict(self):\n        fast_state_dict = self.base_optimizer.state_dict()\n        slow_state = {\n            (id(k) if isinstance(k, torch.Tensor) else k): v\n            for k, v in self.state.items()\n        }\n        fast_state = fast_state_dict['state']\n        param_groups = fast_state_dict['param_groups']\n        return {\n            'state': fast_state,\n            'slow_state': slow_state,\n            'param_groups': param_groups,\n        }\n\n    def load_state_dict(self, state_dict):\n        fast_state_dict = {\n            'state': state_dict['state'],\n            'param_groups': state_dict['param_groups'],\n        }\n        self.base_optimizer.load_state_dict(fast_state_dict)\n\n        # We want to restore the slow state, but share param_groups reference\n        # with base_optimizer. This is a bit redundant but least code\n        slow_state_new = False\n        if 'slow_state' not in state_dict:\n            print('Loading state_dict from optimizer without Lookahead applied.')\n            state_dict['slow_state'] = defaultdict(dict)\n            slow_state_new = True\n        slow_state_dict = {\n            'state': state_dict['slow_state'],\n            'param_groups': state_dict['param_groups'],  # this is pointless but saves code\n        }\n        super(Lookahead, self).load_state_dict(slow_state_dict)\n        self.param_groups = self.base_optimizer.param_groups  # make both ref same container\n        if slow_state_new:\n            # reapply defaults to catch missing lookahead specific ones\n            for name, default in self.defaults.items():\n                for group in self.param_groups:\n                    group.setdefault(name, default)\n\ndef LookaheadAdam(params, alpha=0.5, k=6, *args, **kwargs):\n     adam = Adam(params, *args, **kwargs)\n     return Lookahead(adam, alpha, k)\n\n\n# RAdam + LARS + LookAHead\n\n# Lookahead implementation from https://github.com/lonePatient/lookahead_pytorch/blob/master/optimizer.py\n# RAdam + LARS implementation from https://gist.github.com/redknightlois/c4023d393eb8f92bb44b2ab582d7ec20\n\ndef Over9000(params, alpha=0.5, k=6, *args, **kwargs):\n     ralamb = Ralamb(params, *args, **kwargs)\n     return Lookahead(ralamb, alpha, k)\n\nRangerLars = Over9000 ","execution_count":null,"outputs":[]},{"metadata":{"id":"8AkWJcL89vND","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"## Code copied from Lukemelas github repository have a look\n## https://github.com/lukemelas/EfficientNet-PyTorch/tree/master/efficientnet_pytorch\n\"\"\"\nThis file contains helper functions for building the model and for loading model parameters.\nThese helper functions are built to mirror those in the official TensorFlow implementation.\n\"\"\"\n\nimport re\nimport math\nimport collections\nfrom functools import partial\nimport torch\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom torch.utils import model_zoo\n\n########################################################################\n############### HELPERS FUNCTIONS FOR MODEL ARCHITECTURE ###############\n########################################################################\n\n\n# Parameters for the entire model (stem, all blocks, and head)\nGlobalParams = collections.namedtuple('GlobalParams', [\n    'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate',\n    'num_classes', 'width_coefficient', 'depth_coefficient',\n    'depth_divisor', 'min_depth', 'drop_connect_rate', 'image_size'])\n\n# Parameters for an individual model block\nBlockArgs = collections.namedtuple('BlockArgs', [\n    'kernel_size', 'num_repeat', 'input_filters', 'output_filters',\n    'expand_ratio', 'id_skip', 'stride', 'se_ratio'])\n\n# Change namedtuple defaults\nGlobalParams.__new__.__defaults__ = (None,) * len(GlobalParams._fields)\nBlockArgs.__new__.__defaults__ = (None,) * len(BlockArgs._fields)\n\n\nclass SwishImplementation(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, i):\n        result = i * torch.sigmoid(i)\n        ctx.save_for_backward(i)\n        return result\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        i = ctx.saved_variables[0]\n        sigmoid_i = torch.sigmoid(i)\n        return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i)))\n\n\nclass MemoryEfficientSwish(nn.Module):\n    def forward(self, x):\n        return SwishImplementation.apply(x)\n\nclass Swish(nn.Module):\n    def forward(self, x):\n        return x * torch.sigmoid(x)\n\n\ndef round_filters(filters, global_params):\n    \"\"\" Calculate and round number of filters based on depth multiplier. \"\"\"\n    multiplier = global_params.width_coefficient\n    if not multiplier:\n        return filters\n    divisor = global_params.depth_divisor\n    min_depth = global_params.min_depth\n    filters *= multiplier\n    min_depth = min_depth or divisor\n    new_filters = max(min_depth, int(filters + divisor / 2) // divisor * divisor)\n    if new_filters < 0.9 * filters:  # prevent rounding by more than 10%\n        new_filters += divisor\n    return int(new_filters)\n\n\ndef round_repeats(repeats, global_params):\n    \"\"\" Round number of filters based on depth multiplier. \"\"\"\n    multiplier = global_params.depth_coefficient\n    if not multiplier:\n        return repeats\n    return int(math.ceil(multiplier * repeats))\n\n\ndef drop_connect(inputs, p, training):\n    \"\"\" Drop connect. \"\"\"\n    if not training: return inputs\n    batch_size = inputs.shape[0]\n    keep_prob = 1 - p\n    random_tensor = keep_prob\n    random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)\n    binary_tensor = torch.floor(random_tensor)\n    output = inputs / keep_prob * binary_tensor\n    return output\n\n\ndef get_same_padding_conv2d(image_size=None):\n    \"\"\" Chooses static padding if you have specified an image size, and dynamic padding otherwise.\n        Static padding is necessary for ONNX exporting of models. \"\"\"\n    if image_size is None:\n        return Conv2dDynamicSamePadding\n    else:\n        return partial(Conv2dStaticSamePadding, image_size=image_size)\n\n\nclass Conv2dDynamicSamePadding(nn.Conv2d):\n    \"\"\" 2D Convolutions like TensorFlow, for a dynamic image size \"\"\"\n\n    def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True):\n        super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias)\n        self.stride = self.stride if len(self.stride) == 2 else [self.stride[0]] * 2\n\n    def forward(self, x):\n        ih, iw = x.size()[-2:]\n        kh, kw = self.weight.size()[-2:]\n        sh, sw = self.stride\n        oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)\n        pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0)\n        pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0)\n        if pad_h > 0 or pad_w > 0:\n            x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2])\n        return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)\n\n\nclass Conv2dStaticSamePadding(nn.Conv2d):\n    \"\"\" 2D Convolutions like TensorFlow, for a fixed image size\"\"\"\n\n    def __init__(self, in_channels, out_channels, kernel_size, image_size=None, **kwargs):\n        super().__init__(in_channels, out_channels, kernel_size, **kwargs)\n        self.stride = self.stride if len(self.stride) == 2 else [self.stride[0]] * 2\n\n        # Calculate padding based on image size and save it\n        assert image_size is not None\n        ih, iw = image_size if type(image_size) == list else [image_size, image_size]\n        kh, kw = self.weight.size()[-2:]\n        sh, sw = self.stride\n        oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)\n        pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0)\n        pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0)\n        if pad_h > 0 or pad_w > 0:\n            self.static_padding = nn.ZeroPad2d((pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))\n        else:\n            self.static_padding = Identity()\n\n    def forward(self, x):\n        x = self.static_padding(x)\n        x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)\n        return x\n\n\nclass Identity(nn.Module):\n    def __init__(self, ):\n        super(Identity, self).__init__()\n\n    def forward(self, input):\n        return input\n\n\n########################################################################\n############## HELPERS FUNCTIONS FOR LOADING MODEL PARAMS ##############\n########################################################################\n\n\ndef efficientnet_params(model_name):\n    \"\"\" Map EfficientNet model name to parameter coefficients. \"\"\"\n    params_dict = {\n        # Coefficients:   width,depth,res,dropout\n        'efficientnet-b0': (1.0, 1.0, 224, 0.2),\n        'efficientnet-b1': (1.0, 1.1, 240, 0.2),\n        'efficientnet-b2': (1.1, 1.2, 260, 0.3),\n        'efficientnet-b3': (1.2, 1.4, 300, 0.3),\n        'efficientnet-b4': (1.4, 1.8, 380, 0.4),\n        'efficientnet-b5': (1.6, 2.2, 456, 0.4),\n        'efficientnet-b6': (1.8, 2.6, 528, 0.5),\n        'efficientnet-b7': (2.0, 3.1, 600, 0.5),\n    }\n    return params_dict[model_name]\n\n\nclass BlockDecoder(object):\n    \"\"\" Block Decoder for readability, straight from the official TensorFlow repository \"\"\"\n\n    @staticmethod\n    def _decode_block_string(block_string):\n        \"\"\" Gets a block through a string notation of arguments. \"\"\"\n        assert isinstance(block_string, str)\n\n        ops = block_string.split('_')\n        options = {}\n        for op in ops:\n            splits = re.split(r'(\\d.*)', op)\n            if len(splits) >= 2:\n                key, value = splits[:2]\n                options[key] = value\n\n        # Check stride\n        assert (('s' in options and len(options['s']) == 1) or\n                (len(options['s']) == 2 and options['s'][0] == options['s'][1]))\n\n        return BlockArgs(\n            kernel_size=int(options['k']),\n            num_repeat=int(options['r']),\n            input_filters=int(options['i']),\n            output_filters=int(options['o']),\n            expand_ratio=int(options['e']),\n            id_skip=('noskip' not in block_string),\n            se_ratio=float(options['se']) if 'se' in options else None,\n            stride=[int(options['s'][0])])\n\n    @staticmethod\n    def _encode_block_string(block):\n        \"\"\"Encodes a block to a string.\"\"\"\n        args = [\n            'r%d' % block.num_repeat,\n            'k%d' % block.kernel_size,\n            's%d%d' % (block.strides[0], block.strides[1]),\n            'e%s' % block.expand_ratio,\n            'i%d' % block.input_filters,\n            'o%d' % block.output_filters\n        ]\n        if 0 < block.se_ratio <= 1:\n            args.append('se%s' % block.se_ratio)\n        if block.id_skip is False:\n            args.append('noskip')\n        return '_'.join(args)\n\n    @staticmethod\n    def decode(string_list):\n        \"\"\"\n        Decodes a list of string notations to specify blocks inside the network.\n        :param string_list: a list of strings, each string is a notation of block\n        :return: a list of BlockArgs namedtuples of block args\n        \"\"\"\n        assert isinstance(string_list, list)\n        blocks_args = []\n        for block_string in string_list:\n            blocks_args.append(BlockDecoder._decode_block_string(block_string))\n        return blocks_args\n\n    @staticmethod\n    def encode(blocks_args):\n        \"\"\"\n        Encodes a list of BlockArgs to a list of strings.\n        :param blocks_args: a list of BlockArgs namedtuples of block args\n        :return: a list of strings, each string is a notation of block\n        \"\"\"\n        block_strings = []\n        for block in blocks_args:\n            block_strings.append(BlockDecoder._encode_block_string(block))\n        return block_strings\n\n\ndef efficientnet(width_coefficient=None, depth_coefficient=None, dropout_rate=0.2,\n                 drop_connect_rate=0.2, image_size=None, num_classes=1000):\n    \"\"\" Creates a efficientnet model. \"\"\"\n\n    blocks_args = [\n        'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25',\n        'r2_k5_s22_e6_i24_o40_se0.25', 'r3_k3_s22_e6_i40_o80_se0.25',\n        'r3_k5_s11_e6_i80_o112_se0.25', 'r4_k5_s22_e6_i112_o192_se0.25',\n        'r1_k3_s11_e6_i192_o320_se0.25',\n    ]\n    blocks_args = BlockDecoder.decode(blocks_args)\n\n    global_params = GlobalParams(\n        batch_norm_momentum=0.99,\n        batch_norm_epsilon=1e-3,\n        dropout_rate=dropout_rate,\n        drop_connect_rate=drop_connect_rate,\n        # data_format='channels_last',  # removed, this is always true in PyTorch\n        num_classes=num_classes,\n        width_coefficient=width_coefficient,\n        depth_coefficient=depth_coefficient,\n        depth_divisor=8,\n        min_depth=None,\n        image_size=image_size,\n    )\n\n    return blocks_args, global_params\n\n\ndef get_model_params(model_name, override_params):\n    \"\"\" Get the block args and global params for a given model \"\"\"\n    if model_name.startswith('efficientnet'):\n        w, d, s, p = efficientnet_params(model_name)\n        # note: all models have drop connect rate = 0.2\n        blocks_args, global_params = efficientnet(\n            width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s)\n    else:\n        raise NotImplementedError('model name is not pre-defined: %s' % model_name)\n    if override_params:\n        # ValueError will be raised here if override_params has fields not included in global_params.\n        global_params = global_params._replace(**override_params)\n    return blocks_args, global_params\n\n\nurl_map = {\n    'efficientnet-b0': '../input/efficientnet-pytorch/efficientnet-b0-08094119.pth',\n    'efficientnet-b1': 'http://storage.googleapis.com/public-models/efficientnet/efficientnet-b1-f1951068.pth',\n    'efficientnet-b2': 'http://storage.googleapis.com/public-models/efficientnet/efficientnet-b2-8bb594d6.pth',\n    'efficientnet-b3': 'http://storage.googleapis.com/public-models/efficientnet/efficientnet-b3-5fb5a3c3.pth',\n    'efficientnet-b4': 'http://storage.googleapis.com/public-models/efficientnet/efficientnet-b4-6ed6700e.pth',\n    'efficientnet-b5': 'http://storage.googleapis.com/public-models/efficientnet/efficientnet-b5-b6417697.pth',\n    'efficientnet-b6': 'http://storage.googleapis.com/public-models/efficientnet/efficientnet-b6-c76e70fd.pth',\n    'efficientnet-b7': 'http://storage.googleapis.com/public-models/efficientnet/efficientnet-b7-dcc49843.pth',\n}\n\n## This below function is modified to use the pretrained weight for single channel . Its nothing but summing the weight across one axis .\ndef load_pretrained_weights(model, model_name, load_fc=True,ch=1):\n    \"\"\" Loads pretrained weights, and downloads if loading for the first time. \"\"\"\n    state_dict = torch.load('../input/efficientnet-pytorch/efficientnet-b0-08094119.pth')\n    if load_fc:\n        if ch == 1:\n            conv1_weight = state_dict['_conv_stem.weight']\n            state_dict['_conv_stem.weight'] = conv1_weight.sum(dim=1, keepdim=True)\n        model.load_state_dict(state_dict)\n        \n    else:\n        state_dict.pop('_fc.weight')\n        state_dict.pop('_fc.bias')\n        if ch == 1:\n            conv1_weight = state_dict['_conv_stem.weight']\n            state_dict['_conv_stem.weight'] = conv1_weight.sum(dim=1, keepdim=True)\n        res = model.load_state_dict(state_dict, strict=False)\n        print(res.missing_keys)\n        assert set(res.missing_keys) == set(['_fc.weight', '_fc.bias','fc1.weight', 'fc1.bias','fc2.weight', 'fc2.bias','fc3.weight', 'fc3.bias']), 'issue loading pretrained weights'\n    print('Loaded pretrained weights for {}'.format(model_name))","execution_count":null,"outputs":[]},{"metadata":{"id":"1pqy7kmC9vNF","colab_type":"code","colab":{},"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom torch.nn import functional as F\n\n\nclass MBConvBlock(nn.Module):\n    \"\"\"\n    Mobile Inverted Residual Bottleneck Block\n    Args:\n        block_args (namedtuple): BlockArgs, see above\n        global_params (namedtuple): GlobalParam, see above\n    Attributes:\n        has_se (bool): Whether the block contains a Squeeze and Excitation layer.\n    \"\"\"\n\n    def __init__(self, block_args, global_params):\n        super().__init__()\n        self._block_args = block_args\n        self._bn_mom = 1 - global_params.batch_norm_momentum\n        self._bn_eps = global_params.batch_norm_epsilon\n        self.has_se = (self._block_args.se_ratio is not None) and (0 < self._block_args.se_ratio <= 1)\n        self.id_skip = block_args.id_skip  # skip connection and drop connect\n\n        # Get static or dynamic convolution depending on image size\n        Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)\n\n        # Expansion phase\n        inp = self._block_args.input_filters  # number of input channels\n        oup = self._block_args.input_filters * self._block_args.expand_ratio  # number of output channels\n        if self._block_args.expand_ratio != 1:\n            self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias=False)\n            self._bn0 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)\n\n        # Depthwise convolution phase\n        k = self._block_args.kernel_size\n        s = self._block_args.stride\n        self._depthwise_conv = Conv2d(\n            in_channels=oup, out_channels=oup, groups=oup,  # groups makes it depthwise\n            kernel_size=k, stride=s, bias=False)\n        self._bn1 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)\n\n        # Squeeze and Excitation layer, if desired\n        if self.has_se:\n            num_squeezed_channels = max(1, int(self._block_args.input_filters * self._block_args.se_ratio))\n            self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1)\n            self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1)\n\n        # Output phase\n        final_oup = self._block_args.output_filters\n        self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False)\n        self._bn2 = nn.BatchNorm2d(num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps)\n        self._swish = MemoryEfficientSwish()\n\n    def forward(self, inputs, drop_connect_rate=None):\n        \"\"\"\n        :param inputs: input tensor\n        :param drop_connect_rate: drop connect rate (float, between 0 and 1)\n        :return: output of block\n        \"\"\"\n\n        # Expansion and Depthwise Convolution\n        x = inputs\n        if self._block_args.expand_ratio != 1:\n            x = self._swish(self._bn0(self._expand_conv(inputs)))\n        x = self._swish(self._bn1(self._depthwise_conv(x)))\n\n        # Squeeze and Excitation\n        if self.has_se:\n            x_squeezed = F.adaptive_avg_pool2d(x, 1)\n            x_squeezed = self._se_expand(self._swish(self._se_reduce(x_squeezed)))\n            x = torch.sigmoid(x_squeezed) * x\n\n        x = self._bn2(self._project_conv(x))\n\n        # Skip connection and drop connect\n        input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters\n        if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:\n            if drop_connect_rate:\n                x = drop_connect(x, p=drop_connect_rate, training=self.training)\n            x = x + inputs  # skip connection\n        return x\n\n    def set_swish(self, memory_efficient=True):\n        \"\"\"Sets swish function as memory efficient (for training) or standard (for export)\"\"\"\n        self._swish = MemoryEfficientSwish() if memory_efficient else Swish()\n\n\nclass EfficientNet(nn.Module):\n    \"\"\"\n    An EfficientNet model. Most easily loaded with the .from_name or .from_pretrained methods\n    Args:\n        blocks_args (list): A list of BlockArgs to construct blocks\n        global_params (namedtuple): A set of GlobalParams shared between blocks\n    Example:\n        model = EfficientNet.from_pretrained('efficientnet-b0')\n    \"\"\"\n\n    def __init__(self, blocks_args=None, global_params=None):\n        super().__init__()\n        assert isinstance(blocks_args, list), 'blocks_args should be a list'\n        assert len(blocks_args) > 0, 'block args must be greater than 0'\n        self._global_params = global_params\n        self._blocks_args = blocks_args\n\n        # Get static or dynamic convolution depending on image size\n        Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)\n\n        # Batch norm parameters\n        bn_mom = 1 - self._global_params.batch_norm_momentum\n        bn_eps = self._global_params.batch_norm_epsilon\n\n        # Stem\n        in_channels = 1  # rgb\n        out_channels = round_filters(32, self._global_params)  # number of output channels\n        self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)\n        self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)\n\n        # Build blocks\n        self._blocks = nn.ModuleList([])\n        for block_args in self._blocks_args:\n\n            # Update block input and output filters based on depth multiplier.\n            block_args = block_args._replace(\n                input_filters=round_filters(block_args.input_filters, self._global_params),\n                output_filters=round_filters(block_args.output_filters, self._global_params),\n                num_repeat=round_repeats(block_args.num_repeat, self._global_params)\n            )\n\n            # The first block needs to take care of stride and filter size increase.\n            self._blocks.append(MBConvBlock(block_args, self._global_params))\n            if block_args.num_repeat > 1:\n                block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)\n            for _ in range(block_args.num_repeat - 1):\n                self._blocks.append(MBConvBlock(block_args, self._global_params))\n\n        # Head\n        in_channels = block_args.output_filters  # output of final block\n        out_channels = round_filters(1280, self._global_params)\n        self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)\n        self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)\n\n        # Final linear layer\n        self._avg_pooling = nn.AdaptiveAvgPool2d(1)\n        self._dropout = nn.Dropout(self._global_params.dropout_rate)\n        self._fc = nn.Linear(out_channels, self._global_params.num_classes)\n        # vowel_diacritic\n        self.fc1 = nn.Linear(out_channels,11)\n        # grapheme_root\n        self.fc2 = nn.Linear(out_channels,20)\n        # consonant_diacritic\n        self.fc3 = nn.Linear(out_channels,4)\n        self._swish = MemoryEfficientSwish()\n\n    def set_swish(self, memory_efficient=True):\n        \"\"\"Sets swish function as memory efficient (for training) or standard (for export)\"\"\"\n        self._swish = MemoryEfficientSwish() if memory_efficient else Swish()\n        for block in self._blocks:\n            block.set_swish(memory_efficient)\n\n\n    def extract_features(self, inputs):\n        \"\"\" Returns output of the final convolution layer \"\"\"\n\n        # Stem\n        x = self._swish(self._bn0(self._conv_stem(inputs)))\n\n        # Blocks\n        for idx, block in enumerate(self._blocks):\n            drop_connect_rate = self._global_params.drop_connect_rate\n            if drop_connect_rate:\n                drop_connect_rate *= float(idx) / len(self._blocks)\n            x = block(x, drop_connect_rate=drop_connect_rate)\n\n        # Head\n        x = self._swish(self._bn1(self._conv_head(x)))\n\n        return x\n\n    def forward(self, inputs):\n        \"\"\" Calls extract_features to extract features, applies final linear layer, and returns logits. \"\"\"\n        bs = inputs.size(0)\n        # Convolution layers\n        x = self.extract_features(inputs)\n\n        # Pooling and final linear layer\n        x = self._avg_pooling(x)\n        x = x.view(bs, -1)\n        x = self._dropout(x)\n       # x = self._fc(x)\n        x1 = self.fc1(x)\n        x2= self.fc2(x)\n        x3 = self.fc3(x)\n        return x1,x2,x3\n\n    @classmethod\n    def from_name(cls, model_name, override_params=None):\n        cls._check_model_name_is_valid(model_name)\n        blocks_args, global_params = get_model_params(model_name, override_params)\n        return cls(blocks_args, global_params)\n\n    @classmethod\n    def from_pretrained(cls, model_name, num_classes=1000, in_channels = 1):\n        model = cls.from_name(model_name, override_params={'num_classes': num_classes})\n        load_pretrained_weights(model, model_name, load_fc=False)\n        if in_channels != 3:\n            Conv2d = get_same_padding_conv2d(image_size = model._global_params.image_size)\n            out_channels = round_filters(32, model._global_params)\n            model._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)\n        return model\n    \n    @classmethod\n    def from_pretrained(cls, model_name, num_classes=1000):\n        model = cls.from_name(model_name, override_params={'num_classes': num_classes})\n        load_pretrained_weights(model, model_name, load_fc=False)\n\n        return model\n\n    @classmethod\n    def get_image_size(cls, model_name):\n        cls._check_model_name_is_valid(model_name)\n        _, _, res, _ = efficientnet_params(model_name)\n        return res\n\n    @classmethod\n    def _check_model_name_is_valid(cls, model_name, also_need_pretrained_weights=False):\n        \"\"\" Validates model name. None that pretrained weights are only available for\n        the first four models (efficientnet-b{i} for i in 0,1,2,3) at the moment. \"\"\"\n        num_models = 4 if also_need_pretrained_weights else 8\n        valid_models = ['efficientnet-b'+str(i) for i in range(num_models)]\n        if model_name not in valid_models:\n            raise ValueError('model_name should be one of: ' + ', '.join(valid_models))","execution_count":null,"outputs":[]},{"metadata":{"id":"ef1FmDUd9vNH","colab_type":"code","outputId":"cb697b24-6e8a-467e-b69d-7e5efe85fb5c","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"## Make sure we are using the GPU . Get CUDA device\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{"id":"nqrdveM69vNJ","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"## Now create the model. Since its greyscale , I have  used pretrained model with modified weight. \n##I will make the necessary modification to load pretrained weights for greyscale by summing up the weights over one axis or copying greyscale into three channels\nmodel = EfficientNet.from_pretrained('efficientnet-b0').to(device) ## I switched to effnet-b4 in this version ","execution_count":null,"outputs":[]},{"metadata":{"id":"81pOj1vV9vNM","colab_type":"code","outputId":"c05616df-9ac2-458a-89c7-39d3107ec5d0","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"## A Small but useful test of the Model by using dummy input . .\nx = torch.zeros((32,1, 64, 64))\nwith torch.no_grad():\n    output1,output2,output3 =model(x.to(device))\nprint(output3.shape)","execution_count":null,"outputs":[]},{"metadata":{"id":"L16fEIZZD5WG","colab_type":"code","outputId":"b6207260-2f0a-4152-84f9-c4d5154d9386","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"## This is a placeholder for finetunign or inference when you want to load a previously trained model\n##and want to finetune or want to do just inference\n\n##model.load_state_dict(torch.load('../input/bengef2/effnetb0_trial_stage1.pth'))  \n## There is a small thing . I trained using effnetb4 offline for 20 epochs and loaded the weight\n## I forgot to change the naming convension and it still reads effnetb0 . But this is actually effnetb4","execution_count":null,"outputs":[]},{"metadata":{"id":"AGQiXKUk9vNT","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"n_epochs = 1 ## 1 Epoch as sample . \"I am just a poor boy  , no GPU in reality \"\n\n#optimizer =torch.optim.Adam(model.parameters(), lr=1e-4)\n#scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, 5, 2e-4) ## This didnt give good result need to correct  and get the right scheduler .\noptimizer =Over9000(model.parameters(), lr=2e-3, weight_decay=1e-3) ## New once \nscheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, 1e-2, total_steps=None, epochs=n_epochs, steps_per_epoch=5021, pct_start=0.0,\n                                   anneal_strategy='cos', cycle_momentum=True,base_momentum=0.85, max_momentum=0.95,  div_factor=100.0) ## Scheduler . Step for each batch\ncriterion = nn.CrossEntropyLoss()\nbatch_size=32","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##Local Metrics implementation .\n##https://www.kaggle.com/corochann/bengali-seresnext-training-with-pytorch\nimport numpy as np\nimport sklearn.metrics\nimport torch\n\n\ndef macro_recall(pred_y, y, n_grapheme=20, n_vowel=11, n_consonant=4):\n    pred_y = torch.split(pred_y, [n_grapheme, n_vowel, n_consonant], dim=1)\n    pred_labels = [torch.argmax(py, dim=1).cpu().numpy() for py in pred_y]\n\n    y = y.cpu().numpy()\n    # pred_y = [p.cpu().numpy() for p in pred_y]\n\n    recall_grapheme = sklearn.metrics.recall_score(pred_labels[0], y[:, 0], average='macro')\n    recall_vowel = sklearn.metrics.recall_score(pred_labels[1], y[:, 1], average='macro')\n    recall_consonant = sklearn.metrics.recall_score(pred_labels[2], y[:, 2], average='macro')\n    scores = [recall_grapheme, recall_vowel, recall_consonant]\n    final_score = np.average(scores, weights=[2, 1, 1])\n    # print(f'recall: grapheme {recall_grapheme}, vowel {recall_vowel}, consonant {recall_consonant}, '\n    #       f'total {final_score}, y {y.shape}')\n    return final_score\n\ndef macro_recall_multi(pred_graphemes, true_graphemes,pred_vowels,true_vowels,pred_consonants,true_consonants, n_grapheme=20, n_vowel=11, n_consonant=4):\n    #pred_y = torch.split(pred_y, [n_grapheme], dim=1)\n    pred_label_graphemes = torch.argmax(pred_graphemes, dim=1).cpu().numpy()\n\n    true_label_graphemes = true_graphemes.cpu().numpy()\n    \n    pred_label_vowels = torch.argmax(pred_vowels, dim=1).cpu().numpy()\n\n    true_label_vowels = true_vowels.cpu().numpy()\n    \n    pred_label_consonants = torch.argmax(pred_consonants, dim=1).cpu().numpy()\n\n    true_label_consonants = true_consonants.cpu().numpy()    \n    # pred_y = [p.cpu().numpy() for p in pred_y]\n\n    recall_grapheme = sklearn.metrics.recall_score(pred_label_graphemes, true_label_graphemes, average='macro')\n    recall_vowel = sklearn.metrics.recall_score(pred_label_vowels, true_label_vowels, average='macro')\n    recall_consonant = sklearn.metrics.recall_score(pred_label_consonants, true_label_consonants, average='macro')\n    scores = [recall_grapheme, recall_vowel, recall_consonant]\n    final_score = np.average(scores, weights=[2, 1, 1])\n    #print(f'recall: grapheme {recall_grapheme}, vowel {recall_vowel}, consonant {recall_consonant}, '\n    #       f'total {final_score}')\n    return final_score\n\n\ndef calc_macro_recall(solution, submission):\n    # solution df, submission df\n    scores = []\n    for component in ['grapheme_root', 'consonant_diacritic', 'vowel_diacritic']:\n        y_true_subset = solution[solution[component] == component]['target'].values\n        y_pred_subset = submission[submission[component] == component]['target'].values\n        scores.append(sklearn.metrics.recall_score(\n            y_true_subset, y_pred_subset, average='macro'))\n    final_score = np.average(scores, weights=[2, 1, 1])\n    return final_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Training and validation part "},{"metadata":{"id":"WNPMuR0N9vNU","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"## This function for train is copied from @hanjoonchoe\n## We are going to train and track accuracy and then evaluate and track validation accuracy\ndef train(epoch,history):\n  model.train()\n  losses = []\n  accs = []\n  acc= 0.0\n  total = 0.0\n  running_loss = 0.0\n  running_acc = 0.0\n  running_recall = 0.0\n  for idx, (inputs,labels1,labels2,labels3) in tqdm(enumerate(train_loader),total=len(train_loader)):\n      inputs = inputs.to(device)\n      labels1 = labels1.to(device)\n      labels2 = labels2.to(device)\n      labels3 = labels3.to(device)\n      total += len(inputs)\n      optimizer.zero_grad()\n      outputs1,outputs2,outputs3 = model(inputs.unsqueeze(1).float())\n      loss1 = 0.1*criterion(outputs1,labels1)\n      loss2 = 0.7* criterion(outputs2,labels2)\n      loss3 = 0.2*criterion(outputs3,labels3)\n      running_loss += loss1.item()+loss2.item()+loss3.item()\n      running_recall+= macro_recall_multi(outputs2,labels2,outputs1,labels1,outputs3,labels3)\n      running_acc += (outputs1.argmax(1)==labels1).float().mean()\n      running_acc += (outputs2.argmax(1)==labels2).float().mean()\n      running_acc += (outputs3.argmax(1)==labels3).float().mean()\n      (loss1+loss2+loss3).backward()\n      optimizer.step()\n      optimizer.zero_grad()\n      acc = running_acc/total\n      scheduler.step()\n  losses.append(running_loss/len(train_loader))\n  accs.append(running_acc/(len(train_loader)*3))\n  print(' train epoch : {}\\tacc : {:.2f}%'.format(epoch,running_acc/(len(train_loader)*3)))\n  print('loss : {:.4f}'.format(running_loss/len(train_loader)))\n    \n  print('recall: {:.4f}'.format(running_recall/len(train_loader)))\n  total_train_recall = running_recall/len(train_loader)\n  torch.cuda.empty_cache()\n  gc.collect()\n  history.loc[epoch, 'train_loss'] = losses[0]\n  history.loc[epoch,'train_acc'] = accs[0].cpu().numpy()\n  history.loc[epoch,'train_recall'] = total_train_recall\n  return  total_train_recall\ndef evaluate(epoch,history):\n   model.eval()\n   losses = []\n   accs = []\n   recalls = []\n   acc= 0.0\n   total = 0.0\n   #print('epochs {}/{} '.format(epoch+1,epochs))\n   running_loss = 0.0\n   running_acc = 0.0\n   running_recall = 0.0\n   with torch.no_grad():\n     for idx, (inputs,labels1,labels2,labels3) in tqdm(enumerate(valid_loader),total=len(valid_loader)):\n        inputs = inputs.to(device)\n        labels1 = labels1.to(device)\n        labels2 = labels2.to(device)\n        labels3 = labels3.to(device)\n        total += len(inputs)\n        outputs1,outputs2,outputs3 = model(inputs.unsqueeze(1).float())\n        loss1 = criterion(outputs1,labels1)\n        loss2 = 2*criterion(outputs2,labels2)\n        loss3 = criterion(outputs3,labels3)\n        running_loss += loss1.item()+loss2.item()+loss3.item()\n        running_recall+= macro_recall_multi(outputs2,labels2,outputs1,labels1,outputs3,labels3)\n        running_acc += (outputs1.argmax(1)==labels1).float().mean()\n        running_acc += (outputs2.argmax(1)==labels2).float().mean()\n        running_acc += (outputs3.argmax(1)==labels3).float().mean()\n        acc = running_acc/total\n        #scheduler.step()\n   losses.append(running_loss/len(valid_loader))\n   accs.append(running_acc/(len(valid_loader)*3))\n   recalls.append(running_recall/len(valid_loader))\n   total_recall = running_recall/len(valid_loader) ## No its not Arnold Schwarzenegger movie\n   print('val epoch: {} \\tval acc : {:.2f}%'.format(epoch,running_acc/(len(valid_loader)*3)))\n   print('loss : {:.4f}'.format(running_loss/len(valid_loader)))\n   print('recall: {:.4f}'.format(running_recall/len(valid_loader)))\n   history.loc[epoch, 'valid_loss'] = losses[0]\n   history.loc[epoch, 'valid_acc'] = accs[0].cpu().numpy()\n   history.loc[epoch, 'valid_recall'] = total_recall\n   return  total_recall\n","execution_count":null,"outputs":[]},{"metadata":{"id":"U_DXw7VK1L44","colab_type":"code","outputId":"327ceb49-8300-4fd9-9880-5e8821653448","colab":{"base_uri":"https://localhost:8080/","height":1000,"referenced_widgets":["60a7608bbb994558950656625a80a348","f071169d8d3d43e49622505b453587c2","31a4c071e67c4d9eb417177bbbc06e61","0640f2218c3e486fa7df7877c82da990","7c895c2996164ca984e1f07eea3e9d7e","765e09916e944ccd8c3e94034b5f97d1","5865b44100a746a2832500276d9df9ca","6f3dfe1e04fa4629ae42383564dfca2e","d2b7fe5e528943998a657871fe20525a","f1170bc175bd4bb4a8400f893eb6d85f","a653945e05fe4cbd81f359bfcb025f56","d7d2ab5a2cdc4438b0000d6580c23ed4","4b22d8a5e373440ba3755102b8c085dd","d397610d0fbe43aeb548d1fb127ad4df","33f7a1a0e409420fab7e008b5364164c","370904291b784338b9b0752d27bc61ff","f1246e6a5dbd49c984b8a99c0ffa0dc4","1e29de085df5483597041cb4a95eb939","604a7119c8a04c07837eb9f520a9e8a4","1b54b7a6640043e5883beeb59ab3d403","f18e6cee9e7146f2bd997e5f9dfaf468","a3ea218855e946599fb783a922665ea7","bcc8ebeeb4f14929bcc328fc678c9fbe","6fa82b38d1414e528b86f55c7ae00935","19d88b0a6e6549eab778dda50ccb1d94","d6a168d0800343f8bb19d884a575c081","d1c98826e0754638bf5cbd7e0ec53b35","cd7d2071f5454e2f94c404e1cd02d2a5","fa58e28e6d6d490fa87d369269eb3115","16771e8ab2384fff891d68f90a25224c","a89932cc7ec54d25a6d164dedc26b447","dbb54f3322f748279b3e53cbf0fb7b0f","cb51b398a590470296bc148f5e989738","6f4d2b74df4b46b892804d93561b18a8","2882de2943684097825c37619d2f51ba","8afe693630454028be7a4711b793c6f7","bbb604013fba4f0abd6fa405106b53f5","244b7679206245bd9f3da51c18dae710","180b2f1546304e2ab4e317c0ac7a9da8","85d242efc58e4445a74a3c553f1d4ca3"]},"trusted":true},"cell_type":"code","source":"## A very simple loop to train for number of epochs it probably can be made more robust to save only the file with best valid loss \nhistory = pd.DataFrame()\nn_epochs = 30 ## 1 Epoch as sample . \"I am just a poor boy  , no GPU in reality \"\nvalid_recall = 0.0\nbest_valid_recall = 0.0\nfor epoch in range(n_epochs):\n    torch.cuda.empty_cache()\n    gc.collect()\n    train_recall = train(epoch,history)\n    valid_recall = evaluate(epoch,history)\n    if valid_recall > best_valid_recall:\n        print(f'Validation recall has increased from:  {best_valid_recall:.4f} to: {valid_recall:.4f}. Saving checkpoint')\n        torch.save(model.state_dict(), 'effnetb0_trial_stage1.pth') ## Saving model weights based on best validation accuracy.\n        best_valid_recall = valid_recall ## Set the new validation Recall score to compare with next epoch\n        \n        \n    #scheduler.step() ## Want to test with fixed learning rate .If you want to use scheduler please uncomment this .\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history.to_csv('history.csv')\nhistory.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Inference part "},{"metadata":{"id":"-zIxzyEU9vNZ","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\nimport cv2\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torchvision import transforms,models\nfrom tqdm import tqdm_notebook as tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Load model for inferernce . \n#model.load_state_dict(torch.load('../input/pytorch-efficientnet-starter-code/effnetb0_trial_stage1.pth')) ","execution_count":null,"outputs":[]},{"metadata":{"id":"wFmHJ21S9vNb","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/bengaliai-cv19/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#check https://www.kaggle.com/iafoss/image-preprocessing-128x128\n\ndef bbox(img):\n    rows = np.any(img, axis=1)\n    cols = np.any(img, axis=0)\n    rmin, rmax = np.where(rows)[0][[0, -1]]\n    cmin, cmax = np.where(cols)[0][[0, -1]]\n    return rmin, rmax, cmin, cmax\n\ndef crop_resize(img0, size=SIZE, pad=16):\n    #crop a box around pixels large than the threshold \n    #some images contain line at the sides\n    ymin,ymax,xmin,xmax = bbox(img0[5:-5,5:-5] > 80)\n    #cropping may cut too much, so we need to add it back\n    xmin = xmin - 13 if (xmin > 13) else 0\n    ymin = ymin - 10 if (ymin > 10) else 0\n    xmax = xmax + 13 if (xmax < WIDTH - 13) else WIDTH\n    ymax = ymax + 10 if (ymax < HEIGHT - 10) else HEIGHT\n    img = img0[ymin:ymax,xmin:xmax]\n    #remove lo intensity pixels as noise\n    img[img < 28] = 0\n    lx, ly = xmax-xmin,ymax-ymin\n    l = max(lx,ly) + pad\n    #make sure that the aspect ratio is kept in rescaling\n    img = np.pad(img, [((l-ly)//2,), ((l-lx)//2,)], mode='constant')\n    return cv2.resize(img,(size,size))","execution_count":null,"outputs":[]},{"metadata":{"id":"Kv348NR19vNg","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"class GraphemeDataset(Dataset):\n    def __init__(self, fname):\n        print(fname)\n        self.df = pd.read_parquet(fname)\n        self.data = 255 - self.df.iloc[:, 1:].values.reshape(-1, HEIGHT, WIDTH).astype(np.uint8)\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        name = self.df.iloc[idx,0]\n        #normalize each image by its max val\n        img = (self.data[idx]*(255.0/self.data[idx].max())).astype(np.uint8)\n        img = crop_resize(img)\n        img = img.astype(np.float32)/255.0\n        return img, name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## All test data\ntest_data = ['/kaggle/input/bengaliai-cv19/test_image_data_0.parquet','/kaggle/input/bengaliai-cv19/test_image_data_1.parquet','/kaggle/input/bengaliai-cv19/test_image_data_2.parquet',\n             '/kaggle/input/bengaliai-cv19/test_image_data_3.parquet']","execution_count":null,"outputs":[]},{"metadata":{"id":"DO12892o9vNm","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"%%time\n## Inference a little faster using @Iafoss and  @peters technique\nrow_id,target = [],[]\nfor fname in test_data:\n    #data = pd.read_parquet(f'/kaggle/input/bengaliai-cv19/{fname}')\n    test_image = GraphemeDataset(fname)\n    dl = torch.utils.data.DataLoader(test_image,batch_size=128,num_workers=4,shuffle=False)\n    with torch.no_grad():\n        for x,y in tqdm(dl):\n            x = x.unsqueeze(1).float().cuda()\n            p1,p2,p3 = model(x)\n            p1 = p1.argmax(-1).view(-1).cpu()\n            p2 = p2.argmax(-1).view(-1).cpu()\n            p3 = p3.argmax(-1).view(-1).cpu()\n            for idx,name in enumerate(y):\n                row_id += [f'{name}_vowel_diacritic',f'{name}_grapheme_root',\n                           f'{name}_consonant_diacritic']\n                target += [p1[idx].item(),p2[idx].item(),p3[idx].item()]\n                \nsub_df = pd.DataFrame({'row_id': row_id, 'target': target})\nsub_df.to_csv('submission.csv', index=False)\nsub_df.head(20)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"},"colab":{"name":"Copy of Effnet-starter-kit.ipynb","provenance":[],"collapsed_sections":["ztO_qj4p9vNZ","sH9h392a9vNo"],"toc_visible":true,"machine_shape":"hm"},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"60a7608bbb994558950656625a80a348":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_f071169d8d3d43e49622505b453587c2","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_31a4c071e67c4d9eb417177bbbc06e61","IPY_MODEL_0640f2218c3e486fa7df7877c82da990"]}},"f071169d8d3d43e49622505b453587c2":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"31a4c071e67c4d9eb417177bbbc06e61":{"model_module":"@jupyter-widgets/controls","model_name":"IntProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_7c895c2996164ca984e1f07eea3e9d7e","_dom_classes":[],"description":"","_model_name":"IntProgressModel","bar_style":"success","max":5021,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":5021,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_765e09916e944ccd8c3e94034b5f97d1"}},"0640f2218c3e486fa7df7877c82da990":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_5865b44100a746a2832500276d9df9ca","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100% 5021/5021 [11:53&lt;00:00,  7.62it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_6f3dfe1e04fa4629ae42383564dfca2e"}},"7c895c2996164ca984e1f07eea3e9d7e":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"765e09916e944ccd8c3e94034b5f97d1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"5865b44100a746a2832500276d9df9ca":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"6f3dfe1e04fa4629ae42383564dfca2e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d2b7fe5e528943998a657871fe20525a":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_f1170bc175bd4bb4a8400f893eb6d85f","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_a653945e05fe4cbd81f359bfcb025f56","IPY_MODEL_d7d2ab5a2cdc4438b0000d6580c23ed4"]}},"f1170bc175bd4bb4a8400f893eb6d85f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"a653945e05fe4cbd81f359bfcb025f56":{"model_module":"@jupyter-widgets/controls","model_name":"IntProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_4b22d8a5e373440ba3755102b8c085dd","_dom_classes":[],"description":"","_model_name":"IntProgressModel","bar_style":"success","max":1256,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1256,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_d397610d0fbe43aeb548d1fb127ad4df"}},"d7d2ab5a2cdc4438b0000d6580c23ed4":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_33f7a1a0e409420fab7e008b5364164c","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100% 1256/1256 [00:38&lt;00:00, 32.55it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_370904291b784338b9b0752d27bc61ff"}},"4b22d8a5e373440ba3755102b8c085dd":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"d397610d0fbe43aeb548d1fb127ad4df":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"33f7a1a0e409420fab7e008b5364164c":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"370904291b784338b9b0752d27bc61ff":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"f1246e6a5dbd49c984b8a99c0ffa0dc4":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_1e29de085df5483597041cb4a95eb939","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_604a7119c8a04c07837eb9f520a9e8a4","IPY_MODEL_1b54b7a6640043e5883beeb59ab3d403"]}},"1e29de085df5483597041cb4a95eb939":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"604a7119c8a04c07837eb9f520a9e8a4":{"model_module":"@jupyter-widgets/controls","model_name":"IntProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_f18e6cee9e7146f2bd997e5f9dfaf468","_dom_classes":[],"description":"","_model_name":"IntProgressModel","bar_style":"success","max":5021,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":5021,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_a3ea218855e946599fb783a922665ea7"}},"1b54b7a6640043e5883beeb59ab3d403":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_bcc8ebeeb4f14929bcc328fc678c9fbe","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100% 5021/5021 [11:47&lt;00:00,  7.30it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_6fa82b38d1414e528b86f55c7ae00935"}},"f18e6cee9e7146f2bd997e5f9dfaf468":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"a3ea218855e946599fb783a922665ea7":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"bcc8ebeeb4f14929bcc328fc678c9fbe":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"6fa82b38d1414e528b86f55c7ae00935":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"19d88b0a6e6549eab778dda50ccb1d94":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_d6a168d0800343f8bb19d884a575c081","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_d1c98826e0754638bf5cbd7e0ec53b35","IPY_MODEL_cd7d2071f5454e2f94c404e1cd02d2a5"]}},"d6a168d0800343f8bb19d884a575c081":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d1c98826e0754638bf5cbd7e0ec53b35":{"model_module":"@jupyter-widgets/controls","model_name":"IntProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_fa58e28e6d6d490fa87d369269eb3115","_dom_classes":[],"description":"","_model_name":"IntProgressModel","bar_style":"success","max":1256,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1256,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_16771e8ab2384fff891d68f90a25224c"}},"cd7d2071f5454e2f94c404e1cd02d2a5":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_a89932cc7ec54d25a6d164dedc26b447","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100% 1256/1256 [00:38&lt;00:00, 32.55it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_dbb54f3322f748279b3e53cbf0fb7b0f"}},"fa58e28e6d6d490fa87d369269eb3115":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"16771e8ab2384fff891d68f90a25224c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"a89932cc7ec54d25a6d164dedc26b447":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"dbb54f3322f748279b3e53cbf0fb7b0f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"cb51b398a590470296bc148f5e989738":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_6f4d2b74df4b46b892804d93561b18a8","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_2882de2943684097825c37619d2f51ba","IPY_MODEL_8afe693630454028be7a4711b793c6f7"]}},"6f4d2b74df4b46b892804d93561b18a8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"2882de2943684097825c37619d2f51ba":{"model_module":"@jupyter-widgets/controls","model_name":"IntProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_bbb604013fba4f0abd6fa405106b53f5","_dom_classes":[],"description":"","_model_name":"IntProgressModel","bar_style":"","max":5021,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":2131,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_244b7679206245bd9f3da51c18dae710"}},"8afe693630454028be7a4711b793c6f7":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_180b2f1546304e2ab4e317c0ac7a9da8","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 42% 2131/5021 [04:58&lt;06:29,  7.42it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_85d242efc58e4445a74a3c553f1d4ca3"}},"bbb604013fba4f0abd6fa405106b53f5":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"244b7679206245bd9f3da51c18dae710":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"180b2f1546304e2ab4e317c0ac7a9da8":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"85d242efc58e4445a74a3c553f1d4ca3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"nbformat":4,"nbformat_minor":1}