{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install segmentation_models_pytorch ","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:07:06.267624Z","iopub.execute_input":"2022-05-29T17:07:06.268397Z","iopub.status.idle":"2022-05-29T17:07:23.572210Z","shell.execute_reply.started":"2022-05-29T17:07:06.268351Z","shell.execute_reply":"2022-05-29T17:07:23.571358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport random\n\nimport matplotlib.pyplot as plt\n\nimport segmentation_models_pytorch as smp\nfrom segmentation_models_pytorch.losses import DiceLoss\n\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torchvision\nimport torchvision.transforms as transforms\nfrom torch.utils.data import Dataset,DataLoader\nfrom tqdm import tqdm\nimport cv2\n# import torch_optimizer as optim\n\nimport warnings\nimport os\nimport gc\nimport glob\nimport os.path as osp\n\n\nPATH='../input/airbus-224x224x3/train_v2'\nPATH_TRAIN='../input/airbus-224x224x3/train_v2'\nPATH_TEST='../input/airbus-224x224x3/test_v2'\n\nwarnings.filterwarnings('ignore')\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-29T18:08:43.998849Z","iopub.execute_input":"2022-05-29T18:08:43.999294Z","iopub.status.idle":"2022-05-29T18:08:44.014695Z","shell.execute_reply.started":"2022-05-29T18:08:43.999244Z","shell.execute_reply":"2022-05-29T18:08:44.013836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Albumentations\n","metadata":{}},{"cell_type":"code","source":"# https://github.com/albu/albumentations\nfrom albumentations import (ToFloat, \n    CLAHE, RandomRotate90, Transpose, ShiftScaleRotate, Blur, OpticalDistortion, \n    GridDistortion, HueSaturationValue, IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, \n    MedianBlur, IAAPiecewiseAffine, IAASharpen, IAAEmboss, RandomContrast, RandomBrightness, \n    Flip, OneOf, Compose,HorizontalFlip,VerticalFlip\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:52:05.094455Z","iopub.execute_input":"2022-05-29T17:52:05.094875Z","iopub.status.idle":"2022-05-29T17:52:05.100432Z","shell.execute_reply.started":"2022-05-29T17:52:05.094839Z","shell.execute_reply":"2022-05-29T17:52:05.099769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(DEVICE)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:52:06.099173Z","iopub.execute_input":"2022-05-29T17:52:06.099754Z","iopub.status.idle":"2022-05-29T17:52:06.104893Z","shell.execute_reply.started":"2022-05-29T17:52:06.099721Z","shell.execute_reply":"2022-05-29T17:52:06.104234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" # Ship Detection: Image Visualizations and short EDA","metadata":{}},{"cell_type":"markdown","source":"## Loading the Datasets","metadata":{}},{"cell_type":"code","source":"class AirbusDS(Dataset):\n    \"\"\"\n    A customized data loader.\n    \"\"\"\n    def __init__(self, root):\n        \"\"\" Intialize the dataset\n        \"\"\"\n        self.filenames = []\n        self.root = root\n        self.transform = transforms.ToTensor()\n        filenames = glob.glob(osp.join(PATH, '*.jpg'))\n        for fn in filenames:\n            self.filenames.append(fn)\n        self.len = len(self.filenames)\n        \n    def __getitem__(self, index):\n        \"\"\" Get a sample from the dataset\n        \"\"\" \n        image = Image.open(self.filenames[index])\n        return self.transform(image)\n\n    def __len__(self):\n        \"\"\"\n        Total number of samples in the dataset\n        \"\"\"\n        return self.len","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:52:09.241814Z","iopub.execute_input":"2022-05-29T17:52:09.242081Z","iopub.status.idle":"2022-05-29T17:52:09.248508Z","shell.execute_reply.started":"2022-05-29T17:52:09.242051Z","shell.execute_reply":"2022-05-29T17:52:09.247861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Show data ","metadata":{}},{"cell_type":"code","source":"# functions to show an image\ndef imshow(img):\n    npimg = img.numpy()\n    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n    \nair_img = AirbusDS(PATH)\nprint(air_img.len)\n# Use the torch dataloader to iterate through the dataset\nloader = DataLoader(air_img, batch_size=24, shuffle=False, num_workers=0)\n\n# get some images\ndataiter = iter(loader)\nimages = dataiter.next()\n\n# show images\nplt.figure(figsize=(16,8))\nimshow(torchvision.utils.make_grid(images))","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:52:10.625393Z","iopub.execute_input":"2022-05-29T17:52:10.625885Z","iopub.status.idle":"2022-05-29T17:52:12.187361Z","shell.execute_reply.started":"2022-05-29T17:52:10.625848Z","shell.execute_reply":"2022-05-29T17:52:12.183295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Masks & Albumentations class","metadata":{}},{"cell_type":"code","source":"masks = pd.read_csv('../input/airbus-224x224x3/train_ship_segmentations_v2.csv')\n\n# ref: https://www.kaggle.com/paulorzp/run-length-encode-and-decode\ndef rle_decode(mask_rle, shape=(768, 768),resize=False):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n        \n    img = img.reshape(shape).T\n    if resize:\n        img = cv2.resize(img, (224, 224), cv2.INTER_AREA)\n    return img  \n\nmasks.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:52:12.188781Z","iopub.execute_input":"2022-05-29T17:52:12.189422Z","iopub.status.idle":"2022-05-29T17:52:12.659049Z","shell.execute_reply.started":"2022-05-29T17:52:12.189386Z","shell.execute_reply":"2022-05-29T17:52:12.658353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# an improve\nmean = np.array([0.65459856,0.48386562,0.69428385])\nstd = np.array([0.15167958,0.23584107,0.13146145])\ndef img2tensor(img, dtype:np.dtype=np.float32):\n    if img.ndim==2: \n        img=np.expand_dims(img, 2)\n    return torch.from_numpy(img.astype(dtype, copy=False))\n\nclass AirbusDS(Dataset):\n    \"\"\"\n    A customized data loader.\n    \"\"\"\n    def __init__(self, root, aug = False, mode='train'):\n        \"\"\" Intialize the dataset\n        \"\"\"\n        self.filenames = []\n        self.root = root\n        self.aug = aug\n        self.mode = 'test'\n        if mode == 'train':\n            self.mode = 'train'\n            self.masks = pd.read_csv('../input/airbus-224x224x3/train_ship_segmentations_v2.csv').fillna(-1)\n        if self.aug:\n            self.transform = OneOf([\n                                RandomRotate90(),\n                                Transpose(),\n                                Flip(),\n                            ], p=0.3)\n        else:\n            self.transform = transforms.ToTensor()\n        filenames = glob.glob(osp.join(PATH, '*.jpg'))\n        for fn in filenames:\n            self.filenames.append(fn)\n        self.len = len(self.filenames)\n        \n    # You must override __getitem__ and __len__\n    def get_mask(self, ImageId):\n        img_masks = self.masks.loc[self.masks['ImageId'] == ImageId, 'EncodedPixels'].tolist()\n\n        # Take the individual ship masks and create a single mask array for all ships\n        all_masks = np.zeros((768, 768))\n        if img_masks == [-1]:\n            return all_masks\n        for mask in img_masks:\n            all_masks += rle_decode(mask)\n        return all_masks\n    \n    def __getitem__(self, index):\n        \"\"\" Get a sample from the dataset\n        \"\"\"\n        image = Image.open(self.filenames[index])\n        ImageId = self.filenames[index].split('/')[-1]\n        if self.mode == 'train':\n            mask = self.get_mask(ImageId)\n        if self.aug:\n            if self.mode == 'train':\n                data = {\"image\": np.array(image), \"mask\": mask}\n            else:\n                data = {\"image\": np.array(image)}\n            transformed = self.transform(**data)\n            image = transformed['image']/255\n            image = np.transpose(image, (2, 0, 1))\n            if self.mode == 'train':\n                return img2tensor(image), img2tensor(transformed['mask'][np.newaxis,:,:] )\n            else:\n                return img2tensor(image)\n        else:\n            if self.mode == 'train':\n                return self.transform(img2tensor(image)),img2tensor( mask[np.newaxis,:,:] )\n            return self.transform(img2tensor(image))\n\n    def __len__(self):\n        \"\"\"\n        Total number of samples in the dataset\n        \"\"\"\n        return self.len\n \n \n","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:52:13.714341Z","iopub.execute_input":"2022-05-29T17:52:13.715027Z","iopub.status.idle":"2022-05-29T17:52:13.731517Z","shell.execute_reply.started":"2022-05-29T17:52:13.714987Z","shell.execute_reply":"2022-05-29T17:52:13.730832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Loader ","metadata":{}},{"cell_type":"code","source":"airimg = AirbusDS(PATH, aug=True, mode='train')\n# Use the torch dataloader to iterate through the dataset\nloader = DataLoader(airimg, batch_size=24, shuffle=False, num_workers=0)\n\n# get some images\ndataiter = iter(loader)\nimages, masks = dataiter.next()\n\n# show images\nplt.figure(figsize=(16,16))\nplt.rcParams['figure.facecolor'] = 'white'\nplt.subplot(211)\nimshow(torchvision.utils.make_grid(images))\nplt.subplot(212)\nimshow(torchvision.utils.make_grid(masks,pad_value=25.0))\n","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:52:16.840095Z","iopub.execute_input":"2022-05-29T17:52:16.840630Z","iopub.status.idle":"2022-05-29T17:52:23.868728Z","shell.execute_reply.started":"2022-05-29T17:52:16.840593Z","shell.execute_reply":"2022-05-29T17:52:23.868114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparations before training\n<img src=\"https://memegenerator.net/img/instances/86276885.jpg\" >\n","metadata":{}},{"cell_type":"markdown","source":"## transform","metadata":{}},{"cell_type":"code","source":"train_transforms = Compose([transforms.Resize(512),\n                           Flip(), RandomRotate90(),\n                           HorizontalFlip(),\n                           VerticalFlip(),\n                           transforms.RandomCrop(512)]\n                           )","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:52:23.870113Z","iopub.execute_input":"2022-05-29T17:52:23.870445Z","iopub.status.idle":"2022-05-29T17:52:23.875237Z","shell.execute_reply.started":"2022-05-29T17:52:23.870415Z","shell.execute_reply":"2022-05-29T17:52:23.874462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split, Dataset & DataLoader","metadata":{}},{"cell_type":"code","source":"# Take the individual ship masks and create a single mask array for all ships\ndef mask_as_image(masks, resize=True):\n    possible_masks = np.zeros((224, 224))\n    for mask in masks:\n        if isinstance (mask,float):\n            break\n        else:\n            mask_image = rle_decode(mask, resize=True)\n            if resize:\n                mask_image = cv2.resize(mask_image, (224, 224), cv2.INTER_AREA)\n            possible_masks += mask_image\n    return possible_masks\n\n# source='https://www.kaggle.com/inversion/run-length-decoding-quick-start'\ndef show_image_with_mask(imgs, masks,rows,cols):\n    fig, axarr = plt.subplots(1, 3, figsize=(15, 40))\n#     imgs = cv2.resize(imgs, (224, 224), cv2.INTER_AREA)\n#     plt.figure(figsize=[cols*5, rows*5])\n    for i in range(len(imgs)):\n        img = imgs[i]\n        mask = masks[i]\n        img = img.numpy()\n        mask = mask.numpy()\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        axarr[0].axis('off')\n        axarr[0].imshow(img)\n        \n        axarr[1].axis('off')\n        axarr[1].imshow(mask)\n        \n        axarr[2].axis('off')\n        axarr[2].imshow(img)\n        axarr[2].imshow(mask, alpha=0.8)\n        plt.tight_layout(h_pad=0.1, w_pad=0.1)\n        plt.show()    \n    \n# Dataset class adapted to train, validation and test situations\nclass shipDataset(Dataset):\n    def __init__(self,paths, transforms=None, train= True, test=False, mask_file=None):\n        self.paths = paths\n        self.transforms = transforms\n        self.train = train\n        self.mask_file = mask_file\n        self.test=test  \n        self.permute = False\n    def __len__(self):\n        return len(self.paths)\n\n    def __getitem__(self, idx):\n        p= self.paths[idx]\n        imageId=p.split('/')[-1]\n        image = cv2.imread(p)\n        image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)\n        image = cv2.normalize(image.astype('float'), None, 0.0, 1.0, cv2.NORM_MINMAX)\n        if self.train:\n            mask_list=self.mask_file.loc[self.mask_file[\"ImageId\"]==imageId, \"EncodedPixels\"].tolist()\n            masks = mask_as_image(mask_list, resize=True)\n            if self.transforms:\n\n                image = self.transforms(image=image)['image']\n                masks = self.transforms(masks)['mask']\n            \n            masks = torch.from_numpy(masks).float()\n        image = torch.from_numpy(image).float()\n        \n\n        #swap [768, 768, 3]---TO---> [3,768, 768]\n        if self.permute:\n            image = image.permute(2, 0,1)\n        if self.test:\n            return image, imageId\n\n        else:\n            return image, masks  \n        \ndef get_train (paths, next_path, csv, base_path, ship=True):\n    csv_mask = csv.copy()\n    new_lst = []\n    if ship:\n        df = csv_mask[csv_mask[\"EncodedPixels\"].notna()].copy()\n    else:\n        df = csv_mask[csv_mask[\"EncodedPixels\"].isna()].copy()\n    df = df.drop_duplicates(subset=\"ImageId\")\n    for rows in (df.iterrows()):\n        new_lst.append(base_path+next_path+\"/\"+rows[1].ImageId)\n    return new_lst\n\nfilenames_train = glob.glob(osp.join(PATH_TRAIN, '*.jpg'))\ntest_paths = glob.glob(osp.join(PATH_TEST, '*.jpg'))\nmask_file = pd.read_csv('../input/airbus-224x224x3/train_ship_segmentations_v2.csv')\n\n# reduce train size \nbase_path = \"../input/airbus-224x224x3/\"\ntrain_ship = get_train(filenames_train, \"train_v2\", mask_file, base_path, True)\ntrain_na_ship = get_train(filenames_train, \"train_v2\", mask_file, base_path, False)\nprint(f'len(images with shipe) = {len(train_ship)}\\nlen(images without shipe) = {len(train_na_ship)}')\n# reduce size of train without ships\ntrain_na_ship = random.sample(train_na_ship, int(len(train_na_ship)*0.01))\ntrain_ship = random.sample(train_ship, int(len(train_ship)*0.05))\nfilenames_train = train_na_ship + train_ship\nprint(f'len(data) = {len(filenames_train)}')\n#split\ntrain_paths, validation_paths = train_test_split(filenames_train, test_size=0.3,random_state=42)\nprint(f'len(test)/[len(test)+len(train)] = len(validation)/[len(validation)+len(train)]')\nprint(f'{len(test_paths)/(len(test_paths)+len(filenames_train))} = {len(validation_paths)/(len(validation_paths)+len(train_paths))}')\ncsv_mask_file = pd.read_csv('../input/airbus-224x224x3/train_ship_segmentations_v2.csv').fillna(-1)\n# DataSet\n\ntrain_dataset = shipDataset(train_paths, transforms=False, train=True, mask_file=mask_file)\nvalid_dataset = shipDataset(validation_paths, train=True,mask_file=mask_file)\ntest_dataset = shipDataset(test_paths, train=False, test=True)\n\n# DataLoader\nBS=24\nNW=0\ntrain_loader= DataLoader(train_dataset,batch_size=BS,shuffle=False) #, shuffle = False)#, pin_memory=True)\nvalid_loader= DataLoader(valid_dataset,batch_size=BS,shuffle=False) #, num_workers=NW)#, pin_memory=True)\ntest_loader= DataLoader(test_dataset,batch_size=1,shuffle=False)#, num_workers=NW)#, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T20:00:00.157024Z","iopub.execute_input":"2022-05-29T20:00:00.157347Z","iopub.status.idle":"2022-05-29T20:00:11.534559Z","shell.execute_reply.started":"2022-05-29T20:00:00.157314Z","shell.execute_reply":"2022-05-29T20:00:11.533793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Images & Masks","metadata":{}},{"cell_type":"code","source":"for i,(image, masks) in tqdm(enumerate(train_loader)):\n    if i < 5:\n        show_image_with_mask(image[0:3], masks[0:3],4,4)    \n    else:\n        break\n    ","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:52:41.550362Z","iopub.execute_input":"2022-05-29T17:52:41.550889Z","iopub.status.idle":"2022-05-29T17:52:50.602629Z","shell.execute_reply.started":"2022-05-29T17:52:41.550850Z","shell.execute_reply":"2022-05-29T17:52:50.601911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* As we can see- there are images without ships. I will address the issue later.","metadata":{}},{"cell_type":"code","source":"del airimg\ndel loader\ndel dataiter\ndel images\ndel train_na_ship\ndel filenames_train\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:52:50.604380Z","iopub.execute_input":"2022-05-29T17:52:50.604871Z","iopub.status.idle":"2022-05-29T17:52:50.880240Z","shell.execute_reply.started":"2022-05-29T17:52:50.604834Z","shell.execute_reply":"2022-05-29T17:52:50.879406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model training APIs","metadata":{}},{"cell_type":"code","source":"NUM_EPOCHS = 3\ndef get_train_and_valid_epochs(model, loss, metrics, optimizer):\n    train_epoch = smp.utils.train.TrainEpoch(\n        model,\n        loss=loss,\n        metrics=metrics,\n        optimizer=optimizer,\n        device=DEVICE,\n        verbose=True,\n    )\n\n    valid_epoch = smp.utils.train.ValidEpoch(\n        model,\n        loss=loss,\n        metrics=metrics,\n        device=DEVICE,\n        verbose=True,\n    )\n\n    return train_epoch, valid_epoch\n\ndef get_test_epochs(model, loss, metrics):\n    test_epoch = smp.utils.train.ValidEpoch(\n        model,\n        loss=loss,\n        metrics=metrics,\n        device=DEVICE,\n    )\n    \n    return test_epoch\n\ndef train(model, loss, metrics, optimizer, train_loader, valid_loader, model_name):\n    max_score = 0\n    train_epoch, valid_epoch = get_train_and_valid_epochs(model, loss, metrics, optimizer)\n    df = pd.DataFrame(columns=['train_loss','train_iou','val_loss','val_iou' ])\n    for i in range(0, NUM_EPOCHS):\n\n        print('\\nEpoch: {}'.format(i))\n        train_logs = train_epoch.run(train_loader)\n        valid_logs = valid_epoch.run(valid_loader)\n        \n        df = df.append({'train_loss':train_logs['dice_loss'],'train_iou':train_logs['iou_score'],\n                        'val_loss':valid_logs['dice_loss'],'val_iou':valid_logs['iou_score']},ignore_index=True)\n\n        # do something (save model, change lr, etc.)\n        if max_score < valid_logs['iou_score']:\n            max_score = valid_logs['iou_score']\n            torch.save(model, f'./{model_name}_best_model.pth')\n            print('Model saved!')\n    df.to_csv(f'{model_name}.csv', index=False)\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-05-29T20:00:11.536379Z","iopub.execute_input":"2022-05-29T20:00:11.536660Z","iopub.status.idle":"2022-05-29T20:00:11.549513Z","shell.execute_reply.started":"2022-05-29T20:00:11.536623Z","shell.execute_reply":"2022-05-29T20:00:11.547299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dice Loss\n* Dice coefficient-  \n![picture](https://miro.medium.com/max/321/1*EF3VCtk-VbTIKhriaQF0YQ.png)\n*  measure of overlap between two samples. This measure ranges from 0 to 1 where a Dice coefficient of 1 denotes perfect and complete overlap. The Dice coefficient was originally developed for binary data, and can be calculated as:  \n$ Dice = \\frac{2|A∩B|}{|A|+|B|} $\n  * |A∩B| represents the common elements between sets A and B\n  * |A| represents the number of elements in set A (and likewise for set B)\n\nFor the case of evaluating a Dice coefficient on predicted segmentation masks, we can approximate |A∩B| as the element-wise multiplication between the prediction and target mask, and then sum the resulting matrix.","metadata":{}},{"cell_type":"code","source":"dice_loss = smp.utils.losses.DiceLoss()\ndice_loss.to(DEVICE)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:53:01.527317Z","iopub.execute_input":"2022-05-29T17:53:01.527581Z","iopub.status.idle":"2022-05-29T17:53:01.534742Z","shell.execute_reply.started":"2022-05-29T17:53:01.527551Z","shell.execute_reply":"2022-05-29T17:53:01.533814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## IOU – Intersection over union measure\n<img src = 'https://www.researchgate.net/profile/Swapnil-Bhole/publication/340893451/figure/fig2/AS:921669640126464@1596754574959/Intersection-over-Union-IoU.ppm'\n     height = 550px width=550px>","metadata":{}},{"cell_type":"code","source":"metrics = [\n    smp.utils.metrics.IoU(threshold=0.5).to(DEVICE),\n]\n","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:53:03.878578Z","iopub.execute_input":"2022-05-29T17:53:03.879051Z","iopub.status.idle":"2022-05-29T17:53:03.883489Z","shell.execute_reply.started":"2022-05-29T17:53:03.879012Z","shell.execute_reply":"2022-05-29T17:53:03.882710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Architecture training\n* U-NET\n* FPN\n* PAN\n* DeepLabV3","metadata":{}},{"cell_type":"code","source":"train_loader.dataset.permute = True\nvalid_loader.dataset.permute = True\n\nENCODER_RESNET='resnet34'\nENCODER_WEIGHTS = 'imagenet'\nCLASSES = ['airbus']\nACTIVATION = 'sigmoid'\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ndef run_model(model, model_name):\n    model.cuda()\n    optimizer= torch.optim.Adam(model.parameters(),lr= 1e-3)\n    model_name=model_name\n\n    return train(model, dice_loss, metrics, optimizer, train_loader, valid_loader, model_name)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T20:00:14.975822Z","iopub.execute_input":"2022-05-29T20:00:14.976477Z","iopub.status.idle":"2022-05-29T20:00:14.982160Z","shell.execute_reply.started":"2022-05-29T20:00:14.976438Z","shell.execute_reply":"2022-05-29T20:00:14.981490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## U-net segmentation architecture \n* Its shape is that it uses special architecture to create very few data set segments. \n* The layers he uses: \n    * convolution-layers\n    * pooling-layers\n    * up-convolution-layers (sometimes)\n    * up-convolution-layers: Take each neuron in the input in the layer, and create a neuron of size 2 * 2 (the opposite of polling)\n    \n<img src =\"https://pyimagesearch.com/wp-content/uploads/2021/11/u-net_training_image_segmentation_models_in_pytorch_header.png\">","metadata":{}},{"cell_type":"code","source":"unet = smp.Unet(\n    encoder_name=ENCODER_RESNET,\n    encoder_weights=ENCODER_WEIGHTS,\n    classes=len(CLASSES), \n    activation=ACTIVATION,\n)\nunet = unet.to(DEVICE) \nrun_model(unet, 'U-Net')  \ngc.collect()\ntorch.cuda.empty_cache() ","metadata":{"execution":{"iopub.status.busy":"2022-05-29T20:00:18.459218Z","iopub.execute_input":"2022-05-29T20:00:18.459669Z","iopub.status.idle":"2022-05-29T20:08:23.735410Z","shell.execute_reply.started":"2022-05-29T20:00:18.459632Z","shell.execute_reply":"2022-05-29T20:08:23.734645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## FPN (Feature Pyramid Network) segmentation architecture \n* It combines low-resolution, semantically strong features with high-resolution, semantically weak features via a top-down pathway and lateral connections. \n* FPN provides a top-down pathway to construct higher resolution layers from a semantic rich layer.\n\n<img src =\"https://miro.medium.com/max/1380/1*D_EAjMnlR9v4LqHhEYZJLg.png\" height=500px width = 500px>","metadata":{}},{"cell_type":"code","source":"fpn = smp.FPN(\n    encoder_name=ENCODER_RESNET,\n    encoder_weights=ENCODER_WEIGHTS,\n    classes=len(CLASSES),\n    activation=ACTIVATION,\n)\nfpn = fpn.to(DEVICE)\nrun_model(fpn, 'FPN')\ngc.collect()\ntorch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:21:17.987823Z","iopub.execute_input":"2022-05-29T17:21:17.988367Z","iopub.status.idle":"2022-05-29T17:29:37.500800Z","shell.execute_reply.started":"2022-05-29T17:21:17.988330Z","shell.execute_reply":"2022-05-29T17:29:37.499996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## PAN Pyramid Attention Network for Semantic Segmentation (Semantic Segmentation)\n* Network Architecture:     \n\n<img src =\"https://miro.medium.com/max/1400/1*3_WEElD7UeCGpf2R504Rtg.png\" >\n\n* Feature Pyramid Attention (FPA) Module: \n    * FPA fuses features from under three different pyramid scales by implementing a U-shape structure like Feature Pyramid Network (FPN).  \n    \n* Global Attention Upsample (GAU) Module:  \n    * GAU deploys different scale feature maps more effectively and uses high-level features provide guidance information to low-level feature maps in a simple way. ","metadata":{}},{"cell_type":"code","source":"pan = smp.PAN(\n    encoder_name=ENCODER_RESNET,\n    encoder_weights=ENCODER_WEIGHTS,\n    classes=len(CLASSES),\n    activation=ACTIVATION,\n)\npan = pan.to(DEVICE)\nrun_model(pan, 'PAN')\ngc.collect()\ntorch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T20:08:23.737156Z","iopub.execute_input":"2022-05-29T20:08:23.737723Z","iopub.status.idle":"2022-05-29T20:15:56.203002Z","shell.execute_reply.started":"2022-05-29T20:08:23.737684Z","shell.execute_reply":"2022-05-29T20:15:56.202247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## DeepLabV3\n* The DeepLabV3 model has the following architecture:\n    * Features are extracted from the backbone network (VGG, DenseNet, ResNet).\n    * To control the size of the feature map, atrous convolution is used in the last few blocks of the backbone.\n    * On top of extracted features from the backbone, an ASPP network is added to classify each pixel corresponding to their classes.\n    * The output from the ASPP network is passed through a 1 x 1 convolution to get the actual size of the image which will be the final segmented mask for the image.\n    <img src =\"https://gaussian37.github.io/assets/img/vision/segmentation/deeplabv3/0.png\" >  \n    \n [More about Atrous Convoltion (Dilated Convolution)](https://developers.arcgis.com/python/guide/how-deeplabv3-works/)","metadata":{}},{"cell_type":"code","source":"deep = smp.DeepLabV3(\n    encoder_name=ENCODER_RESNET,\n    encoder_weights=ENCODER_WEIGHTS,\n    classes=len(CLASSES),\n    activation=ACTIVATION,\n)\ndeep = deep.to(DEVICE)\nrun_model(deep, 'DEEP')\ngc.collect()\ntorch.cuda.empty_cache() \n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:53:18.678415Z","iopub.execute_input":"2022-05-29T17:53:18.678692Z","iopub.status.idle":"2022-05-29T18:03:45.363535Z","shell.execute_reply.started":"2022-05-29T17:53:18.678660Z","shell.execute_reply":"2022-05-29T18:03:45.362782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation of training results - model selection","metadata":{}},{"cell_type":"code","source":"unet_results_df = pd.read_csv(\"U-Net.csv\") \nfpn_results_df = pd.read_csv(\"FPN.csv\") \npan_results_df = pd.read_csv(\"PAN.csv\") \ndeep_results_df = pd.read_csv(\"DEEP.csv\") \n\nfig, axis = plt.subplots(2, 2,figsize=(16,16))\nfig.suptitle(f\"Training Results\", fontsize = 18)\ncolumns = 2\nrows = 2\n\n# UNET\nunet_results_df.plot(title=\"U-NET Architecture\",ax=axis[0, 0])\naxis[0, 0].set_xlabel(\"epochs\")\naxis[0, 0].set_ylabel('IOU & Loss') \n\n\n# FPN\nfpn_results_df.plot(title=\"FPN Architecture\",ax=axis[0, 1])\naxis[0, 1].set_xlabel(\"epochs\")\naxis[0, 1].set_ylabel('IOU & Loss') \n\n\n# PAN\nfpn_results_df.plot(title=\"PAN Architecture\", ax= axis[1, 0])\naxis[1, 0].set_xlabel(\"epochs\")\naxis[1, 0].set_ylabel('IOU & Loss') \n\n\n\n# DEEP\nfpn_results_df.plot(title=\"DeepLabV3 Architecture\", ax=axis[1, 1])\naxis[1, 1].set_xlabel(\"epochs\")\naxis[1, 1].set_ylabel('IOU & Loss') \n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T20:15:56.204167Z","iopub.execute_input":"2022-05-29T20:15:56.204563Z","iopub.status.idle":"2022-05-29T20:15:56.860782Z","shell.execute_reply.started":"2022-05-29T20:15:56.204527Z","shell.execute_reply":"2022-05-29T20:15:56.860118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Show Results","metadata":{}},{"cell_type":"code","source":"def show_results(model, loader, num_of_rows=16):\n    fig, ax = plt.subplots(num_of_rows, 4,figsize=(16,16))\n    row = 0\n    model = model.cpu()\n    for image, mask in loader:\n#         image = image.cpu().permute(0, 3, 1, 2)\n        pred = model(image)\n        image = image.cpu().permute(0, 2, 3, 1).detach().numpy()[0]\n        mask = mask.cpu().detach()[0].numpy()\n        pred = pred.detach()[0][0].cpu().numpy()\n        ax[row][0].imshow(image)\n        ax[row][1].imshow(mask)\n        ax[row][2].imshow(pred)\n        ax[row][3].imshow(image)\n        ax[row][3].imshow(pred, alpha=0.5)\n        for j in range(4):\n            ax[row][j].axis('off')\n        ax[row][0].set_title('image')\n        ax[row][1].set_title('mask')\n        ax[row][2].set_title('prediction')\n        ax[row][3].set_title('image with prediction')\n        row += 1\n        if row == num_of_rows:\n            break\n    plt.show()\n    \nbest_model_path=\"./DEEP_best_model.pth\"\nmodel = torch.load(best_model_path)\n\nshow_results(model, train_loader, num_of_rows=8)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T18:04:24.049424Z","iopub.execute_input":"2022-05-29T18:04:24.049684Z","iopub.status.idle":"2022-05-29T18:05:41.281398Z","shell.execute_reply.started":"2022-05-29T18:04:24.049654Z","shell.execute_reply":"2022-05-29T18:05:41.280737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model test APIs","metadata":{}},{"cell_type":"code","source":"def rle_encode(img, min_max_threshold=1e-3, max_mean_threshold=None):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    if np.max(img) < min_max_threshold:\n        return '' ## no need to encode if it's all zeros\n    if max_mean_threshold and np.mean(img) > max_mean_threshold:\n        return '' ## ignore overfilled mask\n    pixels = np.float64(img.T.flatten()) \n    pixels = np.around(pixels, decimals=2)\n    idx = 0\n    res = []\n    counts = []\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1] )[0]\n    zeros_loc = np.where(np.append(pixels[runs],0.0) == 0.0)[0]\n    counts = zeros_loc[1::] - zeros_loc[:len(zeros_loc)-1:]\n    for i in counts:\n        res.append(runs[idx])\n        res.append(i)\n        idx = idx + i\n    return res","metadata":{"execution":{"iopub.status.busy":"2022-05-29T18:10:17.316749Z","iopub.execute_input":"2022-05-29T18:10:17.317419Z","iopub.status.idle":"2022-05-29T18:10:17.325235Z","shell.execute_reply.started":"2022-05-29T18:10:17.317381Z","shell.execute_reply":"2022-05-29T18:10:17.324450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit ","metadata":{}},{"cell_type":"code","source":"itr = iter(test_loader)\nout_pred_rows = []\nmodel.eval()\ncount=0\nfig = plt.figure(figsize=(8,8))\nfor i in range(len(itr)):\n    # current version without real submission\n#     if i > 3:   \n#         break \n    x, imgid = next(itr)\n    imgid = imgid[0]\n    model.to('cuda') \n    x = x.permute(0,3,1,2)  \n    y = model(x.to('cuda'))\n    y = y.cpu().detach().numpy().squeeze()\n  \n    c_rle = rle_encode(y)\n    out_pred_rows += [{'ImageId': imgid, 'EncodedPixels': ' '.join(str(x) for x in c_rle)}]\n                                    \nsubmission_df = pd.DataFrame(out_pred_rows)[['ImageId', 'EncodedPixels']]   \nsubmission_df.to_csv('submission.csv', index=False)  ","metadata":{"execution":{"iopub.status.busy":"2022-05-29T18:10:19.121252Z","iopub.execute_input":"2022-05-29T18:10:19.121826Z","iopub.status.idle":"2022-05-29T19:54:53.349522Z","shell.execute_reply.started":"2022-05-29T18:10:19.121788Z","shell.execute_reply":"2022-05-29T19:54:53.348821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = submission_df.replace(r'^\\s*$', np.nan, regex=True).reset_index(drop=True)\nsubmission_df = submission_df.sort_values(by=\"ImageId\", ascending=True)\nsubmission_df.to_csv('submission.csv', index=False)    ","metadata":{"execution":{"iopub.status.busy":"2022-05-29T20:16:50.778381Z","iopub.execute_input":"2022-05-29T20:16:50.778664Z","iopub.status.idle":"2022-05-29T20:16:50.968497Z","shell.execute_reply.started":"2022-05-29T20:16:50.778634Z","shell.execute_reply":"2022-05-29T20:16:50.967698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![image.png](attachment:9e5647c0-f4de-45be-aa1d-74341868bdf1.png)","metadata":{},"attachments":{"9e5647c0-f4de-45be-aa1d-74341868bdf1.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAA68AAAEDCAYAAADa2TJVAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAGw+SURBVHhe7d0LWFNnvi/+r0YCBAMJRgIoKIiKUlEstoqtVq29WHuzp/bi2HNaPd0zbU/3mV1n9n+m3XPb7czZu3b2Gbed6elT2z1t7cWOWMdbrRWLF0BBwSgCCgRBI8EA0ZgAgcD/XckCwv1ibNP2+3mamrXetd71rnctst5f3netjGgTQEREREREROTHRsr/EhEREREREfmtEeXGCva8EhERERERkd8ZMWIEVMHBCAsLw4jm5mYGr0REREREROR3Wltb4WhogNV6hcErERERERER+bdrdjuDVyIiIiIiIvJvUg8sH9hEREREREREfm3kyJEMXomIiIiIiMj/cdgwEREREdEPkMvlQlNTE5pbWtDWxpCAbizpqcEBo0YhMDAQCoVCnjs0DF6JiIiIiH5gpMBVegBOiErlDiakwILoRpK+IJG+LLE7HBgdEjKsAJbDhomIiIiIfmCkIEIKXIOCghi40jdCOs+k800676TzbzgYvBIRERER/cBIQ4WlHleib5p03knn33AweCUiIiIi+oGRhnCyx5W+DdJ5N9x7rBm8EhERERERkd9j8EpERERERER+j8ErERERERER+T0Gr0REREREROT3GLwSERERERGR32PwSkRERERERH6PwSsRERERERH5Pd8Gr60uOJ3OXl+uVnkZH3NJ+bvkCV9otqKisATmRnn6OjirS3DmvBW+LN7QuGCrKsEpwymcMlrleTdIkwPWqw40+vw4t6Dx6jVYm27QCeRsFOUW+Xe8hrkP7v33wUlDRERE5LdcaLRdRGnhWVTW2NEszx2cZtitVtiHthJRFyOam5uH9wuxvakpwDt/z5Mnukp9YC1mRcgTPmNGwTs7YFn0JO6cpJLnXafLBdi8PQ/RS57EorjrydOBiq8/wlcXU/HQqlnQyXO/Sc7zWfhsXyX0yUnQR03EjBi1nOJ71QeyEVMAvP/kPDyhl2f6RCX+8B9G5M+aiS2LNPI8Hzqdj4B9V+UJ2ciReGLiePz7vXGIVMrzBuDZ/yAc+mkK5srziIiI6HvMKXV4nEJhtWhHKLWImzYL06OH2XaUOoBcCigD5OkuLDizpwYR9073aXvSeuUKdGPGyFODcK0Uhw/k41KzAgGirdTa2gzXSA2m3L4YM8co5IX6U4SMz04DMx7F4kR51nWxorKwBgHxUxAVLM+i7wxLbS00YWHy1OD5tudVNwNPPfVUl9dji6aKhBgEB3kW8XtjZ+Hxp56+zsBVosLEBU/jqce/ncBVUl93Bg0T5uDWW2bc0MD1u0+Jt++/GZfXSK/pyE0dDWtFJeZsLkbFDerwJSIiou8yC07tOQiLbhaW3nkPls6bAsXZvyPrvFNOHyLLKbx/2ixPdCcC24vOb3Ekn6QRp7PzURNyE+5+cAUeeughrFixDKljG3H2aL4II78Nl1B6pgSV3fog6PtN8atf/eo38vvrN2IkFAqF18uBimOZuBx3GxbEq3uPlJ02VJadQmFBPiosLWgLDoMm2PPtje38KZytAbRjRsMzxwnz2TMwXlNBrwkU03ZUnzgLR9x0jLWfxfG8U6iqb4ZCo4Na/ubKWVOCM2UOBIY14vyJfBRUXMXIUB00ykaYzp1AwTEjqtuCENG+DacZ54or4AjWI0zahOCoKcO5wkIcK76Iqy0KaLRqBIzwpPVb/spClF32Lr/Iy1SCk4YC5FdcRtvIMISKjbSnte+vSmFBmeEUjpVVoy0wArrR7UuIzV2tROnpQhScrMBlZxtGazQI6kyWeeqp7MJFXLYrENB8DbYRWuhCxIKtDrHfJ3HKXd42jFCHiv3sKAEqDWdRPUKFUZYi5Gafx4hx4xHW67eAXV2ruID/qAYenhGDGaPlmVersS2jBP+x/zz2XbSLetBgfIjnLLAWnsFfsq1i3lV8tq0YL19UYPVkEWCL8hVnncHvvjRiT5UDcTEBKD5uRXVkJB6Nk78BuXwBHx88h//cX4VDF69i1Bgt4rrnO9aOHV+cxR+OmHChQYGU2NEY5V6im5pq/Gt5Cx5NnYQ5Y5QICgxBZGwUngi/gm2namENisaiKLl++tmuZ/9H4ZmkEcj9ogi/PuzZ7rTxoxHUfq6IfavIPYc/HijFXww1sNhGYnJH+mXs2V6Kwy4lgs+V4t8zSvFZ8TWMjRqD8ar2DIiIiMgv2C/hVE0EZs+KFO0w0f4NUEEXOR6KUQFQyw0zd5utpBKXLNeA0eEY7dU2vdgcisZKuU070oxTRRW4WGdHgMMhGo46eDX9BKm9ew3qGUGwFJ9D1WWv/BwmnDlrQ0iEaJt6FnbnX3JFhbHtjeE+NDY1QaUabGeNGaWFVRg1cSESO0ZSKkV7KxQBbcFi+2HwtNJcuFp+Arl5BThRUgHL1ZHQ6LUIdDeXLDCeEQ1dfRLi2nt2mmtw+lgW8gtEu7WqFo2jIxEhtVfb9ZVuPYuso+dRZ29Ew9UaVF8JwMSoUHkluhHy8vKwc+dOHD9+vM9XW1sboqOj5TX652hoQFDQ0Hs3e40nfcV14QzyLsQgdWp0R4DWhcuCgj2f4lCFOI+nzED0iCocSf8CImZwc9QdxdGqa2jxTAotuFYl5tWJP2wvttMHcNDYBPVYDVzVedi97QDKbJ60FlsVjh4rwMEjZ2ELCkag1YCvdh3EoYPSMipoxjTBeHAHvjht8awgAr2qY0dRJ9++6DAewLa/F6I2JAYpCRo0FO3GtuxKER4KQyq/C+b8v+MjEbg0BOkRqwEqDn6K9Pa8BPfyp4/iYE4VHEEa6JqNOLRzFwpqPN+1uS4X4Isth1DRqseUm0SdXjyCv+05JT4KBqnZjIIdH+HAuQYox8ZC01qBQ1vSkVXVUQLUiX0/c/grfFHaBM04de8B32BcKsZT75Xg5yZgWpSo5/rLuP2jo/jLWU9XZmPNFfy8vBqrPzMhX3xwzg2VTsVGHPhbLmbkXUNjuApxLit+/tlF5LvXkJ09hbs+KsN7l0dh0exwxNXXiulcbJO/rGzP939uvYgihRKJSif+eKwETx0Y4neCU2LwnIil/1AhInLJANv1cOD3nxmRLT7M56k8271/j6gAt2vY80kuJmdfQaNeh6fHAXvyRPqOC2KvJQ0oKLfiz5lF+HmZE1HaIFjNos4+PomcJvcCRERE5C9Em1JjKcVZk6OzR1S03aI1nvuNXDUF2JdTh+DYSZg4tgVlB7q2TQuOHMWlwDEYI/WUKEMxRrR7EKLBGN0YqHttNIsAN78KI8d1y0+lhqLqFKo6eh8dqDpVAcXo6x1B2N0YhAaLmNGYh8pGrz7g4ChMSRJtSnmy5vhe7D1Zi5ERsUiYoIPLlIe9GadF+N2LhrPI3JWJ0kYdxsXHITLEhtKDXyCrUs6/Pd0WgkiRPi7U3pkeqIZOFyJaXCMRpBX1GB7iWYdumNTUVMyePVue6klKk5a50W5g8CpOwNOngJkzkNDXFyF1l1B8WQS381IxKSYWk+bciQceuBUThzjCtVabjKULUjEjORW333c3bg0uQ06pd1ShQtLc25EqpS+Zj6kNZbgatQC33zwDM265E4tSxB+b+Yr4c+/OiZqqMjTMFOslTUT0hOm4/e7/hkWJOvHHIgyl/FdLUXDchhkL7/FsN/lW3H3PrQgsPI4SOdh1a9AjZYlU1hm4dck9SBtXi7yLnvC0XgSENbGpSJszCbExk5C66AHcP28ielav0h1MT5LuPQ2LxXSR19QIJWzGAuRdm4FFd3nynzH3btx9SyDO5JZ0CYAd42/Fo0tuFWWcjuHduuHEgUNmfBw8Fvuevhk/fXAmfvc/UrAlrAUvZp3zGloyEs/dNw9vrxDp8yOB8+X4w0Xgp/Nvds/76YpbsWW2EkZ5aTfxAfWLhVOx5amZeOKWyfjJUxPwi1Yntp31Dk5H4pm7b8W/S9t9SuQl6mGbCGjlMHSQRIAaLv6pd3jWG9R2gYfv9Gz3p0/OwfsiQM2psqJCSmgdiWkzxyN3pUhfOhn3LroZG25SIqfiMorda3pUh0eKbUh1djO23B+ORc1XceCcnEhERET+QRGN1AdnQWk8hPRPP8OugwUoq2lvSYo28AkzEubOQqxGBXXEVNyaokZusakj0FVNnYVZE6I9wa4U9I4RjcfRYxAdrYOq1+C1BdGibdeR30yV3NZVI2ayAhXVcmR8tQqlzaLN6vNOyCDcJNq74xQXcHRHOtJ37EXmsdO4eMUrkHWVoui8E+NuvhNpM2/CTUkpWLgsBRG2Upy8IC/jxXqmFDWqKVgqGuI3Jd2EmfPuxh1xClwsKXJ/se9OV8bhjrvSMFOk33TLUqRN1aK5XrTM3EHzGFEqBULHi7QJN+C5KNRDXwHsNxW4Sm5Y8Dpgr6skPAqJY6uQd/AQ8s5WwGx1QRWhh3qQD8lpN3FchCeYdNMhalIwGizXxEdHOx1Gt38hExQs/syBwFFepeqzFpSIEEFi8MmD+OqYNFzBAptC+oCRI7ohlN9Ra0IVEhDTPgRVMiYKCeG1qLR0lhSROmg6RnkoEeiVlzYqERGVeTh4KA8l582wulTQR6i99r0/DtSaqoDJMdB7jSLRjUvAmLpK1HrdL6Afo+n7mA1KDXIuAYsmjsHEjrodjUVTgkQweBX5HT2JKkyb0Fn51ZVXcEDMu3dG5xCCoJvG4l75vdvYaCyapkJ1oREHjpzBf2y5iI/F7MbO7nnBO98gaIc+IkFwokHqkB4pD9kd8naVmDZWHJmGJk/wO1KFiUkTkNhajQMFZfh4z3H8/pzYQKvU39xpnl76IJaFBHR8k0lERER+JkiP6fPvxqOPrcDS5HDYCv6OA0YpgHWg4YIeGq8AUhGmhe5qAzqaQCOG2tLSQzTPOii0Oujrrrk7XlSiLacqM7nbvbbqCqimxbnbuj43Og633vUQVty/GKnxojB1pcj6cgf25l/yBOUXa1CPYFGIIpwuPO15FVvhHNkMW/uQxg6NOF9rR1AgYGxfVrwuSIvZr4qWpCc9JCqhS1soYsZtWDhznDxF34buAew3GbhK+gzbrs8gel0lCh1m3fsYbp+sREOZAQf+9j7e+/QQKrxiucFQjeo6uDUwSJzmNkfnB8R1UMUtwsMPzIG+2YTiY3vx6V83Y9dJs+ePdCjlb5XWCISiy2dVoDTaA9bGwZVUMXYW7ll5OxICG1BacAB/+/A9fHKwwitI759LKkKAomtgqlSJDzirFGP5kIjIxH+aAPmmYZlmtHScWmC95pnuwR0IjkKw92riDPV+gFzj6Xzc9VY+7j9Qgz2WZmhiRFAsp/mWBQUWIDI8BJFi6rq321qHbf91BOq/leE/TltFQBuCRRHDHpRNRERE3yaXE86On3xRQKmJxax5c+A6XyOP5Ovy7ba7XdStEThETjS13+UlcbWgKSjAc3tXUDRiQkphumqDqVyFGP3gujWGSxE0BrFJt2LhPQ9hWVII7KWnUdQgEhob0SzKabfUorbjZYdSGwF1YPfg9Sqaxf64Gqxey4pXSwgitMGiRj3pCsWweiDoBmsPYL/pwFVyQ4JXT6/rRMyd3k+vazuFCrFTbsXt9z6Ax//7/ZgVUIKvvIf8ij/2zgEJ3f5wZeYr3o8Zc6LWfAnQh/UynHZ4AsdMwoz5d+K+R1bhsTv0uJR7ChXtg/cHKr9MpdaJIMwMq3dRG2thOg/EeX81NwBFSCym3nI77nvwcTy9fBYCz36Fsho5sV8qjA4XYaDZ2iXYddaZUIE4hA/pSdXSw4VO4r3C7lHoSGjckWYoJopdyr5c79Wr2IqC8+LjPECFuD6eyh45RvqwdaBI1EmHqmvIlt+KMB/bjl7FgahYnHvBMzz36Xlh7uDSt1phPXYRf2weiX+aJt107oPtnjmPlfVKvL1iPv7+o5vx03sTMS/sBn13RERERDeWtQS79p+BVQpKZbZLRtjCR4sWlw4RM0txpv0mV9GSNZcWwhU9RqT1YQQQ3Njk1ebt7gzOVnrld+4MQsdHyqPvlIiJ1+FM7lGUhk9CzI2I96QHJGXmoVQKUr2EROsQJFp7jVLRRHs2yKVExC0LsXBh11fa1O5jySIQKo2KDI3rsezChSkYJ9I1oQrY6y90qRNX41VY+UOxfkEKWr/pwFVyA1rPnl5X1c0zMHGAe6elm7g3v7cLBSZPROpyNkD8hwilp+tNGz4VOJ+DvEIzbA4rKo4fhaHOndSFo6gAp2occDodsJTmIf9sMKZ3/EFfDxtKvngHnxwqg036y2l1wiHdpB6sQkDAwOXvImI8kqUhxidKYBF5uKSnFJ8yoCRY1FP0YErqgil3M97bWQCTOyJ0oalBlEf8cSsH2YGnj01GxIU85JVY4Gx2wWmrhKGgBMHJEzFuSJUVjMZ6K57NLMGei9dgvViGPxeJOtBqMM8dh2tw7zTxyXnxAn5/7DIa7Y2oPluMP5a34t4pkZjlzqMXU8fidwEt+FXGGRRcFvleNuG9/Ze9Htg0yjOk2taAiiZxtXBeQ8FXF/GeJ/E6tKKo1IgDBdKrDB+nH8WcIw5Mi5+An0yR/kR8sF3lSLHfLTBelr6PbUXjpTK8W9LLNzFERETk/8ZMx6L4Wuz77DPs2rMLu7Z+gi/MMVg0TXqMrgLRs+6EvuILfLZLpG1PR17TLKRN6Wcw79iJSHVk4bOtR1HZa/NgFuIUBdjVnl/zzZg1obPxpoiKQUy1Gfrx+oE7joZDIxr1V4wwHD6JGjl2dDVewslcI+xBEYiVnkAcMQ2xYVdRll2Eq3LE2XzxKHb8bQfyennwyKRJ40Rb8SSOXmzP8CqKMtKRfuAspDlxEyMxsqYIee3p14zI3rcXWcXej39yodkrmHWZTiLzqyyUyv0r9tIs7Ms8iUtyebpP03ePz4NX5/lT7l7X5CkD//EoolNw9zwVine/j3feeQfvfXwE9ZPuwqIkz7czygmzcNdNGpiyd+DTbftQFT4Lc3rp7kpKmwFX7ja8//5H+PzrSqjn3Y1bvf6gh0+NqfPvQoI9B5++9w7eefd97ChUIO2eWxErsh+o/F3pMOPuZUhqzMfnH76H997/FIdq9LjzgVsR7XUPat+kD8K7kTa6GLs/FGV55z18lFWPhKWLMF16sNBgjJmBe+5JgrPgc7z/1/fw/qeHYNbfiQdSB9FD3sVoPHz/ePxu1DU8sOU4xm65gI9HheLL+yd33JegSZuJfTcpsS37DNRvH0XMLhHETpmE9++UniLVh8BY/HTZWNx77TLmfCjy/ciI6tTxeE5OlrZ779JIPO24jMl/PoSAN4/jz4owPOFJvA4t+MOxStx1QHqJgNsyCv+0aCa+fDBWvv/UB9udMhX/GT8SH2ceR8B/HMLk7VbExfjiHCUiIqJvngKaKbd77nddshRLH3ocj94+tfO5JQE6TF/yKB69S6QtfxT3zY3tuA9VNelO3Dmpex+sBlOXPI7HH/G0MbvSY9baVEyKux333dMzvw6jkhATfUNCV2Ecbp1/E8Y0lSHz88/wmQja03ccRllTBGYuSIXn13NCMDMtFeNaSrA3XaSL1+fZlxA0KRUpvbTfFbG34vakUNRkf47P0tPFay+KHBFInjPF/bM/7vSpIbgkpf9NbHNPHiwhU3DrrPbW5hRMm6jEpeMifY/nt2aba2tQU1+NavlJpNbLNbDWVKNG7rTuPk3fPSOam5vb5PffIhecThcUo5RQXEc47XI6xR/u9eXRp2YnnG0KKJW9fSgMsfxSXlBCOaigtRfSfRauvsoyOD6rqyYHrC1KaEL66P5tdcJ6rQVBo1UIGvS2WtB4tRFiJQT1Gt950hsDVdB4fjjsG+KD7Ur11TQSmlDew0FERETXz1ZTBlNZISrCF+Huqf307nZjvXIFujF93MvVD2norq2xFQEhGunZkr3yLKNAiCak4/dn+9YMu9UOV5AaofJv5HblSZd+Sqiv7dF3j6W2FpqwId236OYnwSsREREREQ2NC47LZlhHev0axiANN3gl8oXhBq/fZLcVERERERH5jAKqsdFDDlyJvqsYvBIREREREZHfY/BKREREREREfo/BKxEREREREfk9Bq9ERERERETk9xi8EhERERERkd9j8EpERERERER+j8ErEREREdEPzIgRI9DW1iZPEX1zpPNOOv+Gg8ErEREREdEPTMCoUWhqapKniL450nknnX/DweCViIiIiOgHJjAwEHaHA42NjeyBpW+EdJ5J55t03knn33CMaG5u5tlKRERERPQD43K53L1gzS0tDGDphpOGCks9rlLgqlAo5LlDw+CViIiIiIiI/B6HDRMREREREZHfY/BKREREREREfo/BKxEREREREfk9Bq9ERERERETk90ZYLLV8YBMRERERERH5tRFtfC42ERERERER+TkOGyYiIiIiIiK/x+CViIiIiIiI/B6DVyIiIiIiIvJ7DF6JiIiIiIjI7zF4JSIiIiIiIr/H4JWIiIiIiIj8HoNXIiIiIiIi8nsMXomIiIiIiMjvMXglIiIiIiIiv8fglYiIiIiIiPweg1ciIiIiIiLyewxeiYiIiIiIyO8xeCUiIiIiIiK/d2OD10Y77I3y+y6csNvt4v/ylPTeJU98gxqri5B9IB9Ga6+F/G6z5OOj/3wDb7z+JjKq5Xn9aoTVakVjizz5TXE5UX+hGIZyM+ztJ8SgdT2P/IbYJ/vQd4aIiIiIiPpxA4PXemT+6XE8/s/bUC7P6WDJxobHNyDbIk0U4/PHH8fnpe6Ub0bFbqxbHIXgqOlIWzwb8dpgxK98E/nX5PTvvHy8cf9svLStCPW2IGg18uz+XEjHGq0Wb+TJ098A87F38fLTz+H3/+8DfPreevzjI2ux4auqwQejXc4jH7haBcO5+usPhks/x+Mbs8VfABERERER+cqNC14v5eHrwjSkabchr0Se5xdEYPfEfdg8/lWcuNyGtrY2NFTtwpNVL2DZL3fDKi/1nXahCFk5wJrfvYNXf7cGKUHyfH9yaR/e2liHu9a/g9f/9TW89q+v453NP8boHeuxq8e3Hd+QS9l4Od0AuzxJRERERET+44YFr+bCLNQ99BieuW0Sdh41+M/QTjmwe+E5EdTpPLOCxi/Duv94B+tipP5iwZqNTa+/gd3evcGlu/HG65uQ7RXdWk+n482fr8WSlS/gjY+zUd1tyO2g059Zhzc/zu8ROFfnfNSx/ivvZXRd32rE7vdewQsrl2Dtz99E+ml5bamcf94Fo3ib/5E0bHi3+701Z1PH+3bGHSL9vexvJWB3Xq5CXtIcJEfIMyShqXjsZ2swM9TrbHHVw/DVB3jr3/6Atzbvg2GA7kz7+Szsem8DXv7Tu9iVXdUzELVXIWvHu9jwLxvw7o4sVMkL2Ev2YVtGEWDKxc70bci75JkvGTDP+mJkpr+FP/zbW9gm0jlgmIiIiIjI925Q8FouAoQ63DErHvpZd2DaVydR7C8tel0kpkUCG/+8Cflew001c9fgpZ8tQ5w0cU0Ehj9f1yVduod03c9F8CcPLTZ+vArTZqxDBqZh2ZwgZP0qDSnPfNQRHA4+fR7WLI1C/obZmPZUZ3r1Z6uQMm8j8mOW4eUnpqH67SVIeVHuGW7Mx2v3xmPNNiDt6ZeweNRuPD9jGd4ocK/aq0bjblH+fHjf/lpdsA7r9hjxbdzxqxwbg9TDX+Prkq6hYMj4ZMTrlJ4JZzm2/epn2FY/CWn33odUfRU+/ec/YJ9XYOmtPnsD/vE/DVAm34HHFiXCefC3+Mf3vb44sWRhw09/i8zmRNzx6B1IRh5++9MPYBAVEDBmEiZN0IoAWo9JCZOgD/GsMmCeNZlY/4+bUCQC7/vuTYP+wod4a1eZnEhERERERD7TdiOUpbe9+GJ6W5l74lrbkf+7vO0vR6+5p9wuf932++W/b/v6sjRR1Pbp8uVtnxa7U74RDUXvtK25KbJN2v24uSvanv/3zW1Zl+RESdXmthUi7dVseVqS/apYfkXb5irxvmF/28uRaFv2drknTXJuc9uapcvaNuaL94NMX5Ne70mTXN7atgaR8jbr27Y+jTb8Yr87ye3ymbb9+XIhc9e3xWFZ2ztGz6TIsK08O6utvD27Xsp/6aMVovyvtmXJ05Ks34ltPLq5zZ1rb/t8g10rTG/7/f9c3rb6+V+2/endL9uOlNXJKR7Xsv7Utvrdk21N8rTEPe+v8rwu51FZW/qLv2n70iS9b9d1XtnWF9te3Oo5Kz2a2soy09u+PitvofjTtuX/5+u2zlIMlGdT28m/rm77zRfV7hQPMe/d1d3yISIiIiKi63VDel6Lj23D6GVpiHdPhWDmLUuxK/uk39xLGJS4Bu+cMqK+aD9efTIO9e+/hLSoeKz9zHtQbT9OZ2NT9QqsWurup/VIeBLvfLkLz88S7weVvhiRLSeQcSDD8zrViMjbqnHCKPWNapB275OI/MNaPPLzN/HRnnwYg6Zh8axId1a4aTHWzN2NV55Zi1feS0d2cSOi5s5D3GAezORHQqY/jF+8vQPv/GoNUmOqkPXHf8Taf9uFcndXsBNlxfswLcyJYoMBBvlVVg/UnzP3fBjSpTLkOWOAy53LGgzVuBaahyqz1E9qRlk+3KMBOikRv+BhLJws9/R2N2Ce9TCf0yJ5st6zvJsSiSl3yO+JiIiIiMhXfB+8Og3I3V0Pw8a1uP/++92vx3+/D/gqE3m+eiqsTwRBk7gYT/6v9dgsAtmsf9Fg04ubkDGYMbQt8Ay/HeWe6mlQ6fUw5ucjP6/9dQnaB9ZjWZzn6UqRj25GfvarSLuWgU0/X4Z4dRSW/CHbM8Q3KAUv7ynHptUaVH+8HqumaRE8eS3SK9yrfucoI+KRduczWLfxdazGW/jgsFnMteOaiFCd5iqUlZZ1vhpj8EyK1rOiN7FwlbIO1d7LlpoxOuUZxAQ3iwXsqD8/GqODPYsPyjDzVCoD5HdEREREROQrPg9encUnsS/+x/jTJ5/gk47X+/jF/Vn4ukAKSr5djftewZK7XkD6BXmGWxCmpcwWEaURVq+nFzV4PSCpsd7rRsvxcViBLJw55xXpNlYj/0AGiqQAfVDpwOyVL+Gln3m9fvoS1szt7D7VzHoSL/15K/afuoTy99OQ8cv1neUOisKyp9fjnS+zUG7LwsujN+GRD7LlxL40yP9KGlF/WX77LTAf24ZdPZ6+pEdMYjzK6qU+ei0iY7Rwxqfh4RUPd3uliiW70Y1Hcnk4Zi7vvuzDWDpVuoFVj/h5BhRVdOv/l36LuK/7sQfMU4vxswwoa3/qk8xcVSy/IyIiIiIiX/Fx8GqH4cgWTLotFfEhIQjpeGmROnclyg4b8G2Hr0EpaZh26k08/9xr2F1cLYJVK4w5b2Ldzzch8iersFgamTt+GtJmAa/97BWki2Wk9Bd+vtWTgWT8Mqz5iUj/5avYXWEVeRix+3crMPvJdM8DlwaVfgnrf/UaMipEgNvSCOOeV5AWMFt+6JIRm5aNQNyzH8Eoxb8tVlRbxJvIKGhGi3jrwCuICl6M1w54Iu1G6yVYq4F5mr7HDUdOS0MKXsNLv0xHkQjSs//zBaz7TE78FmiDruHT9Zuw73xn4Gcv34d0Uc3Lk8e7p+NvWw3l5g+QWeOeFOwo/uRneO6zXoJDbSruWvE1PkgvFkvJpIcpPbseme4e/xAkz3sYeVt3obh9gat5ePeff4kvvb/IuHYN11zy+wHz1GLmbfcha4tXniL9061V8gQREREREfmKb4PXqyeRtTsNd8zq0S8GZeJMLC3fiaxv6zc82+mWYf2BzVjVsgn3TYuCVqtF/LxXYbx/MzL+fRk84V8Knn9vI1ZYXsMjYpklv2vEmn9/wZ3iocGyf8/AOzPSsSZOK/KIx32fRWHjnvVY5v75ncGlvxopyhEXjBEBwYh/JhtpInJ7SbonFnFY8+ddWHXhJcQHjxDpWqT9Ocizvihg0KJXsHtDFDYt1mLEiBEIjnkeRU/swub/NU1auXeznsemDStQ/YdHMD1qCV5tXIP1z8lp3wJl8mN4bW04sl57vHN4+f/JQvhPfomVU+V7UKOWYt3/NwlZr9yPp154Dmvv/wk2VN2BdfcnetK7UCL5idew/NoH+Mn9a/HcC0/h/le+Rvja1Vgo/ySSctaP8No91djwk/ux9tm1uP+Fd3Ht/nW4r/022KkL8QvdNjz30P1465gUjQ6cZ8gtz+A3C8uw/vGnRPpaPPUnM+76h+WeRCIiIiIi8pkR0lOb5Pc/PNessLaIYHC0BkF93Z86kEaRR2MQNBrPvao9XG+6u4x9pTfCam28vvL7AafdjmYEICSkjwcnCe5lAkLQzyKdnHbYm/vLzwm7XWwxJESEp4M0UJ4ukWejCGYHVUAiIiIiIhqqH3bwSkRERERERN8JN+SncoiIiIiIiIh8icErERERERER+T0Gr0REREREROT3GLwSERERERGR32PwSkRERERERH6PwSsRERERERH5PQavRERERERE5PcYvBIREREREZHfY/BKREREREREfo/BKxEREREREfk9Bq9ERERERETk9xi8EhERERERkd9j8EpERERERER+b0S5saJNfk9ERERERETkl0a0CfJ7IiIiIiIiIr/EYcNERERERETk9xi8EhERERERkd9j8EpERERERER+j8ErERERERER+T0Gr0REREREROT3GLwSERERERGR32PwSkRERERERH6PwSsRERERERH5PQavRERERERE5PcYvBIREREREZHfY/BKREREREREfo/BKxEREREREfk9Bq9ERERERETk9xi8EhERERERkd9j8EpERERERER+j8ErERERERER+T0Gr0REREREROT3GLwSERERERGR32PwSkRERERERH6PwSsRERERERH5PQavRERERERE5PcYvBIREREREZHfY/BKREREREREfo/BKxEREREREfk9Bq9ERERERETk9xi8EhERERERkd9j8EpERERERER+j8ErERERERER+T0Gr0REREREROT3GLwSERERERGR32PwSkRERERERH6PwSsRERERERH5PQavRERERERE5PcYvBIREREREZHfu2HBq8tpg8VkhtXhkucMnsvphKtVnvAJF2wmE6zN8uSw+Sqfb1a/9dnqglOkd76GfrxuDF/U9XfzeBERfZ9I16Au15lBXmac1cUorLCKT3I/I66bDqsZplobnLy++MxwzxNv7jz6PSaeNk97m8gn5xjPhy7cx6CPCnW3R+X3N4LDVAzDSQMKTQ55zvXrcV7Kr/ZzyGUzwWT10V65pLwHzsvlsMJsssDmN232b9aINkF+7xvNFhRm7kO+VQ2NSkyLCrYGTcXiO+cgOsizSP8cKP3qQ9iSn0VKhDzrejkrcWTLQWDhSsyPUcozh8FX+XyjBqjPmny8/XmuPCFTaZAwYyFum6nHt7aXw6hrl90Cc3MoojXy8t/J40VE9H1iRv7b29H1KqOCJiEFi29Pgi5AntWDA8YDH2LfxTlY8aMU6OS5w+ZywGK2YlR4NDSDaov0znY+B5mHDOhsG6sRl7YUi2/SQSHPoeEY7nnizdPeyQh/EM/O1svzurGXYt/mDOgektpE13+O8XzoznMMjBN/hKUJUhDgTTrGBqhXLUVCiDzLh5xVR7DlOJB6cwy0YeOgD/XFEZDPqXJ50ssc9znkRGXWFhzEAqxMi73uNrOjdB8+rIjDj+5MEGd/L2yVyMk8CGOLBmqxMddVExxRC3BvWiI0g/ob+X7wcfDqgunYx8gdtRT3ig8Oz0F0wXomA1suJ+B/LIwbxIG9AcHrD9pgglfgwWdT0PFR32hBcfYXyAsQf4y3Xf8f4zfF/Ud/NbnvixYREX3DPEGJZcFKLIz3NMdabOXI2XMQpvh7+m/wSSOD2hRQ+qIN2iVokecNVa0B27eKxveCe3DbFB2UI11wVJ3Avj0l0N3LL0mvz3WcJx2GGryK6es5x3g+9OLbC15vTBuwv/3xvf6DVxuK92yHKeFBLJ6slueJ4PnoTuQFLcCKmdf9Fd93huI3gvzeB+y4aMhHSOJCjO84MUciKDwKEwKVCFQHQSEqv/JkCa6o9AgLlBdxmlF8ug6BkWEIRDPqyg1w6mch+PJpHM8zoKquGaM0Oqjbv1WQlx8V3oZLJ4/hyNlLaA2KgG60C5bSQuSfKIGpOQDh4WoEjJBW6LlNR00pivKPI9d4WWxxNMZogjrGUDuvVuJcwUnkFF/AleZAjNWNlr9B663sVhiLCnAqrxyXmloxWqNFkPwh6KwpRqElEDqYcDovD4ZKO9qCtQgP6edTstUB09kiFBbmotzcjOZATcfyg8vPCVvVOeQX5Ij1WxEkyu6sPC3q82ZE9fZhYa/G8WIg8eYoUQuyUSroxuvRdOIorPpp0Lf/BbnLVgBDwZmex0Tou948nHVGFBtO4VjZJXeda0Sdu9Pbj6emERXHTyDHGoIpEa4udW07b4DRqUWo7Rxyj0v71oxWlQbaYM8WpPSi89WovuJEgMMBhOswWvENHC8iIuqHHdXHS+CYlIIpY8VnvkKBAJUOuuA65B9zYcLN4zDafQ0oh0Ophr3sGHLybAibHAGXuHaV1gBaXTBsFadx9oIL6ghxXZdzhrjmGMT1xhUa4b4Wua9Bp04j/0wV6hwuhLR/vkvLFRlRXV0H58gAOBqkPOXrU7frGtRacb3o/Y4qh8mAbONE3H73JIS52xYjERA2FlpxjWgepYZeI19oOvLMR8XlVowIDRV5el1HOq5DRbjgaENgaDhGd2nf9KwL6TLc5RraEgRNuKrLNfa7bQjnifc13X0OXJKv+Z72ozE4ETNGVSM3L6tHWwHNdSg/ZYQq0dMmsnWcY53tFWnoacHJfOSL9mGrIgyhYmO91bPPzof+0r9z54PnGFg1yZgU7tVAdJOOsRmByZMQLsf1/bXFJe3H4kxlnahTLXTejc4OTphLClF6sb0NaEVT+znSX7u117Zn95Cxv/3xkNqfJVdU7uM9qHaku0y9t/Ob68phsGqRHB/e+TnXzlWL0q8diJo/raP+RBiHMF2U2NcgqEICOuqu33rrr07ccY4RjSGjcLn4OPKMQFRMmLssfbbhvwU+vudVhdAxKhjPGmHzHoatEMFQtEb+1syB2qM5qG10T3g021B1tFakdLIUHkGuVY246UmIUdciNz0DpTY50b18CU4cPwdXZAJmj1Og5O8ZOHgwB1WjdGKdGARWZOCLArM8tr7rNp3njyA9ywxl/BwsnBkNZ8keZLZnXmvAnr+fQYN+OhbemgB1bQ72nOg9H9iNOJieCVNANBJmJSA6wITMLRkw2j3JLVerkHMuF7lFTdDEJ2HqWBsKt+2BodaT3pMFhr9/iNxaJaJvmovp0UDl3j3Iv+zZ+sD5uWA+sQfbzzRAP3k2EsT6pYdzUemUk4ciQI+4KWJ9k8Uz7RJl270dhgapfqVjYsWJv+9D8VVPcv/1Jmqu4iC2flkKl14cr8miYOf3YOsxkyfdfTzPIEccU3f6+FBpjS517RD5lRYeRmY5EC3tm96Jkt2ibmo8WwgMExc5aQxFiAa6seIP0f0XdaOPFxERDYfC3dh3wSV9hLuvATnIO7APJ+qU0IsG5CgxW/rcz6myoUU0kdTKJhiyDKhsv+YIlvN5yCl1QSnam9KQwa2fHEGVawwSpuqBi0ewZXc+LO0Xod40m5EvrrkZ5xqgnhiHMc0VOPjxzo7rSneBIghSoRCFxRY4O54jIcqbmIzkCXJPiFeeyrGxCG4qxcFPOq9VsJUiY8sW5FQDmnF6KK2F2J6+E4Xt15k+6sJWmoEtf8sRTeIIJI5TofZ0OrYcKBVNze+33s6TLu3HxlrRdqiCzfte08ocfHnCApUmFkqbAfv+5lX/3XSeYxKpDbUdH2aUoiFIj9gQJ0ozP+7SlvHmk/OhW7qmVZyDn2zFkSq54fY9Ph/6bYuLGrec3IntpxqgE+2xJFGfVnFs9pX0tofi8yG8axswVAo4urVb43QNMKRv72hT9972HDr3OVTriWAGbkf2387vl0INTYwRpaVmr/NNCNIgemz7FxdyvZ2wQj1B1Fu8FjbvehuoTtzt5lLkZ+fCEhKDpAQdpO8A3G34gyYooqU2vB6uUq82/LfAx8GrAtEpy5GsEB/Gmz7G9q+OwFBqgm0YwZNZlYDFs+IQHR2N2KnzsTjViZwi74pqQnTSHBGgRSM6YQ7mzq6ERTsdKRPFdHQcUmYnwVFdLw5DT/W1hdAnJSNRBNRq8eGWsvBBzJE/aBz1ZpgTU0Q+IgBS65F4671YOr33exfMJUdgmTYf86fGdinnkUKvcjpFHvMSESunz021dQaEPWiReMcT7rHrsWPEH2JMEuakKlB4sV5OF/rL7+o55J/RY/HiFMRJ9RKTiPm3iA+EC57koVIFq2Fp9uyJs+oMDGMXYmnHMZmLu+YGIq/Ys6/915sFpcctSFq6FMnu4xOLpNuWYk6gA/UdF5wWxMxcgCQpvf2e1W5MjdGYK9WNtG8TU7D0Dj0Kj59zf1grNdGebzlH60T+Oqh6OWC+P15ERDRkzVYYK4xAjLgGe31Wq5IWY/mCOUhOjYNGntdOIV03VJWouNTeeLXAdNYC3ZRoz72KYQmY//ByLL01EXExCZiTNhdxNYWoqhNpobFIToqFdJXXTxGBhQhupauMrTwfua65uGf5AiQnJIjP/ntwz60u5BqMvbYdFNGzsXxhLCx56fivd9rbOGY4vBqS7jyvJWPx3QswZ2Yy5iy4V1xnFLhovAjpUTWmohyUjlmA5XfORbKUvlBcK2NMOHLaKNI7dakLlwnFOaWIvesR9/4lJCRjwT1LEXtO5FUjr/B91Md5MiBXHObe7anfuXeKOptq7vOYdiG1ofJsSL7jXixIFedJ6gLcu3gOFNVGXPQOmGXXfz70TE92n4OBKDxWLM7wTt/H86G/tjgaK3HmlA4Lvdqzc8XfTGBuMUydgYBMAdVY7zag5752hzEfOeq5HbFER7sxz9Nu9Bi47SmxVRW6HwTV8SoRAaSc1kO/7chBtPP7pEbiogcRfTkD//XRFuzMzEFxhaXL+eaut6Oh7rpKjBH7JILNOYsWItpZ797nwdWJCeppizEnQbSV3UGx1IZ3IGXxfDnPOCTfuRRJVQac8/oy8Zvk4+BVCNAgYd5y/GjNI1g4XQ9UG7Bzy07knO+slsGIi+waMKrFiamqsqDz8AZilPcCYk/UwV7d/So19Bcaev2w0o5JgvloJg6Ki4XZ6oBLqeroMldp9dCfyMC+E6WolJ4cJy5xqvZxpV3YUF8tyikCJW89yqkMlHucPdSh+o6AsCeFCMLUULifIlaJ0tMGFJ63ihPTa/l+8nNdtaJS/AF2ebBB6DhMHC+/HzYXLOZi6INaRLlMMMkvi6hch+Wqu477rberFlQiAdFjPJNuCh0SZiZ4lTUQgd7l7oUqSgTG8nuJIioWCVW1qB/UlyM34ngREdFgmAsysHP3TuzcuQUfvrcFRyyJWHpbYpfPdH2491R3esTM0KGy/KKnkVVrQmltLKZP8HymK8VndayIeC3i2lRZWojc/GKIMALo86keDrFsJTBWAafXdc0prk0ot3k15LwpoJm6ACuf+B9YuXwOEtRNIojYjg+3HITRvYKcZ6JoGHZczxSInr0cy2+V7tmsh/msA9Fx47z2W4lxE5MA0Rj27iXuUhd1ZpQ4ohHcKgL29rLWuRAcKbZ3dcCQ7DtlMOfJgOL0ni803JTQxySIY+ppvPfHYRHnDqYibnxnm08RnYLly+YitteHfF3v+SCnT46F3qv9oxsv2ka1leLYyjOE7+P50F9b3GUxozhShZbL8v6591FqdJpxVR4t1z8XrLVGJMR0ffCoIiIacV3ajQO3PSVKtQg0x3q9xPHoLTJw67cdOYh2fn+C9CJwfALPPnYv5k7QwFqegfQt+1BYK8cBUr1Nn4hx3udrSDSSZkhf3g22TkR723tYu9SGd4kP12tex8JkQ1NwJaxXv532se+D13YKJTQi4k++7R6svFMP06Hevi3pmzLQu2oF6aQWB8cX1aScMB8rH5iNaHFpKzm8Ex9/5DVMaEwyHvyRdDN5ParyM7B108c4KC5kPTWhoU4PdfcPtOsppzR8ZOeH2JJZiMrLYpth4mI8bvA3njc5xR92wKgef1CjpDEmw2C7aka0SvpCoAlNImuXzQrLZUvnq1mDuePkLwz6q7emBliDle6hB9dDE9QtB4VC5OkY5KPpb8DxIiKiQdGIAG32rNmYPfs2LH7kCaxZuQBxQ4pI5EZ9VQUuika9xVQKS4xopMkj/WzlB/Hxe9uRcUo0hEXjVhul79F72517KKq923VNNNLm3jrGfU9hn+T2TdKti7HysQeRElCMI/LIMHeefbasRKK4lgZ2+ea9vb3jQFOTZ7oHEYCLKx1sFq9yXnYgcMJcxIQO8wLvp3xyngR0rd9AcZ2HqL+mgb7odgcQw6jPYZ8Pcroob5cSS+0SWNHwfTsfxM5671J/bXF3e7bZBmvH/kkvFzQi6O/3b7ODp92qdrdhvbgDy8G2GzsFavSensr2V8cw3SG6znZ+hwA1dBMTMXfxSjx4M3DkmKfntK84wGOYdSK14RUO2LocCxsCx82FRuk5Xt803wav0s3PJ8UFpdu+KML10DtElO4+a1UI7t4T6N3lLTNbuwaMrisWGCdrhvbtWz+UodFIuGkuFixfiScWapF7qLBziIZKh7ipczD/zgfxxMpkXP3K0Mt9o6EYM1FcJOu6JlxPOZ3iYpyrno9H7p2POTOT3N+OqEf19enVk0qtheqCd++04LLBUiG/HwpbKQrP6JAQLe2J+CDVqtCii3MPw+n68gzFcuur3sLGIO5CDSzdht0M9fe+TFdsXZe/Wo+LYpuaQT21zvfHi4iIBqdL42+MGorhtD7GxGH6+EpU1VS6hwzHTW6//phR+lUxdAsfxMq75etnpHaAL0zFdS1cNOJU0Uj0vqbN6HZd6+CCKW8ndh7uNvw0QAudTgQTdtEwbM/T3LWXz2k1wSSCC5fIVR0vkuu6/q6oxVwl9k0nGpXyjO5C1IgT/+gmeZVTLmtiRLcv+r/jBnOetHi1GV1Sa7wbS500+qtTfa1J1K8aqgGqShWqE0fQjHrvoZCNVphMFjh6NFZ8cT70kS7KaxRHfEyYPKM7vz4fVNDoRC12O8fdrBaYRJtN7dVm66st7m7Ptom2o/f+ya/YQd2aKupWHIgqEWh1IdqN5kG3G33vutr50sPJTpu6nm+CWsRYutoG9xcDKun5L93jAHEkPL8dO8w6CdEgulaFcTf1PBbf1vnm2+BV+nbLlIEjxys7x2C7HKgsyEdhfKz81FoVQsWJbTTKy0g/y3IiV3xcdGUV63Tcty2CYkNeKZLix4m1r5f8KPVznR8VDunptGEqd97mEx9j+4nOsexOkeZUqdC9I1jsLMZNTIYxvxDmjoWvr5zKYFEGsb2Oums0obhYfOgO1tgYJCsMMHTsmxPmkydQOoTCSAGlzWTAvl05cM6Zi0T5Q0IXn4pRebmdx0TK+8R2bCnwHLl+6005TnwAiT+6M17pVTnYvqO42x/YAE7mwlAt59BqQ+nxPLhmxngNDxIamzq20ZXvjxcREX2T1Bg3KRbGnIMw1CYiYXz7J7cCijGAzSaCFun6KbU7igpFAOAlQOn+nLdYRCAi/7C/fkIy9CUG5JZaIM1ySe2RzA/xdmbX+089FNCN1cJ65ggOFlTC6nCKBqEN5tM5yClRdVxH3HlW5SG3WNqOE47aYhzeuRO51Q6Rgwqxk5OAglycqLLB1exypxtOWRCbFNf1WuYtJBZTpzvE9SofJulpmK0u2Kpysf2d9B/WAwVFI1ovjnN+Ti6MVgdsNYU4crTLUXZTVRUit8Iqjo/UnslH3lEL9O33RvcnQrShIiqRd7wYFun4OqSfDRQB6jERMPToyvLF+dA13SnOB6etEicLiqGaGYdxfcUFfn4+SO1FnTjHcyo6Oxxc9koRmOZAIdpsnn7G/tviGJuA1BGiXrzSndX52P63fJi9vrzoj27CdKhOGjrbrX21G79B19XODxmFpoIM5BSLc1ue5Y6hDAZIT37SStPSF3wq7zjA83Ol/5Vd6V5nWHWiikXiTBF0n/S6z1d68NwnGSgd1BBu3/PxT+UEQhczHs6yI/j6qyPIOX4cx/NLYRmdhGW3JyLM/cc/EupQDa6cOYh9mTk4bmzDlNREOEuc0Lt/rsXzWOqwlPlwHv0cXxuKcTK7HCNn3o3bp6g90bb7UedWaL0et22/dBzmQK9HWbuXac/T8wh2uB+NHoDwsRqYj38p8q6E0fA1DNdisWR+MnTue72j4Crejx3HynGpOA+HjKOQcvdcxKmkLXvnI/YkNAqxoyqRs3c3jp85i+xTlQideS/SJoW4y9nbI6/d85r0uDmq44dpOoWMhsZSgH1fHIex6jTyKoIxOSEQxhbP8gPnp4J+nGffdp8oRllBIZom34ZpzSUD/FROLkqkYyVe+QVluNwSjAmpi5HmPVYnUIfYCBsK9m1HdnEZCg8bUKlKwuLUOPfDkfqvN3HM9bFAeQb27TWg8Fw2Tl/SYuadNyM2SKT3cjy717V0fEsib0XSpa+wK68IhUfOwDZuARaJ49v+7XqAejRGnj6A3V+XImCi9BM/N/h4ERHRADyfw464vn9qwnMN6Pz5knbuz/2GuC6fyYHBQN2Js7g8NQULErTuAEC69ul0Aag6noHMbHEtK7FAHT8RwecvdOapGA1VsBWnD2cjryoQE6broVLpEa9vQEnOfmTmiOvfySLUaW7B8tum9PqAIIVmHGLDGnHBcARZuQUoKDiNklolEm+/E7fGe64jcOfpQumx/Th0tACGossISlzSca1y5zH6Mo5nfY3s3HyRXoewOcuxaLq8L73WhQJhUbFQVufiyNfZyDmRj9OmkZi0aDFuju6/f/m7YxDniXSc9UGwnj2G4ycMOG8fhzmzdSgpdSDO3X7wtB+bkxZjwoV9SN9/DKfPWhEyo7P+u9dv13NMtKEmirZMaQ72Hz6GAkMRLgdNw5JFYtleqtkX54N3eubRfBScNmFk/B24+5ZYBElPWv4ung+ivRg3aRQu5e3HF1+Ldr7UtiyzYPT0ZVgyo/15Nv23xT3xhB62U3uxPasIZWeOwHBBhaRFtyIuxF2zPfRor3m3W0+fhSH7LBrjF+GuWXqvv7Xubc/uBv6pHPc5hMTBtdOH0c7vMFKNyFgVLp/KxN4D2e56PX7yAtpi0nDHzeM954u73nSwntiHHUdP46whB6XOybjrtmnQSCPKB6qTbm1vD08bflRVDvbuOY5i40kcKWnE+HlzkTz22znfRrQJ8nvfcjnd32QqRimHNzxIJvUE4jrz6It72OoIJZS9nY/NovzSD1cP6perPV3yCqUopzznurjrbrDb7p173xRi33xSoK76zXugehvmvplPvI3tkH94XNrGyOvZNx8fLyIi8gPyZ/sw2gxDvWa6lxdXkP6uZf3nOczrkA/aBz8Y19NWGFIb0Bfngyd9yO1dfz8fBlGP7nrpqy0uGeKx6M2w6vZGut7j5q4TEWP19/khLYO+6/W7fL7duOCVyIe6BK9ERERERPSD4y/fQRD1S6GMRrS/frNIREREREQ3HHteiYiIiIiIyO+x55WIiIiIiIj8HoNXIiIiIiIi8nsMXomIiIiIiMjvMXglIiIiIiIiv8fglYiIiIiIiPweg1ciIiIiIiLyewxeiYiIiIiIyO+NqKur5++8EhERERERkV8b0SbI74mIiIiIiIj8EocNExERERERkd9j8EpERERERER+j8ErERERERER+T0Gr0REREREROT3GLwSERERERGR32PwSkRERERERH6PP5VDRERERETUi3rrVRSdLUVjY5M8Z2iCggIxbUoCtJpQeQ5dD/a8EhERERER9eJ6AleJtK6UB/kGe16JiIiIiIh6kXEw2/3v4gXz3P8O1fWuT12x55WIiIiIiIj8HoNXIiIiIiIi8nsMXomIiIiIiMjvMXglIiIiIiIiv8fg1R+ZzuHNz0tQ5JSnB9BoLMEb6UXIH/6D0IiIiIiIiPyanwWvLWi8YoO1qVWe9oHWK8jeW4D0sgZ5xvA1lhXhzb0VsPqweD3ZsHtvJTYhBNOU8qwu5DqydUa2QXERiLKY8PzeC/IcIiKi7w+n3YxyQznM9kF+qys47fWoKilGVf0g1nHaYbfb4XTJ0104Ya8ph6HcjP42318ZnSJvKf+er8HvD/VuKOfGkI6DOCfM5QaU1/R1XnhxiXNEyqePIjjrq1BcUoX6/srovT15FhH15Gc/lVOB114vw4nZs7F1iVaed50sRVj1ngkZkybj0opYeebw5Kfvx+wyJbY+fTtW6OSZvlZ0ArN32vD8YwuxppfiWg9nYVp2A6rDInHp2SREes3XHh2F/S/cgsWB8kwiIqLvNDPy3tmADdsNqHdPa5H8+Dr8clUyQtzTvem+DqC/7cf45T/eh/ggeUYX5di27h/xbgmwev0OrJwqz5bU5OHdP23ANoOckzYZK//pl1g9y2vrjeXY9aff463DZnmG2N6dv8Br/5gGvXuqHpn/9hTWH3ZPdLMar+9YiUR5ioZiqOfG4I+DveAD/P6PW9B+2BG1FC/+6sdYOr63XgU78v7yE/x2dz3SfvY+frHAq/3aWIV9/++32PBV+7nRexl7bG/qffj1z3+M1Ah5mr5V/Kkc//L9Hzasm4bNz82F8aHrC1wlKQ/djvrn5t24wFXIP3cF+cHhWNxbce3n8OrRBsQF9jxsmhlj8HyrDem5NnkOERHRd5v5q7fw2+2j8djGT7Bjxw588m/L4fxkPf5yuL2V35M9+9Mu6+z45DUsNL2F32wx9NqjZd77Ad4VYUvPANKMfX/+LbaFPoY/fyLy2fEJXl/mxJY//gVZFnkRkaNhy2/w1uWF+PX70jLitWkdEo//AW95BSzzXvgEn3zi/foTfnyLSFkWjxh5KRqaoZ8bgzwOV7Ow6V+2YPRT72CrdDw/fx+/nleGDa/vQrm8iDdnwafYkK1HYrw8w0v57vXYUL4Qr232nBufbHwMoz95GZuy7fISgiUTG/5lH7Sr/oxP3Ofrn/FidBZ+++d94gwkou58H7w665G9vwDrNmXikb8ew6YTnR8i1lOn8EZ6CYzytMSYfQJv7K2EVZ72aIYx5wReeOsQXvjsFDJqWuT5nXkUWSqwaXOW2MYJz5Bgsd2MvcewdmMW1nUZ2luD3XtLsLmwPahrhbXyHDZ9loNHpGV3FqHoitc44P7KX1iCTXvLvMov8jpXhDdFOZa8lYM39lei2uvK2L5vRmm4sbTMJpFfwRU5tTe1KDKJsoxVI06e06kB2TsvYHNYNN5IGiXP8xIWhpRgYGtN3xd0tysmpO+U6knUbXohsr3qts99d1Yj/fMTeDOva97u/dtZhmp5moiIyHfKkbUjD/FPP4b7Jnj6qUKmP4TVjwKZe/P6bNhXnd8H3JaGNHkdhCRjzjyg/lI9vEIGj5pMvLvxGlb/w2rMkWd1KM/CzuPxeObR+xDjzioEiStWYyUyset459Zj5q3Da/97BVLbO9wixPaSgDxL5zVTGRKCEK9XwIU8fHksEQ/fndpPDzL1bXjnxqCOg7ke5lkPY+nNerj7WRVapKbNF5ssRlXHlxayxmJ8+t7XSH52DZZHy/M61KOubjSS75yP5FDPnJAJ4ry8Ddh3vsozQyg/ko6sm1dj9d0xnjKExGDp2tfw2opJUHL8MFEPPg5e65H+/gmkFToQN2U8Vo1pwab9Igg97AkcG2usWFdm6xLsVF+ox7rzNjTK0xJjYSHWFrYgLiIQjeYaLPkgGx9JQZ3gycOENekmGINHQWurxyOf52Ht+wZ8dHUUpmldSDeUYZkI9DwcyC+rx245SGssPIFln1YivVWNNalhCLpgwvS/FiDfnf1Qyt8K4/4jmPa5CRkIwrKIkSgqPIeUt9vzkvetqAxr99bjUnAQUlw2rN13Aq8Z+vo0ssEoNrUiXC1Pd2o0FOKlylF4Y9nUXgJbSQiixKdetaVr/XZhKsSqd4qwzgRMiw5BkKUaaR1128++K8OhaajHC8cueAXuF5B+tB6bWwI7hi4TERH5jKUKxeVAapcuLSUSk+8ACk5CXNp7FT99JbSHs5BXI8+wF8OQByQmxqDrDUlmZH64CdWPr8FDk3sOB62/UCxCpFRM67L5RMxcBBgKy8RV0z0D2snJSB7vFYLWGJBbqMV98X31qdYj+/MPUL/sMdzVS28dDcIwz42u+jgOk+/Da//6TOeXEUK9uQzQToK+y8g7J4o/34Qt0Wuw+jbPAPGuRNArBaH3e2XuNMMs2mBp0e3L16NK7EjyvGTo6w3Yl74N29J3IdMcjuTkeGh7ffYJ0Q+bb4PXumrsFh8YL992C56/fRJWLJ+L3Q9NxJopwfICg5MfEomta27BSytuwTvPTcV6hRMvHamUUyWj8MIDaXhVSn8yGmtanTCOn4p3Hp2Nl1bNx+ZYILv6Sq9BXFHJFWSHRGDjY0lYNjcJr66egaxF4z0B4VDK32TEpgInZs9IwtZVYrtSWX4UjfsaavHmkfZeXsEVjJf/u1TW2Vi/dho2BrfilRLxydWfkd0OS1Ml3vj6CrQzJuPJ6L4OmRra0fLbXjmRkVmNj4IjsH+tVLdSeZKwWdOKzQWiPP3uuxKLk8IQaa9DRvthKKnB5uaRWDVtvDyDiIjI9wIV8huZMli62DWjuY+H6CiTV+NP/6LHl2uewnMv/wxrH9+AopRf45cPdo0U6w9/IK7jd2DNI4meHrZedX+IhFJsX/zT3OyZ9FJ/7F28/PJzeGrNTox++jU8c4tXQOvFWbIP6YfZ6+oLQz03vA36ONRkYtM7RVj47NIuQ8ud5z7Hps2j8eJTC+V7mwcigt30TfggYDUem9seGYtgVroHt/AD/Oyfd0Hqj71mysKmf/oJ1h/koGGi3vg2eA2PxDLx9/jagWys21mI3SUWBE2YhJSIXoa59mPFxEho5PcYOd59/2d1rd2r1y8Yce3dfeHivfhHE9B+gRG71M9eTZsahnn2Gqx97xg2HapEUZMG82ZEQCOtM5Tyl9ZjU+sorEj06ncMH+++H3ZTZXVnT7Jajentn4ojg4fxLVor8veW4RVo8epd19PHWY1sEaOumByNuPb6GRmJJ9csxK5lIgAdaN9visS6gBakF3m+ys4vuYL8EB1WeD/YgoiI6FvmLNmC9Ru/xuhH1+CZx1bjmWcT4dy7AZu8g4Grefj0bQPSXngMyb0+xGnoQuLvwGOP/QirV4Qj748b8O6x3oIP9rr6h0Eeh5osbHhlPczLfokXb/PqikU5dv2/D4C1q7E0Sp7VLxG4fvZb/H63Hut+vhLx3dqChoJwrN7wCzyz4mGsfuE1vP7CJGS+8yX6HKhH9APm2+BVBFgrnpqNrFkhaLxQI4KgUwj+z0y8Zhjaz9REBXYNFjUqMd3Y1O2+2OEJSpqN3Y/F4kllCz46fg7TNx1C/KYiGN1DfYdQfpcUDo5EcJeLXgi0KvFPQ8swy+oJvI12r7tyKgvxSkkrXk6bhDibDdYrNtRLX/i2tqL+ih2N8hBl6YPYKA2TCVF2Bv5diAWlZbv36nYYYN+lLxHiR2J3mRlGXEBGeSuWTdL3MYSZiIjoOikCECD+udbQtQVvv1In/q9FSK+DuqShwB+gatE6/OKphUhNTkba/S/ilz9JRebr25Dnvrw6Yfh8A3bNWI37kkR+7p9KuQbpp9Kbron37d8+j3RvHc4um7fjmnStDQ3p0Vur1MUjOTkNS5/+BX75tBO7Nmb2eMCPp7cvFasfYq/rdRnWudFpUMfhqgEf/PsfUJX6a6x7vGvvvPmrD/CufSVWL9J3nD/un9NpuCbed484najauwG/36HEw795EQu7PEFYC+0s8c/daV2+RNGL8yi5vhzmWnkGEXXwcfAqCUHKotnY+ONFOPPTWdisbsErOeVeQ3jlIMqtAfXeN7vKsizeDzWyIf9iCxAW4rNAKUgfhzWr0rD/fy9Bw1I1NHUmfHRaThyw/LKIYKwQH0hnLnh9SLWakHUJSIlQD/M+UC1SwoH8Wntn8Gttwm7xz2sH8qB9+5j7Nf2U2KatBtPfPoH09nt65PtlF49Ro/cvkcMQFybqtqbO6/5iJ6oLy5AhrejW/76nJOuw2G5F9mFpyLASq2byGe5ERHSDaCdhpmjYf20oFlerdk6UFWYBN0+SH6LUXT3qC4CYaPlhO7KQMKnXrB529/exdvfDm3BwA557/HE87n79FltEypbfiPefF0sLic3PRDK+xslir+t8YxkM2UDqZPnhOs4qZKVvw76Sro+CCtGKVkB9E651Gb4q9/ateBgLB9VbR30a1rnRbhDHwV6MLb97Gfui12Hds6k9hgXXW/KAC1vw8qr28+c5bBDnRdbG5/D4xmyxhU7mAxvw8kYzlr/8CzzcvctVtPv0k8W5aanr8jAxp7kKBnGGhfhoVABdn1GjPOPTpZ+8Gc5LEhTE37H0Fd8Gr5WncN+fDmHN/mpPj+AVOy6JuDMyWOkOqCKjRXAkgqyXPi1BUU0NsnfmY11H8NXpUpkRb5yqh/VKPfL3ncKrdcDzk6P76FEcCicyPj6A4P9XgIx6UbBWp9hGCy6JatBIH3QDlL+LyFis0YmgMqsAuy80oNEuyvqlEeubR+H55B6PnBskNdISgoG6KzghfQUsSU5F28+WdHldmi0+/KTfef3Z7XiyPUquuIKM1pFY1jGeGqg+lIMlb+UhQ/oiUnxALpsu8r5QiVdzatw9uMacAqzYXYH0iyJ5MPs+UY8nQ5x46ajY13Adll3PKGYiIqJ+6ZF2/33AZx/g3YPlqBfX2fKD7+KDdOC+ZZ2/oZr1l5fxh08McuM/HomPamHY/AF2lZg9vWI1efjgXRGaTk1EjPuBO1os/Gf5Z206Xq9jtUiRfud1x+PynY1RaXh4mQho338XmeUi8K0vR+b7H2Ab7sN9t8jhjDIcSvM2bHhjU8f2zCW78MF7WdCumIlEr3synYad2CT19t2T3M89tjQ4wzk3PAY8Do0icP31z/CB/T6svlMLs8EAg/yquupZJPHx7ufP+1h3G9y/87rjnxeKM8zDfHA9fvbHTEx66jEkNhV35GM4Vy8H3Uok3/YwEr96C+s/M8Aszp/68ky8++E2aJfdgZneI5XpWzNjeuJ1BZ/SutOmJMhTdL1GtAnyex9ohfHwMaw9aheBlGdOXNgYbF01Cynub8EakL8zD48UOd33ry6bNAnPu8pwX70IxJ5NQiQq8NrrZcDtk4C8Mrwij1hdMW0qNi8f7w6iqvcfQtSJYGT9LBWen/r1rHNi9mxsXeL5K8/+bD/SuuXZkW6/gDc3l+CF9s7dkaPw0q03Yf1tY8RE/+XvsW27CZu2lOAVS6und3KkEhvvmY3nkzxf+XUth6QeH719Aqu0k9D26ET3nB7s57DurUoELbwdr6b2fnlzl6Ms3CtfwPhFJuLLw1H+3IyOHuqinQcwvWQUNj8qglz378Y2oGhvPh45Lf6V96+zbgc6dh6eOnBizdxb8M7tPZ+KTERE5DtOlO94A79/O8vz8yfaGCx96hf48Z0xcuBRhV3rnsOn4evw+i/lB+e4zMh6/y28m97+kylaxCx4GC/+gwgS5J8s6UkELPf/DBDB60rvZzk0lmPXn36Ptw7LOU1YitX/34+xdLzX9bmxCpkfS4FT+/b0SF3xDH78lAiiOoJXM/b9Zi0+mPAa3nmawatvDOPcGMxxKNmC+9d9IE90JX250eX86FCPzH97Clnz3scvFrRHnJ5566UHMnV32zq87xXk2s9sw4b/+6579J50viY/uAY//tFCxLDnlagHHwev7VrQeKUBjUEh0AQOt3O3FY02OxCoRtCN+JRvssPaOBKasN5ujBhi+aW8nAHQqH1RUBFEfnEI8ZW6LsFpv9wB7wVo75mPl5MGUQapx9nWhKDg3urWF8eOiIjIh1xO2BubERDS817T/jjtoh0hrmfKbk+lHbJGO+yuAISE9Ld1UUb70MtI12mY54Y/8tn5SvQ9doOCV7ou7uCyGUHqEAQNJn7sNxAnIiIiIiL67mPwSkRERERERH6P40KJiIiIiIjI7zF4JSIiIiIiIr/H4JWIiIiIiIj8HoNXIiIiIiIi8nsMXomIiIiIiMjvMXglIiIiIiIiv8fglYiIiIiIiPweg1ciIiIiIiLyeyPaBPm9bzRbYTxlQGH1VQSGxiE5NQn6IDnNm6MSOSeB5HmxUMmzvs8c53NgQDLmTvDd3ubvzMRmTTLW36aV5wxXPXZ/UALcOxfLdPKs61R9OAerCprkqQCkTAzDI7dMxbyIUfK8flhKsG4PsG71VETKs4iIiIiI6IfNxz2vTlQezYR5TAruvfteLJymQOFX+bC45GRvbU7Y7E554geg2QZbs/zeRxobWmBsjw+vk7WhGdYWecIXmpqRMSYaW1fPFq9pWBPdis2fHMJrhgZ5gX60iP0S+0ZERERERNTOx8HrVdiuJiBhghoKhQLKMZOREG6GxS4n98JZZ0ThSQMMJSY4WuWZklYHzKWFMIi04iqbCIvb2VB5slL8v53XtNOM4vNWOKuLxXrty4gguUqaNqCw1OKVTzciuDT1uj1BlMVUIsp4shiVNhds54th7ljAO39z133owSWXzXsbLljLDSit9Y7wrWKfjLD2FvT3p6YSH+08gTfST2F3ZWelW0+JaeMVZO8Vadk1npnOemRI09KyF7oHlA0wnijEm+kife85GK+071QNdu+tRHVNGTa1p/VzbDEqAJowtXhpMW32DGz8UTQu7TuFjPaA216DjP0FogwnsCmnBtZudVd/rkguQxmqvQ9IX+tdqcSm7GpY5fXe3F/ZI08iIiIiIvpu8nHwqkPSsiTx/3b1sNaooept2LDkSiEKTYEYFx8LTYMB2/NMIpST2FCamYFSlw5xk2Ohqj2CL0+Y5TQHao/Wiv+385oWAWhV/mHkVAdCNzYUgWKW5dSXOFIbirhpCdC3GLD1WPs2vLgsyN+VAaO0vWlxCL18BHtOWeRECwx/3wcj9IiL18FZcgRHzlTJvagumE9I+asQOzkOOlcpMjJLvQLrrmxl+Si8ou22Twqog5qQW9a+f0KNEUeaAqFWyNODYSrEqi0mNEwYj1UpQcjfndfRy9lYY8WruwqRoQpDSmSwCMYv4I23CpAujtSKW7SwHinDxo4KdSI/PQerClsx+5bJWDH2Cl74awHy3UGgA/lFZXj+QAvSbhuPtKZqpG0uRLV7vUEIj8OKSBuyS8X7pkq8tqkQ6S3hWLFkPOKqirBs5wXPchJHDd4sUmLxLdGYZr+AtK1laJTm97degw27c0vwalEIlt0WjSgRZC/eWuFZj4iIiIiIvtNu4AObpMAuB5abkhGrlGd1p05C8k3R0Kg1iJ2VgoQCswgVxZqmYuSGzMbcqXqoVVLafCSY83Huqme1fqkSkTIrDtHRGihFsGUTGcbERkOtVEGXuBhPpEaLcLEbhQ4p9y/HfGl7SjWiJydAZba6A2LXhVIYYud60tQ6JKROR3R7oHf1HPLNCZg/SwTfKjX0U+didkguik09wmM3i3oikr33qdrg7rlURMcisfwizO7VXDBVFCNhvL5nOfvkFEFzNeLmzsaaGRGIjJuMl5eFI+twqQi6PaKmTsXLt0/C4jg1GvMuYH1UHDbeHYu48ePx5KNxeEReDqLWUpbPR8aqGZg3Xo242XFYpbSjSO6wRXMI1j06FdMiIjDvgTisc1zBiSty2oCUCJbOBWk/A2Px8j/M95QhLAKL5+sQVX2lMxAOisBLD0zCtPGRWPbQBDx5oR750vyB1lPq8PwDIi0iEiuWRSPtYh2K5CQiIiIiIvruukHBqwhcC/YgF3OwMEEtz+uF2Lp3gNb+KJ8mhxW6cI1XmhpafSUaBtOF1iVPFWKT42D68mNs/+oIcksqYes9rvQMXz56EDt378TO/Sdgluc3NdqgD/XeBxVUYfLbxgZU6rWidO0U0ITrYHX0fiNq3Fht132KNMLmHnarR8y0KlReEoVzmXHRlIS46MGHroAd1TYlpo/3+pYgVo00e1NnUDey81Bbbc1IC2/fCWFkCKK8nyN1tRrpu49h7cZMLNlgwKtdhgaLfDqyktZzwTqI21g9Wt3/ebTCWnoOb27OwpKNB7BkSw2y5BQ3sY1g+W2X90NZLywQUc2t7HklIiIiIvoe6AhDfMlWmolc52zcNXsovYfdtHWLMltFdDVCfj8EijFJWPr4E1i+IBmxbVXY91Vxz2G9dYXYc7IJ+hnzsXzZcixfOluEk7KRCjS1eJdFvPe+j7LbPZWuNrHNPmrV1ugd1LrgatZBIVeQbnwCTBfNcJovwhQX7TX0egi6latBlKOvEdto7bZwx6QJb350HvWTJ2Pjswux/8VZeKWf7x+GpKkSGZeCkTJBvC8zYNkREUQvTcH+FxZh/+MRSPMs1b/hrkdERERERN9pPg9ebeePILMqFgtviUZfo4UHooqIgauwFOb2p/PaSlFYOhX6cGlCRHsqC+rlCNRpquwYGtuTDcYsA0wiH4VSDf2kGOhrG9CjX7TFCbNGB53KE0k6Lps7el5V+jioigtRKQ8VdlQVo7h9CO0YPVJKC1HaHg03m1Fa6EK0rvefw7GcNXrtkxGFFxIQPUaeHhOH6bWFyDxjxtS43kNX6+EszP5UvoeztRbGemB2pPQzOVrMm9CKjUfa7+9shfFAFT6K1mKae7qryMkaNBYZkd3eo1pWhc0dEb0T1qZAzJ6sRZB0AC3VyOrrJt6hqL+Aj7YYsSs+FsukTl97C7LDNEiJkPpJW1F97krXHtS+DHc9IiIiIiL6TvNt8OowIndvIUznMvDx22/jbfmV3x7sDVZoIuanNCF3+3b3MN4te43QL50Nz0haHRJuV6N0+4f4+JMtyGnUIsa9Um/U0I1z4IScz/ZdZ6Ba7P1AKVlEApbaM7Fl507sFK88q7Kz5zUkDgvv0MGUJdJ274OhSY+49h8fVURj9lI9jHu3ePLfnoumlPlIDJXTu4mbpoUpQ8rHs0/Rd3iXRY1xk1wwNidgXB/ra9Km4hVRx7P/byaW/KkAm3ST8FKSJy1u6U14pfG8J21DJu4rU2PrA5N673mNTcLGmU6sefuAe+jtfVkhWDVRTkMsnrylBWv+5ElbssMOTYicNFQVZRjx+n73K/6vlTBOnIqMB8Z70m6Kw9ZrRkwXZZXK+8rlUYPrQR3uekRERERE9J02ok2Q3/shF5xOQKkc9uBj2SDzaXbCOVKJHotJQ2o7wnwbSr/MAeYvRYJXUOfybGBQw6RdThcUvZTFVvIFchQLsDSh957bDk12NI4I8fSMdifSrC2BIuBsv4O4H84GWJsDel9WSmtSQKMebv/5INltsI4KgSZwiN+jDHc9IiIiIiL6TvLz4NUPNFbi4OdnoJ6bghi1CDAvGJBbl4Dli+IwQIg5eE4LTJVVMOQ1IfmRuYgOkOcTERERERGRG4PXwXDaRHBZCYvdBWVYNGJjdVD5ssOv0QpTXRPUOumneuR5RERERERE1IHBKxEREREREfk93jBIREREREREfo/BKxEREREREfk9Bq9ERERERETk9xi8EhERERERkd9j8EpERERERER+j8ErERERERER+T0Gr0REREREROT3GLwSERERERGR32PwSkRERERERH6PwSsRERERERH5PQavRERERERE5PcYvBIREREREZHfY/BKREREREREfs8/gtdmJ5wu+X2vXHD2v8B1cTmdYgs+1GhB5WWnPHEdfJXPULTe2LqWjqXNZIK1uX1SHPsbuj0iIiIiIvo+8Ivg1Xzqv5BpdMhTvbAbkflfBpjlSd8yw/BfmTDa5UkfsJYewRdHzsEqTw+Xr/IZEosB/3XQiH6ORjdOWE0WOAYbfzovwpCRgcJqT1DuMGYOcXtERERERPRDxGHDN4Dmpgfx7ENJ0MjTw+WrfG6sehh35sPUKE8ORBmL+T/6EebHKOUZREREREREA1P8RpDf+4gT1opinDp9Emcq69A8IhTasMAuUbKjphRF+cdhqLRjZKgOAfX5MAcmY1J4gGcBpw2Vpfk4eawcl1qDEDHaifOnndDfHIXRniV6aM8z13gZzWKpMZog9zadNcUovOCCVjcaCs+isJ03wOjUQhcizbGj+rgZgTfFoLkiFyekbTa1YrRGiyB5Bc/yaqiunMWxoydRZRuJ0DFaBDgqcbbgJPIrrmDkaC20wZ4VemyzY3+KcOFKMwLDdRg9yr2o21DK7jAVo+BkvqduR4l9UMt1JrTvV6jtHHKP56Dc3IoRYVqEBfbzHYXDjNIzJ3H8dBXsI0OhC7Ai3xyI5PhwtOfsrDOi2HAKx8ouucunEeXz7JcZxadLcfFiHa4pmmFrUUGvCexcp/AMTp6pQl0zEBoWhkD3SqIuTpbgikovygU015XDYNV2bm+AuiIiIiIioh8mH/e8OlF5eAt2ljihm5KClCla2I5vR2Z556BQW2kG0rPMUE5IQlK8Gta8IyiskxMlzWbk796OMw16JMxKQDRKkZlXiSY5uTfO80c8ecbPwcKZ0XCW7EFmqc2d1nK1CjlVNrS4pzwctTmouuo9pwmVeZliS9HubepbSrBztwio5fsypeVLDXkw1KkRN30qNHVHsPPwQeSedkAVk4CpY20w/E2sLw897rpNCwzy/ky/fS4S1LXI+aIz78GX3QXLyZ3YfqoBunip7nRoOJWO7QWWjvt13eUszEFOdSBiJ89GTGAFDm7PQWVfvaK2UmSk58AcFIOk6XFQW3NxpNAiJ3o4Kg5i60ETFNEJmD1ZD1fpHmw9ZvJsU6GGbqwGKiih1umgk6JRQdqnLbtL4NQlICUlAdor+dh+uH1osAO1R3NQ22uZ+q8rIiIiIiL64fJx8KrEuJnL8eDiFMRFiGAmQgQ8qQkoPW/2BC4uE4pznEhdPB+JMdGIjo5DysIEqKrdK7vZyvNRGLUYS2fFifRoxE6dj7lRTpjk9N7U1xZCn5SMxGgN1JpYkeeDmDNBLacOhgnOqLmYPzXWvc24WUuxOKoQ+eWeIFJiUidgzk1SeiySUudCf06B6JsTEesuYwqSJxthqe/lxk+7FeaaqUgR+6NTq6GfOhf3Lk2CTu7WHHTZ7UbkHw3F3IWibsU23XW3eDH0p/Nw7qq8jGAKnIj5ct3FzVqAuZGFqDL39tAnF0xFOXCmLvba78VIUHnfWWxB6XGH2E7n8Uq+cymSqgyebSpU0EXroEYgtHqRrvEMBVZGJ2P5g0uRMlEP3Rg9Em6eg4RzRpgHurF1gLoiIiIiIqIfLh8Hr1JnnAaqNhssJhOMJQacKBPBkEsO6uxXYQ6Ohi7UM+kWoEfsZPm99CRaayXiInUdw2Ql6uiJiJbf90Y7Jgnmo5k4eFoESFYHXEoVvEbTDoIO0WO9A0YFdJFxqLxcj46wL0DRWSZ3rakQ2HHbpgqqMBF4N/bSPxyigT4iHxlf5aO0ygKbyFAZouzIa7Bld9VbYJwcA32QPEMSIMo9sRK19V7BaVCgV92JvMIBR7N3L3M7B65aVD32Wz8uQX4vXLWg0qUBrplgEsfT87KhKbgS1qv9PKEpQA2NShzLWrF8RTEM+aUwi2PrapPT+zJAXRERERER0Q+Xj4NXG4yZH+Pjv+ei9LIFTYE6xI3zCjubGmANVsIzuLSdCAo7opMmNDnEnM4ZHiNGdVunK+WE+Vj5wGwR4JpRcngnPv5oJ/Jr+gmuelBD1e35QUopMm10dhluPDw6JD/wIyxNCEb9eRGYpW/Cx5lGUVMegy17k1NUjFoEyfK0h6hLEcz2HpwOpAkNdUoouwXKilFedS8dL4UDNnEsLR0vGwLHzYVG2Xf92soP4uOPP0duqVi+KRC6uOh+v3zo1H9dERERERHRD5dvg9daI/ItSbjnvy3G3JnJSJwYDW2QV5ATokH0hVp4dxRKAW99jfxW6inUqmCqr5enPVxXLDDK7/uiDI1Gwk1zsWD5SjyxUIvcQ4WQ7t4MDOplCG6r/G8HIyxXugZjNqsZKp10P6cPjFRBNzERc25bigcfX4lk+z4YzndWQl9l96YK00F33txtvlR3Iu+w4ZRSBc04U9deW0Ha7w7S8apVYdxNyUgWx9P7lRjR19OCxbHKtyDp3pVYfKtYdmocojWqjvtyBzRAXRERERER0Q+Tb4PXIBF8NjjgaI81mq04d65UnhBU45AwXQS4p80dw3Ft5/KR7/XAJl1MMhQnDZCfWQTpibaGgtJ+gkgHSr/6EBnnOvvnHA4HIAI6aR1FqAax5UYY66QtuuCoykVusXsxLyqUFhjQcWuorRS5eS4kx+jkGdehJh8ff57fmbdT1E+DKFuwFPz1X/YuxsRhusoAg9eytnO5yHMmI2asPGNIRFAanyQCzcIu+51f4PWrsqpYJM4UdXGy83i5H/L0SUbHw6k8mtDUMWJaGkIt9rHjJHDBeq4YXmdB3/qtKyIiIiIi+iHz7U/lKMMQNrIEX+/IRlFVMfJP2RGXGInyK8HyT6EoEBYVDVfxfuzIKkLZmXyYwm7BHF0Jatt/Kkelx7gwM45/uRvHz5bhZGETpqRNg7Okr5/KCUD4WA3Mx7/E14ZKGA1fw3AtFkvmJ0MnjTUO1EITaET23q+RfbwElvDZSPbenvuncuyYtCgaFXt24WhxIY4U2jD+9kW4OcozWNl+6ThKkCim5a0316H8VNfySMu0/9xPl59/CYlAVEsh9u84hvJLRcg7ZMSolLsxN06FkQOUvevPyASKwF4P26m92J51GmcN2TjbOAmLls6CXv4pmR7lFLzL1Z1CMw7R7rKJ42UsRP7FUNwyJwIlte0/lTMSan0sRlXlYO+e4yg2nsSRkkaMnzcXyWPbB3KPxmhtNQ7t3IvshrGYEatDeJgCJV/vQHbxBRSfKIB94nREllsRnDwJ4UqpvkuAxJsRFdJtH/utKyIiIiIi+iEb0SbI732n1QVni4hllf08aqfZCSd63nPZSeThdEGhHPwDe1xOJ1wj+stzYFIeouC+f0iQu0763p8hlX3AuhuiwRwvl9imS9H/Mt2463KU2N+hRp4D1BUREREREf3w3JjglYiIiIiIiMiHOBqTiIiIiIiI/B6DVyIiIiIiIvJ7DF6JiIiIiIjI7zF4JSIiIiIiIr/H4JWIiIiIiIj8HoNXIiIiIiIi8nsMXomIiIiIiMjvMXglIiIiIiIiv8fglYiIiIiIiPweg1ciIiIiIiLyewxeiYiIiIiIyO8xeCUiIiIiIiK/x+CViIiIiIiI/B6DVyIiIiIiIvJ7I+rq6tvk90RERERERER+aUSbIL8nIiIiIiIi8kscNkxERERERER+j8ErERERERER+T0Gr0REREREROT3GLwSERERERGR32PwSkRERERERH6PwSsRERERERH5Pf5UDhERERERUS/qrVdRdLYUjY1N8pyhCQoKxLQpCdBqQuU5dD3Y80pERERERNSL6wlcJdK6Uh7kG+x5JSIiIiIi6kXGwWz3v4sXzHP/O1TXuz51xZ5XIiIiIiIi8nsMXomIiIiIiMjvMXglIiIiIiIiv8fglYiIiIiIiPweg1ciIiIiIiLyez4OXlvRaLPBesXrZW+R075pclm+6e032cV+N8gTN1Crs8/6bSwrwpt7K2BtlWf0xtkg1rejsb9lvkl2sS82pzzR06D2iYiIvpecdjPKDeUw2/u+TvTQWI+qkmJU1fezjsuJ+gvFKL5QD6dLnteDE/aachjKzehv84Mq42DKREPCc4Poh8XHP5VTj4/ePoFVV+RJWVyIGq8uTcKTk0PkOQNrNFUiq1qJ2bMjoZHnDY1cFu0ktD06UZ5341XvP4SoE8HI+lkqbuQDsRuP5SA40w6ERKD8uRmIk+dL8tP3Y3aZElufvh0rdPLM7gx5GLG3AZtX344nI+V536Lsz/YjrT4Sl55NQm/FGdQ+ERHR94wZee9swIbtBnFVl2iR/Pg6/HJVMvpuUThRvuMN/P7tLLG2h/7OF/Hrf1iKmCB5huAs34U3/s9byLokz4haihd/9WMsHa+UZwg1eXj3TxuwzeDZOrTJWPlPv8TqWd5bH0wZB1cmGgqeG/TN4E/l+JcbM2x4fCzqn73F/br03ybiZbUdqz4/htcMg++RtBadx5L9F1AkT5M3GzIKReAaMBIp9jpkVMqzZSkP3Y765+Z9r4K87+M+ERFR/8xfvYXfbh+NxzZ+gh07duCTf1sO5yfr8ZfDcsDQC6fhU/zm7Wqk/cv77nV2bFqHxOMigNhRLsIEmdOAT3/zFqrn/Rrvfy6W+fwdrEvMw4b/+znKOxYyY9+ff4ttoY/hz5+IZXZ8gteXObHlj39BlkVeRBhMGQdVJhoSnhtEP0w3JngdFQBNmNr9ioybhDWrU7FL14pXDpfCKC+CVjuMOafwyl8P4ZG/HsObh0wdQ0KN2Sew0SgNh7Vjc/oJbDpl8ySgFdbKc9j0WQ4eeSsHr+wsQtGV6xlH2oLqE54yLNl0DJtyaroNSx3E9mrKsGlzlmf9givoGZ6LPM4V4U15mTf3V3bZhrSvb+ytgFEs89qmTKzr50O3Q90FpIsPx5fS4rAqoAUfnamWEzyshSXYtLdMrusa7Jbq0FCDokN5WLsxB7u9PljhlNLl/RPrVHt9UnrKJsorT0tBc/ZeMS+7Rp72qp+NWVg30PG4Ira19xheeCsTazcXIP2cCMC7afCuzxOdddHrPp264j5+60S9rd18Chk139YQdSIi8r1yZO3IQ/zTj+G+CZ5+qpDpD2H1o0Dm3ryOXqqu7DAc2YL6Ox/DY7doPbMiFmL1U6kofv9rFMvXOHtBFrbUL8Vjj6RCqxAzFHosfGI1Uks+wNdn5IXKs7DzeDyeefQ+xLg3H4LEFauxEpnYdbx964Mp4+DKREPBc4Pohwn4/wEXczp8r2kuWgAAAABJRU5ErkJggg=="}}},{"cell_type":"code","source":"submission_df.sample(2)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T20:22:05.377610Z","iopub.execute_input":"2022-05-29T20:22:05.377908Z","iopub.status.idle":"2022-05-29T20:22:05.391937Z","shell.execute_reply.started":"2022-05-29T20:22:05.377868Z","shell.execute_reply":"2022-05-29T20:22:05.390963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TL;DR\n\n* First we created basic `AirbusDS` class for dataset- with no augmentaions nor masks.\n* Then using `https://www.kaggle.com/paulorzp/run-length-encode-and-decode` ww've got rle decode function for adding masks on the dataset.\n* After that we implemented `AirbusDS` class with albumentations and masks\n* Finally we use the dataloader to iterate and present sample of images and the segmentations.\n\n## work process\n* After the first attempt of training the UNET model, the running time took too long, so after searching I found a reduced Dataset for the competition - thanks to Ofri Harel.\n    * The model training time with the reduced data was significantly improved.\n* Much of the notebook here is taken from the smp documentation examples, see link in references.\n\n1. At the beginning of the notebook I experimented a bit with masks, rle_encode.\n2. In order to train the models, I made preparations - I used transformations and augmentations that do not deal with image blurring.\n    * After looking at the data I saw that there are many pictures without boats - for example pictures of buildings (and not in the open sea). Therefore, I reduced the amount of pictures without the boats in order to create a little proportion in the data and also to reduce training times and prevent the notebook from collapsing. Running time after this reduction decreased from an hour or more to a few minutes.\n3. I divided the data into 3 groups: valid train, test based on the paths of the images and masks and thus I avoided saving them in the dataframe . I made sure that the ratio between train, validation would be the same as the total train data per test.\n    * I created a dataLoader for each group.\n\n4. Before insert the data into the model for training, I updated the dataset department, since a permute must be made for the images in the models - the support is for the dimensions [channels, height, width], so I made sure to swipe the dimensions.\n5. Using trainningAPI I trained the models when after each workout I made sure to clear the cache to avoid CUDA OUT OF MEMORY and notebook crash. I also additionally saved the model results in csv. In order to then load the results and plot on them and thus choose the best model.\n\n6. In the test phase I had to encode the image - the algorithm inspired by other notebooks I saw here, but I had to change it for this notebook:\n    * rle_encode:    \n        * First, if the whole image is zero, it means that there are no pixels whose value is close to 1.0. We will return an empty encoding.\n        * I convert the prediction to np.float64 to 2 digits after the dot.\n        * I am looking for the areas where the image is not symmetrical (the areas where we hopefully predicted correctly that the ships are)\n        * I take out the positions of the zeros in the image I took care to chain between pixels\n        * I create a set of counters that counts the number of pixels (number of members) between the zeros.\n        * Then I extract each time the initial pixel and the corresponding counter does not.\n    And that's it.\n    \n* Note: for better results- \n    1. train on more data -  I reduced the amount of data because of the runtime (more data means about 20 minutes per epoch) in this notebook one epoch ran at 3 minutes.\n    2. Increase NUM_EPOCHS\n\n## References\n\n* https://github.com/qubvel/segmentation_models.pytorch/blob/master/examples/cars%20segmentation%20(camvid).ipynb\n* https://www.kaggle.com/code/alexj21/pytorch-eda-unet-from-scratch-finetuninghttps://www.kaggle.com/code/alexj21/pytorch-eda-unet-from-scratch-finetuning\n* https://www.kaggle.com/paulorzp/run-length-encode-and-decode\n\n","metadata":{}}]}