{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":13836,"databundleVersionId":1718836,"sourceType":"competition"}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport time\nimport os\nimport copy\nimport json\n\n# visualization modules\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom PIL import Image\n\n# pytorch modules\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.nn import functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models\nimport torchvision.transforms as transforms\n\n# augmentation\nimport albumentations\nfrom albumentations.pytorch.transforms import ToTensorV2\n\nimport warnings\nwarnings.filterwarnings('ignore')\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:33:45.808996Z","iopub.execute_input":"2024-05-21T14:33:45.809408Z","iopub.status.idle":"2024-05-21T14:33:53.942837Z","shell.execute_reply.started":"2024-05-21T14:33:45.809375Z","shell.execute_reply":"2024-05-21T14:33:53.942082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!brew install graphviz\n!pip install torchviz","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:33:57.283574Z","iopub.execute_input":"2024-05-21T14:33:57.28411Z","iopub.status.idle":"2024-05-21T14:34:13.585872Z","shell.execute_reply.started":"2024-05-21T14:33:57.284079Z","shell.execute_reply":"2024-05-21T14:34:13.584669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **ВЫВОДЫ ПО ВСЕМ МОДЕЛЯМ**","metadata":{}},{"cell_type":"code","source":"from prettytable import PrettyTable \n  \ncolumns = [\"Эксперимент\", \"train accuracy\", \"test accuracy\"] \n  \nmyTable = PrettyTable() \n  \n# Add Columns \nmyTable.add_column(columns[0], [\"my_model + SGD + sheduler + img_augment\", \"my_model + SGD\",  \"my_model + SGD + img_augment\", \n                       \"resnet + SGD + sheduler + img_augment\"]) \nmyTable.add_column(columns[1], [\"0.2050\", \"0.2023\", \"0.2032\", \"0.8245\"]) \nmyTable.add_column(columns[2], [\"0.6125\", \"0.1067\", \"0.1239\", \"0.8389\"]) \n  \nprint(myTable)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:41:22.749418Z","iopub.execute_input":"2024-05-21T17:41:22.750074Z","iopub.status.idle":"2024-05-21T17:41:22.757387Z","shell.execute_reply.started":"2024-05-21T17:41:22.750041Z","shell.execute_reply":"2024-05-21T17:41:22.756484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = \"../input/cassava-leaf-disease-classification/\"\n\ntrain = pd.read_csv(BASE_DIR+'train.csv')\ntrain.head()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-21T14:34:16.675187Z","iopub.execute_input":"2024-05-21T14:34:16.675566Z","iopub.status.idle":"2024-05-21T14:34:16.725961Z","shell.execute_reply.started":"2024-05-21T14:34:16.675532Z","shell.execute_reply":"2024-05-21T14:34:16.725047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# loading mapping for target label\nwith open(BASE_DIR+'label_num_to_disease_map.json') as f:\n    mapping = json.loads(f.read())\n    mapping = {int(k): v for k, v in mapping.items()}\nmapping","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:34:19.106321Z","iopub.execute_input":"2024-05-21T14:34:19.107129Z","iopub.status.idle":"2024-05-21T14:34:19.117811Z","shell.execute_reply.started":"2024-05-21T14:34:19.107096Z","shell.execute_reply":"2024-05-21T14:34:19.116816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:34:21.440806Z","iopub.execute_input":"2024-05-21T14:34:21.441487Z","iopub.status.idle":"2024-05-21T14:34:21.45642Z","shell.execute_reply.started":"2024-05-21T14:34:21.441455Z","shell.execute_reply":"2024-05-21T14:34:21.455422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DIM = (256, 256)\nWIDTH, HEIGHT = DIM\nNUM_CLASSES = 5\nNUM_WORKERS = 24\nTRAIN_BATCH_SIZE = 32\nTEST_BATCH_SIZE = 32\nSEED = 1\n\nDEVICE = 'cuda'\n\nMEAN = [0.485, 0.456, 0.406]\nSTD = [0.229, 0.224, 0.225]","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:34:23.605345Z","iopub.execute_input":"2024-05-21T14:34:23.605694Z","iopub.status.idle":"2024-05-21T14:34:23.611584Z","shell.execute_reply.started":"2024-05-21T14:34:23.60566Z","shell.execute_reply":"2024-05-21T14:34:23.610643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_test_transforms(value = 'val'):\n    if value == 'train':\n        return albumentations.Compose([\n            albumentations.Resize(WIDTH, HEIGHT),\n            albumentations.HorizontalFlip(p=0.5),\n            albumentations.Rotate(limit=(-90, 90)),\n            albumentations.VerticalFlip(p=0.5),\n            albumentations.Normalize(MEAN, STD, max_pixel_value=255.0, always_apply=True),\n            ToTensorV2(p=1.0)\n        ])\n    elif value == 'val':\n        return albumentations.Compose([\n            albumentations.Resize(WIDTH, HEIGHT),\n            albumentations.Normalize(MEAN, STD, max_pixel_value=255.0, always_apply=True),\n            ToTensorV2(p=1.0)\n        ])","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:34:25.137357Z","iopub.execute_input":"2024-05-21T14:34:25.138019Z","iopub.status.idle":"2024-05-21T14:34:25.144663Z","shell.execute_reply.started":"2024-05-21T14:34:25.137989Z","shell.execute_reply":"2024-05-21T14:34:25.143637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, image_ids, labels, dimension=None, augmentations=None, folder='train_images'):\n        super().__init__()\n        self.image_ids = image_ids\n        self.labels = labels\n        self.dim = dimension\n        self.augmentations = augmentations\n        self.folder = folder\n    \n    # returns the length\n    def __len__(self):\n        return len(self.image_ids)\n    \n    # return the image and label for that index\n    def __getitem__(self, idx):\n        img = Image.open(os.path.join(BASE_DIR, self.folder, self.image_ids[idx]))\n        \n        if self.dim:\n            img = img.resize(self.dim)\n        \n        # convert to numpy array\n        img = np.array(img)\n        \n        if self.augmentations:\n            augmented = self.augmentations(image=img)\n            img = augmented['image']\n        \n        label = torch.tensor(self.labels[idx], dtype=torch.long)\n        return img, label","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:34:27.724399Z","iopub.execute_input":"2024-05-21T14:34:27.724753Z","iopub.status.idle":"2024-05-21T14:34:27.733652Z","shell.execute_reply.started":"2024-05-21T14:34:27.724723Z","shell.execute_reply":"2024-05-21T14:34:27.73255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_train, x_test, y_train, y_test = train_test_split(train['image_id'], train['label'], test_size=0.25)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:34:30.687764Z","iopub.execute_input":"2024-05-21T14:34:30.688683Z","iopub.status.idle":"2024-05-21T14:34:30.704791Z","shell.execute_reply.started":"2024-05-21T14:34:30.688641Z","shell.execute_reply":"2024-05-21T14:34:30.703732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:34:48.302046Z","iopub.execute_input":"2024-05-21T14:34:48.302631Z","iopub.status.idle":"2024-05-21T14:34:48.31054Z","shell.execute_reply.started":"2024-05-21T14:34:48.302599Z","shell.execute_reply":"2024-05-21T14:34:48.309383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:34:52.585449Z","iopub.execute_input":"2024-05-21T14:34:52.586061Z","iopub.status.idle":"2024-05-21T14:34:52.590203Z","shell.execute_reply.started":"2024-05-21T14:34:52.58603Z","shell.execute_reply":"2024-05-21T14:34:52.589255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import WeightedRandomSampler\ndef sampler_(labels):\n    label_unique, counts = np.unique(labels, return_counts=True)\n    print('Unique Labels', label_unique, counts)\n    weights = [sum(counts) / c for c in counts]\n    sample_weights = [weights[w] for w in labels]\n    sampler = WeightedRandomSampler(sample_weights, len(sample_weights), replacement=True)\n    return sampler","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:34:59.642721Z","iopub.execute_input":"2024-05-21T14:34:59.64344Z","iopub.status.idle":"2024-05-21T14:34:59.64921Z","shell.execute_reply.started":"2024-05-21T14:34:59.643409Z","shell.execute_reply":"2024-05-21T14:34:59.648272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sampler = sampler_(y_train)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:35:02.727996Z","iopub.execute_input":"2024-05-21T14:35:02.728935Z","iopub.status.idle":"2024-05-21T14:35:02.754905Z","shell.execute_reply.started":"2024-05-21T14:35:02.728894Z","shell.execute_reply":"2024-05-21T14:35:02.753952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create dataloaders for training antrain_test_splitidation\ntrain_dataset = CassavaDataset(\n    image_ids=x_train.values,\n    labels=y_train.values,\n    augmentations=get_test_transforms('train'),\n    dimension=DIM\n)\n\ntrain_loader = DataLoader(\n    train_dataset,\n    batch_size=TRAIN_BATCH_SIZE,\n    num_workers=NUM_WORKERS,\n    shuffle=False,\n    sampler=train_sampler\n)\n\nval_dataset = CassavaDataset(\n    image_ids=x_test.values,\n    labels=y_test.values,\n    augmentations=get_test_transforms('val'),\n    dimension=DIM\n)\n\nval_loader = DataLoader(\n    val_dataset,\n    batch_size=TRAIN_BATCH_SIZE,\n    num_workers=NUM_WORKERS,\n    shuffle=False\n)\n\nloaders = {'train': train_loader, 'val': val_loader}","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:35:08.151449Z","iopub.execute_input":"2024-05-21T14:35:08.152107Z","iopub.status.idle":"2024-05-21T14:35:08.159645Z","shell.execute_reply.started":"2024-05-21T14:35:08.152076Z","shell.execute_reply":"2024-05-21T14:35:08.158545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset[0][0].shape","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:35:10.789742Z","iopub.execute_input":"2024-05-21T14:35:10.790458Z","iopub.status.idle":"2024-05-21T14:35:10.842547Z","shell.execute_reply.started":"2024-05-21T14:35:10.790425Z","shell.execute_reply":"2024-05-21T14:35:10.841536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass nnUnSqueeze(nn.Module):\n  def __init__(self, dim=1):\n    super(nnUnSqueeze, self).__init__()\n    self.dim=dim\n  def forward(self, x: torch.Tensor):\n    return x.unsqueeze(self.dim)\n\n\nclass nnSqueeze(nn.Module):\n  def __init__(self, dim=1):\n    super(nnSqueeze, self).__init__()\n    self.dim=dim\n  def forward(self, x: torch.Tensor):\n    return x.squeeze(self.dim)\n\n\nclass nnTranspose(nn.Module):\n  def __init__(self, dim1, dim2):\n    super(nnTranspose, self).__init__()\n    self.dim1=dim1\n    self.dim2=dim2\n  def forward(self, x: torch.Tensor):\n    return x.transpose(self.dim1, self.dim2)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:35:13.412253Z","iopub.execute_input":"2024-05-21T14:35:13.41298Z","iopub.status.idle":"2024-05-21T14:35:13.420972Z","shell.execute_reply.started":"2024-05-21T14:35:13.412931Z","shell.execute_reply":"2024-05-21T14:35:13.419903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn.functional as F\nfrom torchviz import make_dot\nfrom sklearn.metrics import classification_report","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:47:51.170482Z","iopub.execute_input":"2024-05-21T14:47:51.171401Z","iopub.status.idle":"2024-05-21T14:47:51.176314Z","shell.execute_reply.started":"2024-05-21T14:47:51.171363Z","shell.execute_reply":"2024-05-21T14:47:51.175334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# target output size of 5\nm = nn.AdaptiveAvgPool1d(1)\ninput = torch.randn(1, 64, 8)\noutput = m(input)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T11:37:09.6782Z","iopub.execute_input":"2024-05-21T11:37:09.678576Z","iopub.status.idle":"2024-05-21T11:37:09.684459Z","shell.execute_reply.started":"2024-05-21T11:37:09.678546Z","shell.execute_reply":"2024-05-21T11:37:09.683574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**NEXT**","metadata":{}},{"cell_type":"code","source":"class getMyModel(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(in_channels=3, out_channels=6, kernel_size=(5, 5), padding=1, bias=True)\n        self.conv2 = nn.Conv2d(in_channels=6,out_channels=12,kernel_size=(3,3))\n        self.conv3 = nn.Conv2d(in_channels=12,out_channels=1,kernel_size=(1,1))\n#         self.pool1 = nn.MaxPool2d(2, 2)\n        self.pool2 = nn.AdaptiveAvgPool2d(output_size=(1, 1))\n        self.fc1 = nn.Linear(1, 64)\n        self.fc2 = nn.Linear(64, 16)\n        self.fc3 = nn.Linear(16, NUM_CLASSES)\n                \n    def forward(self, x):\n        x = F.relu(self.conv1(x))\n#         print(\"SUS1\", x.shape)\n        x = F.max_pool2d(x,kernel_size=(3,3),stride=2) # max pooling\n#         print(\"SUS1_1\", x.shape)\n        x = F.relu(self.conv2(x))\n#         print(\"SUS2\", x.shape)\n        x = F.max_pool2d(x,kernel_size=(3,3),stride=2) # max pool\n#         print(\"SUS2_2\", x.shape)\n#         print(\"SUS1\", x.shape)\n#         x = F.relu(self.pool1(x))\n#         print(\"SUS2\", x.shape)\n        x = F.relu(self.conv3(x))\n#         print(\"SUS2_20\", x.shape)\n        x = self.pool2(x)\n#         print(\"SUS2_21\", x.shape)\n        x = self.fc1(x)\n#         print(\"SUS3\",x.shape)\n        x = self.fc2(x)\n#         print(\"SUS4\",x.shape)\n        x = self.fc3(torch.squeeze(x))\n#         print(\"SUS5\",x.shape)\n        x = F.log_softmax(x, dim=1)\n#         x = x.squeeze(1)\n#         print(\"SUS7\",x.shape)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:35:20.981256Z","iopub.execute_input":"2024-05-21T14:35:20.982091Z","iopub.status.idle":"2024-05-21T14:35:20.99199Z","shell.execute_reply.started":"2024-05-21T14:35:20.982059Z","shell.execute_reply":"2024-05-21T14:35:20.991119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_model = getMyModel().to('cuda')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:35:23.652605Z","iopub.execute_input":"2024-05-21T14:35:23.653196Z","iopub.status.idle":"2024-05-21T14:35:23.85406Z","shell.execute_reply.started":"2024-05-21T14:35:23.653157Z","shell.execute_reply":"2024-05-21T14:35:23.853051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = torch.rand(32, 3, 256, 256)\nout = my_model.forward(x.cuda())","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:35:25.736757Z","iopub.execute_input":"2024-05-21T14:35:25.737391Z","iopub.status.idle":"2024-05-21T14:35:26.460028Z","shell.execute_reply.started":"2024-05-21T14:35:25.737358Z","shell.execute_reply":"2024-05-21T14:35:26.45925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_model","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:35:27.836866Z","iopub.execute_input":"2024-05-21T14:35:27.837501Z","iopub.status.idle":"2024-05-21T14:35:27.843055Z","shell.execute_reply.started":"2024-05-21T14:35:27.837468Z","shell.execute_reply":"2024-05-21T14:35:27.84212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch = torch.rand(32, 3, 256, 256)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:35:29.928821Z","iopub.execute_input":"2024-05-21T14:35:29.92917Z","iopub.status.idle":"2024-05-21T14:35:29.993836Z","shell.execute_reply.started":"2024-05-21T14:35:29.929143Z","shell.execute_reply":"2024-05-21T14:35:29.993041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"make_dot(out, params=dict(list(my_model.named_parameters()))).render(\"model\", format=\"png\")","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:35:31.466908Z","iopub.execute_input":"2024-05-21T14:35:31.467513Z","iopub.status.idle":"2024-05-21T14:35:31.691781Z","shell.execute_reply.started":"2024-05-21T14:35:31.46748Z","shell.execute_reply":"2024-05-21T14:35:31.690936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize= (100, 20))\nplt.axis('off')\nplt.imshow(Image.open('/kaggle/working/model.png'))\n","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:35:33.742988Z","iopub.execute_input":"2024-05-21T14:35:33.743394Z","iopub.status.idle":"2024-05-21T14:35:34.394763Z","shell.execute_reply.started":"2024-05-21T14:35:33.743361Z","shell.execute_reply":"2024-05-21T14:35:34.393839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **my_model + SGD + sheduler + img_augment**","metadata":{}},{"cell_type":"code","source":"import math\ndef my_cyclical_lr(stepsize, min_lr=3e-4, max_lr=3e-3):\n\n    # Scaler: we can adapt this if we do not want the triangular CLR\n    scaler = lambda x: 1.\n\n    # Lambda function to calculate the LR\n    lr_lambda = lambda it: min_lr + (max_lr - min_lr) * relative(it, stepsize)\n\n    # Additional function to see where on the cycle we are\n    def relative(it, stepsize):\n        cycle = math.floor(1 + it / (2 * stepsize))\n        x = abs(it / stepsize - 2 * cycle + 1)\n        return max(0, (1 - x)) * scaler(cycle)\n\n    return lr_lambda","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:35:39.337118Z","iopub.execute_input":"2024-05-21T14:35:39.337818Z","iopub.status.idle":"2024-05-21T14:35:39.343861Z","shell.execute_reply.started":"2024-05-21T14:35:39.337788Z","shell.execute_reply":"2024-05-21T14:35:39.342896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_criterion = nn.CrossEntropyLoss()\n# optimizer = torch.optim.Adam(model.parameters(), lr=0.1)\nmy_optimizer = torch.optim.SGD(my_model.parameters(), lr=1., momentum=0.9)\nmy_step_size = 4*len(train_loader)\nmy_clr = my_cyclical_lr(my_step_size, min_lr=3e-4, max_lr=3e-3)\nmy_scheduler = torch.optim.lr_scheduler.LambdaLR(my_optimizer, [my_clr])","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:35:43.699296Z","iopub.execute_input":"2024-05-21T14:35:43.70002Z","iopub.status.idle":"2024-05-21T14:35:43.706366Z","shell.execute_reply.started":"2024-05-21T14:35:43.69999Z","shell.execute_reply":"2024-05-21T14:35:43.70518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_m_train = []\nloss_m_val = []\netaps = []","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:55:37.89526Z","iopub.execute_input":"2024-05-21T14:55:37.896291Z","iopub.status.idle":"2024-05-21T14:55:37.900899Z","shell.execute_reply.started":"2024-05-21T14:55:37.896245Z","shell.execute_reply":"2024-05-21T14:55:37.899903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def my_train_model(model, dataloaders, criterion, optimizer, num_epochs=5, scheduler=my_scheduler):\n    # set starting time\n    k = 0\n    start_time = time.time()\n    \n    val_acc_history = []\n    \n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n    \n    for epoch in range(num_epochs):\n        print(f'Epoch {epoch}/{num_epochs-1}')\n        print('-'*15)\n        \n        # each epoch have training and validation phase\n        for phase in ['train', 'val']:\n            # set mode for model\n            if phase == 'train':\n                model.train() # set model to training mode\n            else:\n                model.eval() # set model to evaluate mode\n                \n            running_loss = 0.0\n            running_corrects = 0\n            fin_out = []\n            \n            # iterate over data\n            for inputs, labels in dataloaders[phase]:\n                # move data to corresponding hardware\n                inputs = inputs.to(DEVICE)\n                labels = labels.to(DEVICE)\n                \n                # reset (or) zero the parameter gradients\n                optimizer.zero_grad()\n                \n                # training (or) validation process\n                with torch.set_grad_enabled(phase=='train'):\n                    outputs = model(inputs)\n                    loss = criterion(outputs, labels)\n                    \n                    _, preds = torch.max(outputs, 1)\n                    \n                    # back propagation in the network\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n                        scheduler.step()\n                        \n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n                if phase == 'train':\n                    loss_m_train.append(running_loss)\n                if phase == 'val':\n                    loss_m_val.appendrunning_loss()\n                \n            # calculate loss and accuarcy for the epoch\n            epoch_loss = running_loss / len(dataloaders[phase].dataset)\n            epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n            \n            # print loss and acc for training & validation\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n            \n            # update the best weights\n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n            if phase == 'val':\n                val_acc_history.append(epoch_acc)\n                \n        print()\n    end_time = time.time() - start_time\n    \n    print('Training completes in {:.0f}m {:.0f}s'.format(end_time // 60, end_time % 60))\n    print('Best Val Acc: {:.4f}'.format(best_acc))\n    \n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model, val_acc_history","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:58:27.877606Z","iopub.execute_input":"2024-05-21T14:58:27.87843Z","iopub.status.idle":"2024-05-21T14:58:27.893009Z","shell.execute_reply.started":"2024-05-21T14:58:27.878398Z","shell.execute_reply":"2024-05-21T14:58:27.892061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train the model\nmy_model, my_accuracy = my_train_model(model=my_model, dataloaders=loaders, criterion=my_criterion, optimizer=my_optimizer, num_epochs=5, scheduler=my_scheduler)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:37:01.581147Z","iopub.execute_input":"2024-05-21T14:37:01.581762Z","iopub.status.idle":"2024-05-21T14:45:37.909972Z","shell.execute_reply.started":"2024-05-21T14:37:01.581729Z","shell.execute_reply":"2024-05-21T14:45:37.908649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def my_predict(model, dataloader, device):\n    # set mode to eval\n    model.eval()\n    fin_out = []\n    \n    with torch.no_grad():\n        for images, targets in dataloader:\n            images = images.to(device)\n            targets = targets.to(device)\n            \n            outputs = model(images)\n            \n            fin_out.append(F.softmax(outputs, dim=1).detach().cpu().numpy())\n            \n    return np.concatenate(fin_out)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:46:24.278365Z","iopub.execute_input":"2024-05-21T14:46:24.27875Z","iopub.status.idle":"2024-05-21T14:46:24.285466Z","shell.execute_reply.started":"2024-05-21T14:46:24.278716Z","shell.execute_reply":"2024-05-21T14:46:24.284464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# steps for model prediction\ndevice = torch.device('cuda') # if you don't have gpu, set it as cpu\nmy_model.to(device)\npred = my_predict(my_model, val_loader, device)\npred = pred.argmax(axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:46:59.647777Z","iopub.execute_input":"2024-05-21T14:46:59.648144Z","iopub.status.idle":"2024-05-21T14:47:25.283727Z","shell.execute_reply.started":"2024-05-21T14:46:59.648117Z","shell.execute_reply":"2024-05-21T14:47:25.282585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_test.values, pred))","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:47:57.98599Z","iopub.execute_input":"2024-05-21T14:47:57.986328Z","iopub.status.idle":"2024-05-21T14:47:58.005782Z","shell.execute_reply.started":"2024-05-21T14:47:57.986305Z","shell.execute_reply":"2024-05-21T14:47:58.004798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_model_weights = [] # we will save the conv layer weights in this list\nmy_conv_layers = [] # we will save the 49 conv layers in this list\n# get all the model children as list\nmy_model_children = list(my_model.children())","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:28:12.921141Z","iopub.execute_input":"2024-05-21T17:28:12.921941Z","iopub.status.idle":"2024-05-21T17:28:12.926664Z","shell.execute_reply.started":"2024-05-21T17:28:12.921908Z","shell.execute_reply":"2024-05-21T17:28:12.925643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# counter to keep count of the conv layers\ncounter = 0 \n# append all the conv layers and their respective weights to the list\nfor i in range(len(my_model_children)):\n    if type(my_model_children[i]) == nn.Conv2d:\n        counter += 1\n        my_model_weights.append(my_model_children[i].weight)\n        my_conv_layers.append(my_model_children[i])\n    elif type(my_model_children[i]) == nn.Sequential:\n        for j in range(len(my_model_children[i])):\n            for child in my_model_children[i][j].children():\n                if type(child) == nn.Conv2d:\n                    counter += 1\n                    my_model_weights.append(child.weight)\n                    my_conv_layers.append(child)\nprint(f\"Total convolutional layers: {counter}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:29:27.233831Z","iopub.execute_input":"2024-05-21T17:29:27.234735Z","iopub.status.idle":"2024-05-21T17:29:27.242053Z","shell.execute_reply.started":"2024-05-21T17:29:27.234703Z","shell.execute_reply":"2024-05-21T17:29:27.241104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 17))\nfor i, filter in enumerate(my_model_weights[0]):\n    plt.subplot(8, 8, i+1) # (8, 8) because in conv0 we have 7x7 filters and total of 64 (see printed shapes)\n    plt.imshow(filter[0, :, :].cpu().detach(), cmap='gray')\n    plt.axis('off')\n    plt.savefig('/kaggle/working/my_filter.png')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:30:06.197683Z","iopub.execute_input":"2024-05-21T17:30:06.198941Z","iopub.status.idle":"2024-05-21T17:30:07.582192Z","shell.execute_reply.started":"2024-05-21T17:30:06.198899Z","shell.execute_reply":"2024-05-21T17:30:07.580938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **my_model + SGD + img_aug** (2)","metadata":{}},{"cell_type":"code","source":"# create dataloaders for training antrain_test_splitidation\ntrain_dataset2 = CassavaDataset(\n    image_ids=x_train.values,\n    labels=y_train.values,\n    augmentations=get_test_transforms('train'),\n    dimension=DIM\n)\n\ntrain_loader2 = DataLoader(\n    train_dataset,\n    batch_size=TRAIN_BATCH_SIZE,\n    num_workers=NUM_WORKERS,\n    shuffle=False,\n    sampler=train_sampler\n)\n\nval_dataset2 = CassavaDataset(\n    image_ids=x_test.values,\n    labels=y_test.values,\n    augmentations=get_test_transforms('val'),\n    dimension=DIM\n)\n\nval_loader2 = DataLoader(\n    val_dataset,\n    batch_size=TRAIN_BATCH_SIZE,\n    num_workers=NUM_WORKERS,\n    shuffle=False\n)\n\nloaders2 = {'train': train_loader, 'val': val_loader}","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:13:55.80636Z","iopub.execute_input":"2024-05-21T15:13:55.807107Z","iopub.status.idle":"2024-05-21T15:13:55.814181Z","shell.execute_reply.started":"2024-05-21T15:13:55.807074Z","shell.execute_reply":"2024-05-21T15:13:55.813211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_m_train = []\nloss_m_val = []\netaps = []","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:16:00.658832Z","iopub.execute_input":"2024-05-21T15:16:00.659445Z","iopub.status.idle":"2024-05-21T15:16:00.663693Z","shell.execute_reply.started":"2024-05-21T15:16:00.659416Z","shell.execute_reply":"2024-05-21T15:16:00.662752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def my_train_model_without_sheduler(model, dataloaders, criterion, optimizer, num_epochs=5):\n    # set starting time\n    k = 0\n    start_time = time.time()\n    \n    val_acc_history = []\n    \n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n    \n    for epoch in range(num_epochs):\n        print(f'Epoch {epoch}/{num_epochs-1}')\n        print('-'*15)\n        \n        # each epoch have training and validation phase\n        for phase in ['train', 'val']:\n            # set mode for model\n            if phase == 'train':\n                model.train() # set model to training mode\n            else:\n                model.eval() # set model to evaluate mode\n                \n            running_loss = 0.0\n            running_corrects = 0\n            fin_out = []\n            \n            # iterate over data\n            for inputs, labels in dataloaders[phase]:\n                # move data to corresponding hardware\n                inputs = inputs.to(DEVICE)\n                labels = labels.to(DEVICE)\n                \n                # reset (or) zero the parameter gradients\n                optimizer.zero_grad()\n                \n                # training (or) validation process\n                with torch.set_grad_enabled(phase=='train'):\n                    outputs = model(inputs)\n                    loss = criterion(outputs, labels)\n                    \n                    _, preds = torch.max(outputs, 1)\n                    \n                    # back propagation in the network\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n                        \n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n                if phase == 'train':\n                    loss_m_train.append(running_loss)\n                if phase == 'val':\n                    loss_m_val.append(running_loss)\n                \n            # calculate loss and accuarcy for the epoch\n            epoch_loss = running_loss / len(dataloaders[phase].dataset)\n            epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n            \n            # print loss and acc for training & validation\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n            \n            # update the best weights\n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n            if phase == 'val':\n                val_acc_history.append(epoch_acc)\n                \n        print()\n    end_time = time.time() - start_time\n    \n    print('Training completes in {:.0f}m {:.0f}s'.format(end_time // 60, end_time % 60))\n    print('Best Val Acc: {:.4f}'.format(best_acc))\n    \n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model, val_acc_history","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:16:30.064293Z","iopub.execute_input":"2024-05-21T15:16:30.064944Z","iopub.status.idle":"2024-05-21T15:16:30.078858Z","shell.execute_reply.started":"2024-05-21T15:16:30.064915Z","shell.execute_reply":"2024-05-21T15:16:30.077931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train the model\nmy_model2, my_accuracy2 = my_train_model_without_sheduler(model=my_model, dataloaders=loaders2, criterion=my_criterion, optimizer=my_optimizer, num_epochs=5)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:16:33.974987Z","iopub.execute_input":"2024-05-21T15:16:33.975348Z","iopub.status.idle":"2024-05-21T15:25:11.090949Z","shell.execute_reply.started":"2024-05-21T15:16:33.975319Z","shell.execute_reply":"2024-05-21T15:25:11.089827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_m_train_rem = [x for x in loss_m_train if 12000 < x]","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:40:28.108798Z","iopub.execute_input":"2024-05-21T15:40:28.109652Z","iopub.status.idle":"2024-05-21T15:40:28.114133Z","shell.execute_reply.started":"2024-05-21T15:40:28.109609Z","shell.execute_reply":"2024-05-21T15:40:28.113189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot([1.6110, 1.6106, 1.6107, 1.6106, 1.6106])","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:40:57.956941Z","iopub.execute_input":"2024-05-21T15:40:57.95778Z","iopub.status.idle":"2024-05-21T15:40:58.696075Z","shell.execute_reply.started":"2024-05-21T15:40:57.957744Z","shell.execute_reply":"2024-05-21T15:40:58.695127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **my_model + SGD** (3)","metadata":{}},{"cell_type":"code","source":"# create dataloaders for training antrain_test_splitidation\ntrain_dataset3 = CassavaDataset(\n    image_ids=x_train.values,\n    labels=y_train.values,\n#     augmentations=get_test_transforms_no_aug('train'),\n    dimension=DIM\n)\n\ntrain_loader3 = DataLoader(\n    train_dataset3,\n    batch_size=TRAIN_BATCH_SIZE,\n    num_workers=NUM_WORKERS,\n    shuffle=False,\n    sampler=train_sampler\n)\n\nval_dataset3 = CassavaDataset(\n    image_ids=x_test.values,\n    labels=y_test.values,\n#     augmentations=get_test_transforms_no_aug('val'),\n    dimension=DIM\n)\n\nval_loader3 = DataLoader(\n    val_dataset3,\n    batch_size=TRAIN_BATCH_SIZE,\n    num_workers=NUM_WORKERS,\n    shuffle=False\n)\n\nloaders3 = {'train': train_loader3, 'val': val_loader3}","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:52:20.803485Z","iopub.execute_input":"2024-05-21T16:52:20.803868Z","iopub.status.idle":"2024-05-21T16:52:20.811398Z","shell.execute_reply.started":"2024-05-21T16:52:20.803835Z","shell.execute_reply":"2024-05-21T16:52:20.810359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def my_train_model_only_sgd(model, dataloaders, criterion, optimizer, num_epochs=5):\n    # set starting time\n    k = 0\n    start_time = time.time()\n    \n    val_acc_history = []\n    \n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n    \n    for epoch in range(num_epochs):\n        print(f'Epoch {epoch}/{num_epochs-1}')\n        print('-'*15)\n        \n        # each epoch have training and validation phase\n        for phase in ['train', 'val']:\n            # set mode for model\n            if phase == 'train':\n                model.train() # set model to training mode\n            else:\n                model.eval() # set model to evaluate mode\n                \n            running_loss = 0.0\n            running_corrects = 0\n            fin_out = []\n            \n            # iterate over data\n            for inputs, labels in dataloaders[phase]:\n                # move data to corresponding hardware\n                inputs = inputs.to(DEVICE)\n#                 print((inputs.shape))\n                labels = labels.to(DEVICE)\n                \n                # reset (or) zero the parameter gradients\n                optimizer.zero_grad()\n                \n                # training (or) validation process\n                with torch.set_grad_enabled(phase=='train'):\n                    outputs = model(inputs)\n#                     print(type(outputs))\n                    loss = criterion(outputs, labels)\n                    \n                    _, preds = torch.max(outputs, 1)\n                    \n                    # back propagation in the network\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n                        \n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n                if phase == 'train':\n                    loss_m_train.append(running_loss)\n                if phase == 'val':\n                    loss_m_val.append(running_loss)\n                \n            # calculate loss and accuarcy for the epoch\n            epoch_loss = running_loss / len(dataloaders[phase].dataset)\n            epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n            \n            # print loss and acc for training & validation\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n            \n            # update the best weights\n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n            if phase == 'val':\n                val_acc_history.append(epoch_acc)\n                \n        print()\n    end_time = time.time() - start_time\n    \n    print('Training completes in {:.0f}m {:.0f}s'.format(end_time // 60, end_time % 60))\n    print('Best Val Acc: {:.4f}'.format(best_acc))\n    \n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model, val_acc_history","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:59:20.059771Z","iopub.execute_input":"2024-05-21T16:59:20.06016Z","iopub.status.idle":"2024-05-21T16:59:20.074154Z","shell.execute_reply.started":"2024-05-21T16:59:20.060127Z","shell.execute_reply":"2024-05-21T16:59:20.0732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class getMyModel2(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(in_channels=3, out_channels=6, kernel_size=(5, 5), padding=1, bias=True)\n        self.conv2 = nn.Conv2d(in_channels=6,out_channels=12,kernel_size=(3,3))\n        self.conv3 = nn.Conv2d(in_channels=12,out_channels=1,kernel_size=(1,1))\n#         self.pool1 = nn.MaxPool2d(2, 2)\n        self.pool2 = nn.AdaptiveAvgPool2d(output_size=(1, 1))\n        self.fc1 = nn.Linear(1, 64)\n        self.fc2 = nn.Linear(64, 16)\n        self.fc3 = nn.Linear(16, NUM_CLASSES)\n                \n    def forward(self, x):\n        x = x.float()\n        x = x.transpose(1, 3)\n        x = F.relu(self.conv1(x))\n#         print(\"SUS1\", x.shape, type(x))\n        x = F.max_pool2d(x,kernel_size=(3,3),stride=2) # max pooling\n#         print(\"SUS1_1\", x.shape, type(x))\n        x = F.relu(self.conv2(x))\n#         print(\"SUS2\", x.shape)\n        x = F.max_pool2d(x,kernel_size=(3,3),stride=2) # max pool\n#         print(\"SUS2_2\", x.shape)\n#         print(\"SUS1\", x.shape)\n#         x = F.relu(self.pool1(x))\n#         print(\"SUS2\", x.shape)\n        x = F.relu(self.conv3(x))\n#         print(\"SUS2_20\", x.shape)\n        x = self.pool2(x)\n#         print(\"SUS2_21\", x.shape)\n        x = self.fc1(x)\n#         print(\"SUS3\",x.shape)\n        x = self.fc2(x)\n#         print(\"SUS4\",x.shape)\n        x = self.fc3(torch.squeeze(x))\n#         print(\"SUS5\",x.shape)\n        x = F.log_softmax(x, dim=1)\n#         x = x.squeeze(1)\n#         print(\"SUS7\",x.shape)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:59:32.441479Z","iopub.execute_input":"2024-05-21T16:59:32.442306Z","iopub.status.idle":"2024-05-21T16:59:32.452669Z","shell.execute_reply.started":"2024-05-21T16:59:32.442271Z","shell.execute_reply":"2024-05-21T16:59:32.451833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_modelN = getMyModel2().to('cuda')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:59:34.644781Z","iopub.execute_input":"2024-05-21T16:59:34.645155Z","iopub.status.idle":"2024-05-21T16:59:34.65253Z","shell.execute_reply.started":"2024-05-21T16:59:34.645123Z","shell.execute_reply":"2024-05-21T16:59:34.651615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = torch.rand(32, 256, 256, 3)\nout = my_modelN.forward(x.cuda())","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:59:35.820685Z","iopub.execute_input":"2024-05-21T16:59:35.821363Z","iopub.status.idle":"2024-05-21T16:59:35.888661Z","shell.execute_reply.started":"2024-05-21T16:59:35.821331Z","shell.execute_reply":"2024-05-21T16:59:35.887772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_modelN = getMyModel2().to('cuda')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:59:38.278969Z","iopub.execute_input":"2024-05-21T16:59:38.279341Z","iopub.status.idle":"2024-05-21T16:59:38.285976Z","shell.execute_reply.started":"2024-05-21T16:59:38.279313Z","shell.execute_reply":"2024-05-21T16:59:38.285054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_modelN","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:59:39.738376Z","iopub.execute_input":"2024-05-21T16:59:39.739435Z","iopub.status.idle":"2024-05-21T16:59:39.745863Z","shell.execute_reply.started":"2024-05-21T16:59:39.739393Z","shell.execute_reply":"2024-05-21T16:59:39.744921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train the model\nmy_model3, my_accuracy3 = my_train_model_only_sgd(model=my_modelN, dataloaders=loaders3, criterion=my_criterion, optimizer=my_optimizer, num_epochs=5)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:59:43.66999Z","iopub.execute_input":"2024-05-21T16:59:43.670605Z","iopub.status.idle":"2024-05-21T17:06:41.373082Z","shell.execute_reply.started":"2024-05-21T16:59:43.670574Z","shell.execute_reply":"2024-05-21T17:06:41.371806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_model3","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:13:15.362437Z","iopub.execute_input":"2024-05-21T17:13:15.363143Z","iopub.status.idle":"2024-05-21T17:13:15.369258Z","shell.execute_reply.started":"2024-05-21T17:13:15.36311Z","shell.execute_reply":"2024-05-21T17:13:15.368354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(loss_m_train)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:07:41.140465Z","iopub.execute_input":"2024-05-21T17:07:41.14129Z","iopub.status.idle":"2024-05-21T17:07:41.402953Z","shell.execute_reply.started":"2024-05-21T17:07:41.141252Z","shell.execute_reply":"2024-05-21T17:07:41.402038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conv1_weights = my_model3.conv1.weight.data","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:16:06.160747Z","iopub.execute_input":"2024-05-21T17:16:06.161404Z","iopub.status.idle":"2024-05-21T17:16:06.165722Z","shell.execute_reply.started":"2024-05-21T17:16:06.161369Z","shell.execute_reply":"2024-05-21T17:16:06.164785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\n\n# Assuming conv1_weights is the tensor that needs to be converted to numpy\n# Use Tensor.cpu() to copy the tensor to host memory first\nconv1_weights_cpu = conv1_weights.cpu()\n\nnum_filters = 6\nfig, axs = plt.subplots(nrows=num_filters, ncols=1, figsize=(5, 5*num_filters))\nfor i in range(num_filters):\n    axs[i].imshow(conv1_weights_cpu[i].numpy().transpose(1, 2, 0))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:21:09.427611Z","iopub.execute_input":"2024-05-21T17:21:09.428226Z","iopub.status.idle":"2024-05-21T17:21:10.23683Z","shell.execute_reply.started":"2024-05-21T17:21:09.428192Z","shell.execute_reply":"2024-05-21T17:21:10.235903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **RESNET**","metadata":{}},{"cell_type":"code","source":"def getModel():\n    net = models.resnet152(pretrained=True)\n    \n    # if you want to train the whole network, comment this code\n    # freeze all the layers in the network\n    for param in net.parameters():\n        param.requires_grad = False\n        \n    num_ftrs = net.fc.in_features\n    # create last few layers\n    net.fc = nn.Sequential(\n        nn.Linear(num_ftrs, 256),\n        nn.ReLU(),\n        nn.Dropout(0.3),\n        nn.Linear(256, NUM_CLASSES),\n        nn.LogSoftmax(dim=1)\n    )\n\n    # use gpu if any\n    net = net.cuda() if DEVICE else net\n    return net","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:44:09.518051Z","iopub.execute_input":"2024-05-21T15:44:09.518724Z","iopub.status.idle":"2024-05-21T15:44:09.525561Z","shell.execute_reply.started":"2024-05-21T15:44:09.518689Z","shell.execute_reply":"2024-05-21T15:44:09.524513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = getModel()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:44:12.047003Z","iopub.execute_input":"2024-05-21T15:44:12.047426Z","iopub.status.idle":"2024-05-21T15:44:15.141011Z","shell.execute_reply.started":"2024-05-21T15:44:12.047394Z","shell.execute_reply":"2024-05-21T15:44:15.140176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:44:16.201824Z","iopub.execute_input":"2024-05-21T15:44:16.2022Z","iopub.status.idle":"2024-05-21T15:44:16.215156Z","shell.execute_reply.started":"2024-05-21T15:44:16.202172Z","shell.execute_reply":"2024-05-21T15:44:16.214208Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\ndef cyclical_lr(stepsize, min_lr=3e-4, max_lr=3e-3):\n\n    # Scaler: we can adapt this if we do not want the triangular CLR\n    scaler = lambda x: 1.\n\n    # Lambda function to calculate the LR\n    lr_lambda = lambda it: min_lr + (max_lr - min_lr) * relative(it, stepsize)\n\n    # Additional function to see where on the cycle we are\n    def relative(it, stepsize):\n        cycle = math.floor(1 + it / (2 * stepsize))\n        x = abs(it / stepsize - 2 * cycle + 1)\n        return max(0, (1 - x)) * scaler(cycle)\n\n    return lr_lambda","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:44:21.746019Z","iopub.execute_input":"2024-05-21T15:44:21.74676Z","iopub.status.idle":"2024-05-21T15:44:21.753071Z","shell.execute_reply.started":"2024-05-21T15:44:21.746731Z","shell.execute_reply":"2024-05-21T15:44:21.751945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\n# optimizer = torch.optim.Adam(model.parameters(), lr=0.1)\noptimizer = torch.optim.SGD(model.parameters(), lr=1., momentum=0.9)\nstep_size = 4*len(train_loader)\nclr = cyclical_lr(step_size, min_lr=3e-4, max_lr=3e-3)\nscheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, [clr])","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:44:23.341908Z","iopub.execute_input":"2024-05-21T15:44:23.342272Z","iopub.status.idle":"2024-05-21T15:44:23.351136Z","shell.execute_reply.started":"2024-05-21T15:44:23.342245Z","shell.execute_reply":"2024-05-21T15:44:23.350201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# freeze (or) unfreeze all the layers\nunfreeze = True # to freeze, set it as False\nfor param in model.parameters():\n    param.requires_grad = unfreeze","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:44:29.178731Z","iopub.execute_input":"2024-05-21T15:44:29.17935Z","iopub.status.idle":"2024-05-21T15:44:29.187825Z","shell.execute_reply.started":"2024-05-21T15:44:29.179316Z","shell.execute_reply":"2024-05-21T15:44:29.186922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find total parameters and trainable parameters\ntotal_params = sum(p.numel() for p in model.parameters())\nprint(f'{total_params:,} total parameters')\ntrainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\nprint(f'{trainable_params:,} training parameters')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:44:30.816672Z","iopub.execute_input":"2024-05-21T15:44:30.817045Z","iopub.status.idle":"2024-05-21T15:44:30.828032Z","shell.execute_reply.started":"2024-05-21T15:44:30.817015Z","shell.execute_reply":"2024-05-21T15:44:30.826933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_train = []\nloss_val = []\netaps = []","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:44:53.552566Z","iopub.execute_input":"2024-05-21T15:44:53.553461Z","iopub.status.idle":"2024-05-21T15:44:53.557724Z","shell.execute_reply.started":"2024-05-21T15:44:53.553428Z","shell.execute_reply":"2024-05-21T15:44:53.556711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model, dataloaders, criterion, optimizer, num_epochs=5, scheduler=scheduler):\n    # set starting time\n    start_time = time.time()\n    \n    val_acc_history = []\n    \n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n    \n    for epoch in range(num_epochs):\n        print(f'Epoch {epoch}/{num_epochs-1}')\n        print('-'*15)\n        \n        # each epoch have training and validation phase\n        for phase in ['train', 'val']:\n            # set mode for model\n            if phase == 'train':\n                model.train() # set model to training mode\n            else:\n                model.eval() # set model to evaluate mode\n                \n            running_loss = 0.0\n            running_corrects = 0\n            fin_out = []\n            \n            # iterate over data\n            for inputs, labels in dataloaders[phase]:\n                # move data to corresponding hardware\n                inputs = inputs.to(DEVICE)\n                labels = labels.to(DEVICE)\n                \n                # reset (or) zero the parameter gradients\n                optimizer.zero_grad()\n                \n                # training (or) validation process\n                with torch.set_grad_enabled(phase=='train'):\n                    outputs = model(inputs)\n                    loss = criterion(outputs, labels)\n                    \n                    _, preds = torch.max(outputs, 1)\n                    \n                    # back propagation in the network\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n                        scheduler.step()\n                        \n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n                if phase == 'train':\n                    loss_train.append(running_loss)\n                if phase == 'val':\n                    loss_val.append(running_loss)\n                \n            # calculate loss and accuarcy for the epoch\n            epoch_loss = running_loss / len(dataloaders[phase].dataset)\n            epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n            \n            # print loss and acc for training & validation\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n            \n            # update the best weights\n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n            if phase == 'val':\n                val_acc_history.append(epoch_acc)\n                \n        print()\n    end_time = time.time() - start_time\n    \n    print('Training completes in {:.0f}m {:.0f}s'.format(end_time // 60, end_time % 60))\n    print('Best Val Acc: {:.4f}'.format(best_acc))\n    \n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model, val_acc_history","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:46:02.030746Z","iopub.execute_input":"2024-05-21T15:46:02.031765Z","iopub.status.idle":"2024-05-21T15:46:02.04583Z","shell.execute_reply.started":"2024-05-21T15:46:02.03173Z","shell.execute_reply":"2024-05-21T15:46:02.044843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train the model\nmodel, accuracy = train_model(model=model, dataloaders=loaders, criterion=criterion, optimizer=optimizer, num_epochs=5, scheduler=scheduler)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:46:07.337133Z","iopub.execute_input":"2024-05-21T15:46:07.338097Z","iopub.status.idle":"2024-05-21T16:08:47.399371Z","shell.execute_reply.started":"2024-05-21T15:46:07.338061Z","shell.execute_reply":"2024-05-21T16:08:47.398067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![image.png](attachment:123dfe29-0960-4273-a553-1a37a1bdf286.png)","metadata":{},"attachments":{"123dfe29-0960-4273-a553-1a37a1bdf286.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAABS0AAAIKCAYAAADGe2p/AAAgAElEQVR4Aezd768k1Z0naP6Kfn9X6sl39Yp+Za1EadcSkkeFNSv3i4E3rpH6mlVL2CtRs7sqRmXqRXk8agOWTDYSIKurWpZB7bbpNtfuXcAjXG4L6L1usx5+yBiwKTeGYi5l0Ozu9NhndSLiRJyIjIiMvDfvrbxVT0mlm5kRceLEc74ZmfdzT2TeEvwjQIAAAQIECBAgQIAAAQIECBAgQIDABgncskF90RUCBAgQIECAAAECBAgQIECAAAECBAgEoaUiIECAAAECBAgQIECAAAECBAgQIEBgowSElhs1HDpDgAABAgQIECBAgAABAgQIECBAgIDQUg0QIECAAAECBAgQIECAAAECBAgQILBRAkLLjRoOnSFAgAABAgQIECBAgAABAgQIECBAQGipBggQIECAAAECBAgQIECAAAECBAgQ2CgBoeVGDYfOECBAgAABAgQIECBAgAABAgQIECAgtFQDBAgQIECAAAECBAgQIECAAAECBAhslIDQcqOGQ2cIECBAgAABAgQIECBAgAABAgQIEBBaqgECBAgQIECAAAECBAgQIECAAAECBDZKQGi5UcOhMwQIECBAgAABAgQIECBAgAABAgQICC3VAAECBAgQIECAAAECBAgQIECAAAECGyUgtNyo4dAZAgQIECBAgAABAgQIECBAgAABAgSElmqAAAECBAgQIECAAAECBAgQIECAAIGNEhBabtRw6AwBAgQIECBAgAABAgQIECBAgAABAkJLNUCAAAECBAgQIECAAAECBAgQIECAwEYJCC03ajh0hgABAgQIECBAgAABAgQIECBAgAABoaUaIECAAAECBAgQIECAAAECBAgQIEBgowSElhs1HDpDgAABAgQIECBAgAABAgQIECBAgIDQckkN/O53vwsf/OcPw6+uvBveePNX4fWfv+U/AzWgBtSAGlADakANqAE1oAbUgBpQA2pADagBNdBTAzE/izna1Q/2wu9+//slydvwYqHlsE249tuPwhtv/jJ88J/3wofXfhuuffSx/wzUgBpQA2pADagBNaAG1IAaUANqQA2oATWgBtTASA3EHO2Dvb3w81/8Mlz77ccj6dvwIqHlgM2HH/42/NNv3leAIwUoxBViqwE1oAbUgBpQA2pADagBNaAG1IAaUANqQA2M1UDM1z689tFAAjf8sNCyxybOsPyn37wnsBRYqgE1oAbUgBpQA2pADagBNaAG1IAaUANqQA2ogQPWQMzZfrvijEuhZSe0jJ9h+fM3f6UYD1iMYwm7Zf4CowbUgBpQA2pADagBNaAG1IAaUANqQA2ogZurBn7+5i/D71f4jEuhZSe0jB8S+sHeh0JLoaUaUANqQA2oATWgBtSAGlADakANqAE1oAbUgBpYUw3EvC1+2fXUf0LLjlT8diNfunNzJf3+smO81YAaUANqQA2oATWgBtSAGlADakANqAE1cLg1EPO2mLtN/Se07EjFr2X/8LcfSdHXlKJ7wh/uE54vXzWgBtSAGlADakANqAE1oAbUgBpQA2rgONRAzNti7jb1n9CyI/X6z98SWAos1YAaUANqQA2oATWgBtSAGlADakANqAE1oAbUwJprIOZuU/8JLTtSQkt/nTgOf53QR3WqBtSAGlADakANqAE1oAbUgBpQA2pADRy3Gtjo0PIf//Efw7/7d/+u+P+DH/ygExnu725sJ7UZ2z/IP6GlJ/xxe8Lrr5pVA2pADagBNaAG1IAaUANqQA2oATWgBo5DDWx8aDkWVl69ejU8/PDDobvOP//zP4e/+Zu/qcPJX/7ylwvZZNxGaOlJehyepPqoTtWAGlADakANqAE1oAbUgBpQA2pADaiBm60Gjm1o+V/+y38JTz75ZPi7v/u7VmgZH/+Lv/iLpYGk0NKT/WZ7sjteNa8G1IAaUANqQA2oATWgBtSAGlADakANHJcaOLahZQod42zJfKZlnFWZ31+YYlk9kLbvLo/txdmbcRbnsn8uD/dEPy5PdP1Uq2pADagBNaAG1IAaUANqQA2oATWgBtTAcaqBYxlaxmAyXv4dLwPvhpYxjPzRj35UzLaMn105FEAKLT1Rj9MTVV/VqxpQA2pADagBNaAG1IAaUANqQA2oATVwM9XAsQst4+XfMbCMP+O/vtAyXh6elucBZz5zcii0zNdZdttMSyeLm+lk4VjVuxpQA2pADagBNaAG1IAaUANqQA2oATVwVDVw7ELLGFKmb//Of6YZld0wMl7mHT/7MoWYKYjsrpceX+Wn0NIT9aieqPaj1tSAGlADakANqAE1oAbUgBpQA2pADaiBm6kGjl1o2Q0VuzMtuzMru8vT9kOhZVw/BaBp3aGfQksni5vpZOFY1bsaUANqQA2oATWgBtSAGlADakANqAE1cFQ1cMOFljFgjIFkmoWZPvuyGzwKLT3JjupJZj9qTQ2oATWgBtSAGlADakANqAE1oAbUgBpQA6vVwLEPLbth5NT7Q6Hl1O3jemZarlZsnpy81IAaUANqQA2oATWgBtSAGlADakANqAE1oAam1IDQcpWUsrOu0NKTbMqTzDrqRA2oATWgBtSAGlADakANqAE1oAbUgBpQA6vVwMaHluky7zgzch3/8kvH4+dXHuSf0HK1YvPk5KUG1IAaUANqQA2oATWgBtSAGlADakANqAE1MKUGNjq0PEigeBTbHllo+f5/DGcvXAovfPRxuPafngyffeQ/hivx9gb+3/3GhXDbQ0+Ht65D35579Fz44ydf2UiX8bHaDX92/ly48PfTx/Stp74Sbjt3rvp/PvzZi9O3He+LdvioATWgBtSAGlADakANqAE1oAbUgBpQA9e/BoSWB0g3jyq0fOu7Xw13FmHc1fD9Ry7sI6B6JTx+4Vw4+4OrxzDQaz9J3v/BI+G2C0+Gny2EoqsHfxtzAvpPT4Y/PvdI+P777WOd1L+DbLtguI/9a+PYP6cm1ZlxNs5qQA2oATWgBtSAGlADakANqAE1cMQ1ILTc+NDylXDxy4+E77/7cbj29tPhT7/cF9gtCZvefir8ybmvhG++vWS9Iy6+/YQlP/z6ufDJr7+0eKJ47a/DZ/cb/F3n477y3a+G277y1L5mpxbbPvT0xs683c8Y2+b4P0+NoTFUA2pADagBNaAG1IAaUANqQA2ogYPWgNByU0PLIoRLl/92fp6vLhVfFrb9/WPhk/UlxFkbj/zH8H627c+evBBue+Tp8MK3Hgl3ni/X+5On3q6CwQ/D6z/8RvjCl8vHP3n+y+Hsd1/PQsM4+zNruyc4LIK1R54Ju888Fv6kav/Or/94haDtF+GbX8n3kW5fCI//p/IkUAZ/T4bnRvdxJex+t+nDp77yjfBCDIMzi7HbP3z0XPj01x4JXyiO4Wvh4rceC3eeOxc+eeHr4Yf1LMno9fXwp9VxfvLLj4S/fa29jysvPll7fuqhb4Q/e6h7WfvUfpb2x/OS+LbJmLtlrNSAGlADakANqAE1oAbUgBpQA2pADdx8NSC03NTQsgjSYij15XCxCOZeCn92YX+XEL9w8Xz45KM/HgjnroS/fehc+OS5r4QLP3w9XHn3anj/3Svh/b3yyfD6U18Jn/zy18Nzr/0mvP/ub8Luk/GzFL8eftgT9A3N+oufNfmpC18Nf/bD18P7718Nb/3gsfCpOPPzjRWfcO88Hb4wsF3cx23nvlLs48q7v+nZx+vhb792Ifzxo8+En737m/D+27vhq185Fz51cXfApdu38hL7P/nWK+H9d54J//bcufCFp14PV95+OvzbrE+vfyv3ejs89+iFcFs2O/bK3z8WPn3uq+Hi7i9Kz7j+uXPhQv2ZlKv0s7okvt6222f3vaipATWgBtSAGlADakANqAE1oAbUgBpQA8ezBoSWmxxaxsvBqy+1iZ/l+OmLPZdF94SH7Sfjks+zjF/yc+58uP+Hv+kN72KAeaWeRfhxuLZ7KXyqN7QcmvXX81mTb/z1vi5XX+nzLDv7KMLXzhcYxYDxtsEwt/OELtqrQuPi9mPhuWhffKbkY2WIu3spfDrONM1nb6bPnIwh8Ps/DhfOd6w7l7Wv1M/Udj4+S+uhc1zW76379nOIGQ81oAbUgBpQA2pADagBNaAG1IAaUANHXQNCyw0OLX/25JerL895O3zroTTjcsUnSZqdOPR5loMhZLmfK689E776tS+HT+eXmXcuLy+LdmDWXyeUK9b94WMDX6YzfmzFjNGpn2cZL42vL6Mv+9Z823a6vHz6TMsiMK0+dzKfUVpe+h4vtx8IbQvfMtQs2shmXUaL8rL29HmWq/Wzve243VGfWOzPeKgBNaAG1IAaUANqQA2oATWgBtSAGlADB6kBoeXGhZZl+NUXsMXHVv0G8OHZieUTp/w8y/ZnXKaCev/Fr4dPn/tyuPDMy+H1d66EK++8Ei5+5Vz4k2/9YnF22sCsvzJY++vwejajb/xy9aEn9PCM0aX7KILTr4ZvvhaPofN/4izF+HmW6bMj4+3Pfit+rmc5VqVHz4zSjz4OMXi+rfqinMUvEeoEnSv1s7Nt5pvGz8+hWvK42lADakANqAE1oAbUgBpQA2pADagBNbDpNSC03LjQ8uPiMyVf/+7Xwqcv/jhceeft8P1Hz4ezz/yiCNzSZ01OLawYEA5fAl1+nmXzpTv5E7Zc9qffTV/IE5f9MNx/rvnym7wPQ7P+4mdNprCvXP/1cPHLq4ev19KM0Z7PwVzcR/nFPV/47pUyXH3x68VndrY/Q/PDcK363M78OPpvl4Fp+bmTMZw8H/5sN3rE28njx+FC/GzKv88M3/1huP/8+XDh768W/WjCzrTOS+HPzmefZ7lSP6uQ1OdZLgboAlwmakANqAE1oAbUgBpQA2pADagBNaAGjn0NCC03MLS89lEM3b4avhUv6Y6fOXlh4reF9zwhY1B229eeDm+9+5tw5Z1fhLfyz1ssPs8yhW4pSEs/y6Duzos/Lrd97YfhQvEN4tVnObb2NTTrr2f24Uj42B8YVv0pPkfyQvjqi/ELga6EK29fqb4BfcI+iuM8Fz79SPUlPG+8HL716IVw58Xd1reoD+5/yudZfhQv4T8XPpX28dqPiy/6+fSjzbekFwHyhcfCc2/HY3g9/O0jF8Jt+betr9LPgZmtg8fQGq80xn7yUgNqQA2oATWgBtSAGlADakANqAE1oAY2swaElpsYWr54KXy6+tzI+GUx7dmOqxXS+y9eCndWn0f5qQuPhe+/k23/4tCX6pTrxG/5Ttve+bVvhG9+42vhjx/66/CzKgArvsgm/6zLdDt95uUaP88yBrnfeuh88U3bt507Hz775MvlXwz69tH6PMvyWK68+I3wp+fLz7L85Pmvhvu/+3K4MjHIG/o8y7e++9VwWzrW2NZrT4ezRbB7Lnzy/FcW9/HuS0WQGS/z/9SXHwkXv/G1cFv1OZnpBDnez3I25+JHBwwFz9lYTzzW1A8/2akBNaAG1IAaUANqQA2oATWgBtSAGlAD17MGhJabGFoKmI79FObr+aS2by8qakANqAE1oAbUgBpQA2pADagBNaAG1MBxrwGhpdBSQCgkVgNqQA2oATWgBtSAGlADakANqAE1oAbUgBrYqBoQWgotN6ogj/tfAfTfX7LUgBpQA2pADagBNaAG1IAaUANqQA2oATVw8BoQWgothZb+kqIG1IAaUANqQA2oATWgBtSAGlADakANqAE1sFE1ILQUWm5UQfpLxMH/EsGQoRpQA2pADagBNaAG1IAaUANqQA2oATVw3GtAaCm0FFr6S4oaUANqQA2oATWgBtSAGlADakANqAE1oAbUwEbVgNBSaLlRBXnc/wqg//6SpQbUgBpQA2pADagBNaAG1IAaUANqQA2ogYPXgNBSaCm09JcUNaAG1IAaUANqQA2oATWgBtSAGlADakANqIGNqgGhpdByowrSXyIO/pcIhgzVgBpQA2pADagBNaAG1IAaUANqQA2ogeNeA0LLmyW0fPNSuHPrvvD81L8aFOtvha2t6v/Zy8LNqXbWq2rlg/DM2VmYnX0uvHdMTN64dFfYWqXWL9/XPEe2tsKdl94af55cfjD86//t2fCbQ/e4HM5u3RUuvjn1RfqtcPGu7Pm+yrnikI/l+bMTXLM+xPXr89ZKBlOt+tdbtZ/rePNwMx3rUq9Vn4tZzSxte2Tdlc8ZI20dpB/Xb9tNOc+/Fh49tRVOPfba+Dn4GPi/9+x9YTa7Lzxztf9cczRjfeN4HtxrHTXOsxkHno3F9XyOb8q+y/efS98/r+Hc/dNv//fhli/d0v7/F89u1GtG8Z7irkvhjTUcb1NnN87vBM0xTavfm83z+S/dGS5cvrpRNb3qmA2tL7QUWi4t7OIX41WCnLWeaKedlIYK3ON9fkf3BuG4+e8/gJhg+qudcPedD07/w8GBnkervkHJ6qQIf1b4A8eB+pntd6CdfYeBxR9eVglul/dlrJ73188JdTPg0urLsTjWg/m2jnfUZE2mo/tojmX/54ymjenHdgy3OeLavKEtY03yXPqedaUa4Mlz4rl+pbrS5gp1dXSv2f/nX9wSPvU3b6zQt6N/zd2MkC077g37nWDV5+HN5/mj8MD/9G/CX/08G8Mb5HwktBRaLj15Cy1vtCf+0b1BWPXF5Xqvv/8AYrnp81/8o/D5p4/qr19Cy6P+5V5ouSnnyeXPxXWeZ/Z/ztgUr0Puh1Bo6XusleqRJ89N/gVUfa63Pjd5rNfWt6N6zX4n/NWf3xI++3fvbPQY3Xwh2+G+B7kpPS//+3DrF3aO4Kq+wx277nsjoeWmhZbxLxq33x3u+cyJsDW7Ndwzfzzcc/ssbJ24Izxw+YPmRPvmc+GBu24Ns62tMLv1jnDvE91Lol4L3z5zRzhRLL8rPHDpfPhM95LPbhvf6bZRFuOhhpZXXw4Xq35ubZ0Ip85cCj/pXhL16pPh3lPpWO8KZ3e6/fx1eH5+Opw8ES8LPRFO3f14eP5KeiL1vRj2BDrVX5Ke/umlcO+pE+XlpaefbKbnH7Cf7zxxOmydejy80nqRP/rLhYqxTJf8Zz/PXk5e8Wf02Qpnd/KxOR2+WV9q/FZ45sG7wq2z6D0Lt971UHimXpZmgmSX6C7M0q38X7wcHq5q+MSpM+Hbr+Z9GL9dvgh9vnluPDYPnzkRnwt3Z/38IPzk0plwqqqLk6fnWV1U7b/6VDXe5XFcPP+ZhcvD33j2oXDnrbPyWAf72Vdn2TG8vxM+/4dfXJxlubbn+wfhhXk1JifuCPdemod7updGT3y+X9vXX1Wr47/n8+H22VY4ceqh8Oj8M+FErI+7s+fRR8vH5L0X55V3eT54+J7u5eGLbbzQPWfE59kh//K0tJ9LzhlTnovv5eejYlxf7v+ohZFjzds4cfJ0ePjyr5vXkei0pJ/xTcNKx3riZDg9f6nTz/T8uJydO06EB7LzTtnP8jwfX+/uvTRwrK1zaPYcqx9P++r5qIbuc2DhdXPJuS3uY+k5Y3l9pjfRz9fnlq1w4sHsI1hiP0+fLF6/4/uAOx98rnktqo+z79hXeWzsdbNsZ7SfRc0tO89ny+vXm84s7rz+emtnyetRcb5q9tN7ieHouJf1cvbZ18I3s/dMD7+Yvd+aZH5Az4/KfjQfbdFxql6Xm+XpmDvrHYXnR9nzZHZrOHXmyey9zbo8l9fy+LmtfJ/x8BPp9SS+xs9D81pR1tWo55Ian/Q+ZGl9TujHRx+Hpe9DqtftwfewE2r4ensW7z2W/f5zjDy7v3C375f1eXHkfXA6/+aXCTd/HK1e55a+51r+PLrWPT+2fhdc9jwq2x+vnY/DtSuXw8PpNe3EHeFzj10O79Q1WR3Lpedav7d2z8EHf4/wRnj8a7eEcy9MMKn7to91l75+j79WpHF/5jvV7y/xPUD3PdXS8/yS3wmq80X+kXHFfhd+X/s4jP5OMFo7zbYHOS8V9Zlqp/f9EM/8HFGeZ34UHvijO8Nf3mCzLYWWmxhafuK+8PSbvw4vzO8IW/Xt28PWme+VvwhevRzOnpyFOx97Kbxx9YPwxouPhztns/C576RfRj8Iz58/GWZ3PR5eePOD8N6Vl8PFu2Lokr25Ldo4Ge7deS28U7dxMtyfB6PVSfvwQstfh2/fPQsnz3wvvHK17Oc37/5EmJ3JPgPx6vfCvbNPhM898XLWz0+Es1k/f/LYHWHr1EPh+XisV18LT585GbbqzwNJL4b5L7DVm4U8aCtO4LNw4nRldvXX4Z06CFlDP68+VxzHAy9mL4CvPh5ObZ0O36wD1mxZzwvmry7/t/B/nP9dz/9/Dq/3rN9+g5S33WeSLy/fRM9OnA6PvvhWeO/qB+GdK80vcC88mNXW1bfC8w9G/24gW7bXXzux/Vk4ceq+8PSrcczeCs+cPTnYRt9xlC/qj4efXHkrfPvMLGzdVd2+Zyvc/tjLRSjzyqW7wuzkmWofVV2cfCi8kKyuXg73Z8+jd356KdwZg9jsRfu9y/eFk0Ubvy76+cJjsc3z4fm6NpLbuOlvnvnfwx/2fZZlrLv6Ob7f5/vH4Y0n8mOtPPPQcoXne/8blP8v/Ki39n4XfnD5/wnXql+473zs5fDOm0+Fe2dbId2+Z+v28OhPS6elY/Lmk+HOWXleinX3xrP3hZOdzwqNbZxI57bq+T67Kw9GqzEZCfL6amqlx5b2c8I5o6jDkbq58lT4XGbxzk+fDJ/7xCzc+2zzXKz7PHSsr8aabjxf2TkTTm6dDM15aEI/lx7ra+HiXSfq16P3Xv1euDc+r57Iz7nlcc5mt9evOe9d+XUTbFbnx3u/U74epWO9J72mXfmv4QcD9fejhT92DJgufd38OCw9t004Z0ypz+L8NZuF2+Nr35V4Dsxfb8o/ZsVfUuLr+3tvXg4PnNoKJ+flea0e83Qe2+fP8dfN8jk03s907vs49J/npzwPp9TO+OtR47Hfca9q89a7q9e7X4efxNeO2Zn6MyWnvPau07P/HLwhnh99EJ4/m70HePOl8OhdszC7+6kqhFju2YxZU0MrP7b03FbVTXqtWOjnFM+mf301PuV9SHNcA/XZff4W56mtcPLBl+o/Lk16HzL6HnbZ63f8Y8yy14rD9yzqftn7odrreno2ddGM76qPLX8fXNZX+7MNYx2Wf5ipjn/Je66l/Vv63nDCuC+tnZfLzxt+8HL5mla8R0jHEd2qc8btnw8Xi985Fs/B19byHuFH4cJ/uCX8D1+7M9z6pVvCH/yH/zHc/Tf/sObZaMtfv5e9VpSvu7eHey6V7wHK303y937LXzeX/k6wjtByae2k0HLod+sJ56WPeLZymdHzfPsc9PwX/0X47Lc2+6MQlp6f6vN9eWxCy00MLVPgFouzup2/eL3znbsXAp7WTL4Y9G19IvvFtHxDks+0fCUGfSkErYriJ4/dHrbOPle/UUrF1PdmLS076M/WL66xH/GYZ+frGWnFcZ9+MvuL3MfhlUunw+3zNCvlcjg76xzr1efC/bd/Pny7CCX73tjEF+HOZ90VJ4LhAPHg/Yy/ELcD2e79ZZZvXvp9eOK/Cz3/fxd2O0/s8bb6TPKTXfkm5fQTKQTPl30cokUT6DYz2pqZmM36/bVTtn/22Wa9+Ib5M1t3h29PDHCLuqjCxfx2s7+XwgMLAU/7sfd2zoStT2Qh5kextvKZluWL5b07eUj0cnj09q3Q6nthP2764oN/FG598B8WnltFvR/0+f5R2af2eD3XqvFVnu/9vzD/1/D93toL4duX/t/6TWc5Y7e0KG9XY13MqGv7lzXafqw4B3We78/Ub9LjrMAY/jchaNFGcb77TLjYDa+GgryVnitZjWbbLe1nnJ2Yh3Jx2865rTz+sbpp/7Egrh/re3Y+nfuyvg0ca985pvvYsn4uO9b3nj0TZrc/Hn6S+RTPrc9cWph99YnsF/Hy+KtjKPrfPv++89PnwjM/rc5Bb/638O2B+vt+NltzzHTp62Y1ZmPntqXnjIn1WZyzOueexiM+Z9qvae+9ejk8c/m1JuTNrJvtsnpYunzZ62bZ1ng/m/01593msbpfA7UZl0+rnfHXo3o/6Q8nnS9DWz7u5XPwM5fyKzji+bMZg+Wvvev17D8HV7bX2fNa/GPK1h3h0fx8e+XJcLp+bLlnM2Y99bK0dsttuuex2Gb7sfy1p9rPTx8Pt3ffZ4x45v3sq/H8vUd+u2/dFMz0zgSujzkFwvm5c+L7kNH3sMtev7t2fcaH7znl/VAzJmOvnamuDssztX+Qn5XnyPvgoqbS+8OqRmJt5aHl+Huu5f1b/t5w+bi3n3flPluPXb4vzDqvd+89ez7cfs9T1RUEfWPZPgeXV84c9D3Cz8JfPnpnOPeDn4W3r7wTXv/HPw//6kt/EO7+4To/til6Na8dsV7br9/LXyv6xj2+D07vn5a/bi7/naDvNSY/hzXPsxQ8ZpOeqlpcXjtp2/a4NW0vPy+VV/7xrM1Gz/Pt5/vr3/o34Q8f/NHi75316017/XofG7xcaHkMQ8veN0RF6FOdVIo3YWfC0/lssOKx5qRTtFFfrpUuM9qqQ9K8eHv3t66ivvJS+ObZ0+H24vLb1I9OP7OZb3m/ittL33D2vRjGF5W+0LLZ78J+DtrP6NWaWRnDmvaJeGGf6zJeaKfPJD959bxJabXxVnhmfnf4zMnqMvqijjqe1fr9tdPT/tJxzPv3cchfXPPb9f4G2ssDsGK7TnCft5Uuk1+8fCy9acz7NG76/IPxL149n6MTX4DSm9LsdtGP6vH6mPIxyJ/vxWWDnSCveKwZk6KNic/3vjczy2uzPP7RN9ATxiT2M82UTfuMj9W/6BVtpPNE+2e572xMBvaX2j3Iz6X9jGO15JxR7n+8bt558clw9vTt1UcxVMfbdz7sPdaBtp/Nam5CP5cda1GrfbWVz+wfCJSaMYhXB9weZvGysbPzcPHZl1t/qGrWy8Y3fz60bvcfdzyOfMWft+MAACAASURBVBZ10WbreRTbHj+3LT1nTKzP/Pndd2yvPHE6nIiXQZ05Hx594qXwSv5a3jrWKR496/TWy+J6y/qZ+t5rm/o5sq9ptdPzepHabv3c77j3bTd1n5XZyDEmo/hzqufoOXhkX0fh2d+318LFz2yF/Pxfn7OLMVrRszWui3U5GAC2zm19+4yPdd53jXjmY9dX44X36B9P87731Vm+vLpiYtb94215HEvfhxS/zI68hx01HejbEXsWtbXk/VAzJgN9zo6zmG12XTzb49r0OX+8pz47tdh3voh1uM7QsqjrvtfvNA7p46JafxzMn0cD45DVTt9xtE362uj6rPM9QjMOxbeJ//lfh7ezumn3rVl36uOjr9+dMe5rs88rP/8Uy/vGrH7PFe3GfyfoO48X7fa9vxw4tyyvnRRa7ve8VNrzzGZbD4xFXx29HUPL0+ut7b79HOVjQsubOLQ8NX8pvBNnzbX+57PKyhNGfrJcb3GWL1Qn77kUXniz6kec/VafeJdcdhZfZJa+AAy9GDaBTnFMoyeCNfSzeEH8dfjm6a1w6rHXyhkmA5dUDxlPuURtaNv2430m+Qtz981Cviz+5foTYXbqfHjm1WrMXnw8fKYbAldvAPprp6f9peOY9+EoQ8s7wsMvdp8jnZmmxbGOm25CaDn1+d73ZubaR8su4yiPP/+ltbydjfXAGOdBcqyX5aHlmfDtdL7Izl3vVTVX1/rA/url3fVXuL+0n1VIN3ZuK/sxUjdF/0+WlwhVx/n0mZ7gLfa791gH2s5+mUi//I/1c9mxFm90zzwV3sjGonxNyV9LBvrSNb/ycnj+icfDvfGzbk+cDt9Ms7nWcHl477moFVouP7eVx1p9TEvV99Yb/WIcltdnsU39S2H73FbX59XXwgs7l8L9d98RTsxu7/3olnrdruOy+731stiPSf2sZgAvBMKpDyP7Kj2X1U52Dklt9v7sr7Hl4963XXufS197R44xH6Opnv3n4Gp8RvZ1FJ79fRNaFvbVL/y9NbfsDzfVJbb5Rx+VtRNrccL7kNH3sNNev9tB88fhWuu1ov2caPq2vhC4qK10XozHU93uf970PW+zc9ihemb76T0XTVne49l5bvcdd6ytdYeW4+8Ne/pZBJlp3AfGIaudvuPIz4vpfUi7/vr2G/8YvI73CNn4vPDFcMvXvr7iR2xl2w+N/9Drd2eM2w5lu31e+TmlWD76nivaHU1oOV47y0LLZeelzJlnOWNy9DyfeX30cRBaHiDguxE3jYlv3wnnQI9lL9T5C3h+Elt6uVP12R/N55YtXh5eXPKX3hwMnXSrx/OT5YGOrbuf+Flp3bArvtBloWVx3J3LRd+5/Hi4v/6g6J6p9h+9HL59vvkynhiKLF76tUJouZZ+lieTYlr/qYfCw2dmoX05b/tk0+e8/BK15W2U7cbPsUtvfPq2GXizUIzf5XD/LM2oqLYtfvHveI7WTk/7E17Ic5OiLkZnOLQvOy63bT9WjEXnkpX25eHlJRbtN1J9XvGxgTdulcPhXh5eXj7Wrqf25eGrPN/7fylddhlHefyjoeVHbf++MSkuN+k83/NQ81r10ReLv9j1jMuKNZXX17LbS/s54ZxR7mP4uRhniaRf2lJ/okVvODRwrK1LtKpabD02oZ/LjrW8ZPq+ns95zcdk/PlRnNNbH8xfXq5YH+uRXB6+/Ny29JwxsT7z1/M0tvXPV58K959/Kru0/uNQfPzLxNfrup1qvPvvL3/djNuN9jNrf/Q9QnE5cf/rw7Ta6Xm9yPbdHF9/jS19v9R77m7vc/lr73o9+8/B1fPpOntOvTy8/brZ9mzGLD9HrHa7dR7rO7f1zRDruzx8xDPvZ1+NL38fkh9Tf30W+yg+K7f9OZbNvie+Dxn9ZXbZ6/caLw8/gGf+O09+u/88dD0983Hd7+2e50T3dTz+PtT6mJWPQ/N+qDz+8fdcy/u2/L1hTz87z6Olz8Wey8Ov/fSpcH/9mt83lu39ruU9wv/1xfAHX/nz8NPs9eOnf/Mvwy3rnGm59PV7+WtFX73n55/lr5vLfycoP5LrvvBMZtH+/SernYFzy/LaWRZaLj8vxS8+HH8/xLN5rcjG7KOPg8vDb8Tk8QDHdL1Cy2sLXyhwqfiChuaLeOIbkOyD0q+8HOIX3ORhYPHGczYLp84/V34JzpsvhYv3nOyEe+UTID9ZLj45fhQe+MN/0foW2MV12k+kZnkMMbaKb0WLXzjwzqvfC2fjN6VnoWURUsQv4qk+kPiNFy+Fe062vzCo/FDj+8IzxZe6vBaeOX9HOJF9MUf5eZ/VF/Vcqb6QpRuWDpyUy76up595W1vZh/w3HkNO6388vskovrym+AKkX4c3si/aSZdFl2+GuvsuQ5ZPnHmqrptHT8fLxPt/Ke2vnfabkeL4u2/WshfSPp8pvyx0v/TlmbN3hK38i3hiiNb6Ip74RSftUCj+wjub3RHuf/a14guJyvrr+fzE3l98G7vRL+JJYUT2B4vWm5YJz/ein/WXDlVfjpSPSfxFYuLzffQX5sFxmfYGeumYFP1svjjmjcsPhVOdL+KJ57atk80Htr/y7Plw6tb76i/NqOtlSU0d6K+QS/s54ZxRWQ4+F198KHxi667wcPXB9K/s3Fd8M3sd5OVjMXSs3Q/If/a+cKr1RTwT+jnlWE9uhXK2ZvxCtfIcfOvZ7AvVljw/yo/OaL4Y7r03XwoP31XOSq/HMz/e0dt9vwDFz0PtfoFd93Vzyrlt+TljSn22nt/dYyn+6Nh80V70/PaZ9uchlyYHee39OCx73Yz7GO1n1u/+83w6/1U19tjL1Ze6vZV9JnL0XFY7Pa8X2b6b+tjvuPdtN3Wf6RjX6zl+Dr7Onj1fxHMxfoFi54t4poSWBzoHLz23lWP4ibufDD+JX3aVvoin85Ew14o/ppVfHFd+6WBen8349tX4lPchS+uz9sw/x7LZb9x+0vuQ0few7faaPmWPb4BnHlTmt/vPQ33P23g8fZ9jmR3nWj0Pcg7uOcd0X8erz4p94HL8Qsxfh/KL9NKEg/L4DxpaLv9dcMLzaGntVF/Ec7b6ffPV58L9p05kX9bXN5Ydn+LjtQ76HuFH4cJX/iD8q2//KLx99Wr4zf/9jfDZ//AH4bN/1/7opgOdlya8fi977e2r9/b5Z/nr5tLfCapJBJ97Iv1uE7/Qt/37T32eGDq3TPm9Ymjb3tfw9vO02D/Pdi6zgqcv4jlAwHcjbnrdQsv4ZH/zufBAvIRuayvMbr0j3PtE/iHy8Ykff9G5I5wolt8VHrh0PuRfxFOcDLI2tk6cDKcffK76UOR0uWH7M+PKz9TpfjbFQV60Pw7v/fRSuPdUDL1m4dZTZ8LFJ+bh9O33hafzL2R59clw76l0rHeFB57tznD9dXh+fjqcPBH7eyKcOnMp/KT1GWCNxdaJO8K937m08mdarqef5Qn5hQdvrT9QuX5RmHoCX9d6V2MoUJrGsb/nO7lp581Cd59Z3Zw4eTo8sHMpnL39rvobostLVcdqp6f97pu17j4796f9svBB+MmlM+FUVRcnT8/D83ldxTZffaqpv7seChfP51/EU47XG88+FO6sPnO1ON68/ooXkJ5jTUFk6vf7O+Hzf/jF+gum6nGP26d1s9sLb1oy8/7n+wfhhfld5Wcfxhq/NA/35KFl7EfWxsLzvQqV+j43qz+87r65mPoGevmYvPfivPIun8sP35PepKd9/jq8UI9rdd74aXMpcvHmrufzftq/RB/80oll/Zx0zojjMvhczKxmtxbntW/OT4fbz36v/rzHKcfa9GMrxPp9+HL7C7aa5cPn4GXHGj+/82L1erNV9bV9Du77pSSNZ/kzzqT4XPFasBUW67O9bv38Sc+v+HPKczF7DvQ+j7Llvee2uJ+l54zx+ox9X3h+58dR7ON74Ww6P/d6Ro+DvfZe+2jZ6+aSfhbn7J5zX/5Hx+q4mvqJNZi+JK8a06W10/N6kXsdeNz7anPJPvP917cP6LnCOfi6e8bPfn2wer0p6vPJbGbwdM8DhQPxCy7q949957ZyDO95ML2ezMKtdz/eeW9Y1uCg55Ian/Q+ZGl9lv1c9to7+j4k1mCxn+778wnnzbp+r79nHlTmt1vny43yPMg5uOcc0/M++JXvNO9hT515Klxc8+XhxWtp9rq3+No77Xk0/lyMl3VfDg+fPln8Tlr8Hnbp5eyL5aadMw78HiHW+i+fDRe+9i/CH3zplnDLl/q/Pfyg56VrcQLO6Ov3+GtFq96r52c7tCw/M338Pdfy3wma894s3HrXPHxzfld2JU85JsvOS+O/V6znvMQzO49PPs//Q3j4U3eGv/x5tm12ru99D30MlvtMywOkqYcSWh6DomkV+/vPhv/1D/9N+Ktf3VhPjNYxrnNM4myfT3S+eXOd7d8gbQ2FMb2zzI7JMT//pT8Kn396nd9S6Dl3kOfpi/N/GW69wb5Z7yAetj1mzyevvev/eJ5j8lpyozxXD/cc3BMKGd8DPGd4LjzvNv4cXI7Z0uBp9Hlx84374Z6Xjtn7jNHacCwL54RN8br878OtX9gJv9mU/qypH0JLoeUB3sTEv6J8MfzrB//hYG2sqZg39uRRHV+8zO/pMyezy6ic8AfH7GrPl9/EL/pozaA9Zn6/2gl33/ng4mzLm6T+B8f6uhz/z8Jf3vm/hL/1xxbn7utSf2s4d3ntVbvHtXaLfh/2OfjmC1sO9zWW54LvMTgHv7fwBXnle+uFLy4cPJfcbON+2OelNbz2D46Vtheeozel1T+Eh+/8n8Nf3WCzLOPYCi2Flt74H8VJLX72x9ZWOPGZeXjhOAdvR2F1I+/j8oPhX3/xWc+5G3mMHZv6VgNqQA1c5xq42cKWww4seN6cgYhxvznH/bDPJ9o/rLp6/kt3hgcu35hX9QkthZbX+Y2lE9dhnbi0q7bUgBpQA2pADagBNaAG1IAaUANqQA2ogeNaA0JLoaXQ0owINaAG1IAaUANqQA2oATWgBtSAGlADakANqIGNqgGhpdByowryuKb/+u0vV2pADagBNaAG1IAaUANqQA2oATWgBtSAGlhfDQgthZZCS39JUQNqQA2oATWgBtSAGlADakANqAE1oAbUgBrYqBoQWgotN6og/UVifX+RYMlSDagBNaAG1IAaUANqQA2oATWgBtSAGjiuNSC0FFoKLf0lRQ2oATWgBtSAGlADakANqAE1oAbUgBpQA2pgo2pAaCm03KiCPK7pv377y5UaUANqQA2oATWgBtSAGlADakANqAE1oAbWVwNCS6Gl0NJfUtSAGlADakANqAE1oAbUgBpQA2pADagBNaAGNqoGhJZCy40qSH+RWN9fJFiyVANqQA2oATWgBtSAGlADakANqAE1oAaOaw0ILYWWQkt/SVEDakANqAE1oAbUgBpQA2pADagBNaAG1IAa2KgaEFoKLTeqII9r+q/f/nKlBtSAGlADakANqAE1oAbUgBpQA2pADaiB9dWA0FJoKbT0lxQ1oAbUgBpQA2pADagBNaAG1IAaUANqQA2ogY2qAaHlAULLN37xy40aTGn++tJ8lizVgBpQA2pADagBNaAG1IAaUANqQA2oATVw/WrgjTd/NTm1u2XymjfJir+68m748NpvBZf+EqEG1IAaUANqQA2oATWgBtSAGlADakANqAE1oAbWVAMxb3vnyruTE0ahZYfq/at74erenoJcU0H668X1++sFe/ZqQA2oATWgBtSAGlADakANqAE1oAbUwKbUwNW9D8MHH+x1krjhu0LLjs3vfv/78Iu3fhU+/O1HgkvBpRpQA2pADagBNaAG1IAaUANqQA2oATWgBtSAGjhgDcScLeZtv//97ztJ3PBdoWWPzbXffhz+6TfvK8gDFuSmJPn64a9KakANqAE1oAbUgBpQA2pADagBNaAG1IAauH418E/vvh9++9HHPSnc8ENCywGbDz/8bfind98TXAou1YAaUANqQA2oATWgBtSAGlADakANqAE1oAbUwD5roMjXrn00kMANPyy0HLYJv/3tx+Hnv/hl+OA/7/lynn0Wpr9iXL+/YrBnrwbUgBpQA2pADagBNaAG1IAaUANqQA1cjxqIX7oT87SYq606wzJFdULLJDHwM15rf/WDvRC/VTx+LfvrP3/LfwZqQA2oATWgBtSAGlADakANqAE1oAbUgBpQA2qgpwZifha/JfyDDz5c6TMsu9Gc0LIr4j4BAgQIECBAgAABAgQIECBAgAABAtdVQGh5XfntnAABAgQIECBAgAABAgQIECBAgACBroDQsiviPgECBAgQIECAAAECBAgQIECAAAEC11VAaHld+e2cAAECBAgQIECAAAECBAgQIECAAIGugNCyK+I+AQIECBAgQIAAAQIECBAgQIAAAQLXVUBoeV357ZwAAQIECBAgQIAAAQIECBAgQIAAga6A0LIr4j4BAgQIECBAgAABAgQIECBAgAABAtdVQGh5XfntnAABAgQIECBAgAABAgQIECBAgACBroDQsiuyIff3drbDbL57uL3Z2wnbs3k45L0c7jFonQABAgQIECBAgAABAgQIECBA4IYTEFpOHNLd+SzMZvn/ww37DhZa7oZ51teh7DPuY3tnb1GgCDNnhx+aLu7ZIwQIECBAgAABAgQIECBAgAABAgSC0HJiEcTQsjfgm7j9qqvtP7TcCzvbs9AElTHA3A6L2eT44/P5Ecz0XBXF+gQIECBAgAABAgQIECBAgAABAjeFgNBy4jCPhZZx2Xw3n92Yz8IsQ8R6lub2TmjNbUyzGquZkSkYLUPLnSKALLfN2xzp9O48zPJ9xPuznsA1Pt4km3WD6Tj3H5rWTblBgAABAgQIECBAgAABAgQIECBAYF8CQsuJbDHMq4PHImBsQsRyWTObMQV/IZSBZQoi466KdVNYWAWW6W7elSI0nDUzJps287UWb+dhY3F7eyfsLnw+ZuxX09+6lSzwzNupl7tBgAABAgQIECBAgAABAgQIECBA4AgEhJYTkcdCw4Vl9SzGOPuyCTeLXU0MBruhYff+ULfTenk4mh6rt8n6UD8W2n1d2KZZ0S0CBAgQIECAAAECBAgQIECAAAEChyogtJzIuxBMZtstLLuOoWWI++5cDt7tX7zfnd1ZhJTZl/c0s0p7ZmRmx+4mAQIECBAgQIAAAQIECBAgQIAAgXULCC0ninaDv3yz9rL8i3C6l4d37heXh/eHgt2Zjt37+f7btztfsNPdR7yff+Zle+P63vT91Zu4QYAAAQIECBAgQIAAAQIECBAgQGAtAkLLiYwxmGxmH8bbzWXf3WX5Z1iG4rLrbNvuFMdqZmRqO23bDQ2790e7XX1WZtlmOxRtB6zDray0v+FmLCFAgAABAgQIECBAgAABAgQIECCwsoDQcmWyxQ2mBoGLWx7xI0WY2YStR7x3uyNAgAABAgQIECBAgAABAgQIECAwSUBoOYlpfKVjE1qOH4alBAgQIECAAAECBAgQIECAAAECBDZCQGi5hmEQWq4BURMECBAgQIAAAQIECBAgQIAAAQIEKgGhpVIgQIAAAQIECBAgQIAAAQIECBAgQGCjBISWGzUcOkOAAAECBAgQIECAAAECBAgQIECAgNBSDRAgQIAAAQIECBAgQIAAAQIECBAgsFECQsuNGg6dIUCAAAECBAgQIECAAAECBAgQIEBAaKkGCBAgQIAAAQIECBAgQIAAAQIECBDYKAGh5UYNh84QIECAAAECBAgQIECAAAECBAgQICC0VAMECBAgQIAAAQIECBAgQIAAAQIECGyUgNByo4ZDZwgQIECAAAECBAgQIECAAAECBAgQEFpOrYHdeZjNZvX/7Z29bMvdMM+WzbZ3QrN0bNle2Nlu2pzN5mG3bnVsWQh7O9t1X2K/5s2GIYz1dWxZGOvr2LKxvo4tO6zjqBHdIECAAAECBAgQIECAAAECBAgQOIYCQstjOGi6TIAAAQIECBAgQIAAAQIECBAgQOBGFhBa3sij69gIECBAgAABAgQIECBAgAABAgQIHEMBoeUxHDRdJkCAAAECBAgQIECAAAECBAgQIHAjCwgtb+TRdWwECBAgQIAAAQIECBAgQIAAAQIEjqGA0PIYDpouEyBAgAABAgQIECBAgAABAgQIELiRBYSWN/LoOjYCBAgQIECAAAECBAgQIECAAAECx1BAaHkMB02XCRAgQIAAAQIECBAgQIAAAQIECNzIAkLLG3l0HRsBAgQIECBAgAABAgQIECBAgACBYyggtDyGg6bLBAgQIECAAAECBAgQIECAAAECBG5kAaHlBozu7nwWZrN52N2AvugCAQIECBAgQIAAAQIECBAgQIAAgestILScPAJ7YWd7FuaHkCyuO7Tc29kOs8Po6GSrVVbcDfNZDG1XtN3bCdvVdrPZdtjZ69lnWqdjUfjEbbd3QnuzcoxjX8r/A+327Co+lNrt7G5g7cN9OPWlOI6F4xzadzMWtUHrYNrLW4uy4++6tvpS2K7mOtRbjxMgQIAAAQIECBAgQIAAAQI3roDQcvLYHl5oObkLE1csQqJuojRx2yNdrQgVY4BVhmHTuxzHogm+iuNdCOZim9thPm8HuEVAPN8tA8bONrGd7Tz93J2vEP7GPs3D7krbHJJ27EN2bNProTTLCZoeduu/ve4y1+MTojdH7BYBAgQIECBAgAABAgQIECBw/QSElhPsi9Cnnn3XMwuvCqrKGZOdWYNptl/aPguTQr4sf7yatba9s1PPQlzl8vHRkCrf52zWDulCGUzVs+xal6yPLYuI1fLOcUzgDSGsGlp2Wi2OqX15fRyLGEAOWRSPd/sax7GetVkeTyvE7Oy2dTf2oUhd47G0+xLXy2ujO5Zjy5LN9NCv7HcTAKdxW+xTq//FnXYQ2VreCUJDYdWtn2q2acd1aAxa7btDgAABAgQIECBAgAABAgQIEMgEhJYZxvjNbhiUrV0FOHVQ1A14Wqv2XAbds34R9GSXTKcQLmtq8OZgSFSEe80MxRSItfpd3+k0H/s4tKxYtQrHOoFVp5WBuwcLLReON/NcWFb1oHi8r69ZqDt6uJ0jie2l9eNYpdtxtSKU7NvXkmXlLkqbcfu8M3nwmFzzx/J1u7erfaWAvQ5wqzCyOqhkt9vzMQRpWX7ZffFY3eZsSR11++Q+AQIECBAgQIAAAQIECBAgcDMKCC0nj/qS0HIglErBYDN7cXF2WjFrrbN9EfRkyVf3/li3B9ftCR5bYWgd2PXMyhtbNtaZSctSuDZp5fZK8ZhaMxtjW03/hyyKxzvm5ezBKtStjnfaTMtOKNhy7ixr9X5sWWvFFe5Ube7Gz/xMAfX+9lMYVZbJsQhg8/Ayq9HYyV7XVu9jX3qeA6113CFAgAABAgQIECBAgAABAgRudgGh5eQK2F9omYc8cVetkDDtO5sZmB5KIdHQ/fR438/utvU6rTCtfLS3P9Xl2s2l0nUL8QiqS9ZTIJYv2+/tss1O/rW8sYXAsvkynDwkLm+3+9sXri1Y9IxLb6eKfqSPDUg/U3A6FhiOLevd04QHqxmvWXBbfgxB6s+EJtIqRXBbbVcdYx7iLnhNCi0HngNpn34SIECAAAECBAgQIECAAAECBEIIQssVyqAvpCk2Hwy3ygCpDnqGZu/1bN8NHrv3x7o9uG6x/yy8697vNBqPdyhIXFxWhWXd2YudNvvvjoSWPWFZbKM4xjyY6294pc+0jMeUX4Zd7GPC8fTVRe7TbTfv6tiycr3SJu9Xvn3f7e749+2j9Bse39hue7tOwDpQO0vNBrbrOw6PESBAgAABAgQIECBAgAABAjevgNBylbEvApc0ky4L/3pCx7rZKnQrZ/vNw07+DdWtZVW7VUjWDZ669+v2e26kQCqfbViHj5191o/XQWA6vp4Ab/RzCfcTWlaBXN5ufUlzdWBVf+vgt3i4b7v+S467bn02TSCYZikmgymzEzthXtXtYj914DnW7tiy2Fh1rPlAVfsY+1EEjsm1Z9vk0F7UcW0vjElx2E5tdsYptZfXXHJtL8ueN2MHYBkBAgQIECBAgAABAgQIECBwUwsILW/q4XfwBAgQIECAAAECBAgQIECAAAECBDZPQGi5eWOiRwQIECBAgAABAgQIECBAgAABAgRuagGh5U09/A6eAAECBAgQIECAAAECBAgQIECAwOYJCC03b0z0iAABAgQIECBAgAABAgQIECBAgMBNLSC0vKmH38ETIECAAAECBAgQIECAAAECBAgQ2DwBoeXmjYkeESBAgAABAgQIECBAgAABAgQIELipBYSWhz38u/Mwm83K//Pdw96b9gkQIECAAAECBAgQIECAAAECBAgcewGh5REN4d7OdpitFFruhvlsO+zsHVEHD7ib4vhSOLu9E6Z0u7VN2rZ1zHthZ7sMfLe7EHkY3NomO5C9nbAd213JPe1zHjYhYt6dV4H35ONI/c+2i9vWY9Jd3tTY8vEIIeTudZuZuZsECBAgQIAAAQIECBAgQIAAgTUICC3XgDiliRs6tIxBVhZgrX6slWARMqawMIa2szDfLUO2VmhZrNeEbaG1XRqNMvSdz1cMi2Nb890Qw8KVss602zX+7DrGPrUcJu6r206+2diyrmux7iyNT96K2wQIECBAgAABAgQIECBAgACB9QoILSd4FmFNFsrFTVoBUhGaZTPbOuvG9UfDod4+LJlpmc94K8K9rJFuf/L0bWxZ0UQZFk6fnViGis0u0ky+1cOtlml9OD2hZTz2ZodRN+xsZyFmNj6rusf1i6YX9hE7lI6tGuvWOI8ta2YotrpdH2Pfje74V+PS2mffdt3Huu20l/ebl+u0ly0at1tyjwABAgQIECBAgAABAgQIECCwPgGh5STLbvDTvd9uJIY93XBq1fAshJF9FIFlFgoWQWQT2vXtP/VwbFm5TtzvKpdU5/0st53vQIQiJgAAIABJREFU5o+lPS/5WRxDdkz16mUY2Jph2F23CnBr83i/CvdWc4/7Sn2Ix5Bux85UoWS9k7qDS5ZV63X7mG/edzs/xjS+u/Fy97xPfRu2H+s9/qK9vuA12zbff3y4ur8TP+Yg1sdKNZK16yYBAgQIECBAgAABAgQIECBAYIKA0HICUlwlhj91cBYDqFZ4VYZ1dZgzW7yMtzc8Gt33cPDX6kvRRhmopS4V+xoIlcaWjXZncGHVzyJQS8HpcN+HmmnP6svX6gktq/Govec72UzLuO8m2FvJPQs7Yw9aAW83xGt1cfUwMd+893baXxF2VseTHuvdoO/B5eNQ+PTM3lwYj2LfeV23a65v7x4jQIAAAQIECBAgQIAAAQIECOxXQGg5VS6GNlW40wqzqnArDzEXAp8UsqVUcdI+hwOnGDTVAWrR1kCAVM3uS7MOW7sdW9Zacdmdct+zLChMs/Imf5FNHswt7K4/tGyv1gSVdSibZgPWP1Og2t4yvxfHrQ5C03ZpzMYCw7Fl+Q5Wuh2PKf8CneoS856AcajZ4nhS/4dWKmb0NiFvsVrfePQcY1+dD+7GAgIECBAgQIAAAQIECBAgQIDACgJCyxWwducx+NoN81Zw1AnVinAnn5FW7mClGX/FJsOhZfkNzlnQ1Bcy1cfVBHr1Q/WNvmVVWLY07KobWfi8zsWwrCfYrDfv+NWPpxsHWz7dvc879xnrx9iy6jiqkHgF1vbnplaXp/eF1a3AuGaLsz+XB7WFz1g9p/aq/Tf97/OqV3aDAAECBAgQIECAAAECBAgQIHAgAaHlKnxDwVP1eDlLbx7i5/6lcKl35l+T/IzsvQoP04y/zqy7drt5OFUGaPmMwWZ3Y8tSV6r9NhulBaM/W7MUF7ZN+82C1tTaYOCatmnPfiybbi9b2F1qe5UZrrEfrfAuNlLuJ41l+TmjWX9a63fGq7VsP1/E0+w/jWXTj3SAyWHRdTE4rrapQvXUZj5DuFhjcDwKzLCd1eOYe+qhnwQIECBAgAABAgQIECBAgACB/QgILfejZhsCBAgQIECAAAECBAgQIECAAAECBA5NQGh5aLQaJkCAAAECBAgQIECAAAECBAgQIEBgPwJCy/2o2YYAAQIECBAgQIAAAQIECBAgQIAAgUMTEFoeGq2GCRAgQIAAAQIECBAgQIAAAQIECBDYj4DQcj9qtiFAgAABAgQIECBAgAABAgQIECBA4NAEhJaHRqthAgQIECBAgAABAgQIECBAgAABAgT2IyC03I+abQgQIECAAAECBAgQIECAAAECBAgQODQBoeWh0WqYAAECBAgQIECAAAECBAgQIECAAIH9CAgt96NmGwIECBAgQIAAAQIECBAgQIAAAQIEDk1AaHlotBomQIAAAQIECBAgQIAAAQIECBAgQGA/AkLL/ajZhgABAgQIECBAgAABAgQIECBAgACBQxMQWh4arYYJECBAgAABAgQIECBAgAABAgQIENiPgNByP2q2IUCAAAECBAgQIECAAAECBAgQIEDg0ASElodGq2ECBAgQIECAAAECBAgQIECAAAECBPYjILTcj5ptCBAgQIAAAQIECBAgQIAAAQIECBA4NAGh5aHRapgAAQIECBAgQIAAAQIECBAgQIAAgf0ICC33o2YbAgQIECBAgAABAgQIECBAgAABAgQOTUBoeWi0GiZAgAABAgQIECBAgAABAgQIECBAYD8CQsv9qNmGAAECBAgQIECAAAECBAgQIECAAIFDExBaHhqthgkQIECAAAECBAgQIECAAAECBAgQ2I+A0HI/arYhQIAAAQIECBAgQIAAAQIECBAgQODQBISWh0Z7sIb3drbDbL57sEaWbb23E7Zn83DIe1nWC8sJECBAgAABAgQIECBAgAABAgQItASEli2O4Tu781mYzfL/hxv2HSy03As722Vft3f2Bg8q7qN3eRFmzg4/NB3smQUECBAgQIAAAQIECBAgQIAAAQI3s4DQcuLox9CyN+CbuP2qq+0/tNwN89kszHfL4HK4z3G97bCYaZaPz+dHMNNzVRTrEyBAgAABAgQIECBAgAABAgQI3BQCQsuJwzwWWsZl890yLCxnY+azMJtZj8Wy7Z3QmvuYZjVWszhTyFiGljv1jMnZypdxLwktd+e9MynTce4/NJ0IajUCBAgQIECAAAECBAgQIECAAAECAwJCywGY7sMxzBu6PLxc1sxaTMFfCIvBYbFu+qzKKrBMd/N9FqFhMWOyfLRpM19r7Pbivpu147Kmv/XjMcisQlWhZa3iBgECBAgQIECAAAECBAgQIECAwBELCC0ngo+FhgvL6lmMcfZlPusyhDAxGOyGht37y7s9ElpmfWjaafd19f01LblFgAABAgQIECBAgAABAgQIECBA4CACQsuJegvBZLbdwrINDy1jf7uzO4uQsvVFQ2lmac+MzOzY3SRAgAABAgQIECBAgAABAgQIECCwbgGh5UTRhWAy2669rJzhWIaC3dmOnfvF5eH9oWB3pmP3frb7gZudfaW14j67n6uZlmU/V99ftrGbBAgQIECAAAECBAgQIECAAAECBA4gILSciBeDyfHPtGyWpy/TKZvOv6BntvjlN3FWZjbDMW3bDQ2794e7XYaVeZvxdppZ2Q5YR1rZ8e3hwzqWECBAgAABAgQIECBAgAABAgQIHKaA0HINulODwDXs6mBNFDM7O5+xebAWbU2AAAECBAgQIECAAAECBAgQIEBg7QJCyzWQHpvQcg3HqgkCBAgQIECAAAECBAgQIECAAAEChy0gtFyDsNByDYiaIECAAAECBAgQIECAAAECBAgQIFAJCC2VAgECBAgQIECAAAECBAgQIECAAAECGyUgtNyo4dAZAgQIECBAgAABAgQIECBAgAABAgSElmqAAAECBAgQIECAAAECBAgQIECAAIGNEhBabtRw6AwBAgQIECBAgAABAgQIECBAgAABAkJLNUCAAAECBAgQIECAAAECBAgQIECAwEYJCC03ajh0hgABAgQIECBAgAABAgQIECBAgAABoaUaIECAAAECBAgQIECAAAECBAgQIEBgowSElhs1HDpDgAABAgQIECBAgAABAgQIECBAgIDQcmoN7M7DbDar/2/v7GVb7oZ5tmy2vROapWPL9sLOdtPmbDYPu3WrY8tC2NvZrvsS+zVvNgxhrK9jy8JYX8eWjfV1bNlhHUeN6AYBAgQIECBAgAABAgQIECBAgMAxFBBaHsNB02UCBAgQIECAAAECBAgQIECAAAECN7KA0PJGHl3HRoAAAQIECBAgQIAAAQIECBAgQOAYCggtj+Gg6TIBAgQIECBAgAABAgQIECBAgACBG1lAaHkjj65jI0CAAAECBAgQIECAAAECBAgQIHAMBYSWx3DQdJkAAQIECBAgQIAAAQIECBAgQIDAjSwgtLyRR9exESBAgAABAgQIECBAgAABAgQIEDiGAkLLYzhoukyAAAECBAgQIECAAAECBAgQIEDgRhYQWt7Io+vYCBAgQIAAAQIECBAgQIAAAQIECBxDAaHlMRw0XSZAgAABAgQIECBAgAABAgQIECBwIwsILTdgdHfnszCbzcPuBvRFFwgQIECAAAECBAgQIECAAAECBAhcbwGh5eQR2As727MwP4Rkcd2h5d7OdpgdRkcnW62w4t5O2J7F0Db+3w47exO23Z1X66ftyp/pkIvjr9tcbLf0rrZNGxW7Lce47MtsZcO031aTEw7nMFZJfSmOZXsnTGFtbVP7ZWMyOla7YV5vE23bIfyw+WEcvTYJECBAgAABAgQIECBAgACB4y4gtJw8gocXWk7uwsQVi/BpE5Kzpf2NQVc3FGuHXUubKFZotzN2/N1lMUzbrpLSIljL3PJly/sR62MedmOgmrWxfLtDWCP2IQsqu8c8eY9FSJnGIx5fM1ZFm9k+olV+2HF5cu3ufzXXyb21IgECBAgQIECAAAECBAgQIHADCQgtJwxmEbq0ZpF1Zu9VQVURelXr1QFOa3barBUmhXxZFgDFLpWhz042ey2FR8s73A2JWlvk+5w1gV25TmemYWu23NiyosfFTNQ8LGvtt+dOt5/Jubbr2abvod52ehtph5shVLMDC/t2KFfsZ5UAMroW+4xtLo5VXhujsxAXtq362Hs8vRqdGcFp3Bb71Ld1/thouFjUUdNmOXbpftnnssvxdhN2ts3zvblNgAABAgQIECBAgAABAgQIEGgEhJaNxZJbZfjTmx3FcGuWzTTrzHTLG45B0EIbPet3A7zRACnfQRV49s72K4KmxQCp7s9YSDe2rNh/FY51wtdO11p382OKt2Of88daKw/e6YZiZeBbX+IdQ+R0gHnQlix24+XpZdjW3vdqxxPHK+0mtpNux24XxzbgMrasPORVQ8vco9x2vps/NgjZXpBbtZcU97pBcfFgrJEitM9qLG+nx7ynaQ8RIECAAAECBAgQIECAAAECBILQcnIRLAktB0KpemZZNlMzXTZb73ootMySr96QqG6gfWNw3Z7gsRXUFaHS4ucRFq2PLWvvfvK9ct+7xczAZNLqz4SWBo+13rYM7or2U4BWhGvVrMD0WLF+uW4dvO1MvdS7Ewq2nDvL6n7FG2PLWiuucKdqswhjU3i4+n5GxyH3q3pWjEOaJVosr4Lb5Jtvkx5b4aisSoAAAQIECBAgQIAAAQIECNxcAkLLyeO9v9Ayhj/1TL9q1l0K6Opdx0CnE3p2w7ju/XrbnhuD68b9ZEFo3LQ/nIohV+cS+Ho/Y8vqlSbdKIOufFbiiHFfi0X4lYK5vhXKx5pjrPqeW/fYp5Zi/xbGKi3MfxaBXPTK/+eXSg/1cfUwMd9t/+1qhmgKEONKq4aExfGk/nf20rtscdyaGlzNvLM3dwkQIECAAAECBAgQIECAAIGbVEBoucLAN+FXZ6PB4KsMc+rgqwiPup8jWSSHRxNadkO+7v2Fw8oDxfbCaNHOP6uwLA8E25ss3uuGaX2BWPFYj1kVuHZD2IWddI6xPYad8ck37utLvjy73W6zXJD7xNtD/RxbVrVUBsht7GzvizebwLDpS3f/i4FxamfYpNymL8xc3CY/rni7fg6ExXXTnv0kQIAAAQIECBAgQIAAAQIECCQBoWWSmPKzCMDSbLps9txgaFkFkvUMvHnYyWfvVYFca4ZeFfp1g6fu/bHupkAqb7fOvDr7rB9Pn4VZ97UdtC20mW9YdKYMo7ozRsf6WSxr9acnEKuWN6FX1WInjMz30+5rNk7FSlU/q+NstZv3ZXL42j9bsuhD3UZ7n+0v4hlbFjtczVRc8M6PePF2ERqmsezZNhktLCoMesYh9SO1ueBX9TMtr4899q19jC3zxa57hAABAgQIECBAgAABAgQIECDgMy3VAAECBAgQIECAAAECBAgQIECAAAECmyVgpuVmjYfeECBAgAABAgQIECBAgAABAgQIELjpBYSWN30JACBAgAABAgQIECBAgAABAgQIECCwWQJCy80aD70hQIAAAQIECBAgQIAAAQIECBAgcNMLCC1v+hIAQIAAAQIECBAgQIAAAQIECBAgQGCzBISWmzUeekOAAAECBAgQIECAAAECBAgQIEDgphcQWh52CezOw2w2K//Pdw97b9onQIAAAQIECBAgQIAAAQIECBAgcOwFhJZHNIR7O9thtlJouRvms+2ws3dEHTzgborjS+Hs9k5YpdutbVtGe2Fnuwx8t1sQzeN1INzaZ3t5e9tlB5q2nYdNiJh351XgHW1bNkPHkfqfbRe3zXxa3rPOce7thO00jt36ywP4yf0Z6qfHCRAgQIAAAQIECBAgQIAAAQLDAkLLYZu1LrmhQ8sYZnVDsUkBWwhFKJdt26DH0HYW5rtlCNcOHsvHhnYRrev1ixBuhfA3rj/fLfo11H7Tx8O91a2ZaFUf1wq7brVTBI9ZUNkau05QXtildeOydDvuPI7BCq4r9NeqBAgQIECAAAECBAgQIECAAAGh5YQaKEKfTrDWCpBas9Pas9pS863gKD04+rMTIHXX7cx6awVs3f7kC8eWFfsow8Jps/riBt0Asbw/awVc3c6n+90gLD2e/yzba4d13X3m63dvr7JuCHGcCq7om7sVzaZjq2YxtmpibFmI6WzxEQELTXa7W9/vjn81Lq191iuP3Gi3E4+vbdmMQbdGi/tFcBybr44vHUBxPHmIOdIFiwgQIECAAAECBAgQIECAAAECKwoILSeBtYOfELr3243EQDNlO2lJNxBKjw//HNlHNzAqgshm1lvf/tN+xpaV68T9Tr0UOW6R97Pcdr6bP5b23PMzHsf2TtjJLoFuB2pxmzIsaz/eCQjrYK1nH4XN1HAttpvWjceQbjf9WAwyly2r+rRqaJn3O43vbrx0O+9Tz/F2Hlqou8o8Xb5fBpNl7eRBfLwdjzV/LDZdPN653LyzS3cJECBAgAABAgQIECBAgAABAgcWEFpOJIzhTh2cxeCnlUqWYV39+Yqzxct4F8KjpfsdDv5afSnaKUO81KUyiOoPHseWLe1S7wpVP4tALQWnw31vNbEQ5PVt1xdatlqpZjGmfefLYnuLAXK+Rut2J9CLAV0yDXmI2Noo5qqrh4ndJhbupzYLoyqoTI8trDz0QJ9nFjwWdbpTh7NlQLlbfI5oqvUmtKyC4gqkDC9XC1CHeulxAgQIECBAgAABAgQIECBAgEBXQGjZFRm6HwOj6tLcVpiVZp/V6VYZCqXQJzV3lKFl2me6JDn/vMlJy+qVptyowqx8BuDUcK0TEqZZlRnlwEzLbr/6wrkVA8s0jvWX0FSXgKfOjB3T2LJuVyffL/vfGrsFr/HGimAx9X9o1ayuU6DdbJKF4QvHmC0batvjBAgQIECAAAECBAgQIECAAIF9CggtV4DbncfZfLth3vpcwTK8qUPKItw53JmWZRiZzXLLZ+MtHE8Mv7J1W8v7llVhWZNctbbou9MNZBfDsp5gs2go7iubIbkQjMWVOr49HSj2n49JNQYrHELnMve0k9xnrB9jy6q2ijHKZm6mXYz8jI51XfU6DLmm2Z+Zbe9+yrGunbr+eV0Vy/L2Otv2tu9BAgQIECBAgAABAgQIECBAgMD+BISWq7gNBU/V4+Xl4fOwk11Knmav5ZeOty8tH+pAGQq1tsuCuXa7eZiUgqxqpmDr8uixZakf1X7rJCs9Pv6zCCrTLMWFbdN+e8LTlt34cUSLsunUXt8X4rQvf679MrveI4n9WFin3E8THHbGpLX+2LJiGueKX8QTe9k+zqYf6QjS8kXXxeA4bZP3M/eulrfGo9Nua1keqKa2/SRAgAABAgQIECBAgAABAgQIrEdAaLkeR60QIECAAAECBAgQIECAAAECBAgQILAmAaHlmiA1Q4AAAQIECBAgQIAAAQIECBAgQIDAegSElutx1AoBAgQIECBAgAABAgQIECBAgAABAmsSEFquCVIzBAgQIECAAAECBAgQIECAAAECBAisR0BouR5HrRAgQIAAAQIECBAgQIAAAQIECBAgsCYBoeWaIDVDgAABAgQIECBAgAABAgQIECBAgMB6BISW63HUCgECBAgQIECAAAECBAgQIECAAAECaxIQWq4JUjMECBAgQIAAAQIECBAgQIAAAQIECKxHQGi5HketECBAgAABAgQIECBAgAABAgQIECCwJgGh5ZogNUOAAAECBAgQIECAAAECBAgQIECAwHoEhJbrcdQKAQIECBAgQIAAAQIECBAgQIAAAQJrEhBarglSMwQIECBAgAABAgQIECBAgAABAgQIrEdAaLkeR60QIECAAAECBAgQIECAAAECBAgQILAmAaHlmiA1Q4AAAQIECBAgQIAAAQIECBAgQIDAegSElutx1AoBAgQIECBAgAABAgQIECBAgAABAmsSEFquCVIzBAgQIECAAAECBAgQIECAAAECBAisR0BouR5HrRAgQIAAAQIECBAgQIAAAQIECBAgsCYBoeWaIDVDgAABAgQIECBAgAABAgQIECBAgMB6BISW63HUCgECBAgQIECAAAECBAgQIECAAAECaxIQWq4JUjMECBAgQIAAAQIECBAgQIAAAQIECKxHQGi5HketECBAgAABAgQIECBAgAABAgQIECCwJgGh5Zog193M3s52mM13191su729nbA9m4dD3kt7n+4RIECAAAECBAgQIECAAAECBAgQWCIgtFwClBbvzmdhNsv/H27Yd6DQsggjU1+3w85eOor2z7iP7b6FafvDDk3b3XGPAAECBAgQIECAAAECBAgQIECAQCEgtJxYCDG07A34Jm6/6mr7Dy33ws52E1QW7WzvhMXccjfMZ816Tf/Kx+fzI5jp2ezULQIECBAgQIAAAQIECBAgQIAAAQK1gNCyphi/MRZaxmXz3Rj2pdmN+SzMGCKmx2dh1g0Q06zGatsUjJah5U62bd7meF9bS4cuAd+d915+no5z/6Fpa+/uECBAgAABAgQIECBAgAABAgQIEFhZQGg5kSyGeUOXh5fLmlmLKfgLoQwsUxAZd1Wsmy67rgLLdDfvShEazmIYWj7atJmvtfx2f/jYno1ZtxKDzCpU7d+uXtMNAgQIECBAgAABAgQIECBAgAABAocmILScSDsWGi4sq2cxxtmXnRmSE4PBbmjYvT+p23Ff3f3HDbM+NO20+7qv/TWNuUWAAAECBAgQIECAAAECBAgQIEBg3wJCy4l0C8Fktt3Csk0ILYcCy2q2Z5rBmQ6jCCnry9vzWaXNDNK0rp8ECBAgQIAAAQIECBAgQIAAAQIEDlNAaDlRdyGYzLZrLysvCS9Dwe7l4Z37xeXh/aFgd6Zj9362+4WbZQDZmeGZ1or77H6uZlqW/Vxlf9lmbhIgQIAAAQIECBAgQIAAAQIECBA4sIDQciJhDCbHP9OyWZ5/hmUI+Rf0zBa//KaYEbm4bTc07N4f7nZnf50v+GkHrMOtTN/fcBuWECBAgAABAgQIECBAgAABAgQIENiPgNByP2qdbaYGgZ3Njv7u0DeJH31P7JEAAQIECBAgQIAAAQIECBAgQIDAoIDQcpBm+oJjE1pOPyRrEiBAgAABAgQIECBAgAABAgQIELhuAkLLNdALLdeAqAkCBAgQIECAAAECBAgQIECAAAEClYDQUikQIECAAAECBAgQIECAAAECBAgQILBRAkLLjRoOnSFAgAABAgQIECBAgAABAgQIECBAQGipBggQIECAAAECBAgQIECAAAECBAgQ2CgBoeVGDYfOECBAgAABAgQIECBAgAABAgQIECAgtFQDBAgQIECAAAECBAgQIECAAAECBAhslIDQcqOGQ2cIECBAgAABAgQIECBAgAABAgQIEBBaqgECBAgQIECAAAECBAgQIECAAAECBDZKQGi5UcOhMwQIECBAgAABAgQIECBAgAABAgQICC2n1sDuPMxms/r/9s5etuVumGfLZts7oVk6tmwv7Gw3bc5m87Bbtzq2LIS9ne26L7Ff82bDEMb6OrYsjPV1bNlYX8eWHdZx1IhuECBAgAABAgQIECBAgAABAgQIHEMBoeUxHDRdJkCAAAECBAgQIECAAAECBAgQIHAjCwgtb+TRdWwECBAgQIAAAQIECBAgQIAAAQIEjqGA0PIYDpouEyBAgAABAgQIECBAgAABAgQIELiRBYSWN/LoOjYCBAgQIECAAAECBAgQIECAAAECx1BAaHkMB02XCRAgQIAAAQIECBAgQIAAAQIECNzIAkLLG3l0HRsBAgQIECBAgAABAgQIECBAgACBYyggtDyGg6bLBAgQIECAAAECBAgQIECAAAECBG5kAaHljTy6jo0AAQIECBAgQIAAAQIECBAgQIDAMRQQWh7DQdNlAgQIECBAgAABAgQIECBAgAABAjeygNByA0Z3dz4Ls9k87G5AX3SBAAECBAgQIECAAAECBAgQIECAwPUWEFpOHoG9sLM9C/NDSBbXHVru7WyH2WF0dLLVCivu7YTtWQxt4//tsLO3wra782q7WZht74R808Igttl5PITdMK/3l4fF5fiW/Uj96dt+uH9pn5tAn/pSHM+CQf8xtLapjaoxya3rZfnzYcg1hMV2Vxzn/u56lAABAgQIECBAgAABAgQIELiBBYSWkwf38ELLyV2YuGIREm1Ccra0vzHoygKsIsCcNuO0DML61y1C4PluGZZ1Aru4LKeJ7WwPJKWrOcb6mIfdGO7lO1hqcAgrxD5kx73acWT9GR2P9tiNue57/1lX3CRAgAABAgQIECBAgAABAgRuLgGh5YTxLkKXbHbZwqzAKqgqZ0yWs/Tq3Ko1k7Azcy9floVMsUtxn9s7O9mswP6Arq/7oyFRvs/ZrBPYdWcb5vscW1b0uJiJmodlfX3LH+v2MznXdvnKrduxL1nY2VrW3Cna63FtLsUvZwf2768dyjWtDtyKrkVDcbvcrVw/r41m/8uX1TND+zvZ05lynJrV07gt9qln49ZDsc9TA91y7NI+2q7dcW7txB0CBAgQIECAAAECBAgQIECAQI+A0LIHpf+hbhiUrVVdOlsHRZ2ZbtmaoTsjrVjWs343wBsLkPL24+3BkKgILPOwrx0uhSp87bZX97E+wL41qnCsExL2rZkey48p3o4zFPPH0noLP4vjmIedeBl8CpN7+lY49PWnGq+xy9EHDRc6Uz4Q109diMeQbselxbH19WPJsrLlcoymz96M66cxTuObPzZwAN2HK+P+T0MYaG/AtbBM4xR/5jjd/bpPgAABAgQIECBAgAABAgQIEAghCC0nl8GS0HIglKpnymWhzcLstaHQMgt3VgnRBtftCSVbIWERVOWf85jhjC3LVlvlZrnv3WKGZjJp9WeosaovaZsQ+semcOiMSxmgVTMCq5AtY672OBDKDfWn+JzMFBQWSWQWzI21NbZscGdLFlRt7sbPCk19Wn0/Y+PQV1/TXGPXY1+GZ3DcOk3SAAAgAElEQVQuOTiLCRAgQIAAAQIECBAgQIAAgZtEQGg5eaD7g7Fi857QMTUbw598ZllvGNSzfTcY6t5P7ff9HFx3WWhZN1YGS/0zEceW1Q1MulEGXfmsxBHjvMWeWYB9rkX7rdBysf0+q+6Y5bvuvV2Fn/WszyKgzi+VTuFhd+vVw8RuC4v3y2NsXYLe47W4XfZIcTyp/9nj8WbRVvd4prmmlvrGKi3zkwABAgQIECBAgAABAgQIECAQBYSWK9TBYNjSEzqWzZZhTj0jcGGGYLXznu27YVr3/li3B9ftBk7d+51G4/EuzkIsV1pcVoVlrZCw02D3brH/LBzrC8uKx7oz87ohWX/4Vzi0+tMZj3R5dn6QS0y6hxDv99VF7hNv58F13sbYsnK9KiTO+5g30HO7O/59+yjWmfWN76JRvou+ttJM17rO+1xTI/vwTZv6SYAAAQIECBAgQIAAAQIECNw8AkLLVca6CFzKL9ppzULsCR3rZqvQrZyFV34OYx3utJZV7VYhWzd46t6v2++5kQKpfOZfnXl19lk/nj4LM7uMPQ/aFtrMNyz6sI/QMm7X6k8WYKbjqpbXZunx1li0w7eFvrY+RzHNFG17p2b7Q7m0tO/nlMC0sqlt8+McWxb3t3poWWwVg9K0v4Wxqj73tC+0LLzz/mXHPBo4Dru2x6M7SzNr300CBAgQIECAAAECBAgQIECAQCUgtFQKBAgQIECAAAECBAgQIECAAAECBAhslIDQcqOGQ2cIECBAgAABAgQIECBAgAABAgQIEBBaqgECBAgQIECAAAECBAgQIECAAAECBDZKQGi5UcOhMwQIECBAgAABAgQIECBAgAABAgQICC3VAAECBAgQIECAAAECBAgQIECAAAECGyUgtNyo4dAZAgQIECBAgAABAgQIECBAgAABAgSEloddA7vzMJvNyv/z3cPem/YJECBAgAABAgQIECBAgAABAgQIHHsBoeURDeHeznaYrRRa7ob5bDvs7B1RBw+4m+L4Uji7vROmdTseYxXopp9dozz07Wk37be7WQh526s47oWd7dinediEiHl3nvksHuTAqKVjSNt2jn9vJ2wn706NJc8UtG/3FGBaZ3J3BnrpYQIECBAgQIAAAQIECBAgQIDAkIDQckhmzY8XQc9KKc8xCi1jsJgFitOPdfwYy3BsJDws9jsP8+1ZaNFWoVzrsanjGbed74YYFu5r+6n7mbBe1zH2qS9E7DYVt2utF53qg+mYF1aVcbydjWMZ/Hb8h8y7nXCfAAECBAgQIECAAAECBAgQIHAAAaHlBLwiPGqFOaEItepgqArJ0uy0PMBLzXcDqPT48M9OuNRdMYZH9Wy5TsDW7U8dWIUQxpYV+4j7nWUhV3fH3fvlrL5mF2mWXyfs6m5W76szC7BeL7YztCyulHy6+4+H2Ant6jaX34jbFsfSCvrSdunYqhmMrZoYWxa7W45X45TaHPqZji8tr8altc+0rPOz2FeyK/uVarVbh8X9Waqfch+tdVv7S31aNO/0wF0CBAgQIECAAAECBAgQIECAwIEEhJaT+FJYk1bu3k+Plz/7Zul1w6L2Fn33RvZRhFJZKFgEkSmkKgPVoXCsr2/tvcf9rhJa5v0st53v5o+1W2/fq/ZVh6/NMZTh6jzsxMvq0/LsoJpZh90Arbq/k18CnVm1O9C5F7dN68a+pdtxtbLdZsZivunYsmq9VUPLYkyr/afx3Y3HlPcp70PndhZOZ2ytsD0axuNpLGMb1bH01ECzXrlO3m5n7+4SIECAAAECBAgQIECAAAECBA4kILScyBdDxzQDrZg110psuuHb4mW86wwtW30p+t8OkYp99YROcdWxZRMpOqtVAWURqKXQsXps2gdb1u2VfcuDutwxO8YYANb+2eNFS+X9fLZr0W69fr27xRux3WxmYQzp6s3yELG75diy7rpT76c284A6PbasjWKbaiyKbRrHMnjcLT63M9VzE0aWdVw+3nEcNV/WIcsJECBAgAABAgQIECBAgAABAqsJCC2nesXwpwq0WmFWceVve2ZiEwI1jU8OzupNhoO/2FYKnMrVu8Fd1UgRXs1aQVzTfHV5eRbS1ctWulGFW/kMwKnhWnc/+Xb57Wq90vWt6oty0pfMZD+LY+mxiA4TjjO2X8/q7M7u7OlP3f2xZfVKq94oA8RWv1c4jlZ9ZNuVwXAWxlYzK4twNluv7G2qwTTGPT4TXFc9cusTIECAAAECBAgQIECAAAECBISWK9TA7jzOXtsN81ZQUwY6dUhUBFjNzLbU/DpDy/LzEbPLhGPYlIeGaafFzxg8ZesuXVaFZfUUw9YGvXe6x1aEf63tU+g11I+y2fZ23fAxBWjdLnTXq2aTZvsvw85l0z772s/tOuPc6sbYsmrFYozysLDVQO+ddr/79lE+1h37tmPlkWq2G7DmtZPfjj0q1k2zZ/MulvvNiPOFbhMgQIAAAQIECBAgQIAAAQIEDiwgtFyFcCh4qh4vZ+mVn8OYQsw0s601g29S2lOFh2nGX/yZgqf8Mu9ieR4spSCrmRXX7G5sWYKo9ttslBaM/iyCstTXhW3TfruhZecYu9tVAXCy6y4uO9QXoKX9VQb9G7aPJ45h5pu3ncay/PKfxrW9fudYum0N1U67F5177eNo+pFWS8u7runx1NfO8k697qbmFupqKGQt25/CmjXtJgECBAgQIECAAAECBAgQIEBgsoDQcjKVFQkQIECAAAECBAgQIECAAAECBAgQOAoBoeVRKNsHAQIECBAgQIAAAQIECBAgQIAAAQKTBYSWk6msSIAAAQIECBAgQIAAAQIECBAgQIDAUQgILY9C2T4IECBAgAABAgQIECBAgAABAgQIEJgsILScTGVFAgQIECBAgAABAgQIECBAgAABAgSOQkBoeRTK9kGAAAECBAgQIECAAAECBAgQIECAwGQBoeVkKisSIECAAAECBAgQIECAAAECBAgQIHAUAkLLo1C2DwIECBAgQIAAAQIECBAgQIAAAQIEJgsILSdTWZEAAQIECBAgQIAAAQIECBAgQIAAgaMQEFoehbJ9ECBAgAABAgQIECBAgAABAgQIECAwWUBoOZnKigQIECBAgAABAgQIECBAgAABAgQIHIWA0PIolO2DAAECBAgQIECAAAECBAgQIECAAIHJAkLLyVRWJECAAAECBAgQIECAAAECBAgQIEDgKASElkehbB8ECBAgQIAAAQIECBAgQIAAAQIECEwWEFpOprIiAQIECBAgQIAAAQIECBAgQIAAAQJHISC0PApl+yBAgAABAgQIECBAgAABAgQIECBAYLKA0HIylRUJECBAgAABAgQIECBAgAABAgQIEDgKAaHlUSjbBwECBAgQIECAAAECBAgQIECAAAECkwWElpOprEiAAAECBAgQIECAAAECBAgQIECAwFEICC2PQtk+CBAgQIAAAQIECBAgQIAAAQIECBCYLCC0nExlRQIECBAgQIAAAQIECBAgQIAAAQIEjkJAaHkUyvvYx97OdpjNd/ex5Qqb7O2E7dk8HPJeVuiQVQkQIECAAAECBAgQIECAAAECBAiEILScWAW781mYzfL/hxv2HSS07PZ1KPuM+9je2VsUKMLM2eGHpot79ggBAgQIECBAgAABAgQIECBAgAABoeXUGohBYG/AN7WBFdc7SGjZ2tXuPMy2d8JiNLkb5rPtsJhZlo/P50cw07PVUXcIECBAgAABAgQIECBAgAABAgQIlAJmWk6shLHQMi6b78awL83EzGdh7oWd7fT4bDFATLMaq21TMFqGljvZtnmbEzsdQhgMP2OY2TMFMx3n4HbTd21NAgQIECBAgAABAgQIECBAgAABAvsSEFpOZIth3tDl4eWyZtZiCv5CKAPLFETGXRXrprCwCizT3bwrRWg4i2Fo+WjTZr7W0O2hADWtH/vV9Dc9GrJZmULLWsUNAgQIECBAgAABAgQIECBAgACBIxYQWk4EHwsNF5bVsxhjeNiZITkxGOyGht37E7sdU9IwG+lD0067r/veX9OgWwQIECBAgAABAgQIECBAgAABAgT2JSC0nMi2EExm2y0s26TQspjt2Z5VGfubZnCmwyhCyvry9nxWaXvbtL6fBAgQIECAAAECBAgQIECAAAECBA5LQGg5UXYhmMy2ay8rLwkvQ8Hu5eGd+8Xl4f2hYHemY/d+tvvxm92ZlnGfvV/M025m3/trN+MeAQIECBAgQIAAAQIECBAgQIAAgZUFhJYTyWIwOf6Zls3y/DMsQ8g/X3K2+OU3Rai4uG03NOzeH+52Z3+dgLIdsA63Mn1/w21YQoAAAQIECBAgQIAAAQIECBAgQGA/AkLL/ah1tpkaBHY2O/q7xczOzmdsHn0v7JEAAQIECBAgQIAAAQIECBAgQIDAqIDQcpRn2sJjE1pOOxxrESBAgAABAgQIECBAgAABAgQIELiuAkLLNfALLdeAqAkCBAgQIECAAAECBAgQIECAAAEClYDQUikQIECAAAECBAgQIECAAAECBAgQILBRAkLLjRoOnSFAgAABAgQIECBAgAABAgQIECBAQGipBggQIECAAAECBAgQIECAAAECBAgQ2CgBoeVGDYfOECBAgAABAgQIECBAgAABAgQIECAgtFQDBAgQIECAAAECBAgQIECAAAECBAhslIDQcqOGQ2cIECBAgAABAgQIECBAgAABAgQIEBBaqgECBAgQIECAAAECBAgQIECAAAECBDZKQGi5UcOhMwQIECBAgAABAgQIECBAgAABAgQICC2n1sDuPMxms/r/9s5etuVumGfLZts7oVk6tmwv7Gw3bc7+//bu7rpVHAoDaDmU5pJcistwKSmBWfoDSQiCE5KbZPbDnQBC0tGW5+VbcjLd5+cy6lHbPL89bkstoa772nGej2o9apuPaj1qO6r1qO2r1rEguiBAgAABAgQIECBAgAABAgQIEPiFAkLLX7hpSiZAgAABAgQIECBAgAABAgQIECDwlwWEln95d62NAAECBAgQIECAAAECBAgQIECAwC8UEFr+wk1TMgECBAgQIECAAAECBAgQIECAAIG/LCC0/Mu7a20ECBAgQIAAAQIECBAgQIAAAQIEfqGA0PIXbpqSCRAgQIAAAQIECBAgQIAAAQIECPxlAaHlX95dayNAgAABAgQIECBAgAABAgQIECDwCwWElr9w05RMgAABAgQIECBAgAABAgQIECBA4C8LCC3/8u5aGwECBAgQIECAAAECBAgQIECAAIFfKCC0/IWbpmQCBAgQIECAAAECBAgQIECAAAECf1lAaPkDdvd5n+Zpus/PH1CLEggQIECAAAECBAgQIECAAAECBAj8awGh5ekdeJsft2m+f0GyeHVo+fa4zdNXFHra6oUX3x7zbQqhbfh3mx9vZ/qmvUh9ct/bY267Puf7cNz6+TYsTnuRx3zRMLpPX/MZOaNSv1NqiUYbm/rN9brpM7J73vM+jfbq2HUZ+2Qta1WuCBAgQIAAAQIECBAgQIAAgf+jgNDy9K5/XWh5uoSTL8aA6MXA7eTQF78Wgq4qqIwB5pkTp+/sRQ5CRwQhlKyfB6tbSUpDKLc0phBuuX135aGm+/xsxni309e8EGqowsEPfx7q/YjX+3t15BqD4PtzjnVUdX3N4o1KgAABAgQIECBAgAABAgQI/AUBoeWJXYxhy3LyLJ/Cq8O2HFTVp/SWsCsHaMupwDq0qdvq5/McA57b41GdFjwT5qXFHIZU9ZzTtAZ2sWt/grGe86gtdM7t3TqOePs6i/Nit9s5zbX3XhhnCSK7MdIcZV3HwWTYz71xumHDhs23WFAYs4y/vlV/NvpfBXDUNs+pxjVMXcccX/U2Zd+2NY37r0+b9W/C2DDuGmKecY3vvPD5WCtxRYAAAQIECBAgQIAAAQIECPzfBISWp3e8D4OqjiHQqb8W3J10q96cQxC0CdsG76cQaH23CZDqAQfXse9mkpiEzrc6bM2B2PLqJpiqBj9qi6/lcOyFUKpeU7gOwVz9rJq9uyxBXAmQV6cSnt4f9dfOu8AurOXdr6OHsHAN5boCNrfBvDiGNZTr8GJc247LUVua5NXQsq479b0/62eb0scPYrhdufX32bBe5/yOq9ByTO0pAQIECBAgQIAAAQIECBAgsBUQWm5Ndp68E1ruhFLLSbnqpObm9F4Ie7r+ffDY3+8UGR/vvhvmaVKmFKgt9SynMKuwqkx01FbeefFnCiif8XeFlhrSs/a3U747bAzLSsC4DU9rj3hdTkLmkK0jWU6NlprenT+Gv2X+mFJWzkeB4VHb+7OO38hjPkNoW2p6fZ7RPiS78vs+H/snLXdcY//ucz5eg6cECBAgQIAAAQIECBAgQIDA/11AaHn6E/Cx0DKepKtSsVEYFE+odWFOHbSFEvv7o7J3330vtFwGDSHX6I+thBeO2pYBTl3EOusTqvkr5hXXqXFSTSWgG+xTWHf03bZtrdI75wPLHFJWoXQ6xVmC36PA8Kjt5NI3r6X6m6+g96ckN326BzF0LPV3bcttqL28c8Y1f4a7z/kynAsCBAgQIECAAAECBAgQIECAQCUgtKww3rscBo6h0xKK9SN0AVg+rbgJxAb9+zCtv+9nqu93343zl3Bv9HXxepR0CnMvQAwWbVsOy14JpfowbRSWxWfHv1syrreat1//um/dfsStS19LTytPgexmf1qWzd06/tpU+4Tr/oRrefOoLb2TatrrX8apf47W3/eP7zSBcRlha1Ra1p/9O/19/kp8+wHxh3hWQFcECBAgQIAAAQIECBAgQIDAOwJCy3eAmuYcOm5+H+IgdFz65dCtnL571H8kpmnLX7vN4VsfPPX3y/iDixJIpTnTuEt+1M25PC+nOesTg1XjZsyqLZWQgqv+a+6D8tpHTT3l5F71Sm5vg8Q8V6m1CiybWkp7U2sOAUtb1XezxvjOoKaqvPaU59oQx1rG7updTiiG94/aQvvroWXsFYLSssZm/anGstZNU/Qerbmtc9Ov1FnmXNaeT1iW5+XndoBUmP8SIECAAAECBAgQIECAAAECBOZ5Flr6GBAgQIAAAQIECBAgQIAAAQIECBAg8KMEhJY/ajsUQ4AAAQIECBAgQIAAAQIECBAgQICA0NJngAABAgQIECBAgAABAgQIECBAgACBHyUgtPxR26EYAgQIECBAgAABAgQIECBAgAABAgSElj4DBAgQIECAAAECBAgQIECAAAECBAj8KAGh5Y/aDsUQIECAAAECBAgQIECAAAECBAgQICC0/OrPwPM+T9OU/t2fXz2b8QkQIECAAAECBAgQIECAAAECBAj8egGh5Tdt4dvjNk8vhZbP+T7d5sfbNxX4yWni+ko4e3vMr5b9vIdgt1tvHfgetk1j27fHfAs1veT+Nj9uoZb7/BMi5uTyauhd1pD7NXZHbfPc7OM0zbf6A3i0H5/8/OhOgAABAgQIECBAgAABAgQIEKgFhJa1xhde/+nQMoRZVVD56lrT+/c2pI2BYxVixvsSJIZAt1yHTQtBXPVu3McU+t7vL4bFYZ77cw5h4UtZ5xd8dnrHUFMTIu7MGfo174X92VlMM0dYe7WP81w5H+7HTiEeEyBAgAABAgQIECBAgAABAgQ+KCC0PAEXg50mzJljqLUEQzHQKafapibAK8M34VB5ePjznZOWzam3LmDr66kDq6O2WE+Y95XTienk3jpFOclXh4oHC431hHe79W6CtjqYzHOUSaNFO18J+F51D+/HYTfzhzWUteW9bj4TR21z+MDEXxFQSj4QyU2dR/QZf7Y2Y8W5Soib6lo+q93LxSk9Tntf3m0+9xuPMG6ZoxvULQECBAgQIECAAAECBAgQIEDgkwJCy1OAowBpP7AZndJ7NTxLp9x25oihVBXSxeBvfXc0f1nmUVt6JwVXeyfzyjjrz9om9b0/62frm9urFKilIK/rs4SZudcg9Atrib8vtAkPc0CYn73mHuoprqGech1qSLWOXY7a9uvfelRP6vWX/X2Gr7vXNVXv95exT/LZBKVVW31CNg2R19IH13U94cXBfvQluCdAgAABAgQIECBAgAABAgQIfFRAaHlSLoRf5QRaDGyaJCiFdcsf3Ol/F2CIvC78nZZNLbH+FDSVkuJcfeiU13nUdpKiey2HjTFQK8FpF0B2Pcptu45tn6XWuJZHdbKvDQlTeFnCvDBOuX7RPQRxVQDaBLx9aFcWEX4etdXvvXJdxqwD6vLsvXFin7wXsc/+18qj8bLmYFfezcZLW/f7Lu/1frxXkHYCBAgQIECAAAECBAgQIECAwGsCQsuzXiH8yQFOE2bFQ2ft16lD+xJw5vFjOFRSxVNzbkO80q0N+8LTNrQs75XTcHUQd6pteenMRQ63qqDwXIiXArI66F2uh05VGLkJ79b1N0FnCDuXfyVQ3V9TCj/rPtW+buasxjlqq1577TL7VKFh3M/6fmfAzeevC2PbbpXr5r3QtudW9WsHdEeAAAECBAgQIECAAAECBAgQ+LSA0PIFwuc9BDjP+d4ERykwW0LKGGB9bWiZwsj1NOHmvlnTUbg0agvPqrCuGWt80weyMfxrgsdBsLkZKsy7F5CNjOt3U83NlHn8vrbNtMuD0fzhWXHualj6hYujtvxiPP3Y/e7RZoztTRs+juZIz/q/dN77R4PmM7vO1bTFGst6ywnS2rn0G9VS2vwkQIAAAQIECBAgQIAAAQIECHxeQGj5iuFe8JSfp1N99/lRfZU8hkLLab98im+Urm3qSEHcelKw/SMs7bh1sFSCrPXE4DrdUVspIM+7dioNhz9jUFbWuelb5q0Csc1ofWhY+qR1bIZszLchcRk+Om06l9bqZxhvE+ylGpZAuvwxnLLO5v1uv5q2j/4OyNZgraPUXdp71/K8fAaq9hyqL5+rzqb9XNUhaztm160U5CcBAgQIECBAgAABAgQIECBA4BIBoeUljAYhQIAAAQIECBAgQIAAAQIECBAgQOAqAaHlVZLGIUCAAAECBAgQIECAAAECBAgQIEDgEgGh5SWMBiFAgAABAgQIECBAgAABAgQIECBA4CoBoeVVksYhQIAAAQIECBAgQIAAAQIECBAgQOASAaHlJYwGIUCAAAECBAgQIECAAAECBAgQIEDgKgGh5VWSxiFAgAABAgQIECBAgAABAgQIECBA4BIBoeUljAYhQIAAAQIECBAgQIAAAQIECBAgQOAqAaHlVZLGIUCAAAECBAgQIECAAAECBAgQIEDgEgGh5SWMBiFAgAABAgQIECBAgAABAgQIECBA4CoBoeVVksYhQIAAAQIECBAgQIAAAQIECBAgQOASAaHlJYwGIUCAAAECBAgQIECAAAECBAgQIEDgKgGh5VWSxiFAgAABAgQIECBAgAABAgQIECBA4BIBoeUljAYhQIAAAQIECBAgQIAAAQIECBAgQOAqAaHlVZLGIUCAAAECBAgQIECAAAECBAgQIEDgEgGh5SWMBiFAgAABAgQIECBAgAABAgQIECBA4CoBoeVVksYhQIAAAQIECBAgQIAAAQIECBAgQOASAaHlJYwGIUCAAAECBAgQIECAAAECBAgQIEDgKgGh5VWSxiFAgAABAgQIECBAgAABAgQIECBA4BIBoeUljAYhQIAAAQIECBAgQIAAAQIECBAgQOAqAaHlVZLGIUCAAAECBAgQIECAAAECBAgQIEDgEgGh5SWMBiFAgAABAgQIECBAgAABAgQIECBA4CoBoeVVkheP8/a4zdP9efGo3XBvj/k23ecvnqWb1C0BAgQIECBAgAABAgQIECBAgACBYwGh5bHP0vq8T/M01f++Nuy7IrSMY0zTvJd9hvbb421Z43IRw8zp60PTZUIXBAgQIECAAAECBAgQIECAAAECBFYBoeVqcXgVQsthwHfY6+ONnw4tn/d5ut3n+20vtHzO9+k2bzPL9Px+/4aTnh/n0ZMAAQIECBAgQIAAAQIECBAgQOAPCwgtT27uUWgZ2u7PEPaVk5j1Kcy3+XErz6d5uj3m5mxjOdWY+5ZgNIWWj6pvPeZ7RZdAMs09PGkZQs1BQ1nnp0PT90rUToAAAQIECBAgQIAAAQIECBAgQGBHQGi5A9M/DmHe3tfDU9t6arEEf/OcQsMSRIYx47slLMyBZbmt54yhYfXV7nXM+q3x9fruXmgZnq/1LqPE05kpVBVaLiouCBAgQIAAAQIECBAgQIAAAQIEvllAaHkSfA0Ctx02bcspxnDisTsheTIY7EPD/n5bRX6yzB3ud0LLqoZ1nLbW0/OtA7giQIAAAQIECBAgQIAAAQIECBAgcImA0PIk4yaYrPpt2pbgsA0CY5cqMDwKBvu2/r6avrpMIWV7IjSfEK2+lh7q7U93xvGXr7fXp0oHJzKrGV0SIECAAAECBAgQIECAAAECBAgQuFpAaHlSdBNMVv3atvp0Y7pevx7e3cevh49DwT6k7O+r6Q8u61rya2HOKsDc6/yx+fZG85wAAQIECBAgQIAAAQIECBAgQIDAeQGh5UmrEEy2JxjXr333bWtIGQYPpy2rvv0Rx3DysmovffvQsL8/V/Y2tGwD1v1RPjbf/nhaCBAgQIAAAQIECBAgQIAAAQIECJwVEFqelTp472wQeDDE9zTFk51r2Po9k5qFAAECBAgQIECAAAECBAgQIECAwGsCQsvXvIZv/5rQcli9hwQIECBAgAABAgQIECBAgAABAgR+loDQ8oL9EFpegGgIAgQIECBAgAABAgQIECBAgAABAllAaOmjQIAAAQIECBAgQIAAAQIECBAgQIDAjxIQWv6o7VAMAQIECBAgQIAAAQIECBAgQIAAAQJCS58BAgQIECBAgAABAgQIECBAgAABAgR+lIDQ8kdth2IIECBAgAABAgQIECBAgAABAgQIEBBa+gwQIECAAAECBAgQIECAAAECBAgQIPCjBDTMwMMAABqQSURBVISWP2o7FEOAAAECBAgQIECAAAECBAgQIECAgNDSZ4AAAQIECBAgQIAAAQIECBAgQIAAgR8lILT8UduhGAIECBAgQIAAAQIECBAgQIAAAQIEhJZnPwPP+zxN0/Lv9nirej7ne9U23R7z2nrU9jY/buuY03Sfn8uoR23z/Pa4LbWEuu5rx3k+qvWobT6q9ajtqNajtq9ax4LoggABAgQIECBAgAABAgQIECBA4BcKCC1/4aYpmQABAgQIECBAgAABAgQIECBAgMBfFhBa/uXdtTYCBAgQIECAAAECBAgQIECAAAECv1BAaPkLN03JBAgQIECAAAECBAgQIECAAAECBP6ygNDyL++utREgQIAAAQIECBAgQIAAAQIECBD4hQJCy1+4aUomQIAAAQIECBAgQIAAAQIECBAg8JcFhJZ/eXetjQABAgQIECBAgAABAgQIECBAgMAvFBBa/sJNUzIBAgQIECBAgAABAgQIECBAgACBvywgtPzLu2ttBAgQIECAAAECBAgQIECAAAECBH6hgNDyF26akgkQIECAAAECBAgQIECAAAECBAj8ZQGh5Q/Y3ed9mqfpPj9/QC1KIECAAAECBAgQIECAAAECBAgQIPCvBYSWp3fgbX7cpvn+Bcni1aHl2+M2T19R6GmrF158e8y3KYS24d9tfryd6Zv2IvXJfW+Pue6aTHNbZxF9wnxdnzDz0rbTflRd6dtNd9Tly9pKLdFosM69iZt+0zTf6g15Z6+Wvpv5nvN92eOwJwL6PX/PCRAgQIAAAQIECBAgQIAAgSQgtDz9Sfi60PJ0CSdfjOHRT0jO3q03hFlVUBlDsTOB1vFe9OsPAWYJ32KYeX+mcLIP1573JsjsxzleTqjpPj/DGP/a/qPrCP6NSdifsh9hfeteRZvq3SPX0FaThL5lP45NtRIgQIAAAQIECBAgQIAAAQL/VwGh5YmdjwFNc1KsOxWYg6oY3OT3lpCmOZ3Wne6r26oAKJSUgp1HdUKthEfvFxzrXQro3q/n7E/SzSkMXE8w1nMetcWK40nU0enFroLltq+zOO+VvnTMdY7f64LQOZ/yG/i2tab1rWOW9dYGawWbq+AaO9dB3/pW/dnoTxoetc2l/rWwddDh1WfWkaxKoBj3o3Nbpoyfo63NqE/a1/JumuP0cpYJXRAgQIAAAQIECBAgQIAAAQL/JwGh5end7sOgqmMILafqNFl30q16c+5PncW2wft9gBf6lTCpHm90HfuOUqEYNK2n5Uogtryaw9fRmPNRW+yQQ769kGswaL2mGNzdn9Hn/XWWQDF//bu2r8O0st5n+Ap6Cc1SIdtwLYRpxaYEa/WzwQKqR2G84tjvcVzbjstRWxo+1XL+9GZd8+vrmOvguiyoWme53PuMbV1zj/D5iYF+MS4j+UmAAAECBAgQIECAAAECBAgQ2AoILbcmO0/eCS13QqkSDK6nFwfh415oWYVGeyHRqNjddwfBYwjNlpAwhnw7v3PwqG1UxIlnae5nPKFZamjqOTFGfCUGYjkMK6FlfJaDyvKsGm8bruWwLwacJVirA8Cq8+aye69x7tqavkdtzYsv3OQxP7yO8nk4CKFr266yrWv5PaF5L2LfKuDv+rslQIAAAQIECBAgQIAAAQIECAQBoeXpz8HHQst4kq4KH4ehXAhyutCzDx77+6Oyd99twrQ0wrCe8pXk5eRhPVsIxbqvx9fNL1zHOutTkodf+z4auA7/cn21555v/c5ywrA6kTkIO4dV5CCuDqbXr4DXtfW9j9r6d8/el1OoH1xHYzKoL661GrsrK+5pM8b2/5vdz2c3llsCBAgQIECAAAECBAgQIEDg/ysgtHxh78cB3zzHr043QU0ZNAU25RThnE8rLvfltb1QrQo7Xwl6dt+N85dThPEXZ863YTCZCgvrrUoo1caf27Yclg0dmq7rTR8KjgKxHAhuzNZRNn9Up92nbg9yv224lk8EVgsO45z5WnY7X5qg9jka56gtj5RC4qquaunDy37/R3PEd5rAOH+O66/Rd5+X1Gc/sAzFbF23/qN6hgvxkAABAgQIECBAgAABAgQIEPjfCggtX9n6HDpufjffIHRchs2hW+pznx/1X05u2vLvZ8yhXx889ffL+IOLFC6tv+8xzL1kXt2cy/MSONV/cKhq3IxZtaUSPhBaho5NPYNALLe3oWWeq9S6CUrb9rrvZh1hjGotMVAr41bPB8z50eA0YrFc6mrrWU9hhiGO2iLQy6Fl7BUC14N1FId+ieV56bu2h3VWY+brYtv3i/2Xzl3fxSUT+kGAAAECBAgQIECAAAECBAgQ6ASElh2IWwIECBAgQIAAAQIECBAgQIAAAQIE/q2A0PLf+pudAAECBAgQIECAAAECBAgQIECAAIFOQGjZgbglQIAAAQIECBAgQIAAAQIECBAgQODfCggt/62/2QkQIECAAAECBAgQIECAAAECBAgQ6ASElh2IWwIECBAgQIAAAQIECBAgQIAAAQIE/q2A0PLf+pudAAECBAgQIECAAAECBAgQIECAAIFOQGjZgVx++7zP0zSlf/fn5cMbkAABAgQIECBAgAABAgQIECBAgMBfExBaftOOvj1u8/RSaPmc79Ntfrx9U4GfnCaur4Szt8d8ruywxhzolp+1UR34NhZv8+PW9evmXOrpnr+/zDL2ff4JEfPzXq2ztjlcSFlD6dt/jlb3zZCNeW+wjnv7LR/MQyeNBAgQIECAAAECBAgQIECAwE8VEFp+08786dAyBF1VOHh+rQfB7NtjvtVBZbwvIVoKzzaBW97LGPTdn3Oso6rr1FaHee7POYyxN/6pcS54qXcMNZ0JC0O/5r2wP2Uxi2sKLsvjVG54VoznaLD0m8v7yb4Z/4K1GoIAAQIECBAgQIAAAQIECBAgUAsILWuNnetR+NUESDEIKqfapibAK0P2AVR5vv/zINALnZoTcV3A1tdTJ1NHbbGYFE6tYdV+hamlDxDLabw1/Nof4WCNddAWBwjjlhOD/ZzjGUb7Nn5zfRr6RK7N/OGdsra8100getS27le9Feuso6veJu9LM+eoX5mrtdqGjGm8w3qCwWa+tM7teDu1eEyAAAECBAgQIECAAAECBAgQ+ICA0PIU2ihAKqHQdoDRKb1LQ8sYWFah4HJ6LtUymr9UedSWe6evbB+mWWW08LO2KUFY/ax+t79O7y+/83P3ZGUJ4ko42wWEU3nejv96aBnGLa6htnIdxs1zDl2O2nJNcc/GdbZV57u4p3n+sr/PcPq0rmnYMz2swulhycvJyf0xmmB+eS2tVWi5gLggQIAAAQIECBAgQIAAAQIEvkBAaHkSNQRgS1CzOYXXh2/br/FeGVo2tcT6U5BUwqk4V/gdkeVBtcajtuq1Fy5zQBkDtRLk5mfnfrHlMleqbQ3lllrjWh7VSculS7qIgWCZe22L/TcnBdf2zVUYp3q/CXjrELHveNTWv3v2vowZ11aHl6vP7lC1Rw4vl8/u0il9ZgcfkfjGvp3QciF0QYAAAQIECBAgQIAAAQIECHyZgNDyLG0If3Kg1YRZ8RBgGxCOTqjFEGgvIRrWsB/8hbHaEKoNLZfhYng1/rr68vXyKqRb+r10keae6hOAJXB7aZxwmPHoJGHw2Avsxlb7wdu4sLBv66nPfF327Ki2o7bxVCeeplCxDlHjnp3Yr83nrwtj0+Rp/LK8uqBjN6FlbeWaAAECBAgQIECAAAECBAgQ+BoBoeULrs97OM33nO9NcNSFODHA+tqTlilwrAK8+jTeZj3vhX3VOLFvDstGadZm7PSgD2Rj+Nf0HwSbg7G2/cpLnXF5nH/uhWx7z7vu+Tasuz+tWdsd1XDUVoa/x0C0YRkXsjxtw8fRHOlZExgPQvSxQ9rnvp64B83neyknX4zq6N9xT4AAAQIECBAgQIAAAQIECBD4nIDQ8hW/fHKxD3qWU4vha8zTfX5UJyFjYBSfV6f4NgOMisjhYd23CpPaceuwrQRZ63zrdEdtpYY879qpNBz+jGFXqXXTt8y7E5AO+5U+aR3tkG1bcxoxHti87Z+Y3FvF8DRimmc91drtSbUf6Xd7ruZ9TeUz0q5jr5jyvF3nWkff3ru2/dpQs1tDtM+fnxy496dNU839mKN9KXX5SYAAAQIECBAgQIAAAQIECBD4nIDQ8nN+ehMgQIAAAQIECBAgQIAAAQIECBAgcLGA0PJiUMMRIECAAAECBAgQIECAAAECBAgQIPA5AaHl5/z0JkCAAAECBAgQIECAAAECBAgQIEDgYgGh5cWghiNAgAABAgQIECBAgAABAgQIECBA4HMCQsvP+elNgAABAgQIECBAgAABAgQIECBAgMDFAkLLi0ENR4AAAQIECBAgQIAAAQIECBAgQIDA5wSElp/z05sAAQIECBAgQIAAAQIECBAgQIAAgYsFhJYXgxqOAAECBAgQIECAAAECBAgQIECAAIHPCQgtP+enNwECBAgQIECAAAECBAgQIECAAAECFwsILS8GNRwBAgQIECBAgAABAgQIECBAgAABAp8TEFp+zk9vAgQIECBAgAABAgQIECBAgAABAgQuFhBaXgxqOAIECBAgQIAAAQIECBAgQIAAAQIEPicgtPycn94ECBAgQIAAAQIECBAgQIAAAQIECFwsILS8GNRwBAgQIECAAAECBAgQIECAAAECBAh8TkBo+Tk/vQkQIECAAAECBAgQIECAAAECBAgQuFhAaHkxqOEIECBAgAABAgQIECBAgAABAgQIEPicgNDyc356EyBAgAABAgQIECBAgAABAgQIECBwsYDQ8mJQwxEgQIAAAQIECBAgQIAAAQIECBAg8DkBoeXn/PQmQIAAAQIECBAgQIAAAQIECBAgQOBiAaHlxaCGI0CAAAECBAgQIECAAAECBAgQIEDgcwJCy8/56U2AAAECBAgQIECAAAECBAgQIECAwMUCQsuLQT8y3PM+zdN0n5+vdn7e52ma5vvLHV+d6N+8n1yCzS9Z4x/fj3/zKTArAQIECBAgQIAAAQIECBAg8H8UEFqe2PW3xy0GZyE8q//dHm8ner//itDyyOhtftxeDy3jnn13mnt5aJnWXj5z/eetDnWni9eaxr7N9Ue8me+3BMlHHy1tBAgQIECAAAECBAgQIECAwI8VEFq+tDXP+T61Qc5L3b38AYFfFFp+YHVHXULwugSVb4/5Vn/2QkC6BJXhc/l6sLs3dwp878ef9TD/7TFfE9vvVeI5AQIECBAgQIAAAQIECBAg8H8VEFq+tPM7oWUOkOqTaEueFMOm6oRmHfTUbfXzeZ5TYPWIYVQ6aVd/fbw+gdeHqKHG+/yoTocuwVdYaz1nOTm6FPsORj5JWE7+1d3qta9fdW9ruT9TuLa0h1puj/m5V2ssZy+0LGMl22WNo/VN0xr+zfPc1Nq5N211SHhI84n9OBy3btxzSO+EupPBO+b1kKPr6Bc+a2Gc/rO1dtieZK0Nwp7Un9e1nysCBAgQIECAAAECBAgQIECAwBkBoeUZpeWdnSAnh3lLiHdwCi2ES8t7ZdzB+zEUqk7PraFU6RR+juoJz6b1FNwSQpX3q/kH89ajN9dxjeMQKwZ91aJi7TEMzLXcnzGEnWIIVtVcAsbSt6m1zD4K68Kzupb+PoW+60nEMlYJg9fzgSkczvdh/i7EXHueuarWtrx+tB/LS+9fDG1Kt3rePN+eeeky/Flb12OWl/PYMezuQsnw+Sj7WF73kwABAgQIECBAgAABAgQIECDwQQGh5UtwoyAnHt1bQ8LNeHXQ050KLO8OwsP+JFt/n7qO6umf1fepliVbikFkFz6Vmrqf49A0vLQNDNNpzva03lp/Vc8miBuMFcevgtY4ZfiqdHV6NV8v68onVbchWgrlyknR5efSMfmkcLUDOHVbrW15v3/W3y8vHlykupYymzfTmpaTplWQPTRv+rY34f3ROO1b+a7/7MS9dMJyaOUhAQIECBAgQIAAAQIECBAg8LKA0PIlsp3AaRA6lmH7U4jD8G/Qfw2c0kj9fXo6qqd/1t7HepbArz6tWCoe/xzWHV8dBI1LGLnOvda/PlvDzTLnYKy90PKdE5HrfGXs8HM0ft1erlMQGELNcVBY3ut/Vmtbmvpn/f3y4s5FeH+vjlTnGjSGIdbxV4P12c4kud82CI7B7hBhzzLV+/Hgd79CLQQIECBAgAABAgQIECBAgMD/R0Bo+dJe74Q/g9AxDduFSvk0WhsyjU9qroFTHin83sdNeDSqp39W3Yf53wn7djn6k3XVi6NgNtW6zr2uZ322CS2HjsmwXXoYo/wOx6qQ+nI4Vv59licNQs2bvarn2FxXa1va+mf9/fLi9iJ/Xtq1l9f2DNbxh+al+7s/13GGrx58HsL74TMxrns4mocECBAgQIAAAQIECBAgQIAAgUZAaNlwvHezE+TsBGRxtBjulBNs6Q/kLEFY05bfyYHaGjilmtr7FFgtX2+OJyfLqcm+xva+PWkZ5iz93lt7/j2RyynNOpRKweJSzxIKrnOv9a/Ptn8UqP6qejdm/xXwHOgtc27+8EvbfzHPJzfXftU6+v1Y1vGeTVhT2ePys7hW600fiMM/cFPPtN2r9XeVRs/NnCe+kl9PcHg9qrusba2jDLGpR2JZaPwkQIAAAQIECBAgQIAAAQIEPiAgtPwA2q/tEoO+OhjMJw//VcA0qOfX2iqcAAECBAgQIECAAAECBAgQIEDgMgGh5WWUv2CgeJKwDi3TacT1FOI3r0Fo+c3gpiNAgAABAgQIECBAgAABAgQI/A4BoeXv2KfLqtx85fhfnbIMKxJaXravBiJAgAABAgQIECBAgAABAgQI/CUBoeVf2k1rIUCAAAECBAgQIECAAAECBAgQIPAHBISWf2ATLYEAAQIECBAgQIAAAQIECBAgQIDAXxIQWv6l3bQWAgQIECBAgAABAgQIECBAgAABAn9AQGj5BzbxlSXUv9PyX/46y1dq9i4BAgQIECBAgAABAgQIECBAgMD/S0BoeWq/n/N9muap+nfZX9wOf9H79pjfTtURXkq19IHj2+P2wjjpr4b3Y7xfQnZ4qd73R/3YG+2enF9L22+aqr+mHv8wULXP3Tqj8c5noG2rxvzY4vQiQIAAAQIECBAgQIAAAQIECPyvBYSWp7Y/BF23+VGSxSv/6vXLoeU8x9OSTUr3agj56vsZ6Xmfb4/n/LhVFqf8rn6pr7/bn4Ppgl1NF8LGcQDdzRH2vAkxw5w5nAx7WK7D3B/Y04OSNREgQIAAAQIECBAgQIAAAQIE/ncCQstTW96FYpsAKweJ5RReE26l8Gs9pVlO4YUxq1N95bpO1PZq60OyLkRtT/1Ng1CuC+T25umeP+8prBwGfbGGdT1NEHjUNs9zqveFILQPBaPHaJ3dApa52j0Yk3d7nk+4lnXFmvM+bz2qQHNbgicECBAgQIAAAQIECBAgQIAAAQLvCAgt3wFKzX3A2AZsfWjV3IdAbZyK5aFf/Xp46JbqKcPGAK3cbNYzCtA+Elo+53sJY/vQNoeSwxKO2nKtsf76JOtmDe2Der3x+vaYn+Hr8cMC2r7xLoec02DOVEsKX0tAuY5QBdD1XF2I+up61vFdESBAgAABAgQIECBAgAABAgQIBAGh5anPQQj+qqAyBnHlvgqyymnJ8LOEWjm0a74+XM/ZBV5109H1+hXxQQC5hHLl5GOptYw46FOadn7WQeE8h/7rmG1bO8BRW/vm+bsy5mqQT2sW84OhYt/yVe7sNO6WjNbgMgXF6T611b+LNNaS9//2eKxfHT+oRRMBAgQIECBAgAABAgQIECBAgMBYQGg5dumedqFlDO3K70ZsA7yuY3WbQq/N6b4PhpZzDEPv87P8XGZK86xBXF97eDGFbus7S+edixzS1aHstH4du4SIo85HbaP3Tz3LYeMaKKav59f343G26z6sL8xTkDb7NHLNs4Y9KadSx4V4SoAAAQIECBAgQIAAAQIECBAgcCAgtDzAWZu6gCoGhetJw3jK7mRIFd4tOVgcfxM6rrMeX6UA7nbrvhbd1ZZOFq61pjG34d3hXKMaw7Oy5m7OZqyjtvziuMZmlO7meD/Sy+GdqfuL6tls+YtKoz9qVKbq3o1BafldmPGXY863+vRt6dZ9dX957IIAAQIECBAgQIAAAQIECBAgQOC0gNDyFFUOwJaThuMQcP1jO2swmQK58jXt6mvj1bz1V4uXk31V+95lGbsJQeMfr67ne1Rf5U5BXF1nuO779/PFeTYvJZPlcT79WMZuTj0etS1/HKc37avo7mMYWtY56jsKLcMY3V6W4HWpo4y5dSneZY3L2psxR7V0tbslQIAAAQIECBAgQIAAAQIECBA4FBBaHvJoJECAAAECBAgQIECAAAECBAgQIEDguwWElt8tbj4CBAgQIECAAAECBAgQIECAAAECBA4FhJaHPBoJECBAgAABAgQIECBAgAABAgQIEPhuAaHld4ubjwABAgQIECBAgAABAgQIECBAgACBQwGh5SGPRgIECBAgQIAAAQIECBAgQIAAAQIEvltAaPnd4uYjQIAAAQIECBAgQIAAAQIECBAgQOBQQGh5yKORAAECBAgQIECAAAECBAgQIECAAIHvFhBafre4+QgQIECAAAECBAgQIECAAAECBAgQOBQQWh7yaCRAgAABAgQIECBAgAABAgQIECBA4LsFhJbfLW4+AgQIECBAgAABAgQIECBAgAABAgQOBYSWhzwaCRAgQIAAAQIECBAgQIAAAQIECBD4bgGh5XeLm48AAQIECBAgQIAAAQIECBAgQIAAgUMBoeUhj0YCBAgQIECAAAECBAgQIECAAAECBL5bQGj53eLmI0CAAAECBAgQIECAAAECBAgQIEDgUEBoecijkQABAgQIECBAgAABAgQIECBAgACB7xYQWn63uPkIECBAgAABAgQIECBAgAABAgQIEDgUEFoe8mgkQIAAAQIECBAgQIAAAQIECBAgQOC7BYSW3y1uPgIECBAgQIAAAQIECBAgQIAAAQIEDgWEloc8GgkQIECAAAECBAgQIECAAAECBAgQ+G6B/wAq/cZ81QeobQAAAABJRU5ErkJggg=="}}},{"cell_type":"code","source":"plt.plot(loss_train)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:15:11.352834Z","iopub.execute_input":"2024-05-21T16:15:11.353335Z","iopub.status.idle":"2024-05-21T16:15:11.553867Z","shell.execute_reply.started":"2024-05-21T16:15:11.3533Z","shell.execute_reply":"2024-05-21T16:15:11.552931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save the model and model weights\ntorch.save(model, '/kaggle/working/best_model.h5')\ntorch.save(model.state_dict(), '/kaggle/working/best_model_weights')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:08:49.452219Z","iopub.execute_input":"2024-05-21T16:08:49.452956Z","iopub.status.idle":"2024-05-21T16:08:50.664626Z","shell.execute_reply.started":"2024-05-21T16:08:49.452917Z","shell.execute_reply":"2024-05-21T16:08:50.663357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unfreeze = True # to freeze, set it as False\nfor param in model.parameters():\n    param.requires_grad = unfreeze","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:08:50.666372Z","iopub.execute_input":"2024-05-21T16:08:50.666797Z","iopub.status.idle":"2024-05-21T16:08:50.675479Z","shell.execute_reply.started":"2024-05-21T16:08:50.666759Z","shell.execute_reply":"2024-05-21T16:08:50.674384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_params = sum(p.numel() for p in model.parameters())\nprint(f'{total_params:,} total parameters')\ntrainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\nprint(f'{trainable_params:,} training parameters')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:08:50.676768Z","iopub.execute_input":"2024-05-21T16:08:50.677096Z","iopub.status.idle":"2024-05-21T16:08:50.693683Z","shell.execute_reply.started":"2024-05-21T16:08:50.677069Z","shell.execute_reply":"2024-05-21T16:08:50.692713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:08:50.694862Z","iopub.execute_input":"2024-05-21T16:08:50.6952Z","iopub.status.idle":"2024-05-21T16:08:51.003754Z","shell.execute_reply.started":"2024-05-21T16:08:50.695174Z","shell.execute_reply":"2024-05-21T16:08:51.002744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(model, dataloader, device):\n    # set mode to eval\n    model.eval()\n    fin_out = []\n    \n    with torch.no_grad():\n        for images, targets in dataloader:\n            images = images.to(device)\n            targets = targets.to(device)\n            \n            outputs = model(images)\n            \n            fin_out.append(F.softmax(outputs, dim=1).detach().cpu().numpy())\n            \n    return np.concatenate(fin_out)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:08:51.005222Z","iopub.execute_input":"2024-05-21T16:08:51.005535Z","iopub.status.idle":"2024-05-21T16:08:51.015494Z","shell.execute_reply.started":"2024-05-21T16:08:51.005508Z","shell.execute_reply":"2024-05-21T16:08:51.014673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# steps for model prediction\ndevice = torch.device('cuda') # if you don't have gpu, set it as cpu\nmodel.to(device)\npred = predict(model, val_loader, device)\npred = pred.argmax(axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:08:51.016486Z","iopub.execute_input":"2024-05-21T16:08:51.016764Z","iopub.status.idle":"2024-05-21T16:09:26.557869Z","shell.execute_reply.started":"2024-05-21T16:08:51.016741Z","shell.execute_reply":"2024-05-21T16:09:26.556526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test.values[:5]","metadata":{"execution":{"iopub.status.busy":"2024-05-21T07:55:02.054508Z","iopub.execute_input":"2024-05-21T07:55:02.05522Z","iopub.status.idle":"2024-05-21T07:55:02.061624Z","shell.execute_reply.started":"2024-05-21T07:55:02.055188Z","shell.execute_reply":"2024-05-21T07:55:02.060702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred[:5]","metadata":{"execution":{"iopub.status.busy":"2024-05-21T07:56:05.263069Z","iopub.execute_input":"2024-05-21T07:56:05.263766Z","iopub.status.idle":"2024-05-21T07:56:05.270378Z","shell.execute_reply.started":"2024-05-21T07:56:05.263733Z","shell.execute_reply":"2024-05-21T07:56:05.269408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test.values","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:00:33.821199Z","iopub.execute_input":"2024-05-21T08:00:33.821581Z","iopub.status.idle":"2024-05-21T08:00:33.828783Z","shell.execute_reply.started":"2024-05-21T08:00:33.821551Z","shell.execute_reply":"2024-05-21T08:00:33.827549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mapping.keys()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T07:59:38.365071Z","iopub.execute_input":"2024-05-21T07:59:38.366059Z","iopub.status.idle":"2024-05-21T07:59:38.371795Z","shell.execute_reply.started":"2024-05-21T07:59:38.366018Z","shell.execute_reply":"2024-05-21T07:59:38.370855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:09:26.562229Z","iopub.execute_input":"2024-05-21T16:09:26.562631Z","iopub.status.idle":"2024-05-21T16:09:26.567557Z","shell.execute_reply.started":"2024-05-21T16:09:26.562601Z","shell.execute_reply":"2024-05-21T16:09:26.566619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_test.values, pred))","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:09:26.599092Z","iopub.execute_input":"2024-05-21T16:09:26.599734Z","iopub.status.idle":"2024-05-21T16:09:26.621922Z","shell.execute_reply.started":"2024-05-21T16:09:26.599703Z","shell.execute_reply":"2024-05-21T16:09:26.621009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Given groups=1, weight of size [256, 3, 3, 3], expected input[32, 256, 3, 256] to have 3 channels, but got 256 channels instead","metadata":{}},{"cell_type":"markdown","source":" only batches of spatial targets supported (3D tensors) but got targets of size: : [32]","metadata":{}},{"cell_type":"markdown","source":"Given groups=1, weight of size [256, 3, 3, 3], expected input[32, 256, 256, 3] to have 3 channels, but got 256 channels instead","metadata":{}},{"cell_type":"code","source":"model_weights = [] # we will save the conv layer weights in this list\nconv_layers = [] # we will save the 49 conv layers in this list\n# get all the model children as list\nmodel_children = list(model.children())","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:24:00.428651Z","iopub.execute_input":"2024-05-21T17:24:00.429048Z","iopub.status.idle":"2024-05-21T17:24:00.434172Z","shell.execute_reply.started":"2024-05-21T17:24:00.429019Z","shell.execute_reply":"2024-05-21T17:24:00.433068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# counter to keep count of the conv layers\ncounter = 0 \n# append all the conv layers and their respective weights to the list\nfor i in range(len(model_children)):\n    if type(model_children[i]) == nn.Conv2d:\n        counter += 1\n        model_weights.append(model_children[i].weight)\n        conv_layers.append(model_children[i])\n    elif type(model_children[i]) == nn.Sequential:\n        for j in range(len(model_children[i])):\n            for child in model_children[i][j].children():\n                if type(child) == nn.Conv2d:\n                    counter += 1\n                    model_weights.append(child.weight)\n                    conv_layers.append(child)\nprint(f\"Total convolutional layers: {counter}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:24:20.617151Z","iopub.execute_input":"2024-05-21T17:24:20.617529Z","iopub.status.idle":"2024-05-21T17:24:20.626495Z","shell.execute_reply.started":"2024-05-21T17:24:20.617499Z","shell.execute_reply":"2024-05-21T17:24:20.625448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for weight, conv in zip(model_weights, conv_layers):\n    # print(f\"WEIGHT: {weight} \\nSHAPE: {weight.shape}\")\n    print(f\"CONV: {conv} ====> SHAPE: {weight.shape}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:24:25.108535Z","iopub.execute_input":"2024-05-21T17:24:25.108928Z","iopub.status.idle":"2024-05-21T17:24:25.117896Z","shell.execute_reply.started":"2024-05-21T17:24:25.10887Z","shell.execute_reply":"2024-05-21T17:24:25.116748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 17))\nfor i, filter in enumerate(model_weights[0]):\n    plt.subplot(8, 8, i+1) # (8, 8) because in conv0 we have 7x7 filters and total of 64 (see printed shapes)\n    plt.imshow(filter[0, :, :].cpu().detach(), cmap='gray')\n    plt.axis('off')\n    plt.savefig('/kaggle/working/filter.png')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:26:34.848031Z","iopub.execute_input":"2024-05-21T17:26:34.848914Z","iopub.status.idle":"2024-05-21T17:26:58.693745Z","shell.execute_reply.started":"2024-05-21T17:26:34.848859Z","shell.execute_reply":"2024-05-21T17:26:58.69285Z"},"trusted":true},"execution_count":null,"outputs":[]}]}