{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":39763,"databundleVersionId":11756775,"sourceType":"competition"},{"sourceId":11568812,"sourceType":"datasetVersion","datasetId":7253205},{"sourceId":11569667,"sourceType":"datasetVersion","datasetId":7253605},{"sourceId":11569755,"sourceType":"datasetVersion","datasetId":7253661},{"sourceId":11887338,"sourceType":"datasetVersion","datasetId":7377931},{"sourceId":11910577,"sourceType":"datasetVersion","datasetId":7487776}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"RUN_TRAIN = False # bfloat16 or float32 recommended\nRUN_VALID = True\nRUN_TEST  = True\n\nimport torch\nif not torch.cuda.is_available() or torch.cuda.device_count() < 2:\n    raise RuntimeError(\"Requires >= 2 GPUs with CUDA enabled.\")\n\ntry: \n    import monai\nexcept: \n    !pip install --no-deps monai -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:48:31.063057Z","iopub.execute_input":"2025-05-22T15:48:31.063292Z","iopub.status.idle":"2025-05-22T15:48:38.957893Z","shell.execute_reply.started":"2025-05-22T15:48:31.063268Z","shell.execute_reply":"2025-05-22T15:48:38.957131Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ConvNeXt Baseline Notebook\n\nThis notebook builds on [HGNet-V2 Starter](https://www.kaggle.com/code/brendanartley/hgnet-v2-starter) notebook. I recommend reading that one first, as many components are reused here. The differences here are that we train on the full-size seismic data and adapt a ConvNeXt model for the non-square input shape. \n\nIn addition, I provide 2x pretrained model checkpoints (trained for 50 epochs), which both achieved a validation MAE of ~34.\n\nOther stuff:                                                                 \n- Activation swaps\n- Normalization swaps\n\nNOTE: ConvNeXt is unstable on float16 (loss becomes NAN). Use bfloat16 or float32 if possible.\n\n#### Updates\n\nV2: [@Harshitsheoran](https://www.kaggle.com/harshitsheoran) shared some further finetuned weights with the community. Cheers! 🔥","metadata":{}},{"cell_type":"code","source":"%%writefile _cfg.py\n\nfrom types import SimpleNamespace\nimport torch\n\ncfg= SimpleNamespace()\ncfg.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ncfg.local_rank = 0\ncfg.seed = 123\ncfg.subsample = None\n\ncfg.backbone = \"convnext_small.fb_in22k_ft_in1k\"\ncfg.ema = True\ncfg.ema_decay = 0.99\n\ncfg.epochs = 1\ncfg.batch_size = 16\ncfg.batch_size_val = 16\n\ncfg.early_stopping = {\"patience\": 3, \"streak\": 0}\ncfg.logging_steps = 100","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:48:38.958869Z","iopub.execute_input":"2025-05-22T15:48:38.959445Z","iopub.status.idle":"2025-05-22T15:48:38.965658Z","shell.execute_reply.started":"2025-05-22T15:48:38.959420Z","shell.execute_reply":"2025-05-22T15:48:38.965054Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Dataset\n\nAlmost the same as the starter notebook. \n\nThe input dataset changes to Egor's [openfwi_float16_1](https://www.kaggle.com/datasets/egortrushin/open-wfi-1) and [openfwi_float16_2](https://www.kaggle.com/datasets/egortrushin/open-wfi-2) datasets.","metadata":{}},{"cell_type":"code","source":"%%writefile _dataset.py\n\nimport os\nimport glob\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\n\nimport torch\n\nclass CustomDataset(torch.utils.data.Dataset):\n    def __init__(\n        self, \n        cfg,\n        mode = \"train\", \n    ):\n        self.cfg = cfg\n        self.mode = mode\n        \n        self.data, self.labels, self.records = self.load_metadata()\n\n    def load_metadata(self, ):\n\n        # Select rows\n        df= pd.read_csv(\"/kaggle/input/openfwi-preprocessed-72x72/folds.csv\")\n        if self.cfg.subsample is not None:\n            df= df.groupby([\"dataset\", \"fold\"]).head(self.cfg.subsample)\n\n        if self.mode == \"train\":\n            df= df[df[\"fold\"] != 0]\n        else:\n            df= df[df[\"fold\"] == 0]\n\n        \n        data = []\n        labels = []\n        records = []\n        mmap_mode = \"r\"\n\n        for idx, row in tqdm(df.iterrows(), total=len(df), disable=self.cfg.local_rank != 0):\n            row= row.to_dict()\n\n            # Hacky way to get exact file name\n            p1 = os.path.join(\"/kaggle/input/open-wfi-1/openfwi_float16_1/\", row[\"data_fpath\"])\n            p2 = os.path.join(\"/kaggle/input/open-wfi-1/openfwi_float16_1/\", row[\"data_fpath\"].split(\"/\")[0], \"*\", row[\"data_fpath\"].split(\"/\")[-1])\n            p3 = os.path.join(\"/kaggle/input/open-wfi-2/openfwi_float16_2/\", row[\"data_fpath\"])\n            p4 = os.path.join(\"/kaggle/input/open-wfi-2/openfwi_float16_2/\", row[\"data_fpath\"].split(\"/\")[0], \"*\", row[\"data_fpath\"].split(\"/\")[-1])\n            farr= glob.glob(p1) + glob.glob(p2) + glob.glob(p3) + glob.glob(p4)\n        \n            # Map to lbl fpath\n            farr= farr[0]\n            flbl= farr.replace('seis', 'vel').replace('data', 'model')\n            \n            # Load\n            arr= np.load(farr, mmap_mode=mmap_mode)\n            lbl= np.load(flbl, mmap_mode=mmap_mode)\n\n            # Append\n            data.append(arr)\n            labels.append(lbl)\n            records.append(row[\"dataset\"])\n\n        return data, labels, records\n\n    def __getitem__(self, idx):\n        row_idx= idx // 500\n        col_idx= idx % 500\n\n        d= self.records[row_idx]\n        x= self.data[row_idx][col_idx, ...]\n        y= self.labels[row_idx][col_idx, ...]\n\n        # Augs \n        if self.mode == \"train\":\n            \n            # Temporal flip\n            if np.random.random() < 0.5:\n                x= x[::-1, :, ::-1]\n                y= y[..., ::-1]\n\n        x= x.copy()\n        y= y.copy()\n        \n        return x, y\n\n    def __len__(self, ):\n        return len(self.records) * 500","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:48:38.967116Z","iopub.execute_input":"2025-05-22T15:48:38.967317Z","iopub.status.idle":"2025-05-22T15:48:38.980829Z","shell.execute_reply.started":"2025-05-22T15:48:38.967301Z","shell.execute_reply":"2025-05-22T15:48:38.980105Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model\n\nThe model uses the `ConvNeXt` backbone from timm. See more info on this backbone [here](https://huggingface.co/timm/convnext_small.fb_in22k_ft_in1k) and the original paper [here](https://arxiv.org/abs/2201.03545). We modify the stem to aggressively downsample the height and we replace normalization layers with `InstanceNorm2d`.\n\n# ![convnextblock.png](attachment:ccaa394e-27dd-4fd2-8054-7a02bf0dd129.png)\n\n### Encoder\n\nFor the unet, we typically want the encoder to downsample by a factor of 2x at each stage. This works best when the input is square so that we can use as little padding as possible. In the original notebook, we did this by interpolating the input data. This worked okay, but we lost a lot of detail as a result. Here we rely on the stem to downsample using convolutions. See the `update_stem()` function for more details. \n\n#### Normalization\n\nMost CNNs use `BatchNorm2D`, which relies on batch statistics when computing normalization. However, replacing this with a batch-independent normalization layer like `InstanceNorm2D` or `LayerNorm` can improve convergence speed and stabilize validation performance. ConvNeXt uses `LayerNorm` by default, but we use `InstanceNorm2D` instead.\n\nSince normalization is now independent of batch statistics, smaller batch sizes can be used without a drop in performance. For example, a batch-size of 16 uses ~9GB of vRAM during training.\n\n### Decoder\n\nThe decoder is mostly the same. One small change is that we did not use intermediate convolutions here.","metadata":{},"attachments":{"ccaa394e-27dd-4fd2-8054-7a02bf0dd129.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAZ4AAAGCCAYAAADQaKXvAAABUmlDQ1BJQ0MgUHJvZmlsZQAAGJV1kEEog3EYxn/fTCtNHBwUagfJYaRR3LTtgHL4Mordvn1m1Pbt37dJ7pTj4qQcxF1OXB3cHBwsZS1XF06ixPq834Zt5F9Pz6+n5//29oIHQ6m0F8hYeXtuKhJYXIoHfA+04KedHjTDzKmwrs9KhW9vfq+3aK7fDLmzHu+Cscnd8pO66ns78RRLf/tNr205mTPFP0SDprLzoPUL6xt55bKILluWEt5xOVXjA5cTNT6tdubnosKXwp3mqrEsXBQOJhryVANn0uvm1w7u9v6ktRBzXdTLNDoB4oQY/6c3Vu1FyaLYxGaNFKvk5VdYEkWapPAMFibDBIVDjIhG3fv+vls9yx7BxAu0FOpZYh/Ot6G7VM/6D6FjC86ulWEbP9fUXr25ldFQjf0RaL13nOcB8O1BpeA470eOUzmW+WW4sD4Bo4NkHvd1zqAAAACKZVhJZk1NACoAAAAIAAQBGgAFAAAAAQAAAD4BGwAFAAAAAQAAAEYBKAADAAAAAQACAACHaQAEAAAAAQAAAE4AAAAAAAAAkAAAAAEAAACQAAAAAQADkoYABwAAABIAAAB4oAIABAAAAAEAAAGeoAMABAAAAAEAAAGCAAAAAEFTQ0lJAAAAU2NyZWVuc2hvdHQQPy8AAAAJcEhZcwAAFiUAABYlAUlSJPAAAAHWaVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA2LjAuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOmV4aWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vZXhpZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjM4NjwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj40MTQ8L2V4aWY6UGl4ZWxYRGltZW5zaW9uPgogICAgICAgICA8ZXhpZjpVc2VyQ29tbWVudD5TY3JlZW5zaG90PC9leGlmOlVzZXJDb21tZW50PgogICAgICA8L3JkZjpEZXNjcmlwdGlvbj4KICAgPC9yZGY6UkRGPgo8L3g6eG1wbWV0YT4K4PyUdQAAABxpRE9UAAAAAgAAAAAAAADBAAAAKAAAAMEAAADBAAA69lHfjXoAADrCSURBVHgB7F0JvE5F/5+y71oQEaFIlkKikqVs9dKCLFlSaZGUNiX1CkWiRSTbq5KWN1siWyFlK0IIWVNKi/Lal2r+v+/UnP+c85zn3uee+5zlufc3n8+95zxzZjvfmTnf+f1m5jenSHKCHSPACDACjAAjEBACpzDxBIQ0Z8MIMAKMACOgEGDi4YbACDACjAAjECgCTDyBws2ZMQKMACPACDDxcBtgBBgBRoARCBQBJp5A4ebMGAFGgBFgBJh4uA0wAowAI8AIBIoAE0+gcHNmjAAjwAgwAkw83AYYAUaAEWAEAkWAiSdQuDkzRoARYAQYgdCI5+DBg2Lv3r1xa+CUU04RRYsWFWeccYbAfRhu69atVrZ58+YVZcqUsX7rm+PHj4vdu3ernyhn+fLlxamnnqof+349cOCA+Omnn+LmkzNnTnHOOeeIHDlyuIZBXKQBd/rppyu8XQP65Pn777+LEydOqNRR33ny5PEpJ06WEWAEooJAaMTz+uuvi1tuuSVdHPAxvP3228XAgQNF7ty50w2fzAAm4eGD+PXXXytiMfNYs2aNqFmzpuW1efNmUalSJet3Rm527Nghjhw5oqIUK1ZMlChRIt3o48ePF927d08zHEizSpUq4t57743BvGvXruKNN95Q8Vu1aiXef//9NNNK9sOmTZuKBQsWqGT79OkjhgwZkuwsQknvl19+EV999ZXYsmWLOPvss8VFF10kypYtG0pZMptpkAOwffv2id9++80qcv78+RV+lsc/N99++601YIEXBnzxBlfOuM7fPHhzIhLAb5jMCcO99tprsBGX8F+9evXkyZMnAy2qs3ytW7eOyf/LL7+0vQMRT0yYRD2uuOIKKy36CCcUbdy4cVYcZ3ndfj/wwAO2dLt06WLFb9mype1ZED+aNGli5f/QQw8FkaWveWzfvl02bNjQeiezDk477TTZq1evwNtxZl/YfAcagEm8o9Mlqx/MnTvXhl2BAgXkjz/+aMuOBnuStApWOJKU5c8//6zCoGzr169Xf6RRscWL9yORPkSDN0kDTDlx4sSYZMw+RIO3mOd+e5h9KNHvht9lSi99kV4Av547iad06dLy3HPPVX80MpS5cuWyGpZu+M8//7xfxXFNV+drXhcvXmwLm6wOh0STQTzlypWzcDzrrLMkSW02HNGBDh8+bL2D2WmYeCxYPN1MmTJF4kNpthe3+0aNGslff/3VUx5hRHK+g98DsHbt2tkwvOOOO2yv3bhxY9vzsWPHWs+T0Yec7+v8zYM3C27PN5Ehnp07d9peguZO5Jtvvmkb2Vx99dW2MPrH/v375ZIlS+SiRYvk//73P+3teoXUBKlk1qxZcvbs2XLt2rUSebk5Z4PD74svvlj++eefVvBEiAfhv/nmG5UfqV7kX3/9ZcU3b5LRaY4ePWomKUl1J4cPH27rqKROs8IkQjzHjh2TeM+33npLklpMJjqSJLWJ/OSTT+SKFStiRq26AOZoLZUlHlKtSVIL23AuWLCgkn4uvfRSCcI321OtWrXitgONTVSuZrn1vZ8DMEg4RYoUsfAiFZokNbeCA21XlwHXK6+80oZjMvoQD978b3mRJR796jVq1LAaWoUKFbS3ukKsbtCggfUcDREj/AsvvFB97GyB6cfUqVMlGpXZcHFP8ylywIABzuAx4XS8CRMmWGHTIh4QzEsvvSQLFSpkSwudqkePHpbK5b333pOlSpWSNIdlhcNHC37maM7K1LhxqgmcxIOgtPjBShfvAELXLi3iAeGgnOj4+t319YILLpBQebg5DADOO++8mDj4AM+fP98WJR7xYFBQuHBhCRzwR3NU0u3dbImF+IPmK23v+69//csmWdI8iaxataotDOrdze3Zs0dC5fT2229LmieKOzByiws/1MuyZctcpSq0Sf3nFl8/w1U7Xefm1c8BGPJ95ZVXbFhBGqdFKPL888+3/KH227Rpky6muiaDeJztjAdvNoiT8iPyxEMT9VZDa9OmjfXS+ODjg2R2BvOeJiWVVKMjYKTkVDuZ4XH/5JNP6uDqaj6Hflf/hgqLJiRVmLSIp2PHjlYcHde8XnPNNfLQoUNyzJgxccO98MILtjI5f6RHPBiJ33XXXVb6tMpN0uStlUw84oE6Dh8Xs7zOe3R8U3pCorTYwSalOuOAXD/77DMrfzfiwYfTrFta2RjzgbESiMDN999/b2tbGNy4zUd+9NFHNjxvu+02W+mXLl2q1KROzKB2hu7emeZNN92kBgUYGDzxxBPy0UcflbSQwcoD8yCoX60FgJRipt2/f39b/iA88zkt9FDPTT/zPtkDMLMw0BLUrVvXVp4bbrjB9vupp56yovDgTShsTK1BlAdvkSEefIwwMsffTlK7YdR8zz332BoaRkHaXX755bZnkHwwsYcPq+4ckCz06KVatWqWP0jk1Vdflf/+979tEhAmKf/44w+dhRUe6WFOqmLFipbfY489psLFIx6M7HU5cMXHk1aQ2fKD/zPPPCOXL18u0WDMjwbKC7/PP//cKo/bjZN4aMm3pOXT6g8Sk1kG3N988822ZOIRD60itMWFFNm7d28J/b5J4MhPY0xLo23qJpDMVVddJTFnZ5ajTp06VhmcxPPdd98pSU+Hx5wJVHVRdlA/6vLimtZcJNoR2h7+EE87Wlloa7tmevq+fv36NvJBXehn5mS79tPXbt26qWzwMcdcqvaHBGo6DBr0M1yhFoYz/fwcgJllwf26deviYgIJ2FSR8+DNTjxRH7xFhnjMxu12D2LRHzjnyG3UqFFWm3V+BN555x2l8jDT7NmzpzVPs3r1aqWaw4cVf7t27bLSMuNgfsPUL0NnD4KMRzzmyiYQgS47VBimOqBy5cpWfqZ/oqtTnMRjltl5D9UVLUO18sONG/FA2jHVg/jg6fIjjvMDNXLkSHjLwYMH2z5S7777rvIHmWOCWJcHI3Q9CjeJB2GqV69uhcNIHyqnqDu8v343XL/44osMFRmEjYGJTgODJwwQsAKOlglb/nhutnWTePAMUuKNN94ooRkwiQjpQU0F9/DDD1vpIYy5yAFxdRkuueQS6x20H65+DsCsDI0bs7y6HBj4mFIzgvPg7f+JJxUGbylBPGeeeab84YcfrOY4evRoq4OgMWpSADHMmDFDQgWkG2mHDh2UTttU3eBZ8eLFJVQVL7/8sty2bZuVtnmj09B54BkWOGj/tm3bxiUe2oNjhWvevLkiLZQPf5B8dBq46pGl38SDvIAlyFY7N+JZtWqVrXzOjz9GzqYUg/eBw8dSvxekQ9OB0KG60X96+atJPDquvqJuUsHR/ijrvVF25/Lf9N4B84v6nXGFikQ7EDQGRPo52q2efzGJB2S+cOFCHU1J8zoOrnouBFKE6Y95JDgQEwYm+hnmJrXTfrjqvqb9/BiA6XxxxSAIc7A6P1yhcovn/O5DPHiLh3zG/CNDPFjlA/UZ/mrXrq0+kGZjgxpKz6tgOaP5LK17fPThnGo7ZxxIVHrljIbQDIMOB4c9AuZkOzqoGQ4r5lBO0y+9e61KSkanwUSoniAGQWABxrBhw2xlxoIB7dyIZ/LkybbyYw7D6TA/pd9Lq2xMnXyLFi2cUVx/p0U8nTp1co0TNc+7777bwgKYYF4tIw4LEZxYmvExl6Kf4wpJG84kHqieTAcSMuOYKltT7Yz6h1tEqm0dHhISWbSwktP+uOp+4OcAzMqYblauXGmT3lAGqNDjkXsy+pD5vm73PHgza8jbfWSIR3cm/RqYSHWuFJo2bZp6DFWZ2SAwlxDvD2v+4aDuwRyROY9ipoF7qDvMpcLmc93hkJZJYqZKA+FBPPjwmHERJl754I9OD5eMTmOqxFSi//wz08bKIO3ciGfSpEm28pvSpo537bXXWmH0nA0+fvq9obZJxKVFPEjLqVJJJM2gwziXq0P9mhFnrgAkKx0xUUEaGldc58yZo8KYxOPE26kCNokHUqdOD5I5BiqmSguDCtPpsLjqfuDnAEznjW+AuarVLAe0FW7ObOde1dU8eHNDNrl+kSUevCb25piNTc8lmFIGJksTcVoKAAGhE2JSH6M2czEC8nrxxRet5My8dYfDQ+jFsQvdfK7vteUCc08H5kQSccnoNPGIx1yhhjkn7dyIx/mhMyfBEQ9YmnMPnTt3VsmZZIS5GtNBndivXz/rT4+oTeKBJAm1k1knZGrGtuDDTDMq9+bcH9oBJrrdHD6kwAoqWvzp+RpsE9Dtx7lZEulgzkg/x/XDDz9UyZvE41w0khbxYAGPuUAEYU11ntnWkZGZt/nMrwGYejn69+yzz9ryNsuBe1MlqeP42YfMtHnwphH3do008eBjZTY2qIzg5s2bZ/M35yCg5sLkKj6C+MNH3zmCN+d0MPdgfugefPBBC0kzb7PDIYBJfmY4TTww8aP9nauRyE6dVT6UUZOF2bCxgiwR51xcQMZX1confOSw6ge6/ccff9wqC8pEtt2spN2IB2nky5fPioOVaXpyGhGxD0i/G656yff9999v+QNTc5SNiXIdB1In1IBwJvFg2TccVIE6LK76A60eRvAf2pNZXhAJ8Hc6ZzvUAylTbQlVs9M5rXxokzVeiQfpm4tf7rzzTqv8WFRiWrZAWPPdzH7g1wAMeeIdzTYIbQSIxiwLlq07y2r2Ia8Sj+6PKIfpePBmopG5+0gTD9RvZkMbNGiQeluIwuZGMnQWqCjwgTXVFoiL0eHGjRtt6UA1hHmMDz74wKZiQHh8HLQz8zY7HJ7jw4JNlGYY3GvicerlkSf2HYAUoV7T8bSaCmliF7b2h0Tx9NNPqw2EeBbPOYlHx0/rOnPmTCs5N+LBw759+1plQVqYd8NeESwkMOe4sKcJe5HgoH4xVY9Q40At2qxZM5s/0tbOJB69BwELD8yJbkiXGZ030ekHdW3fvr0NL8zbwKKGdpirMNW8kDiwbwYOkqBZXx9//LGOpj6sproJkrQm7cwQj3Nlos5fLxSxCkA3+hmuzn7gxwAMeZPxWFu+WH4O5zSno9uMekj/TOLhwdvfK91Qb1EbvEWaeJyT9Fqlg0aG5ZPmB9DsHPr+1ltv1e1RYqJa+8e7grT0AgZENMM5OxyeQ9duhsG9Jh48N1VPznD4jX1DWGWkHUZoznBamtBhnNeMEs99991nSyIe8WA1FZZ6O8tj/oZUg+XqpgPmZhjnPVbDQdWjnRvx4Jk5D4E03FRQOo0oXCE5O83i4DcWXmDuyyRkvI/eB4ayg1RNEzFYlQkpBARtzpshHuaTtMsM8WAJt7n6U9eTU62KvPQzXJ39wI8BmFOiNq0kkAV3W7nRBk0LGjx4+9vQbtQHb5EmHjR65yjRXGEF3bRzIyk6BzYd6hES0oDDqBwTqKb4rjsURp+YnEWjNp2pgnN2OB3OVJMgPdhk0w4mZyC1uBmOhKrD/AAjDsyqmKNbpJcZ4sF7IW+ofjACh8kgpzMXcDiNhAIzPDfnAzRmMOgK8nc6jMax+dRtUIDRqJ7b0fFM/MzRK7AzzRvhww3JNcoOJGxKahor5xVzi+ZGZbwT2quTnJzxIHWamyYzQzzI04yPvLDhWEtTeK6dWQ63fpDMARjs+zmXT8Oig+nQTswyQWugy82Dt7+JB3hFefAWGvGYDSmz9xhtwuintm1lzkc408ZID8uXsbkRu8Wxogwbrvx0+IBjvgMjOZhNcX58nXmjjJibARGZEpgzXFC/UQaQzMSJExXOwBsLDNJymCeCegnzE1B3pvfOaaWVSs8weMHSfDeyhlQzYsSIGNLR74e2UbJkSdtHFR9YpIW9QiBj05nE4VSRORcX4LfTTZ8+3ZaXOb9phg1yAGYuWMC7uy2pd264RTi9gIcHb/9PPFEevIV2EBw1FnaMQJZFgAYbgua8xIYNGwQOMyOVmSDVpSCJO813pg++oLlNFY8+sCoe4pLkmma8VHhICwHUYYqkFRA0NyhoP5GgDbG+FJ3m19QJx0Sa6kBFmgf2JZ9EE8WJyySxC1LFCxzySAZj1cnANKiImwTaEA6fpEGowon2OvqGV9xC+PSAiccnYDlZRoARYAQYAXcEmHjccWFfRoARYAQYAZ8QYOLxCVhOlhFgBBgBRsAdASYed1zYlxFgBBgBRsAnBJh4fAKWk2UEGAFGgBFwR4CJxx0X9mUEGAFGgBHwCQEmHp+A5WQZAUaAEWAE3BFg4nHHhX0ZAUaAEWAEfEKAiccnYDlZRoARYAQYAXcEmHjccWFfRoARYAQYAZ8QYOLxCVhOlhFgBBgBRsAdASYed1zYlxFgBBgBRsAnBJh4fAKWk2UEGAFGgBFwR4CJxx0X9mUEGAFGgBHwCQEmHp+A5WQZAUaAEWAE3BFg4nHHhX0ZAUaAEWAEfEKAiccnYDlZRoARYAQYAXcEmHjccWFfRoARYAQYAZ8QYOLxCVhOlhFgBBgBRsAdASYed1zYlxFgBBgBRsAnBJh4fAKWk2UEGAFGgBFwR4CJxx0X9mUEGAFGgBHwCQEmHp+A5WQZAUaAEWAE3BFg4nHHhX0ZAUaAEWAEfEKAiccnYDlZRoARYAQYAXcEmHjccWFfRoARYAQYAZ8QYOLxCVhOlhFgBBgBRsAdASYed1zYlxFgBBgBRsAnBJh4fAKWk2UEGAFGgBFwR4CJxx0X9mUEGAFGgBHwCQEmHp+ADTLZTZs2iWHDhomhQ4eKM844I8isOS9GIDII7N+/Xzz44IOid+/eomrVqpEpFxckFgEmnlhMUs7n448/FldffbX49ttvxTnnnJNy5ecCMwLJQOCHH34QZ599tpg7d65o1qxZMpLkNHxCgInHJ2CDTJaJJ0i0Oa+oIsDEE9WaiS0XE08sJinnw8STclXGBfYBASYeH0D1KUkmHp+ADTJZJp4g0ea8oooAE09Uaya2XEw8sZiknA8TT8pVGRfYBwSYeHwA1ackmXh8AjbIZJl4gkSb84oqAkw8Ua2Z2HIx8cRiknI+TDwpV2VcYB8QYOLxAVSfkmTi8QnYIJNl4gkSbc4rqggw8US1ZmLLxcQTi0nK+TDxpFyVcYF9QICJxwdQfUqSiccnYINMloknSLQ5r6gikCjxHD9+XOzdu1eULVvW9VUOHjwofv75Z1GhQgXX526esJowefJk0b17d5E7d263IOxnIiDZpTwCH330kaQ6lWS5IM13OXTokNyyZYukjpdmOLeH3333nfzll1/cHsX1W7lypZw5c2bc5/yAEUgmAnv27FH9gCwXxE22f//+MmfOnCpcqVKl5LRp06ywJ0+elG3btpV58uRRz6+44gr59ddfW8/Tuvnqq69UHCKgtILxs38QEIxE6iOQHvGAaNq1aydz5MihOkfevHnl2LFjrRcn8yLKH+Sl/9CR4HCtUaOG8keHvP322+Uff/xhxU3r5oknnpBNmzZNKwg/YwSShkB6xLNz507VjgcMGCC/+eYbed9998mCBQtaA7aBAwdKMrkj169fLzdu3Chr1qwpb7vttoTKx8STEExWICYeC4rUvUmPeF566SVZqFAhuWDBAonO17lzZ1m4cGF55MgR9dKkUpDPPvusXLJkifUH6QgjQHS+1q1by99//11Onz5d5sqVS6Y1ojRRZOIx0eB7vxFIj3gGDx4sS5cuLf/66y9VlP/973+S1GLyzTffVL+LFy8uH3/8cauYIJ/Zs2dbv82bEydOyLvvvlueeeaZklR2slu3borUWOIxUYp/z8QTH5uUeZIe8dx0002yffv21vts2LBBdZJZs2YpcoHqgSxcW8/1zeLFi1W4tWvXai85Z84cifhubvv27bJ+/fqyQIEC8pJLLpHNmzdniccNKPbzBYH0iIestyvi0ZlDfXzqqafKxx57TIKEIO3feeedqu1edNFFEuEx4HJzI0aMUIO5IUOGSJrbkWXKlGHicQMqjh8vLqDWluouvcUFpFYQJPGIkiVLqlcl6UaQukFQRxW//vqrOO+88wTpswXps0XDhg3VBCmRhhg/frx44IEHBKnXBJGUqFSpkujatato06aNK2RXXnmlwCTrk08+KYicBKkulNXsefPmuYbPqp7Hjh0TRNYY1GXVV4zke6Ett2rVSkydOlXceOONMWVEe4f1dpJUlPXqCRMmiPfff1+Qyk2QFkDUrl1bkCZAEBGpdvzqq6+KRo0aiYkTJ4pFixZZ6bVo0UJcd911guaI1DM8eP7559WRDGj/RYoUscLyjTsCSSWeTz75RNBcgvjzzz/dc2NfXxAglZigEZtYsWKFuPTSS+PmceDAAUGqBDFq1ChFDjTRqkzIX3vttcqfpBQxadIk8d577wlSMQhSvQmQVN26dQWpEgQIhCZj1eqda665xtYZEYZGfeKDDz4Q6Jhw9erVUx05uxHP8OHDxUMPPRS3HviBvwg8/fTTom/fvq6ZYDCFs6tAUo0bNxarVq1SRHPxxRcLtP9nnnlG/UZkDJyQ1qeffmrrV7t27RLnn3++6kc0B6TyWb58ubjssssUYTHxuEJv80wq8fTp00cdRoYDydgFh8DWrVvFuHHjBOmkRZUqVVwzXrp0qaC5GkGTqarDmOeVYHkpLRxQ8Uh3rc40admypShWrJiqT6RfsWJF9bxatWrqkC18WNFRtRszZoy44447BJaiIg84SEsoU3YjHny8QPA4oI9dcAhgCXSDBg3EjBkzlETizJlWdAqa4xSQ5uEgnZQoUUINoCD10xyPeP3110WXLl3U85EjR4pBgwYpzQAtzFF++h8ko549eyqigh/6H9o/SzwaoXSucVRwnrwffvhhecEFF3iKy5G8I5DeHA8WFeTLl0/SwCBmKTWekZRjy5xGc7Jfv36SOp5aTAD9t3akTlOTqvq3vmLuh5qa/PLLL7WXpMPpsuUcD42SJak1LRz4JhgE0pvjIY2MaqOYu9y3b58ktbG88MILJalGVQGJtCSp29SqNr2ak04zdS08qZtluXLl5MKFCyUNzGSdOnVU2kQ8ruHZ045AUhcXMPHYwQ3qV3rEc/3116sORqo4af6hk+i4pBeXWMlGaiLVgbB/AfuCsIQay06xh2fKlClqSfayZctiXu23336TRYsWlaRbl7t375akclNxs+NyaiaemOYRiEd6xINCdOrUSS0KwECM5nvkmjVrrLJt3rxZDZwxgMJf9erVraXWVqB/btDG6XhtFe6UU06RN9xwg7ondbYzKP92QYCJxwWUVPPS5BFvAymWierOZF5BNnDYm4OVaNjno1f5aAzeeOMNmT9/fis+qRP0o5grllsjHXREdGzsD2LiiYGJPXxCIBHiQdYgByyVdnM0P602jUKK0cuu3cLBD/vZIBmRxYR4Qdg/DgJJneN55JFH1OonrI5iFxwC6a1qS6QkR48eFVj9BjMheo5Gx6P9PoJGg2pFEO1b0N6uV8zxYE6ocuXKggjLNUxW98QcD+YHYMKFXXAIJGoyJ7gScU7xEGDiiYdMCvkng3hS6HUjX1QmnnCqiIknHNy95MrE4wW1iMVh4olWhTDxhFMfTDzh4O4lVyYeL6hFLA4TT7QqhIknnPpg4gkHdy+5MvF4QS1icZh4olUhTDzh1AcTTzi4e8mViccLahGLw8QTrQph4gmnPph4wsHdS65MPF5Qi1gcJp5oVQgTTzj1wcQTDu5ecmXi8YJaiHFoWbw4fPiwbclzPOKBbTaY9mAXLAJMPMHgvXr1alGrVi0rs3jEQ5tEBZ0pJWiPmhWWb0JGIM7+Hk/ebLnAE2wZjgQzHvPnz7fiuW0ghRkcnLHDLngE2HJBMJhjczMsZdAx1ipD5wZSGqDJ+++/X5mKCqZEnEuiCLDlgkSRilC4devWKQsDsCIAO2om8dCoT5JpeHWyIhlNjFCps09RmHiCqWtYDoBdQRzG9vbbb0uTeMiitCTDn8qSxk8//RRMgTiXhBFg4kkYqmgFxKmgJCyr0w8fffRRdY8P3umnn67uYRCUXTgIMPEEhzukHvQD/NHRHOoKQ8Uw/QQ/aGHYRQ8BnuOh1pmKjmxECTolEQOHmOLD5M2OHTvUsQYxD9nDdwR4jsd3iK0McPYXjgKBuSenI7uBqh/guAN2EUMgmVzIczzJRDP9tKDfpuYU88fSTvrY+RmCJR4/0Y1N25R6zP7A0k4sVlHxYYknYgOBjBTHTephaScjCPoTliUef3CNl6qb1MPSTjy0IuKfTAZkiSeZaCaWllPqYWknMdz8DMUSj5/ouqftlHpY2nHHKSq+LPFEZADgtRi0wk3gvHhqUGpvD8/teEUyefFY4kkelommZEo9LO0kilqI4ZLJgCzxJBPNxNPSUg9LO4lj5mdIlnj8RDd+2lrqYWknPkZRecIST4ikn6ysIfXUr19fbN++nVeyJQvUTKTDEk8mwMtEVEg9l1xyiZg7d67glWyZADKAqFmKeGhDmcDpp/gAE7MHAF90stiwYYOgM+CjU6AASkJHbAvat6TMoRQtWjSAHBPLIkzigTmlWbNmiS+++ELs27cv2/WDY8eOibx58yZWUVkoVJEiRVT/v/7661Ni8JkliOe3334TZBpDTJ06VeCYZnbZC4FcuXIJ2lAryESQOp477LcPi3imTJki7r33XkEWK0SlSpXUBwjkzC7rI0AWTMTGjRtF7ty5Rf/+/cWDDz4oolz3KU88MBTYsmVLNbJ7/PHHRb169USFChUiDXrW7wbBveEvv/wiyGSQGDx4sMBoF2oWLLYI04VBPBMmTBDdu3cX7du3F0OGDIkEAYdZB9kx799//10Nvp577jk1AHnxxRejC0MyJ5uCXlxA0o2sXLmyvOyyyyR9gJL5KpxWiiFAUq+kQYc855xzJFnlDrX0QS8uINWyzJMnjyRpJ9T35syjgcC4cePUpvIPP/wwGgVyKQUkhaS5oIln4MCByhgmLSFO2jtwQqmLwPfffy/z588vSfoJ9SWCJp6ePXvK0qVLy6NHj4b63px5dBBo3LixbNCgQXQK5ChJSqvarrrqKnH22WcLWkYZXZGSSxYoAl27dlW67lWrVgWar5lZ0Ko2MoopmjVrJiKtWjEB4nvfEfjPf/4jyHq9OHjwoMiXL5/v+WU4AwcRZepnkBLPX3/9JWklh6TOlqkyc+SshcALL7yg2kWYbxW0xFOoUCH50ksvhfnKnHfEEFi0aJFSt+3cuTNiJfu7OCmraqOJZAXs5MmTIwksFyocBMaOHStz5swZTub/5Bo08dDyYfnKK6+E+s6cebQQwHlEJIVIstodrYL9UxomnkhWCxfKKwJMPF6R43hZCQEmHp9qkyUen4BN8WSZeFK8Arn4SUGAiScpMMYmwsQTiwn7SMnEw62AEZCSicenVsDE4xOwKZ4sE0/0KpDM+EgsdSdbatErXBYtUdSJJ2WXUx8/flzZZKLFBaJjx44ZXs0XdIRdu3aJu+66S9DeI2XIMDP5Y/n4tGnTxIwZMzwlA+xoc5mYPn26Ohq4YsWK4oEHHhDVq1d3TY82ZArYgIItuBEjRriGiYonbZ4TPXr0ECdPngytSEEvp8Zy2eeff17cfffdob1zWhm//vrr4pZbbhErVqwQ5cqVE4MGDUoruNp9T/ux0gyDh7BSMXv27LjhzjvvPNGrV6+4z50PYG5r2LBhys5djhw5BLZr3HTTTaJEiRK2oFiqD/NcixcvVrYCyTq8uO2222xhwv7x2WefKcPBOBIcOETOJZPwg1xOnWoSD30U1CoT6iiZgnz//v2STAJJOnPEczr0YVZlQTqw/ECNUu1837Nnj2uaROwqTIsWLVyfR8mTJZ4o1cbfZXnttddU+1m+fLnctGmTukebi/eXqBWSAQMGxE0DaTds2DBhMGAFhQZWKj1sQi5WrJi6r1GjhoTEpt3ChQsl2UBTS/ZhKePUU09V4UaPHq2DROIadYmHV7X52Ez27t0raSQkq1SpYnUQr8RDozvZtm1bSdaYVVpeiQfr+9FxOnXqpFQf2A9Fdr5UmkOHDo1BQ59xgo7MxBMDj6sHL6e2w2ISD1mQV+atQC7m35w5c1QbJGO/9shp/AIhmGnoe9IsqCX1n3zySRqx7Y/Ivp3K/9Zbb5Vk80ySxCxJ+lF+JDWpwIcOHZLly5eXJLUp1SE8sVyZJCJJxzHYEwz5FxOPTxWQChLP1q1bZc2aNdUfWVhQjdhJPLArhkbidGj48+fPt7zHjBljpYURmVfiwaFxZMFW/vTTT1baIB9svJw5c6blh5tt27ZJbE7s3LmzMkXDxGODJ+6P7Ew8pMaVMOFTtmxZSUdVyFatWsl+/fqptg+Jx82BQMiatqxTp4764CNMov3CmR6pmCSpyTJsNqlJkyaqjOa+F0hBBQsWlJB64N5++20VBrbQTEfHUMjhw4dHag6LicesoSTepwLxmK9LFmNdiadDhw6KCNB4tQPptGnTRoX/+OOPtbd1RQf1SjzVqlVTNpzIhLrEKA+jOain0MlMd+LECfUhwOiOTK4z8ZjgpHOfnYmH5gJVu73iiisk1MuwLqJVavGIB6pfSOFkad5C1ku/AFlBIiETQhaBWQmmc0MWzZW6Ge3edNofkppW7YGc0GdglBV2ATHAjJpj4vGpRrIK8UCqgIFHSCEgH5N0aELWdRTllXgg2UBaKlWqlLrig4DRIa4YoZpzPI8++qh6pqUxxGOJJ7HGnF2JZ+XKlaotYW4FbQ0Oc5KQZtDG3IiHFhyoZ2jrpvPSL+gMGpWWU6tgphvvHkSHMppahm+//VbSwg3lD3Lp0qWLutdzQbrvwDL4u+++Gy/pUPyZeHyCPasQD+AxOxlGiugA8UgH4b0SD4gFaeMPCwaQLywaP/LII8qvXbt2SF5iAhWTplCRaMfEo5FI/5pdiQfW4tG2nPYT6YA+5e9GPNddd52EyR9z0KMRzki/wLEYUIvRSjQdPUNXECDaPNJA34M0g7kb3V8w73P55Zer3+eff76kM6AkpKAFCxaoQVzhwoVDP47DfGEmHhONJN5nJeIBLFjto1fIwKR5WnsevBIPHYWsOg5Gaj/88INVG5g0ReeH5IWRKuajoCKh5doSo0f8YVRXq1YtdU9Lw624UbvhVW3h1QgW0uBDbarMUJqJEycqfyfx0DH1SsV28803xy10ov1Cq8HQZr26999/X60YxTtAAwEJ/+qrr1b9AWmC1PBs1KhRtiwwbwp/5/vZAgX8g4nHJ8CzEvFAvYYVa2i8IAU6ylmiE8RzXokH6WFEh8PSnA5LqzFqA+GhHGn9jRw50hk9Mr+ZeMKrCq3qoj1mtkLcc889rh9mMtuv/CFhu7lE+wXCYfnzWWedleG5Hbd8MVeERRJwtMdNNm3aVN1369ZNlRcqRdM99thjyh8rT6PimHh8qomsQjxm56KzZCRGgSVLllTkE2/0lhni0aq8tWvXWjUDlQaIBie5wm3YsCHmD6RUt25d5Q/JKaqOiSe8mqEzYFQ7wqo20+m9Yk6JAJJ1mTJlrPkgM05G+sWyZctUvr179zaTyNA9+h4GfdgCoR0W9qBfPPHEE8oLR0/gd9++fXUQdcUCBPjDOkNUHBOPTzWRVYine/fuqtGi4Wv12ubNmy3yQQNyunjEg/0SWO2mO4ozHn5DN41OAqkHK3Jox7taCQS/tKSs0047jRcXuAHq4pdd53gwX4h2BYkdy4uxtFlLO2hfJvHojaRo924uI/3iqaeeUm3aXBlqpplIv9DHRWPOac2aNWqhD0gRmgCoouGw8hNkiQUHt99+u8QeNzqAT+WNfXFRckw8PtVGqhKPUxxfunSp2vegSUfDBfLBKhrnMmc8j0c8esTZp08fnYzrFVIBFgvgY4A/zN+kpz4D8Vx77bWu6UXJkyWecGsDH20swddtC3OFbvt4cH4Qwrz66quuBc5Iv2jUqJFKK54knki/QP/D5lFdblyx8RuLDkwHTYG5IRzhoCbHloMoOSYen2oj1YjHJxhiksVIcfz48TH+Tg9s2oOKAh1Lj+icYVLxNxNP+LWGvTCYB4GEo+dKwi5Vov0CC2cwONyyZUvcIoOkoBKHKu7HH3+MGy7MB0w8PqHPxBMLLPYgYC5mZ0SPu40tcfJ9mHiSj2mqp5gd+wUTj0+tloknFliMwmCRIDs7Jp7sXPvu754d+wUTj3tbyLQvE0+mIcySCTDxZMlq5ZfKIAJMPBkELNHgTDyJIpW9wmVH4sFCEeemxuxV6/y2TgQ08UTRjhzKmrLHImCHPazfwqoyO0ZAI4D2gJVUYbqgl1NjiS+WFLNjBDQCdMhjmiv9dLiwril7AiktYxRkzkLQZksxadIk/GTHCAhagi5oKbr4/PPPQ0Mj6BNIae+JIFtiYsmSJaG9M2ccLQRoE686ZXjHjh3RKpguTTIZL8gTSFFujCyxYRI779kxArt371b7k2CUMkwXtMSDjb/Un5UdvTDfm/OOBgJYCo5NrjDaGlWX0hIPzfMIMlwp6LAyQYeYieLFi2s+5Ws2Q4A2Dwoy6ijoFEpBJn8EDUhCQyBoiQcvSufgCDpxU9BhZaJ58+ahvTtnHC4C69evV22B5v0E7dELtR+khURKEw9ejHYSi5YtWwqQENlQEnQOuiAzF4IsPaf13lnqGc13CdqoJ8jCtKADtbLUu6X1MjSaU0RDZoAESTmCNvaJefPmierVq6cVzfdnYRAPbQIWZOVZDcDIirKgEzUFGc7MVu2BNq6q7wGd/yNons/3eo5SBmQ5QZCJIkH2HVX7x0CcrM1HqYj2siRTFAta1abLjrMyYO4Clpfp7fgvm2EAkz84njsqRhqDVrXpfoAFN1OnTlU29c4880x15AD3h+zxPcCCGhgAHj16tHSeoqrbR5SuKS/xmDSKkT/pN8X27duxWs98lKXv161bJ8gwqJgwYYIa5Wbpl3W83BlnnCHoOG+lbnU8Cu1nGBJPaC8boYzpjClBK/wEmbwRZLwzQiXjojgRyFLE43y57PKbbEapFX50VK8g68DZ5bUj+55MPOFUDRNPOLh7yZWJxwtqEYvDxBOtCmHiCac+mHjCwd1Lrkw8XlCLWBwmnmhVCBNPOPXBxBMO7l5yZeLxglrE4jDxRKtCmHjCqQ8mnnBw95IrE48X1CIWh4knWhXCxBNOfTDxhIO7l1yZeLygFrE4TDzRqhAmnnDqg4knHNy95MrE4wW1iMVh4olWhTDxhFMfTDzh4O4lVyYeL6hFLA4TT7QqhIknnPpg4gkHdy+5MvF4QS1icZh4olUhTDzh1AcTTzi4e8mViccLahGLw8QTrQph4gmnPph4wsHdS65MPF5Qi1gcJp5oVQgTTzj1wcQTDu5ecmXi8YJaxOIw8USrQph4wqkPJp5wcPeSKxOPF9QiFoeJJ1oVwsQTTn0w8YSDu5dcmXi8oBaxOGkRDw5FW7lypVXiHDlyqPOK6tatK3LlyqX858yZI37++WdBRwvYzjFatmyZOHjwYLqWftHhkYbp6AREUbNmTVG5cmXTO+494uPwqgYNGsSEWbhwocBZK84DzuCPc5iuueaamDhhejDxhIN+IsSDY9FxQNott9wSU0g8W7p0qTK4W7ZsWes5HbehznnCseq6z1gP+cYbAsk8oyGs83iS+Q6pmBYdhKbOICLr1DHFHzp0qDqX5eKLL5b4ow4l6ZA8dXbHH3/8ocLXr19fxR85cqQtfrdu3SSZl7f5uf2gw9dU/IoVK8oLLrhA/dFxBcqvT58+blFi/FCGTp06xfjDo3Xr1rJhw4Yxz9q0aSOJqGL8w/YI6zyesN877Pz37Nmj2hwdixC3KCNGjJC5c+d2fY5n9BWNaWs0KFL+Bw4ccI3HnhlHAOfWJM0x8SQNygwllB7x0EmUtvRARuhgdFSy8tfEU7hwYUmjRitsRoln165dVlzctGrVSpKEJel0RJs/DixzOiYeJyL8O6MIJIt46BRfOXHiRCt7Jh4LiqTdMPEkDcrwEsoo8cyaNUsRD6ngVKH1R58O0ZJt27a1XiSzxDN+/HiVj5bESHWnpK6cOXNKUsHJN99808pLl8HyMG5Y4jHA4Nu4CCSDeDBIw2nGOMH1119/VXl9+OGHLPHERd3bAyYeb7hFKlZ6xFOgQAE5aNAg9XfffffJMmXKyEaNGlnvgI9+jx495PTp01UHmz17tnqWUeKhuSZJenK5adMmlVaNGjXkRRddpNLau3evLFSokGzXrp0kHbscNmyYUgEuWbJEPWfisaqDbzwikCzi2bdvnyxevLikeSBVEiYejxWSRjQmnjTASZVH6REP1F033HCD+mvatKk866yzJJ1UqggC76iJB/cIV65cOXn48GGZUeKB+s7599577yFZRTR58uSRNFEroSvHHx1ZLUGEcEw8Cgb+lwkEkkU8KMJbb72l2vLixYslE08mKiVOVCaeOMCkknd6xOOc4wGpVK1aVfbq1Uu9pkk8IAbM9TzyyCMZJp6vv/5aHjlyRJHW1q1bZfny5dUfMrnrrrtUR4b+XP+BpDp27GiVId7igp49e8patWqpcOa/Fi1ayPbt25tekbjnxQXhVEMyiQdvgPYFlfCMGTNU2+XFBcmrVyae5GEZWkoZJR4UFKvVIP3AmcSD36NGjZKYh6ldu3aGVrU5FxdglRzI5fjx47Jfv34SK93+/PNPZKHc6tWr5bZt29R9WhLPmDFjJC1jlUePHv0n5t+XUqVKSXzko+aYeMKpkWQTz86dOyXU1LT1gIknyVXKxJNkQMNILj3iKVq0qJpXwdwKlj5DsgCxjB07VhXXSTwgh3r16qnOppdTb9y4Ufbt21f+8ssvMa+ol1M7iWfKlCkqDcT57LPP1D3mdg4dOiSnTp0qoXqDSgMOZWjSpImEasP8Q9z169cr4oHOfffu3XLHjh1qTgqSk54jiilUiB5MPOGAnyjxYBBjtjHcQ1rHcmqndgDtVauPWeJJXr0y8SQPy9BSSot4nnvuOavjoANhvqdSpUqSNjlKvazZSTx4Ef2xh7oB7t1331XpYPGA02ni0avX9HN0aOSpCW7w4MEqf+wjwt8dd9yhgyri0R3cvELNAfff//5Xnnbaada7QB04btw4K36Ubph4wqmNRIjn5ZdfttqQ2c46dOgg8QyLCkyHvW60EVrFYeIxkcncPVsuoNaX6i4tywXJejeSgpRVgQULFghYJfDqfv/9d0HzP4IWNwha5JChZKipC1LNCfoYKIsIJPFkKH5QgdlyQVBI2/NJxHKBPQb/CgsBJp6wkE9ivkEQD0zawNTIU089lcSSZ82kmHjCqVcmnnBw95IrE48X1CIWJwjiidgrR7o4TDzhVA8TTzi4e8mViccLahGLw8QTrQph4gmnPph4wsHdS65MPF5Qi1gcJp5oVQgTTzj1wcQTDu5ecmXi8YJaxOIw8USrQph4wqkPJp5wcPeSKxOPF9QiFoeJJ1oVwsQTTn0w8YSDu5dcmXi8oBaxOEw80aoQJp5g6gNL82lvl5VZPOKhYzkEGai1HXJoReKbcBDI3DYge2w+j8eOhx+/sOkTGzFNCwJuG0i3b98unQe7+VEeTjMWAd5AGouJHz7Dhw+Xo0ePtjZCu20ghaV1bZPQjzJwmt4QYMsF3nALNRaOL8AOa5ikgTOJB+ZusAMbRxDA+gC74BFg4gkGc5J4JMxBXXXVVRJ21UziwTOYWIKFjHXr1gVTIM4lYQSYeBKGKjoBIfXgnBuSkdX5Nu+88466X7RokToKGv7mgW7RKXn2KAkTT3D13L9/f9X2MdB68skn1T0kHBxqiH5w4403BlcYzilhBHiOJxwNZ6ZzJRtmgs7OcU2HRnli7dq1gs67cX3Onv4iwHM8/uJrpr5//35x7rnnClydDiaV0A+qV6/ufMS/Q0aAiSfkCvCaPQ0tBBkvVB3LmQZJO4KMajq9+XdACDDxBAT0P9nAjBNJPjGZkrQjyAp6jD97hI8AE0/4deC5BG5SD0s7nuFMWkQmnqRBmVBCblIPpJ01a9YIOn49oTQ4ULAIMPEEi3dSc3OTeljaSSrEnhJj4vEEW6YiQeIxDdiytJMpOH2PzMTjO8T+ZmBKPSzt+It1oqkz8SSKVPLCmVIPSzvJw9WvlJh4/EI2oHRNqYelnYBATycbJp50APLpsZZ6WNrxCeAkJsvEk0Qww0oKUk/r1q15JVtYFeDIl4nHAUhAPyH1lC9fXtC2Ap7bCQhzr9lkSeI5dOiQoL0uXjFJuXiQemgXt3jooYdSruyZKTBUKgULFhS4RsmFTTxk1UKsXr1a/Pbbb9inFyVofC/LN998I84//3zf84laBkWKFBFVq1YV5cqVi1rR3MtDDTNpLkyTOW+88Ya85pprZIkSJdTGMXpbvmYDDPLkySMvu+wyOWHCBElHYietLWcmobA2kO7YsUNtmMyRIwe3/WzQ9t2+cZdffrlcuXJlZppvIHFTXuL59ddfRbdu3QTZZBItW7YU9BESFSpUiNwo2J322TczCFAPEaj/BQsWCKgb69evL8ickM1wZGbS9xo3DIkHEk6zZs0ERr79+vUTTZo0EcWKFeN+4LUSUyweDKEuXbpUkB1HpXKfPHmyaNOmTXTfIpn0FobE06pVK1myZElJFpqT+SqcVoohsGzZMmW/Dna7YK8uTBe0xHPgwAFZunRpidEufYDCfHXOO2QETp48Kbt06SLz5csnN2/eHHJp4mef0rbaiNUl6fcljXjjvyE/yTYILFmyRLWHt956K9R3Dpp4YK28QIECykhmqC/OmUcCgePHj8uKFSvKjh07RqI8boVIaVUbbJWdOHFCqdmiK1NyyYJEAOomuHnz5gWZrS2voFVtderUEZUrVxY0z2krB//IvggMGzZMbajFmUU5c+aMHhBubOTVL2hVG9QLAwYM8FpcjpcFEcDon+Y2Qn2zoCWe008/XdKHJtR35syjhcD8+fPVApPvv/8+WgX7pzQpq2qDOEk0LqFuY8cIaATGjh0raYSnf4ZyDZp48ubNK1955ZVQ3pUzjSYCn376qfo+0vLySBYwZYnn2LFjTDyRbFLhFoqJJ1z8OfdoIMDE41M9MPH4BGyKJ8vEk+IVyMVPCgJMPEmBMTYRJp5YTNhHSiYebgWMgJRMPD61AiYen4BN8WSZeFK8Arn4SUGAiScpMMYmwsQTiwn7sMTDbYARAAJMPD61g1QhHthN6t27t2zUqJHs0aOHJNM+8siRI55Qee2119Su5MaNG8uBAwfKL7/8UpIx1AynBeymTZsmO3furHa7d+3aVa5bty5uOtgNj/Lfe++9ccNE5QFLPFGpCSlff/11ed111yWlQJlNK5F+uHXrVnnnnXfKK6+8UjZt2lQ+8cQTaVqCiHK/YOJJSrOLTSQViIfMs8v8+fOr5b1kOVaSJWW1Eq9KlSry8OHDsS+Vhg9ZnlZxCxcuLJEWLDZgOTn8M+pAgIhLNu0kbTxU9zC2uWfPHteksAMa4Vu0aOH6PEqeTDzRqA06okC1L1hUyKzLbFqJ9MP169crMzNo52XLllX9FvfoH2Tt3vUVotwvmHhcqyzznqlAPM2bN1emTFatWqVe+OjRo2okhQY9ZcqUhEFAx6PTRWW9evXkwYMHVbzt27erjpI7d25J1hsSTgudEKTVqVMnZdMMEhMsO6NMQ4cOjUkHVr/xDH9MPDHwuHpk5308c+fOlXQgocSmVrSZzBBPstJKpB+2b99elXfSpEmqTkE2dMaV8kP/cLqo9wsmHmeNJel31IkHhirJUrAkEy62N4YdMXTIkSNHKn8YeEQjcToY+8PuY7g5c+aoOOPHj7cFI5NByv+nn36y+af1g05nlCArMw7I54UXXpAzZ860Rd22bZssVKiQUslBcmPiscET90d2Jp4xY8bImjVrqj+0GTfiSaTNA9xE0opbCf88SLQfYlCHAZlpZHXixImqf9FZV7ZsUqFfMPHYqix5P6JOPPiYr1ixQuKMFO0gmeDDD+KBxALXoUMHRQSzZs3SwSRIh0yaq3Cwur1v3z65fPlyW6dAurDKXatWLSteIjfVqlWTDRo0kBs3bpRDhgyRvXr1UkuQnfNOKCvZAJN0sJTKl4knEXT/DpOdicdECe3HjXgSafNmOriPl5YznPN3ov3w2WefVf3t/vvvl3TUhlyzZo2sXbu28tuyZYuVbKr0i6gTz/8BAAD//0XuvtkAADjqSURBVO1dCdxNVffelVmSRkPIGBUlEtGfBkRChkhIJU0alJBQGj5fEpXhI0NRERooEpUiiRAyVSQqQqRCIn37v571dU7n3vcO59733HP2vXft3+++55x99nSevff77GHttZT20D3wwAO6atWqHqYYPak//vhDK6X0K6+8Ej2QIW+++uorfd555+lChQpxme+77z67ZJs3b9ZnnHGGzpcvn549e7b+888/ddu2bTlc165d9V9//WWHxc3w4cN15cqV9bHHHqvz5MmjFy5cGPI+1sN///tfLkPJkiXtshx33HGcV9myZfX27dvt6H379tV49/HHH7Mfyt60aVP7vak3zz//POMSZPmeeOIJXaJECd+KUKBAAT169Gjf8nObUe3atXXhwoVzBE+0zSOBaGnlSDyGR6x+uG/fPn3hhRdyX7D6BP6/9OrVKyTFdOkX6Lco/9dffx1SflMelJcFEeKJjOY333yjW7Vqxf+M0BiKFy+ulyxZYgd2dsT69etHJR1EmD59ur700ktt4mjYsKHev3+/nVasGxAL8sevY8eOGvkeOnRI9+7dm/3at2/P0RcsWMDE1r9/fzs5IR4birg3Qjz/gygWWSTS5pFarLTiVsjfAWL1wxEjRvCABQPA5s2b65o1a3KfKFq0qF63bh2nkE79QojHbatIMFw6zXicn/bss8+G/JO33m3cuJH/2YMULrvsshwzHSucdT1y5IgG6SD8a6+9ZnnHvO7du5fDY0S3Y8cOO+yBAwc0Rs2YeWFWVKpUKY0ON3PmTD1nzhz+5c+fnzsjnrdu3WrHNe1GZjzm1Eg8skikzcdLK9GvdvZDrCqccsopOm/evPr777+3k5owYQL3l549e6ZdvxDisavR2xvTiefDDz9kYpgxY0bIhx88eJCX1UqXLm37Y3mtXbt2NimgA8yaNct+//DDD3NaP/30k+2HmylTpnAc59JdSIAID8cff7wuU6ZMjjcVKlTQJ5xwAhOeNSuKdh05cmSO+KZ4CPGYUhOxZynx2nz4VyRLPG764Zo1a7gfNWvWLCRbrAYULFhQly9fPu36hRBPSFV692A68axYsYIbc5s2bUI+etGiRezfoEED9nd2wBtuuEFv2LCBl+RAPphxwFlLYdOmTeNn689dd93FaSVCBNZS3urVq61keMkNJHPxxRezH5YWwn8gpTp16rA/Zk6mOiEec2omGlm4afPhXxEtrfBw4c9u+uGuXbu4H2Gmf/ToUTsJi5Dq1avHfuF9As+m9gshHrsavb0xnXgwfYdAwTHHHKPvv/9+/c477+hHHnlEn3nmmdzIX3rpJQbklltu4WeQjiVI8OWXX9rkgwaEBo7lMSwHjB07lpfWunXrxstjJ510kr1s9uKLL/Jm7oABA6KC/f7773N+mPUMHjxYDxs2jEd0IB7nLCs8gWLFiolwQTgoUZ5lj+d/wEQjCzdtPhzaaGm1bNmS2/zatWvDo/Cz235oLVtjmRuDlzFjxugaNWpwXxk/fnzEtOFpar8Q4olaZbl7YTrx4Oswe6levTo3XmvZCktdQ4cOtT/+k08+0T169LBJx3oB8unSpYv+/fff2Wvy5Mm6SJEiIWlVqVJFL1682IqiJ06cyO/79Olj+0W6QceyJOxQLuzfxJs1oYNdddVVkZIzyk9mPOZURzSycNvmnV8SLa0mTZpwm1+5cqUzeMi9m364c+dOHlhZ/RRXrDo4BWxCEv37wdR+IcQTqbY88EsH4sFnYuoOMc53331XY9rvVgItEkQQAli+fLmeN2+eBjE5lwWs8BhNxhqhWeGw1wTJuqVLl2qkmylOiCdTatLddxw+fJilRCEOHcu57YdbtmzRc+fO1R999JEO31ONlb5p74R4UlQj6UI8Kfr8iMnOnz+f15y//fbbiO+zwVOIJxtq+Z9v7N69e1rMxP8psT93QjwpwlmIJyewWFJYv359zhdZ5CPEk0WVTZ+KZTscARAXioAQTygenj0J8XgGZUYlJMSTUdUpH5MkAkI8SQIXLxrWbLEpjn804gQBCwEISUBwIkjnt1QbpB2HDBkS5CdL3oYhgD1lCEg4D4qbVMRjUBgqoCeOzpso0jemaMnHk/TiJUK6lRRJu6hRo0bFCyrvswQBOmWuSHRdkUBHYF/8r3/9SxEBKur0vpThkksuUaR/T9E5L1/yk0zMR4AGP4qkZ9WePXsUHcUwr8BesqCfutpQ7jvvvJNP4edGUszL75e0gkUAouc4BIiDtUE6v2c8UP8CHWObNm0K8rMlb0MQgJQq1F/dfPPNhpQoZzHSVkkoPgV6lU488USNw5SywZizcrPJB/UPCSfonAtaqs9v4oFqF2gsx5mxH3/8MZuqXb41DAEMvq6++mrWtWiyTsW0XmrD/JHMIig6aKnoxLEikwHqnHPOUaQtwLyppZQoZQiQskn14IMPqrfffltNmjRJderUKWV5uUnY76U2lAlLi40bN1YkdKOw3Ij7U089Nav6Ag0+1K+//qrokLaiw59uqipjwuC7ScKPl9foMKx644031BVXXGHs96U98QBZAE62axSpWlekVVmVK1dOkb0aY0H3umA0ylE0+1OkzDCrOhwN9hQd8lM//PCDIntCimzSKFL06DW8CacXBPGgkMBi4MCB6uWXX1a03JJwuSVCeiMAsqXZjiJVWIpmwEZ/TEYQDxCmE8yK1GYo0g6gyO4GlhCNBt7LwoF0SKEoky+p1fEyaaPTwsyWdNWpWrVq8eiOpByNKG9QxGN9PGY9mAWSMtes6gf43uuuu04Bf7SJbHIYcJ911lk88E6H784Y4kkHsFNVxg8++ID/8W7btk2R8s9UZSPpukQgaOJxWcyMCwYpQhIuUSRKrEiHW8Z9XyZ9kBBPBtSmEI9ZlSjEE0x9CPEEg3syuQrxJIOaYXGEeMyqECGeYOpDiCcY3JPJVYgnGdQMiyPEY1aFCPEEUx9CPMHgnkyuQjzJoGZYHCEesypEiCeY+hDiCQb3ZHIV4kkGNcPiCPGYVSFCPMHUhxBPMLgnk6sQTzKoGRZHiMesChHiCaY+hHiCwT2ZXIV4kkHNsDhCPGZViBBPMPUhxBMM7snkKsSTDGqGxRHiMatChHiCqQ8hnmBwTyZXIZ5kUDMsjhCPWRUixBNMfQjxBIN7MrkK8SSDmmFxhHjMqhAhnmDqQ4gnGNyTyVWIJxnUDIsjxGNWhQjxBFMfQjzB4J5MrkI8yaBmWBwhHrMqRIgnmPoQ4gkG92RyFeJJBjXD4sQjnnXr1qlly5bZpYYp3AoVKqg6derYZhTmzp2rdu/erTp37hxiUmLJkiWKLLzGVbqITo80nK5gwYLqggsuUFWqVHF6R71H/EKFCqkGDRrkCLNgwQJ15MgRdeWVV4a8gz+0MZtgDsEqmBCPhYS/V7fE8+eff6oZM2aoVatWcfuvW7cut7kTTjjBLjDsOh09etR+tm7QPqEB+8svv1RLly5ljfDWO+tKBtgU+iTsQjk1pn/33XfqvffeU9dff70ig4VW8Oy8hhmwy9Wj36avc1XYDIr8/vvvwwaEJu3UEb9qyJAhmkwI6Bo1avCPbNdoslek69evr6lzcZxLLrmE0xg5cmRIGjfeeKMmTb8hfpEe5s2bx/ErVqyoq1atyr+TTz6Z/fr06RMpSg4/lIE6aw5/eLRp00Y3bNgwx7u2bdtqIqoc/kF6+G2BNMhvNSnv7du3c3sj7dRRi0W2mzSZTNA0KNJkKE3Xrl1b00BMo91u2bLFjgdT4rBuDFPqzt/555/PYZ577jk2N25HcNxMmzaNy0H2kRy+Wk+fPp39aYAX4p+ND2lt+jobKyzSN7shHrJGGRIVZASyWrhwIftbxEOjPk0jRztsosQTbm63RYsW3LHJQqKdJm4imSoX4gmBSB4SRMAN8WAAU7p0aU32iuzUyX6XLly4sKaZiO0H4nn++eft5/AbIZ5wRBJ7FuJJDC8jQydDPLNnz2bioSU4/ibrnz5Gd+3atbO/M7fEM378eM7Hmo3R0h3PuvLkyaNpCU6TtUw7L6sMtofjRmY8DjDkNiIC8YiHlse4LU6ePDlH/Ndee00PGDDA9hfisaFIyY0QT0pg9TdRN8SDEd3jjz/Ov3vuuYdHfZdeeqldUPzTv+OOO/Sbb77JnXPOnDn8LlHiobVtjQ6OESXSOu+887S1PEG24DVZSNXt27fXtD6uhw4dykuAixYt4ryEeOzqkJskEIhHPK+//jq3bYSDw6wbbdL5s7IF8WAAhjbq/FkDNZnxWEgldxXiSQ43o2K5IR6sY19zzTX8a9y4sS5evLgma6X2koNFPPgwhDvzzDP1wYMHdaLEg+W78B9t5DJe6MC02aqxzv7bb7/xr1q1ahpECCfEwzDInyQRiEc8I0aM4LZpLfuSqewcbdXa5wHxkACOrlevXsjvhRde4NIJ8SRZSX9HE+LJHX5GxHZDPOF7PCCVc889V9999938DU7iATFgr6d3794JE8+GDRv077//zqS1adMmXb58ef4hk9tuu407OgQdrB9IqmPHjnYZogkX9OjRQ9esWZPDOf80bdpUd+jQwekV+L0IFwRTBfGI56233uL2hyscZjw//vgj/1555RV+h/0euNwstWHfFO3aIjFOkP5MnDiRB14kVWd5Ze1ViCcDqj4Z4sFnQ1oNsx84J/HgedSoURr7MJAASkSqLVy4AFJy6ISHDx/W/fv315B0++uvv5AFu5UrV+rNmzfzfawZz9ixY3XevHn1oUOH/o75v0vJkiU1/tGb5IR4gqmNeMSDfUZIc9511105Cti3b1/PiOfnn3/mtKZOnRqSDwZ5WHoWp7UQTwa0AjfEA9FQ7KvgB9FnzCxALJbkTjjxgBzofAN3IIt41q9fr/v166fDxUQBoSVOHU482LQF8SDO4sWL+R5LbgcOHNBYc8fS25QpU7gWUIZGjRrpjz76KOSHuGvXrmXi6dq1q6bzEDyaxJ4UZk7WHpEpVSnEE0xNxCMelOrOO+/UdIZGYyADsebVq1fzzB6DGrRT54ynV69eIe3QapdIB0ttiGP5WVfM+OGwhFypUiW9YsUKvWfPHm7jEOG2lpU5UBb/EeLJgMqPRzxPPfUUdyp0LPyw33PWWWdpOuhoizWHEw9gsf7ZYzkLzjqfAOGBcGcRjyW9Zr1Hh0SeFsENHjyY88fIE7/u3btbQXnWZZXReZ05cyaHwTmIYsWK2d+C5cBx48bZ8U25EeIJpibcEA8dhuYlX7Q9q41hORjCNBicWcSDAZH13nnFQAfO2i9yvsM9HS7l9xgcVa9e3U4DgzwIK9AhaH6f7X8YRQLME0d7AorEdBWxvifpSSLuEIinucBdKvFD0SyIT3jj9DW0EiTr9u3bp2j/R5FwgyIhh4SSoQ6raGmOT5VDIwL9I0govh+BRXOBHyjnzMOt5gLEpBk3ax+gMz3q9NNPz5mYRz4021G0CqDOOeecXPUZj4pjTDJCPMZURfIF8Yt4oNIGakIGDRqUfGGzIKYQTzCVnAjxBFNCydVCQIjHQiKNr34RTxpD5GvRhXh8hdvOTIjHhsL4GyEe46sofgGFeOJj5GcIIR4/0f4nLyGef7Aw/U6Ix/QaclE+IR4XIPkYRIjHR7AdWQnxOMAw/FaIx/AKclM8IR43KPkXRojHP6ydOQnxONEw+16Ix+z6cVU6IR5XMPkWSIjHN6hDMhLiCYHD6AchHqOrx13hhHjc4eRXKCEef5CGaL1TnD4W8YSH9aeEkktUBLw8yCSG4LxEM3Ja0C8FXVNOezaRDpDioJqlYTpySuKbKgTkAGmqkA1NF4c4oU3DcpEOkEIXG4n/h/QXK7xcg0NANBcEh33SOT/99NOsPw2no+HCiWfNmjWaTE4zQSWdiURMGgEhnqShSyjiV199xQbcoA0DijfDiQeqmKAbEOY5xJmFgBCPWfXhqjTQLE2nrXXRokVZZQxpEmDVHFD38dhjj7FmXWhyds6KXCUsgTxBQIjHExhdJdK5c2du+zBhbfWDSZMmsWkPWuZho4PSD1xB6WsgIR5f4fYuM8x60LHwg5E3XGFjx/KzVL97l6Ok5BYBIR63SOU+HGY90D1otfvwq8x2co9xKlIQ4QJqqenoyOaNIuWGateuXTmKT7MdtXz58pCN1xyBxCNlCIhwQcqgjZhwly5d1EsvvZTjXY0aNRSZ3ZB+kAOZ4D2EeIKvg6RLMGzYMHX//ffniE+zHXX11Vfn8BcPfxAQ4vEHZyuXr7/+Wp199tkKSmydjmY7qlWrVk4vuTcEASEeQyoimWJEmvXIbCcZJL2NI8TjLZ5uUguf9chsxw1qwYUR4gkOe09ypr0eRQar7LRktmNDEdiNEI//0IfPemS2438dJJKjEE8iaBkY1jnrkdmOGRUkxBNMPVizHpntBIN/IrkK8SSClqFhrVmPzHbMqCAhnmDqwZr1kLl12dsJpgpc5yrE4xoqcwNi1kNmddn6q1OFiLklzuySmUA8dKBSwdJrtrkxY8aoW2+9Nesk2cgMvCpQoEDaVHfGEA/0lU2bNo3FJ2EaWVx2IEAn01WtWrXUzTffrJo0aWLERwdFPCCb8ePHqwkTJqhVq1YpOjhpBB5SCH8QqFSpkrr++uvVfffdp4oUKeJPpknmkvbEQ6f4WaT4+eefV6QmRl188cV8vkVG/km2iDSL9tNPPylSGaSWLVumOnTooCZOnBi4bfsgiGfv3r2qZcuWbJr82muvVY0bN1annnpq1o3806z5elbcX375RX3yySeKtDYo0mqiSE+jqlKlimfpe56Ql6dSg1ASeuONN7LqmMmTJ3v5KZJWmiHw+uuvaxrlaVpyDLzkfmsuoPMrumHDhqy5gg4OB/79UoDgENi2bZs+77zzdLly5TQttQZXkDg5p7XKnNmzZ7OqDFpii/OZ8jobEJg5cya3h7fffjvQz/WbeKAMk2b4+uOPPw70uyVzMxD44YcfeDDet29fMwoUoRRpvdSGJQUstXz44YeezwQlwfRE4JJLLlHY9yESCuwD/F5qa9q0Ke/nzJs3L7BvlozNQoBWn9TUqVPV999/b+ZyawQyStrL76U2TCcHDBiQdHklYuYhAO3cUJYapPN7xlOiRAmNPMUJAhYCUBJMVKh3795teRl1TdultsOHDzOwpBzQKEClMMEiMHbsWDYLEWQp/CYeEqPVo0ePDvKTJW/DEMCyK4iHzjYZVrL/FSdtieePP/5gYF955RUjgZVCBYMASTfqPHnyBJP537kK8QQKv2ROCAjxpKgZCPGkCNg0T1aIJ80rUIrvCQJCPJ7AmDMRIZ6cmIiP1kI80goEAZnxpKwNCPGkDNq0TliIJ62rTwrvEQIy4/EIyPBkhHjCEZFnICDEI+1AEJAZT8ragBBPyqBN64SFeMyvvm7duumhQ4fmuqDffvutJv18+rPPPks6rVdffVWTfjN96aWX6oceekhH0vzw448/6hEjRnBe9erV0zg2Qipqks7Tj4gy40kRyiYTD+lL0qQ3y5Mvz20nfe+99/Qdd9yh0WGaN2+uH3/8cX3o0KGQsvXs2VP36NEjx2/Dhg0h4dApcRq6Tp06ulmzZpoUUoa8N+FBiMeEWohehgULFrA0ateuXaMHcvnm9ttv57RIL5nLGKHB0JYhcoxf2bJl+ZovXz69dOlSO+CBAwdY/cxxxx2nL7roIn3iiSdyuMsuu8wOY+KNEE+KasVU4sFIqEKFCrpw4cK5/vLcdlKccUKHQWeC/iZc0clIm7MmTcZcPtL8wH5WB3Re3333XfsbUBaoZSlatKiuW7euPvbYYznef/7zHzuMCTdCPCbUQs4yYMCDf9YQdUcbS5Z4du7cqUkTuT777LPtdpsM8Xz11VfcN8466yy9fv16LvDq1av18ccfr8844wwN/XdwGJCh3UMXINz+/ft127Zt2W/Lli3sZ+IfIZ4U1YppxIN/0lBQedJJJ3GHyA3xeNVJy5Qpo0877TS9adMmroU9e/ZoUinD5cOsDO7TTz/lZ+g3O3LkSMiP1OpzGIz6ypcvr88880wNPVBwOJhGWnD1hRdeyM+m/BHiMaUmQsvRunVrTdrjbcKIRDzoQ1abc8Ymkyca/R0ObRnp4FeqVCluu8kQD9oJCDBc4wNpOGd/tG8cUsfArVGjRs7icBmeeuop/eWXX4b4m/QgxJOi2jCNeHBi3uoQhQoVijjj+fzzzzXWi8MdRk4bN260vd10UjtwlBsQBDoWltmcDrMg+N91113sbT3HGr2RzieOM27cOGdSGkpayfqpPToMeRnQgxBPQMC7zHbr1q3clsKJ54033mB/MuIWQj7QOo/Z9W233ZYjB/zzR1tOhngGDx7McclaaUi6lj/KA2JB+iNHjtQYmPXp00c/+OCD2rkSEBLZoAchnhRVhmnE4/zM2rVr5yCeX3/9lWdDZCMjhHy++eYbXbp0aR69WaM6K61ondR6H+uKjVesYZONjpBg/fr14840aNAg9h84cCCP6kAq3bt352UMkChmP5Z79NFHOQ5GgfjHDtJCB7VmUlY4E65CPCbUQvQyRGvTWPpt06YNtzOLfCzSqVixoj3TdqacG+KxiA57qE4HIQOQDSl6ZbLBPfLHFbMfXPGDQILJTognRbWTbsQDGLC8hdFb1apVmXws0smbN69GRwh30TppeDi3z3PnzuU9Guj2spYJOnbsaHcma5kQHatatWq25E6XLl04zLnnnstXqwPmz59fm2aSQojHbWsIJlysNg3ywWwf7Q/2hdBXopEOSp8b4sEyWuXKlXmvBnlB2XDNmjXtvoB29Mwzz/Az9njIpLb+7bffmACxR4oyhs+WgkE0cq5CPJFxybVvOhIPPvrFF1/kDoVGj03MaKSDsLE6Kd67ddBQe8stt3BnAbk4SQ4dHcIQixcv5uQwi4FEHjqWtUwHiTg8o8xk7VMfPXpUQ1oOS4pk6507pNuypDqcEE+qEc5d+vHaNGba1j92DJC+++67qBnmhniQKJaXyVKrLXRDpqP1Pffcw219yZIlTDZo9w0aNAgpg7X0DGlQU50QT4pqJl2JB3BAeAANGr9Ro0ZFRSheJ40a0fECSlQtEVBI40TaY3IE59t169Zx2SDxA3f55ZdHLKs1OoWAgilOiMeUmohcjnhtGnuOmOlYs2pIsEUSOEDquSUeq4QgOyyFw2EPB/0S0qnvvPMO3/fq1csKylfs1SIMymaqE+JJUc2kK/FgeQ3SZuhcmMJjFrF9+/aIKMXrpBEjOTyHDx/OHQQzmo8++sjx5n+3WG7ADGfv3r0h77DkAak8SLHBwbw4OtqyZctCwlmd1KTNViGekCoy7iFWm7ZIB8tr6CdXX321/Q8+EvnkhnggLIA+CMEBy+F/SsmSJTX2YeEw24L4N2ZCTof9H/QHLMWZ6oR4UlQz6Ug8mNqDdLC8hnMB+CcZi3xiddJ4sOK8AfKBUbRwYrHiomOhA6GDOx2IBP7WIdhnn32WnyGY4HQ1atRgf0vE2vkuqHshnqCQd5dvtDaN/UdrT4esZnJiGBhZ5BM+60CAaMSDvRgMnM4///yohdqxY4cuWLAgC/WQ5VaNWUyLFi24DM5BGgQd0BewD4QD0xCsQdo4pmCy9gIhnqhVn7sX6UY8mMo7Scf6ekiTWeSDb3K6aJ0UYUAK6ABr1651RrHvsQeDDoODo3fffXeOH0Z8cNBCgHA4KIfzEliag5Qd/LDODff7779zB0VHhRQQpI2gqgRhOnXqxGFM+SPEY0pNRC4HSAXt5qabbgoJgP4BQReLdKyXIB+EXbNmjeVlXy3iCZ9xIy3kgbNnsRz2arBHibD4kcl0PWTIkJAoILHOnTvbYRAOM7KVK1eGhDPtQYgnRTWSbsQDGCCWbJ2AdsKCkRRmFeEuWidFOOsff7QOABPQVoeKdL333ns5O+TRvn17XaRIETt8sWLFckir4VS387Q40sSBWWttPLzsQT0L8QSFvFn5zp8/n/tIvFJh1oJ/0osWLWKhmWjhoTEBMyFIg0K4xnQnxJOiGjKZeFL0ySHJYiSIZbR9+/aF+Cf7ADwXLlzIB1ktdTrhaUGNCPS3YWbkRkghPL4fz0I8fqBsdh5oyyVKlNCmqXPyEzUhnhShne3Eg8OeV111VYrQTd9khXjSt+68Kjmk1GbNmuVVcmmZjhBPiqot24kHGgkiSfqkCO60SVaIJ22qSgqaQgSEeFIELpaDIOo4ceLEFOUgyaYjAqNHj9Y4eBikg+JJLPX45aAx3GTRXr9wkHz+QQDL4diHhYCSie4YFIoK6Inr3bu3IsWRivYBPEkvXiIkLqlIzFFRp4sXVN5nCQKkyFHNmDFDkeh6YF9M5zwUnQ9RJLLrSxlIvZEireOKSNeX/CQT8xEgFT+KtDAokspTpNrKuAKnNfGQaK8iVS/qiy++UGRrxjhwpUD+IkDSRooO/ylS9KhITN3fzB25+U08GPCRKiYmW7In4yiJ3GYjAphLkM0sRRpLFImamwmBl9MwmISFAky/HEQbcbYk/GCjX/lLPmYhAEWPULViGfYKqnR+L7VB8wUMmEFjsmXALKhvl3yDRwCa43E2ECLipjrlZcH8Jh6U3dIge8MNN2gYOhOXfQig3i0zyLAPFLTzm3jwvTgfhj1PGC1bsWJF0BBI/gEgADVDlnor6IM02aX1Ups1hyT15Io0KStSDaNI95ki3WSK1G9Yr+WaoQhQx1JkulutWrVKkaZsRSfZFZZfg3Z+L7VZ3/vhhx8qImBFZp0VncJXp556qqKRr/U6469YaiVJT0WqorLqu1GxdJCb9xRJlQ/3AzJlYnR9ZwTxAGGQDklyKBrt8Vo3GmC2ONIqoGjUr+rXr69IoitbPpv/uZCZB0XmtxVp3la4N8EFRTz4dvzzJVFatXz5cu4TIOdscbS0pEiRLbeFcuXKZctn83eSZKMie1mKZrw8CDP94zOGeEwHOpXlI/ME6s0331QkOqnKli2byqwkbRcIBEk8LoqXsUFoj0/REpMihaPqyiuvzNjvzIQPE+LJgFrs0KGDIkugQjyG1KUQTzAVQboQ1cMPP6zIjo5q2rRpMIWQXF0hIMTjCiazA5H2XPXyyy+rb7/9VpENHbMLmwWlE+IJppJJmkuRhKuaM2eOatasWTCFkFxdISDE4womswOR2nj1wgsv8N5Wtq1tm1gzQjzB1MrQoUMVSdbyIXbSYxhMISRXVwgI8biCyexAZKxKkY4yReKUimyQmF3YLCidEE8wlfzcc8/xaX1oTxHiCaYO3OYqxOMWKYPDkRE3NWrUKCEeQ+pIiCeYiiAzCHysgowcqubNmwdTCMnVFQJCPK5gMjtQz549WV/d5s2b+QyT2aXN/NIJ8QRTx2RQUd1yyy3qrbfeUmQyO5hCSK6uEBDicQWT2YGgqwuHJzdt2qTILK/Zhc2C0gnxBFPJkyZNUl27dlVki0e1aNEimEJIrq4QEOJxBZPZgfr3769ITYv6+uuvVaVKlcwubBaUTognmEqeMmWKIn11QjzBwJ9QrkI8CcFlZuBBgwapRx55JCbxrFu3jk91W19AyjR5Wa5OnTqsYgT+OHi3e/duBfFsp8qhJUuWqP3796smTZpY0eUaAwEhnhjgpPAVVGe1a9dOzZw5U7Vs2TKFOUnSuUbAS0VyQSgJ9bL86ZoWlFJSQ9CkoyvqJwwZMoQ11taoUUPjRxoONJGLJjU7mtSscDyy6cLpkC2ZkHSgeJBIJ8RPHqIjEISS0OilyZ43RDjcfkmLR/Z8dJp+adprp05T3D0tNkgFxAMzEdEcwpDSyJDXVryFCxeyv0U8J5xwgiYjZnZYIR4bClc3QjyuYPI8EB0c5X7wxhtveJ62JOgtAkI83uIZSGrDhw/nDrdx48ao+UciHjrvwPFIsSLHA/F06tRJlypVStOShZ2WEI8NhasbIR5XMHkeaP78+UI8nqOamgSFeFKDq6+pYmkMM554xFO4cGENOx34kVlcXbp0aU3WOu2ygnjIvITGUgXSwwgSTojHhsjVjRCPK5g8D0RmIbjdwjaROLMREOIxu35clY60FnCH27BhQ9TwmPHAOuc111zDv8aNG+vixYvrMmXK2IRlEQ8SQTjS+6YPHjwoxBMV1cgvhHgi45Jq38WLF3M/ICGDVGcl6ecSASGeXAJoQnTS08YdLpbJ50hLbSAVsuGh7777bv4MJ/H88MMPGns9dEZIiCfBShbiSRAwj4J/9tln3A9mzJjhUYqSTKoQEOJJFbI+pkuaqbnDkch01FwjEQ8CQ1oNsx84J/HgmdTwsDnlWrVqiVQbAHHphHhcAuVxMLJEK8TjMaapSk6IJ1XI+pgu2eJxRTwnnniiXrp0Kf/mzZvHggR58uTRWKqDCyeev/76S9etW5fTFnFq9xUqxOMeKy9DYuCFvcnp06d7mayklQIEhHhSAKrfSUJ8FB1u7dq1UbMmlTocBuHww37PWWedpemwoyYz4RwvnHjgiTTJhr0mw1pR05YXoQgI8YTi4dcTzrGhbWMgJs5sBERzAbXUdHdQAw+liF988YWqVq1aun9O2pdfNBcEU4UwhAizIK+++qpq3759MIWQXF0hIMTjCiazA9GyGduYX7NmjapevbrZhc2C0gnxBFPJJBCj6IhADuKhJWN1zDHHhKiBCqaEkquNgJcTMlGZ4yWakdPCshjERp1uwYIFvMRAxGN7Q2Jt6tSp9rPc+IeALLX5g/XEiRP1tm3b7Mx27drF/cDZ7sk4osb/JXFmISB7PGbVh6vSYF+GlhL0zp07OfzHH3/MHW716tX8DE0ElStX1qT001V6EshbBIR4vMUzWmo4twaBmWeffVZDEObnn3/mfkBaqjnKhAkTdJEiRTQZhouWhPgHhIAQT0DA5ybbAwcO8OHPk046SaNzffrpp9zhli9frklLNYtAX3DBBbbQQG7ykriJIyDEkzhmycYg+zvc9knLuiYt6nz/zDPP6FatWvE9LT1LP0gW3BTGE+JJIbipTHrEiBHcsWjNVJ9yyil8f/zxx9t+cogulejHTluIJzY+Xr4lgQKdL18+bve0j2O3f/QL/KzZj5d5Slq5R0CEC+zdrvS6OXLkiKpSpYqCJE+4IzFpRcsQspkaDoxPzyJc4BPQf2dDmjcUDcRyZFqhQgVFItYKtqfEmYWAEI9Z9ZFQaSxTv+GRaNNVkWLPcG959gkBIR6fgP47GxIqYKOGJFATkvHYsWNV9+7dQ/zkwQwEhHjMqIekSgExUYhPY3ZjOYiTbt68WdHyg+UlV58REOLxGXDK7qGHHlLA3XIlS5ZUW7ZsUfnz57e85GoQAkI8BlVGMkUhEwaqdevWdlSS8FFYehAXHAJCPP5j/8svv/Dh0X379nHmQ4cOVffff7//BZEcXSEgxOMKJnMD0TafuuiiixRJtCmyMKq2bt2qChUqZG6Bs6BkQjzBVPK///1v9eCDDyqS9lR0vkeRsE0wBZFc4yIgxBMXIvMDvP/++6pRo0aKDLzxkoP5Jc7sEgrxBFO/2OOpWLGiuvXWWxUdKwimEJKrKwSEeFzBZH4gMtymyC6PogN15hc2w0soxON/BWPmTwdI1ZgxYxRZ1VVVq1ZVxYoV878gkqMrBIR4XMFkXqDvv/9ezZo1S5G5XwUdVT/++CMvsWFTtVKlSqp58+bqiiuuUAULFjSv8BleIiEefyr40KFDiqyNqpkzZ6r58+crOlgdkjEGYVdeeaVq2bIl74OKwE0IPIE+CPEECn/imZOZAtW3b1/1zjvvqAIFCqj/+7//U+XKlVMlSpRQ6Ig7duxQZBBLkW0SRepCeIMVm6yy3p041snGEOJJFjl38UhfocJRgoEDB/KA6+KLL2bt7Di/hn6A2Q/6AaQ9SV2OIsukisy4KzrYy1qroTBUXMAI5P4M6j8piJLQf7Dw+u7o0aOs7BB2dGgZgU9k79+/385m0KBBeuHChfYziVRr2mjVJGjA6nU++ugj+53cpBYB0VyQOnx//fVXfdVVV+ljjz1Wd+7cWUNzgdPBj0SrnV4adnratGnDmgyuvfZaDQW64oJFQFTmBIu/q9xBMDDEBmuhw4cP13/++WeOeKeddpqGsbdwR8twmpYb2JjbuHHjwl/LcwoQEOJJAaiUJLRPn3322frkk0/W0MgeyTVs2FDfdNNNkV7pt956S59wwgm6Zs2amsSvI4YRT38QEOLxB+ekc4HW3RYtWmjaKNW0nxM1ndNPP10PGTIk4nvMlu655x4eJcJaqbjUIiDE4z2+hw8f1vXr19elSpXSmM1HcyRYoElrR7TXbFEXug0xkEO/EBcMAkI8weDuOldax2Yz1bR5GjNO8eLF9ZNPPhkzzPXXX6+hSHT9+vUxw8nL3CEgxJM7/CLFvu+++zQJymhoYI/lQDzQWB3LYQAHc+6PPfZYrGDyLoUICPGkENzcJg0jVtC8i39k8RyIhw7QxQxGwge8P4SlN3GpQ0CIJza2MGaYiLP6QaSl5PB0LrvsMn3DDTeEe+d4xj4QBmGWTascAcQjpQgI8aQU3tDEYbDtyy+/DPWM8YQZSpkyZTQII54jaR49ePDgeMF4nZvkWaKukcdNQALERUCIJzZEmHH3799f79mzJ3bAv99i6cxtPwDxdOnSJW66EFIgTR+6Z8+eccNKAO8REOLxHtOoKcKAG4l86k6dOsUloN9++41nO7C748a5JR6kdd5557FEkJt0JUziCAjxxMesd+/ebB0UkpexCIjMf7CVUdLKET9RCnH55Ze7btu9evXSpFRXDMW5QtbbQEI83uIZNzWIPEMUFGLRsQho2rRpGoatIJXmxtHBUU3nR9wE1RC9hmRQJOk4VwlIoJgICPHEhIdfWsu+mH3DPHU0AiJ1UCwGTefS4idKIUA86FdunGUyfsWKFW6CSxgPERDi8RBMt0mR9mjuTOh00QgIYc4991y3SWoQD/7huXGWqewvvvjCTXAJkyACQjzuAFu6dCm3f/SDaASE4wMQgXbrSFuHxhK1G4eBF/rfiy++6Ca4hPEQAdFcQC3ebwdlhrTcpWjT1M6aOoC67rrrFK19K5zAbtu2rfrjjz/U7Nmz7TCxbs444wx1++23u1ISCvU6UK0zd+5cVikSK115lzgC0FwA+zA0ok48cpbF6NOnj1qyZEnIV0PjRo8ePVjrBklqch+AJo41a9ZgoBzzR8cGWF8htBqEh0Umlh+0F0CnGzQdwIwItFqL8w8BT4kH/zRptCfK+VzUn2U3JDwoCIiWCpiUKleurGjzU9GZA7vDWB0n/AoFiTB6Bd1s4e+czyC8RYsWKRInZaWiJAEUXgR5ziUCNIIWC7C5xBDRMQCrVasWG3SjA6Oe6h1EP/n9999ZmSidk1MgOHH+IeAp8cAE7auvvqpgGVNcbAQwwqKN05BAtPejSKWHGjBggCL1Q6z0E7bkoXUaDqM0S89U+BWEX6dOHTaP4HznvEcapOGAiQyae+kwqYJWa3HeIwC7SHRA0fuEMyxFmDAAqTgdZuOYCcFsNdr11KlT1aZNm2xLu8427bxHGt26deMZz7BhwyL2FSs8+lrZsmUV7XWqfv36idE4ZwX4cU+jYXE+I4BNTQgYUP3yD/cdOnQIOdhJHUiT8kPXJYN0DoQG3DiIsyJvrLGLEwSCQgCHookI7H6AfUqyoBtyfGDUqFF8cNStloHGjRtzX3LzTVBFhX4wZcoUN8EljIcIYFlGnI8IQEEhmS2ISjhWUWjEpgsXLqxpOcDyinkF8ZDxq5hhrJcTJ05kvW+0PGd5yVUQ8BUBnKOhGQf3g0iEYxXm888/5zBuldw2adJEt2/f3ooe80pm4zntcEWjMSPJS08QEOLxBEb3ieDAWqQZTngK0EeF0RipdQ9/FfEZB+wefvjhiO/CPck+iYYyRXGCQFAI0DIaS2KGz3DCywMtB2jb0DXoxoF4oIHajcNBU5xpE+c/AkI8PmJO0ju6Y8eOIUtqsbKvUaMGa5aOFcZ6h84JvW7x3JYtW/hgKpYwxAkCQSCAMznPPfdcyJJarHLQnqdr9TZQB9WuXbtYyfE7Sw0PVhbE+Y+Ap8IFNEIXFwMBWlNm42wxgoS8IjXubD3xgw8+UKQKJORd+AMMXUFCjfZ5wl+FPNMZB7V48WJFNkrYkFzIS3kQBAxEgJblVMWKFdmK6NixY2OWsFmzZoqWqNWMGTNihoMQz7Jly6QfxEQphS/95zrJMREEGjRooIlU9O7du2NGQxiMDGM5kg7i5buXXnopVjB5JwgYh8D48eO57ZLl0Zhlg7kDGH2L5Z555hlOi8xmxwom71KIgCy1pRBcL5Levn07r4Vfcskl2mlxNDxtEA8UL0ZzkKSDNdKbb745WhDxFwSMRoAOSGs6q6YhFBDN0YxHt27dOtprPXnyZNZWQCLUUcPIi9QjIMSTeoxznQPZjGdFidWrV9fRJHDKlSuXw+SvlTE6W4ECBXSjRo00DGqJEwTSEQEoDMUeKYRzcHQgUlsG8dDZtByfB91wUApKi0caBAYDi+KCQ0CIJzjsE8oZ5hRIkwErVHz00Uc1NF07HYgnfBSHTdzmzZtzZ7vttts0Oq44QSCdEYCUG5Thwgx8hQoV9MsvvxzSF/r27Rsy84fYNpbpcNwAs6WRI0em8+dnTNlFuCCF+2deJ02mEhT0gJFEEKvHgSodbKYS6aiiRYsqUnqoSMW8WrVqlZo1a5aiA6KKzgwpMhCnWrVq5XVxJD1BIDAEIBwDjQPQvgH1NxC+gYodaD0gclLQR7hhwwZF53+4X0APIlkc5b4SWKElYxsBIR4bivS5IVMJigQE1MyZMxWZAsasNaTwULJIYqUsBQRlozQ6DHkvD4JApiCwbds2HmTNmzdPfffdd4pWBph4SLO7on1P7gfQxVaqVKlM+eSM+A4hnjSvRlpyUzt27GBlhxA5heg1mcEWsknzepXiJ4cAlOBiNgTN7uLMRUCIx9y6SahkOLuAZbXVq1cnFE8CCwKZhEDt2rXZfAIJHmTSZ2XctwjxZEiVnnTSSYo2ULnTZcgnyWcIAgkjQMcO+GBouOb3hBOSCClFQIgnpfD6lziW12DygKyK+pep5CQIGIYAWSBVZF6eBQoMK5oUx4GAEI8DjHS+hW0RMhGs1q5dm86fIWUXBHKFAKQ8ydyC2ELKFYqpjyzEk3qMfckB1krz5cunYCJYnCCQrQjAsCF0HIoxSrNbgBCP2fXjunTVqlVjMVIy8hYxDiTf5s6dG/IO5x8uuOACVaVKlRD/SA+QEiKDWQpLGaQJO0cQWJHEujrEuJ0O/oiLkag4QSDVCJBBRTV9+nTuC6nOS9JPHgEhnuSxMyombNPDhjwOzUVyWH4gWyWs5Tdv3rwchBSPqr1797KZYRwyjeV27typSpQowWcmcC4i3OG8ENL68MMPQ16Rinr1008/8UG+kBfyIAikAAFoaCcVUTnOtqUgK0kyFwgI8eQCPJOi1qtXT5FFUbVx48aIxbKIZ+vWrWxr3gpERuHUnDlzOC72iCyHQ6mWfXr4CfFYyMjVZATIwJwaN24cz3ic7dfkMmdl2TJG+U+WfwgsipLKkKgo0Mlu1tlGxBMSxlI3TyfA2R/G6mCADrqwaAmOdWHhBakg4fikiickvvUAVfSRrJrSTEjDtIM4QcAPBHr06MHtVJSA+oF28nnIjCdDhhvYWyHN1XxqO9InWTMeaDaA+hBqMqxe5JFHHuGZDfS77dq1iw+hYj+GTHSzwbgHHniAxVNxOFWW2iIhK34mIUAaqNXTTz/NUm3HHXecSUWTsjgRSJ6zJKZJCNC+iyZyiFoka8ZDdc8jQueVrDVyvKFDh7IGX9IFp0khKf9IaIHt3cuMJyq08sIgBB588EFu36Qw16BSSVHCEcDIV1wGIAA786SrLeqXWMRDwgeahBD0wYMH9aZNm3T58uX5h4gwnQBCorVx+4dn2ECJRzxY4qhZs2aO/GERkiSNcviLhyCQCgRoBs9tWEyApAJd79IU4vEOy0BT6tSpE9sniVYIi3jC93hgnwTkAqNasGB68sknhxjJWrlypd68eXNc4hk7dqwmaTkNg1tOR2rq9RNPPOH0kntBIGUIDB482G7PKctEEs41AkI8uYbQjARuuukme+YSqUTRiAd250E8JPKsFy9ezPdYcoOhuddff52X3uj8jk08IBGycWL/SD0JZ0caE5h4unbtqkk9vd6yZYu+4447eOa0aNGiSEUSP0EgVwjAKFy4GzZsGLdhOjsW8gozfHHmICDEY05duC7J9u3b9YoVK0LCw5wvrJA63bJlyzQd4GQvi3gs6TUrHEgExPP888+zF0aMtCnL5oVhYpjEU9mfxKk5HMI6f5jlWI4O7ulixYrZ70k8W5Noq/VaroKApwhgyRgmsH/55Rc73VGjRnH7c868J0yYoF988UU7jNwEj4AQT/B1kFQJGjduzDbkrZHcvffeq8nwFaeFjVWsdZOGgRxLX24yo/NAGqSFfZ1EHUahX3/9tcZeUqQRaaLpSXhBIBYCIBos55IlUg5mHQ8AKdEBaU2WdzUpz+V9zVjpyDt/ERDi8Rdvz3Ijs9a8jAXhAMxm+vTpw8SDf/oXXXQRj/qeeuopz/KThAQBExHA3iT6AGbhrVu31k8++STfT5s2TZPGdr4nc/EmFj2ryyTneKjFpquDQkSYv3Y6GISjWZCCfR5oKYAZbHGCQCYjADPwXbp0ifiJRYsWVTCPjas4cxAQ4jGnLhIuCTRRw9QvLWnliDtw4EBF6985/MVDEMg0BKCJ+vzzz4+omZ3O9Sia8WTaJ6f99wjxpHkVYqSHEZ/TYdaDUR6JRju95V4QyFgEMPPHCoDTQfs6Zv0wkCjOLASEeMyqj4RLQ2LLqmrVqmySwIoMdTckVmo9ylUQyHgEaMNE1a1bl81eWx9Lh5rViBEjrEe5GoSAEI9BlZFsUe688041evRojp4/f371zTffsD62ZNOTeIJAOiIAPYSwFwUH0x+kmSNEE3s6flOmllmIJwNqlsSe2c4O7PF069aN1cJnwGfJJwgCCSMA4gEB0UFm9cILLyQcXyL4g4AQjz84pzyXvn37KtI4wPZ4oElanCCQjQjQ+TMF21QQvHFjWTcbMTLhm4V4TKgFD8oAI3D9+vVTY8aM8SA1SUIQSF8EJk2apGCJVJy5CAjxmFs3CZcMYqVigyRh2CSCICAI+IyAEI/PgEt2goAgIAhkOwJCPNneAuT7BQFBQBDwGQEhHp8Bl+wEAUFAEMh2BIR4sr0FyPcLAoKAIOAzAkI8PgMu2QkCgoAgkO0ICPFkewuQ7xcEBAFBwGcEhHh8BlyyEwQEAUEg2xEQ4sn2FiDfLwgIAoKAzwgI8fgMuGQnCAgCgkC2IyDEk+0tQL5fEBAEBAGfERDi8RlwyU4QEAQEgWxHQIgn21uAfL8gIAgIAj4jIMTjM+CSnSAgCAgC2Y6AEE+2twD5fkFAEBAEfEbg/wG+iJ+eRV4NUQAAAABJRU5ErkJggg=="}}},{"cell_type":"code","source":"%%writefile _model.py\n\nfrom copy import deepcopy\nfrom types import MethodType\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport timm\nfrom timm.models.convnext import ConvNeXtBlock\n\nfrom monai.networks.blocks import UpSample, SubpixelUpsample\n\n####################\n## EMA + Ensemble ##\n####################\n\nclass ModelEMA(nn.Module):\n    def __init__(self, model, decay=0.99, device=None):\n        super().__init__()\n        self.module = deepcopy(model)\n        self.module.eval()\n        self.decay = decay\n        self.device = device\n        if self.device is not None:\n            self.module.to(device=device)\n\n    def _update(self, model, update_fn):\n        with torch.no_grad():\n            for ema_v, model_v in zip(self.module.state_dict().values(), model.state_dict().values()):\n                if self.device is not None:\n                    model_v = model_v.to(device=self.device)\n                ema_v.copy_(update_fn(ema_v, model_v))\n\n    def update(self, model):\n        self._update(model, update_fn=lambda e, m: self.decay * e + (1. - self.decay) * m)\n\n    def set(self, model):\n        self._update(model, update_fn=lambda e, m: m)\n\n\nclass EnsembleModel(nn.Module):\n    def __init__(self, models):\n        super().__init__()\n        self.models = nn.ModuleList(models).eval()\n\n    def forward(self, x):\n        output = None\n        \n        for m in self.models:\n            logits= m(x)\n            \n            if output is None:\n                output = logits\n            else:\n                output += logits\n                \n        output /= len(self.models)\n        return output\n        \n\n#############\n## Decoder ##\n#############\n\nclass ConvBnAct2d(nn.Module):\n    def __init__(\n        self,\n        in_channels,\n        out_channels,\n        kernel_size,\n        padding: int = 0,\n        stride: int = 1,\n        norm_layer: nn.Module = nn.Identity,\n        act_layer: nn.Module = nn.ReLU,\n    ):\n        super().__init__()\n\n        self.conv= nn.Conv2d(\n            in_channels, \n            out_channels,\n            kernel_size,\n            stride=stride, \n            padding=padding, \n            bias=False,\n        )\n        self.norm = norm_layer(out_channels) if norm_layer != nn.Identity else nn.Identity()\n        self.act= act_layer(inplace=True)\n\n    def forward(self, x):\n        x = self.conv(x)\n        x = self.norm(x)\n        x = self.act(x)\n        return x\n\n\nclass SCSEModule2d(nn.Module):\n    def __init__(self, in_channels, reduction=16):\n        super().__init__()\n        self.cSE = nn.Sequential(\n            nn.AdaptiveAvgPool2d(1),\n            nn.Conv2d(in_channels, in_channels // reduction, 1),\n            nn.Tanh(),\n            nn.Conv2d(in_channels // reduction, in_channels, 1),\n            nn.Sigmoid(),\n        )\n        self.sSE = nn.Sequential(\n            nn.Conv2d(in_channels, 1, 1), \n            nn.Sigmoid(),\n            )\n\n    def forward(self, x):\n        return x * self.cSE(x) + x * self.sSE(x)\n\nclass Attention2d(nn.Module):\n    def __init__(self, name, **params):\n        super().__init__()\n        if name is None:\n            self.attention = nn.Identity(**params)\n        elif name == \"scse\":\n            self.attention = SCSEModule2d(**params)\n        else:\n            raise ValueError(\"Attention {} is not implemented\".format(name))\n\n    def forward(self, x):\n        return self.attention(x)\n\nclass DecoderBlock2d(nn.Module):\n    def __init__(\n        self,\n        in_channels,\n        skip_channels,\n        out_channels,\n        norm_layer: nn.Module = nn.Identity,\n        attention_type: str = None,\n        intermediate_conv: bool = False,\n        upsample_mode: str = \"deconv\",\n        scale_factor: int = 2,\n    ):\n        super().__init__()\n\n        # Upsample block\n        if upsample_mode == \"pixelshuffle\":\n            self.upsample= SubpixelUpsample(\n                spatial_dims= 2,\n                in_channels= in_channels,\n                scale_factor= scale_factor,\n            )\n        else:\n            self.upsample = UpSample(\n                spatial_dims= 2,\n                in_channels= in_channels,\n                out_channels= in_channels,\n                scale_factor= scale_factor,\n                mode= upsample_mode,\n            )\n\n        if intermediate_conv:\n            k= 3\n            c= skip_channels if skip_channels != 0 else in_channels\n            self.intermediate_conv = nn.Sequential(\n                ConvBnAct2d(c, c, k, k//2),\n                ConvBnAct2d(c, c, k, k//2),\n                )\n        else:\n            self.intermediate_conv= None\n\n        self.attention1 = Attention2d(\n            name= attention_type, \n            in_channels= in_channels + skip_channels,\n            )\n\n        self.conv1 = ConvBnAct2d(\n            in_channels + skip_channels,\n            out_channels,\n            kernel_size= 3,\n            padding= 1,\n            norm_layer= norm_layer,\n        )\n\n        self.conv2 = ConvBnAct2d(\n            out_channels,\n            out_channels,\n            kernel_size= 3,\n            padding= 1,\n            norm_layer= norm_layer,\n        )\n        self.attention2 = Attention2d(\n            name= attention_type, \n            in_channels= out_channels,\n            )\n\n    def forward(self, x, skip=None):\n        x = self.upsample(x)\n\n        if self.intermediate_conv is not None:\n            if skip is not None:\n                skip = self.intermediate_conv(skip)\n            else:\n                x = self.intermediate_conv(x)\n\n        if skip is not None:\n            # print(x.shape, skip.shape)\n            x = torch.cat([x, skip], dim=1)\n            x = self.attention1(x)\n\n        x = self.conv1(x)\n        x = self.conv2(x)\n        x = self.attention2(x)\n        return x\n\n\nclass UnetDecoder2d(nn.Module):\n    \"\"\"\n    Unet decoder.\n    Source: https://arxiv.org/abs/1505.04597\n    \"\"\"\n    def __init__(\n        self,\n        encoder_channels: tuple[int],\n        skip_channels: tuple[int] = None,\n        decoder_channels: tuple = (256, 128, 64, 32),\n        scale_factors: tuple = (2,2,2,2),\n        norm_layer: nn.Module = nn.Identity,\n        attention_type: str = None,\n        intermediate_conv: bool = False,\n        upsample_mode: str = \"deconv\",\n    ):\n        super().__init__()\n        \n        if len(encoder_channels) == 4:\n            decoder_channels= decoder_channels[1:]\n        self.decoder_channels= decoder_channels\n        \n        if skip_channels is None:\n            skip_channels= list(encoder_channels[1:]) + [0]\n\n        # Build decoder blocks\n        in_channels= [encoder_channels[0]] + list(decoder_channels[:-1])\n        self.blocks = nn.ModuleList()\n\n        for i, (ic, sc, dc) in enumerate(zip(in_channels, skip_channels, decoder_channels)):\n            # print(i, ic, sc, dc)\n            self.blocks.append(\n                DecoderBlock2d(\n                    ic, sc, dc, \n                    norm_layer= norm_layer,\n                    attention_type= attention_type,\n                    intermediate_conv= intermediate_conv,\n                    upsample_mode= upsample_mode,\n                    scale_factor= scale_factors[i],\n                    )\n            )\n\n    def forward(self, feats: list[torch.Tensor]):\n        res= [feats[0]]\n        feats= feats[1:]\n\n        # Decoder blocks\n        for i, b in enumerate(self.blocks):\n            skip= feats[i] if i < len(feats) else None\n            res.append(\n                b(res[-1], skip=skip),\n                )\n            \n        return res\n\nclass SegmentationHead2d(nn.Module):\n    def __init__(\n        self,\n        in_channels,\n        out_channels,\n        scale_factor: tuple[int] = (2,2),\n        kernel_size: int = 3,\n        mode: str = \"nontrainable\",\n    ):\n        super().__init__()\n        self.conv= nn.Conv2d(\n            in_channels, out_channels, kernel_size= kernel_size,\n            padding= kernel_size//2\n        )\n        self.upsample = UpSample(\n            spatial_dims= 2,\n            in_channels= out_channels,\n            out_channels= out_channels,\n            scale_factor= scale_factor,\n            mode= mode,\n        )\n\n    def forward(self, x):\n        x = self.conv(x)\n        x = self.upsample(x)\n        return x\n        \n\n#############\n## Encoder ##\n#############\n\ndef _convnext_block_forward(self, x):\n    shortcut = x\n    x = self.conv_dw(x)\n\n    if self.use_conv_mlp:\n        x = self.norm(x)\n        x = self.mlp(x)\n    else:\n        x = self.norm(x)\n        x = x.permute(0, 2, 3, 1)\n        x = x.contiguous()\n        x = self.mlp(x)\n        x = x.permute(0, 3, 1, 2)\n        x = x.contiguous()\n\n    if self.gamma is not None:\n        x = x * self.gamma.reshape(1, -1, 1, 1)\n\n    x = self.drop_path(x) + self.shortcut(shortcut)\n    return x\n\n\nclass Net(nn.Module):\n    def __init__(\n        self,\n        backbone: str,\n        pretrained: bool = True,\n    ):\n        super().__init__()\n        \n        # Encoder\n        self.backbone= timm.create_model(\n            backbone,\n            in_chans= 5,\n            pretrained= pretrained,\n            features_only= True,\n            drop_path_rate=0.0,\n            )\n        ecs= [_[\"num_chs\"] for _ in self.backbone.feature_info][::-1]\n\n        # Decoder\n        self.decoder= UnetDecoder2d(\n            encoder_channels= ecs,\n        )\n\n        self.seg_head= SegmentationHead2d(\n            in_channels= self.decoder.decoder_channels[-1],\n            out_channels= 1,\n            scale_factor= 1,\n        )\n        \n        self._update_stem(backbone)\n        \n        self.replace_activations(self.backbone, log=True)\n        self.replace_norms(self.backbone, log=True)\n        self.replace_forwards(self.backbone, log=True)\n\n    def _update_stem(self, backbone):\n        if backbone.startswith(\"convnext\"):\n\n            # Update stride\n            self.backbone.stem_0.stride = (4, 1)\n            self.backbone.stem_0.padding = (0, 2)\n\n            # Duplicate stem layer (to downsample height)\n            with torch.no_grad():\n                w = self.backbone.stem_0.weight\n                new_conv= nn.Conv2d(w.shape[0], w.shape[0], kernel_size=(4, 4), stride=(4, 1), padding=(0, 1))\n                new_conv.weight.copy_(w.repeat(1, (128//w.shape[1])+1, 1, 1)[:, :new_conv.weight.shape[1], :, :])\n                new_conv.bias.copy_(self.backbone.stem_0.bias)\n\n            self.backbone.stem_0= nn.Sequential(\n                nn.ReflectionPad2d((1,1,80,80)),\n                self.backbone.stem_0,\n                new_conv,\n            )\n\n        else:\n            raise ValueError(\"Custom striding not implemented.\")\n        pass\n\n    def replace_activations(self, module, log=False):\n        if log:\n            print(f\"Replacing all activations with GELU...\")\n        \n        # Apply activations\n        for name, child in module.named_children():\n            if isinstance(child, (\n                nn.ReLU, nn.LeakyReLU, nn.Mish, nn.Sigmoid, \n                nn.Tanh, nn.Softmax, nn.Hardtanh, nn.ELU, \n                nn.SELU, nn.PReLU, nn.CELU, nn.GELU, nn.SiLU,\n            )):\n                setattr(module, name, nn.GELU())\n            else:\n                self.replace_activations(child)\n\n    def replace_norms(self, mod, log=False):\n        if log:\n            print(f\"Replacing all norms with InstanceNorm...\")\n            \n        for name, c in mod.named_children():\n\n            # Get feature size\n            n_feats= None\n            if isinstance(c, (nn.BatchNorm2d, nn.InstanceNorm2d)):\n                n_feats= c.num_features\n            elif isinstance(c, (nn.GroupNorm,)):\n                n_feats= c.num_channels\n            elif isinstance(c, (nn.LayerNorm,)):\n                n_feats= c.normalized_shape[0]\n\n            if n_feats is not None:\n                new = nn.InstanceNorm2d(\n                    n_feats,\n                    affine=True,\n                    )\n                setattr(mod, name, new)\n            else:\n                self.replace_norms(c)\n\n    def replace_forwards(self, mod, log=False):\n        if log:\n            print(f\"Replacing forward functions...\")\n            \n        for name, c in mod.named_children():\n            if isinstance(c, ConvNeXtBlock):\n                c.forward = MethodType(_convnext_block_forward, c)\n            else:\n                self.replace_forwards(c)\n\n        \n    def proc_flip(self, x_in):\n        x_in= torch.flip(x_in, dims=[-3, -1])\n        x= self.backbone(x_in)\n        x= x[::-1]\n\n        # Decoder\n        x= self.decoder(x)\n        x_seg= self.seg_head(x[-1])\n        x_seg= x_seg[..., 1:-1, 1:-1]\n        x_seg= torch.flip(x_seg, dims=[-1])\n        x_seg= x_seg * 1500 + 3000\n        return x_seg\n\n    def forward(self, batch):\n        x= batch\n\n        # Encoder\n        x_in = x\n        x= self.backbone(x)\n        # print([_.shape for _ in x])\n        x= x[::-1]\n\n        # Decoder\n        x= self.decoder(x)\n        # print([_.shape for _ in x])\n        x_seg= self.seg_head(x[-1])\n        x_seg= x_seg[..., 1:-1, 1:-1]\n        x_seg= x_seg * 1500 + 3000\n    \n        if self.training:\n            return x_seg\n        else:\n            p1 = self.proc_flip(x_in)\n            x_seg = torch.mean(torch.stack([x_seg, p1]), dim=0)\n            return x_seg","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:48:38.981638Z","iopub.execute_input":"2025-05-22T15:48:38.981837Z","iopub.status.idle":"2025-05-22T15:48:38.997786Z","shell.execute_reply.started":"2025-05-22T15:48:38.981815Z","shell.execute_reply":"2025-05-22T15:48:38.997202Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Utils\n\nSame as starter notebook.","metadata":{}},{"cell_type":"code","source":"%%writefile _utils.py\n\nimport datetime\n\ndef format_time(elapsed):\n    elapsed_rounded = int(round((elapsed)))\n    return str(datetime.timedelta(seconds=elapsed_rounded))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:48:38.998436Z","iopub.execute_input":"2025-05-22T15:48:38.998675Z","iopub.status.idle":"2025-05-22T15:48:39.011544Z","shell.execute_reply.started":"2025-05-22T15:48:38.998659Z","shell.execute_reply":"2025-05-22T15:48:39.010864Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train\n\nSame as starter notebook. ","metadata":{}},{"cell_type":"code","source":"%%writefile _train.py\n\nimport os\nimport time \nimport random\nimport numpy as np\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nfrom torch.amp import autocast, GradScaler\n\nimport torch.distributed as dist\nfrom torch.utils.data import DistributedSampler\nfrom torch.nn.parallel import DistributedDataParallel\n\nfrom _cfg import cfg\nfrom _dataset import CustomDataset\nfrom _model import ModelEMA, Net\nfrom _utils import format_time\n\ndef set_seed(seed=1234):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = False\n    torch.backends.cudnn.benchmark = True\n\ndef setup(rank, world_size):\n    torch.cuda.set_device(rank)\n    dist.init_process_group(\"nccl\", rank=rank, world_size=world_size)\n    return\n\ndef cleanup():\n    dist.barrier()\n    dist.destroy_process_group()\n    return\n\ndef main(cfg):\n\n    # ========== Datasets / Dataloaders ==========\n    if cfg.local_rank == 0:\n        print(\"=\"*25)\n        print(\"Loading data..\")\n    train_ds = CustomDataset(cfg=cfg, mode=\"train\")\n    sampler= DistributedSampler(\n        train_ds, \n        num_replicas=cfg.world_size, \n        rank=cfg.local_rank,\n    )\n    train_dl = torch.utils.data.DataLoader(\n        train_ds, \n        sampler= sampler,\n        batch_size= cfg.batch_size, \n        num_workers= 4,\n    )\n    \n    valid_ds = CustomDataset(cfg=cfg, mode=\"valid\")\n    sampler= DistributedSampler(\n        valid_ds, \n        num_replicas=cfg.world_size, \n        rank=cfg.local_rank,\n    )\n    valid_dl = torch.utils.data.DataLoader(\n        valid_ds, \n        sampler= sampler,\n        batch_size= cfg.batch_size_val, \n        num_workers= 4,\n    )\n\n    # ========== Model / Optim ==========\n    model = Net(backbone=cfg.backbone)\n    model= model.to(cfg.local_rank)\n    if cfg.ema:\n        if cfg.local_rank == 0:\n            print(\"Initializing EMA model..\")\n        ema_model = ModelEMA(\n            model, \n            decay=cfg.ema_decay, \n            device=cfg.local_rank,\n        )\n    else:\n        ema_model = None\n    model= DistributedDataParallel(\n        model, \n        device_ids=[cfg.local_rank], \n        )\n    \n    criterion = nn.L1Loss()\n    optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n    scaler = GradScaler()\n\n\n    # ========== Training ==========\n    if cfg.local_rank == 0:\n        print(\"=\"*25)\n        print(\"Give me warp {}, Mr. Sulu.\".format(cfg.world_size))\n        print(\"=\"*25)\n    \n    best_loss= 1_000_000\n    val_loss= 1_000_000\n\n    for epoch in range(0, cfg.epochs+1):\n        if epoch != 0:\n            tstart= time.time()\n            train_dl.sampler.set_epoch(epoch)\n    \n            # Train loop\n            model.train()\n            total_loss = []\n            for i, (x, y) in enumerate(train_dl):\n                x = x.to(cfg.local_rank)\n                y = y.to(cfg.local_rank)\n        \n                with autocast(cfg.device.type):\n                    logits = model(x)\n                    \n                loss = criterion(logits, y)\n        \n                scaler.scale(loss).backward()\n                scaler.unscale_(optimizer)\n        \n                torch.nn.utils.clip_grad_norm_(model.parameters(), 3.0)\n        \n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad()\n    \n                total_loss.append(loss.item())\n                \n                if ema_model is not None:\n                    ema_model.update(model)\n                    \n                if cfg.local_rank == 0 and (len(total_loss) >= cfg.logging_steps or i == 0):\n                    train_loss = np.mean(total_loss)\n                    total_loss = []\n                    print(\"Epoch {}:     Train MAE: {:.2f}     Val MAE: {:.2f}     Time: {}     Step: {}/{}\".format(\n                        epoch, \n                        train_loss,\n                        val_loss,\n                        format_time(time.time() - tstart),\n                        i+1, \n                        len(train_dl)+1, \n                    ))\n    \n        # ========== Valid ==========\n        model.eval()\n        val_logits = []\n        val_targets = []\n        with torch.no_grad():\n            for x, y in tqdm(valid_dl, disable=cfg.local_rank != 0):\n                x = x.to(cfg.local_rank)\n                y = y.to(cfg.local_rank)\n    \n                with autocast(cfg.device.type):\n                    if ema_model is not None:\n                        out = ema_model.module(x)\n                    else:\n                        out = model(x)\n\n                val_logits.append(out.cpu())\n                val_targets.append(y.cpu())\n\n            val_logits= torch.cat(val_logits, dim=0)\n            val_targets= torch.cat(val_targets, dim=0)\n                \n            loss = criterion(val_logits, val_targets).item()\n\n        # Gather loss\n        v = torch.tensor([loss], device=cfg.local_rank)\n        torch.distributed.all_reduce(v, op=dist.ReduceOp.SUM)\n        val_loss = (v[0] / cfg.world_size).item()\n    \n        # ========== Weights / Early stopping ==========\n        stop_train = torch.tensor([0], device=cfg.local_rank)\n        if cfg.local_rank == 0:\n            es= cfg.early_stopping\n            if val_loss < best_loss:\n                print(\"New best: {:.2f} -> {:.2f}\".format(best_loss, val_loss))\n                print(\"Saved weights..\")\n                best_loss = val_loss\n                if ema_model is not None:\n                    torch.save(ema_model.module.state_dict(), f'best_model_{cfg.seed}.pt')\n                else:\n                    torch.save(model.state_dict(), f'best_model_{cfg.seed}.pt')\n        \n                es[\"streak\"] = 0\n            else:\n                es= cfg.early_stopping\n                es[\"streak\"] += 1\n                if es[\"streak\"] > es[\"patience\"]:\n                    print(\"Ending training (early_stopping).\")\n                    stop_train = torch.tensor([1], device=cfg.local_rank)\n        \n        # Exits training on all ranks\n        dist.broadcast(stop_train, src=0)\n        if stop_train.item() == 1:\n            return\n\n    return\n    \n\n\nif __name__ == \"__main__\":\n\n    # GPU Specs\n    rank = int(os.environ[\"RANK\"])\n    world_size = int(os.environ[\"WORLD_SIZE\"])\n    _, total = torch.cuda.mem_get_info(device=rank)\n\n    # Init\n    setup(rank, world_size)\n    time.sleep(rank)\n    print(f\"Rank: {rank}, World size: {world_size}, GPU memory: {total / 1024**3:.2f}GB\", flush=True)\n    time.sleep(world_size - rank)\n\n    # Seed\n    set_seed(cfg.seed+rank)\n\n    # Run\n    cfg.local_rank= rank\n    cfg.world_size= world_size\n    main(cfg)\n    cleanup()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:48:39.012443Z","iopub.execute_input":"2025-05-22T15:48:39.012751Z","iopub.status.idle":"2025-05-22T15:48:39.024343Z","shell.execute_reply.started":"2025-05-22T15:48:39.012730Z","shell.execute_reply":"2025-05-22T15:48:39.023682Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if RUN_TRAIN:\n    print(\"Starting training..\")\n    !OMP_NUM_THREADS=1 torchrun --nproc_per_node=2 _train.py","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:48:39.025153Z","iopub.execute_input":"2025-05-22T15:48:39.025396Z","iopub.status.idle":"2025-05-22T15:48:39.035893Z","shell.execute_reply.started":"2025-05-22T15:48:39.025375Z","shell.execute_reply":"2025-05-22T15:48:39.035309Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Pretrained Models\n\nNext, we load in 2x pretrained models. These models were trained with a batch size of 16 for 50 epochs.","metadata":{}},{"cell_type":"code","source":"import glob\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom _cfg import cfg\nfrom _model import Net, EnsembleModel\n\nif RUN_VALID or RUN_TEST:\n\n    # Load pretrained models\n    models = []\n    for f in sorted(glob.glob(\"/kaggle/input/simple-further-finetuned-bartley-open-models/*.pth\")):\n        print(\"Loading: \", f)\n        m = Net(\n            backbone=\"convnext_small.fb_in22k_ft_in1k\",\n            pretrained=False,\n        )\n        state_dict= torch.load(f, map_location=cfg.device, weights_only=True)\n        state_dict= {k.removeprefix(\"_orig_mod.\"):v for k,v in state_dict.items()} # Remove torch.compile() prefix\n\n        m.load_state_dict(state_dict)\n        models.append(m)\n    \n    # Combine\n    model = EnsembleModel(models)\n    model = model.to(cfg.device)\n    model = model.eval()\n    print(\"n_models: {:_}\".format(len(models)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:52:33.493740Z","iopub.execute_input":"2025-05-22T15:52:33.494010Z","iopub.status.idle":"2025-05-22T15:52:38.114174Z","shell.execute_reply.started":"2025-05-22T15:52:33.493991Z","shell.execute_reply":"2025-05-22T15:52:38.113515Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Valid\n\nNext, we score the ensemble on the validation set.","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm\nimport numpy as np\n\nimport torch\nimport torch.nn as nn\nfrom torch.amp import autocast\n\nfrom _dataset import CustomDataset\n\n\nif RUN_VALID:\n\n    # Dataset / Dataloader\n    valid_ds = CustomDataset(cfg=cfg, mode=\"valid\")\n    sampler = torch.utils.data.SequentialSampler(valid_ds)\n    valid_dl = torch.utils.data.DataLoader(\n        valid_ds, \n        sampler= sampler,\n        batch_size= cfg.batch_size_val, \n        num_workers= 4,\n    )\n\n    # Valid loop\n    criterion = nn.L1Loss()\n    val_logits = []\n    val_targets = []\n    \n    with torch.no_grad():\n        for x, y in tqdm(valid_dl):\n            x = x.to(cfg.device)\n            y = y.to(cfg.device)\n    \n            with autocast(cfg.device.type):\n                out = model(x)\n    \n            val_logits.append(out.cpu())\n            val_targets.append(y.cpu())\n    \n        val_logits= torch.cat(val_logits, dim=0)\n        val_targets= torch.cat(val_targets, dim=0)\n    \n        total_loss= criterion(val_logits, val_targets).item()\n    \n    # Dataset Scores\n    ds_idxs= np.array([valid_ds.records])\n    ds_idxs= np.repeat(ds_idxs, repeats=500)\n    \n    print(\"=\"*25)\n    with torch.no_grad():    \n        for idx in sorted(np.unique(ds_idxs)):\n    \n            # Mask\n            mask = ds_idxs == idx\n            logits_ds = val_logits[mask]\n            targets_ds = val_targets[mask]\n    \n            # Score predictions\n            loss = criterion(val_logits[mask], val_targets[mask]).item()\n            print(\"{:15} {:.2f}\".format(idx, loss))\n    print(\"=\"*25)\n    print(\"Val MAE: {:.2f}\".format(total_loss))\n    print(\"=\"*25)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:52:39.651106Z","iopub.execute_input":"2025-05-22T15:52:39.651390Z","iopub.status.idle":"2025-05-22T15:57:14.988037Z","shell.execute_reply.started":"2025-05-22T15:52:39.651370Z","shell.execute_reply":"2025-05-22T15:57:14.987127Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Test\n\nFinally, we make predictions on the test data.","metadata":{}},{"cell_type":"code","source":"import torch\n\nclass TestDataset(torch.utils.data.Dataset):\n    def __init__(self, test_files):\n        self.test_files = test_files\n\n    def __len__(self):\n        return len(self.test_files)\n\n    def __getitem__(self, i):\n        test_file = self.test_files[i]\n        test_stem = test_file.split(\"/\")[-1].split(\".\")[0]\n        return np.load(test_file), test_stem\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:57:38.374809Z","iopub.execute_input":"2025-05-22T15:57:38.375442Z","iopub.status.idle":"2025-05-22T15:57:38.380140Z","shell.execute_reply.started":"2025-05-22T15:57:38.375415Z","shell.execute_reply":"2025-05-22T15:57:38.379536Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import csv\nimport time\nimport glob\nfrom tqdm import tqdm\nimport numpy as np\nimport pandas as pd\n\nfrom _utils import format_time\n\n\nif RUN_TEST:\n\n    ss= pd.read_csv(\"/kaggle/input/waveform-inversion/sample_submission.csv\")    \n    row_count = 0\n    t0 = time.time()\n    \n    test_files = sorted(glob.glob(\"/kaggle/input/open-wfi-test/test/*.npy\"))\n    x_cols = [f\"x_{i}\" for i in range(1, 70, 2)]\n    fieldnames = [\"oid_ypos\"] + x_cols\n    \n    test_ds = TestDataset(test_files)\n    test_dl = torch.utils.data.DataLoader(\n        test_ds, \n        sampler=torch.utils.data.SequentialSampler(test_ds),\n        batch_size=cfg.batch_size_val, \n        num_workers=4,\n    )\n    \n    with open(\"submission.csv\", \"wt\", newline=\"\") as csvfile:\n        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n        writer.writeheader()\n\n        with torch.inference_mode():\n            with torch.autocast(cfg.device.type):\n                for inputs, oids_test in tqdm(test_dl, total=len(test_dl)):\n                    inputs = inputs.to(cfg.device)\n            \n                    outputs = model(inputs)\n                            \n                    y_preds = outputs[:, 0].cpu().numpy()\n                    \n                    for y_pred, oid_test in zip(y_preds, oids_test):\n                        for y_pos in range(70):\n                            row = dict(zip(x_cols, [y_pred[y_pos, x_pos] for x_pos in range(1, 70, 2)]))\n                            row[\"oid_ypos\"] = f\"{oid_test}_y_{y_pos}\"\n            \n                            writer.writerow(row)\n                            row_count += 1\n\n                            # Clear buffer\n                            if row_count % 100_000 == 0:\n                                csvfile.flush()\n    \n    t1 = format_time(time.time() - t0)\n    print(f\"Inference Time: {t1}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:57:38.655923Z","iopub.execute_input":"2025-05-22T15:57:38.656378Z","execution_failed":"2025-05-22T15:58:07.562Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We can also view a few samples to make sure things look reasonable.","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt \n\nif RUN_TEST:\n    # Plot a few samples\n    fig, axes = plt.subplots(3, 5, figsize=(10, 6))\n    axes= axes.flatten()\n\n    n = min(len(outputs), len(axes))\n    \n    for i in range(n):\n        img= outputs[0, 0, ...].cpu().numpy()\n        img = outputs[i, 0].cpu().numpy()\n        idx= oids_test[i]\n    \n        # Plot\n        axes[i].imshow(img, cmap='gray')\n        axes[i].set_title(idx)\n        axes[i].axis('off')\n\n    for i in range(n, len(axes)):\n        axes[i].axis('off')\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-05-22T15:58:07.562Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}