{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Use Hugging Face's Tensorflow 2 transformer models for NQ\n\n![logo.png](attachment:logo.png)","attachments":{"logo.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAArwAAACvCAYAAAAfZw1fAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAAFiUAABYlAUlSJPAAAKK+SURBVHhe7Z0HgBPF/se/uUtyud4bHE06ioqooKKI2Hvv9Vn/9vbsPnt56rN3n72XJ/bewI5iwYKA9HIc1/ul3f1/39kNhJDNJbnjGvOBvexudmdnZyaz3/ntb2ZsbQI0Go1Go9FoNJo+SoL5qdFoNBqNRqPR9Em04NVoNBqNRqPpAXj8sribza118bhb4G01NzQxowWvRqPRaDQaTTciOheoW4rmz/+FOb/9rPaFMvf3X9D8xfWoK18GrXtjRwtejUbTLn//8hXuuu9hte5pqEZFVZ1a12g0Gk38sBNVVVU1Gn96AW1vHo3ML25AVYPH+DKEyrpmZHx+DdxvHoum2S+jqrpWC98Y0IJXo9FE5KOnb8FZt7+KCWP6oWDIVlg2+3O8+PoX5rcajUajiQfK2orvnkDSG/vC9cmpsC39GnCIMEtINA4IwZZoB5xAfuUMOD8+GUlv7YOqmc+ocDTto0dp0Gg0ERm/+RaYNftXtb5o/lxUL/oZf3s2wY/TbsbcylY0uQbhuasOxuHn/RtV9V789uWL2G7KIagsr8D3f81Bdvi6W6PRaDZaqmrqkf31ufDOfg7OVp9hfrTJ4gamT5mByTvtqI4LZvqMLzF5+k6ASzao3GTxJDrg3Px4rNr6PhTlJKvjNOHRFl6NRhORxNQMcw0YMnyk1LVetDQ344Jr78BV/7wAi/6chTm/zEL+8K3w0vOPo/T3H9DozMPzL7+ATC12NRqNZg1VfuCvt29DzsMZsP36FJxtInbt8kVA8EYDj+NCg2+bF/jlcRQ9nYK579+FOu3jYIm28EZLawvQMB+onyvLPKBxEeCrNxav+Rm8TRzpUiCDluDt1CFA+ghZRgJpw6Wws8mm0fQ8dhq/OWbMmq3Wb7jgFGw3dS80OYtw6yWn4ZZHXsB1V/4Tn33yEX77YQYOPfhIvPXLQiRV/I3TjjsUJ93xJo7aScq4RqPRbMQ0iy5N/P42OD+/VHSC7HDKEjA5BpseRWq0a+GlIZfKLXBeQMXJNVpkffn4OzFotwvg0CbNddCCNxxNS4HyL4Canwxx2yAil/s2VFLZpKmWMlCErwgDiuCscUD+FGOfRtPNLP3pA2x/2D9x4M6bYXZtPp47bxK+qsjFHZeegv0OPxTX3/48Pn3hKtzy4ky0rJiP+++5HkeddROGpbfgkoffwqQReWZIGo1Gs3HhF9ngWfUH/NOvR9q8VwyRyjdflBNcD3wGiFfwBtZNC2/j8KPg2/pqpBSP1sLXRAte4i4zBO7qz+TzcxG4C80vupm0TQzhy6VAlqRC8wuNpqvxYdGyCgwZUGRuG5RV1qEw13B5qC5bjozCElWXextr0GRLQ2YK39VpNBrNxgU7krUs/RWJf7+F1F/vAWoq5RkuOwNuCxtK8BJu+4GKxALYtz4PicP2h7P/ZuryGzMbr+Ct/QVY+jyw6gOgbo65s4eTMRoo3AMYeIxhBdZoNBqNRtNjoIG1ZvlcuOa9hJafnkVO3QI18oISpKGilNuhYrWzBC+3GRkfUJ0+As6xx8Iz9Chk9RsWtatwX2PjErwtK0XkviDLcyJ4fzd39lIyNgUGHSfi92jA1c/cqdFoNBqNpjugVbftjzfgmX4D0st+MlwXKDy5WInS0P2dKXgJ17mI+G3I3xr27a9B24h9VRAbG8FJ1DfxNxuW3K/2At4bDPx2We8Xu6TuD+NeeE9f7WncI+9Vo+lkHnvwQXNtfZ7473/NtfV5/PEnzTWNRqPpu1DoLp49HY3PHgjnu8cjvULELq26PWWUGsbDDqRV/Yikj4+B+5VDsOSPrza6SSv6roXXVwf8/YAs9wDuCnNnB7FLqXFKKXaan4nSXkiwGZ/seJYQtE38UpxaJXlb5ZPJHNjmp8crCyfNlk+fmlSw4yTlAsPOA4aeJT+2THOnRtMxJm+9Nab/+KO5Bcyf/T0WVduw++Rtsd8+e+Gmm2/A8LFbI1mK/apFf2JBTSJ2GDcSu0zZHZ99/pF5lkaj0fQ9yhv8yH//SPj/eE1ZEG38o1b4rUCFxe3AZwCr/RvCwhu8Lgs/GM8FOUcg45CXkJ+qvu3zBCdR38BTCfxxNfDeEOMzXrGbJII2Ow3olwMMKhAh2Q/YpAgoEVFZkAVkSQlJl1KX6gJcTuN4hwjhgNglXOc+fsdjeCzP4bkMg2ExTIbNa/BavCaPjwc37/1fwPu896skLTpJ6Gs2apKT1778mv/lK7jknmmY8cLNePnrpWha9AvenzEL48dPgrd8Lo69+E589vRNeOP7ucjN1o0ujUbT9wiYqBZ/9wbyb7ADv76GRBG4NlpSKXQDYrenYcZNiXJRvUMrXkb+vTYs+el9JYI7yfTWY+k7grelFJh9sSH2/roF8NaaX0SJUwothWixiM6hxYYAzZcHdpo87ClAabndUDBsXoPX4jWVwJY4UAAzToxbLHjrJA1ulbTYRNLkIsN3WaPpBIbveDi2LPTiu9kLULlqJQZvvRsuPed0nLDLWNx+27/RWLMav/1divfefg9Jzo29T7BGo+lLKFHYXIdlXzyFpocnYPA7BwEpsjMw+sIGlAmdSiCulBYS/0Gf7o2FD26Hld8+C7+7oc+6OvR+wdvqBuZcB3wwDJh/N+BrMr+IAorMPBGYtLIOLjSsrrTABltpuwu6R1AAM06MG+NIMRyL9ZdpMf8eSZvhwJ/XGmml0cSIz7+23X/nxUdg5D7n4cGbL0JjUzPKli1R+xcsX4msjExceffTeOWdj3HJ/x2BmjpzAhaNRqPp5XhEBdbOnQHf68dj8GcnIWXlTKwzeURvhcJX7mOo+zsM+P54eN4/HjULvoK3D6re3p1Vq94HPh4rYu4GaXZFKeYoZuk2QCsqlxxZp29uT4dxDI4316MV5kybOTcaabXqPXOnRhMd40YWYupue2Cnybthh8POwrWn7I0bHn0PuZkOTNxyKLafsDVcY/bAmVffhX+ftBNGjhyNVkc6Nt10tBmCRqPR9E7YIa184Wy0fXw5El49BElz3jSUE21PvcWi2x68D96TLMkLp8H+wSHAN1diyfzf1KRwfYXe2WmteRnw64XAimnmjihIcYlITDX8aPsSjS1ATaPxGS39DwS2uEtK9gBzh0aj0Wg0mmAqlsxBxs93ofnv6cjkrKtUS3QD4GeovSmwL/S7WPdvyE5rJLAdaT8REbzaMQLZw6egbtQFyO3f+6eID77dnk+bH5h3O/DRZtGLXboFDMo3Ooj1NbFLeE/95d54j7zXaFjxhqThpsDc24w01Wg0Go1Go6D5yP3rK8j4375w/vgYMutE7AasoBsD5n0WeObB8dsjcH6wL1b+9L9eb+3tPdnXvJzNHuC3ywFfo7kzAvTF5at/dvxKoqNNH4f3GBhRIiMK4Uv/3t+vMNKUaavRaDQazUYMzT8Lf5mO5ge2R9K0I+CsXmhYdAOjL2xMBAS+3Ht6498o/vZQND63AxbM/qrXCt/eIXjpd/rpeKDyG3NHBGjlZCcvjrYQ7/BevRnec5Hc+xBJg2gsvkzTT7aSNH7X3KHRaDQazcYD3+JXly7G8runYJPndkb2ym+hhiqgQtrYhG44RPTaJJEyq7/B0I93xIqndkN95fJeN5pDzxa8fN3+26XANwcYY8xGgkN38dU+rZyxDuPVF3FIGjAtmCbtpYenStL4QEnrSyTN+5KLukaj0Wg04XHL485dX4nV3zyH7LuHYNDqL4xhxmgr6z3vv7sOWrpdwKC6T5D+9AAsnfkyPI1V8PQS2dBzs5Qd06bvDMz7j4iwgBd1GNj6ykmXHCjsmz66HYVpwrTJlTSK1FJlGs+700jzpqXmTo0mOqTJhAN22MHY0Gg0mh4MLZO+ujLUfXYHKh7eE4VvHmcMMbYRvhSOGeoI2tAkvQZ/dyQqXtoLFd/ciea68jX93XoqPVPwVn0PfLo1UPmtucMCjrxAMZeXoV87RIJpkytpRFcPplkkKr8z0r5KPjWaKFjk8+FGmw1vfvMNTs7KQoxTvmg0Gk2XwfF0a35+C22vHYv86f9E/+ofDbGrLbqxYQrffs0zkffzRfB+dCzq5r7bo/17e14Wc2zdL3eL7MLAhOYkDBx5QbsvRA/dHJhmTLtIDQS6OHy5ux6zV9Muv1RX4QmHA3eqLRser63FFQkJ0HP7aTSankatG0h851wkv3k8HPM+MRSQlhDxQx2RKDLMAWSUfQTXjGNR8f4FqOPgxT2QniV4lz4HfHuQMYKAFZyAoSTfmHhBEx9MuwGShg465FjAPPj2YMmTZ80dGs26fLl4ET7PycUNaivQgrLhgbY23G+zYV5jFKOpaDQazQamqhlY9uG9SHtoUyTOvA/JnlpD6EYy/GhiQ9RkkrcGOX/fjbTXN8PyL+9DRQ97BPSciSfm32l0UIsUHY46UJQtCatLaafQKmm9qhpokNrAChEu2OxWYMRF5g7NxgSrh/LyciRwqmsTp9OJz2bNwrJddsE5ak+432Mbrpa/e/76KzYdOBA+n+4MqYkev9+PwsJCc0ujiQ/WOst+noGij05CUvVCw8IXELqUGoFqLZp1EtgO3U+svot1f0+YeCKW/SR0XRb6SbekDEPFhGdQMnq7dU7tLnqG4KXQZec0K1g48zK1VXdDUd0AVEiLN1JJGHEhMPY2c0OzsTBq1CjMnTvX3AqPUWzWit4FsmeYua7RxMvdd9+N8847z9zSaKLD1FtoWDEHmHY6MhZ+afjoBl5oWom09tZJYDt0P7H6Ltb9fUHwBo6n6pWltnBnJO78KFLyh6snRTgTSVfQ/YL3t3+K2L3L3AgDLYwcXkuPwLBh4dTEpVWG1deK4fLw2TxCw0TT5xhYUoxLbn0B2+04BR729jBhheVwJuDdn2bi/IMnICOoCvtFarqfvluFsQWF8PtaVb2n0URLekYCzjruMOwxeQtcceVV5l6Npn3cUtkkLJuFRV88if6/P4TUNqmzgocYY2UU7zoJbIfuJ1bfxbq/rwhewnU+GvzSAGlLRGn//8OQHU6GL2dLuCJ4VG4ogqPf9cy7I7LYTZToDcjTYrcrYBqXSFoHvbpej/n3SH7dbm5oNhbYJm6V54bf71uz+GRplUqsTb5gnRYMt7nf729TxwWfpxe9tLewrLXJP9o6NJpoaaytRs2n98Fx39YYMfsBpCZKQaJlV9O98HdsB9LsfgxfcT/sr45DxfcPobmxzvi+C+k+wcvOUL9dZm6EgZ3TKHZdusR2GUzrge10ZuPUzkueNjc0GwMUr0qEUNwGLdzXKn84Trspc9VC4wP3t4oiDj1HL3ppf5FS1AM87TS9g9raOjT+8Bw8Lx2Pwo/ONVRNkizda87ThGIKX7qWlPx6JryfnoCGP19CZU3X9WzrniLB4a5mnWpuhIFDjVF4cawLTdfCtOcIDpGGe/vpdD0VsUZVYCmiTm6W1QdkuU+Wh2S5X5YUZZ7TJjqNRrNhYIe02u+fg+OFA2F//WRk//2O4aerhW7Phvkj+ZSx8g0kfnUSkr44CHV/vESvhw1O1/vwVn0LzNgd8FuMDBAQXHRn0HQf/lZgWTks5wxMTAZ2/BDI3d7coemL0If34puexYRJu8Lv88HusMNPVwapNVSDXX6uTVIU2oK0baJ8l9REFwjD5kvty+M8bo+y/BKbLQHOJOPtTSyymOHx2h6P21jpZBIT7eoeY5XqjIlffjM+rzEApU1u2uFM6pTX8q2tbfDyfgWG65RwGcF40o3hdFaV35G48A/zMFxcMjJdOPvEQ7D3lHG4/Artw6tZnzp5LDU+fQwK/n4ViT6vIaK4sCCySAXLh+DtjqyTwHbofmL1Xaz7+5oPb6RzmF/ySPDbnSgvOAbpez+B1AgvmDtK1wrepsXAp9saExuEg6/SKXbpzqDpfryiWCh6fRZtL2c2sMtMIHWIuUPT1wgI3u0n74qVy1biyQcuRlFx6jp9Gyl8QgmuVmy2Nqxc0YRzL38EyclJqp5r89vw6nMPoLG+Juz5VrS2taKweDAOOPwk+HydJ96I3eFE6bJleG/aUxLHNqmLo4sXY0D3jW222w3jt9sRXhG9CQl2vPHS46gqXwlbJL/4SEjADHf46K0wZc/9RUzLg10aCq8//yhqqlZJukUfbpukW3ZuMQ455nQR5p4OpxvzzCb3+PoLEpfK+OJy8NGnyf1514uLFrwaK1Y3i4z46GYMnnUTWpuajC4ngaIX+GRxCi6OwdsdWSeB7dD9xOq7WPdvTIKX6/yUhc+UhOR0LBl0DZK3vggFqTyoc+k6wdsmlfUXklFVP5g7QmDJVW4MEV6la7oeWngpemnxDUf21sCUL6UkafeTvggF7z9vfg7b7TQVt119Iu554gq43RZlwQLq2cTENpx/yp24+NrHlMXypssOw+U3noys7BzZjj68xMRELJw/H88+9jUuuPoBEYGG5bOj0LK7YtlSvPnyZbjs+gslTtwbfdVot9vxv+dfk5/JOOx76JFyfyfhjAv2Rr8BA0RgxvuyzgaHhPvNjK/w00wP/nH2Zbjp0hNwzqUHorBfv5jSjeMor1q5Eg/c/iYuvfEpeD3yVI0bmzQOknDrlcfj3EsPiiMuiRKX5bj/39NwmTSmvN5146IFryYU/oJKv3wB+W8djyS3bAX76AZEU6iQCmD1XazrJLAdup9YfRfr/o1N8BKu077AT5EcHmcSqrZ+EYVbHMRvozQ9tE9wVDYsHGvXUuzK7ehpgnsmzJN+kjdWk31wHvLZl5gbmr4IxdLq0goMH52PlhY/GhuaYloa6pvAeSf6D0hGU2MzZs74BEf/Y1f0KxkkYTtFLLqiXmzSsNpi620wZmw6VixZjAQRwJ0BBe+bLz+EW+67TtZ5raT1rh1pEcmLU887HYsXzsCfs//E9jsPwqjNNpewkuB0psS5JMOW4MA+Bx8Ah2MFvv/yK+yy1ygMHTkm5nTj8cPkvB2nDsP8P39X9xsvbHTM/eNXTN5tWJxxcUhcNsXk3UfiLwmnI3HR9F2ofZpbWtD413S03LMjSl47BkltInbZS5Y/+85SQZqeAfOTitQhsqPNjaLvD0bla7uhZtE3cLvdqjx0lK4RvCvfBObfa26EwJssztGjMfRkkiVvmEdWFczf90kev2FuaPoarGgaG+tEXCZEfBUeyTWBp/Fbd0sTli9fiOL+/eFxG6+zY108LR7kF+ajrHS5hGl9zVjwiiJPsHnUUGu0Voa7bntLS7MbGekuzPvjFwwcPEgqacN1oKMLw83JzcQfs2ehZNAA5Qsd7rj2Fo/Hg6LiApSWLouYV+3Bc5cunIcBgwbFHxc5r7h/P6xctqhDcdH0Terld9j2+xuofO402B6bitTFX+mRFzYmmM/SqMmr+gSuT3ZG1cf/Byx/Fw1e4+t42fDFp2kJMOtkcyMM+Vl6nN3eAPOIeWXFrFMMH21Nn4Oikh2yIjWx0zPS0dLUjDT5jCB7RVD6laCkL+562BKQnJKMlNRk+YxcJ9AP1N/q65RWvyguNRyWuscwOF0uFacUiZu9nc60fgnH42XnvHVj5kxKMsKQpb0w1kfSX8JlhzjRiuvhCA7b7P9Av+Fw12mTeDEPOpJuFKzGeLkh6SX5xzRiPJKT180/R+iIO1JIVDlQgzl3Si5q+gj1NRVwv301Ep48CCW/PYsUm5QR2sN0u2jjgvkt1VlyohfFK5+E7eN94fn+OtXvI142rOBt8wHfHwV4LCKYngxkbQDPZM2GgXnFPAsH85h5zTzX9D0iiJL0zAxce9pZuO2Ge3DovmciKdV6CnAVjPoT8vQSsdTWVI27brwXj9//FO687cl2RC8VU+cIJcakTUnA9cOj2H3n6efw8D1P4Z6b78WcRRURBKthb6YYD747it0PXnwJD9/9JO69+T78Pm9VTKI3EBbFaigUu19Mm4YH73oC9916P2b9uhhOEZcNq5fj1zkrw18nXGOjo0j+2ZqqcMcNRv7de+fTSEgy8o9i94uPvlxf9JLOyUJNH6BBluYfX4L9mYOQN/1Go+DTqhv8Y9JsfDD/WXVItZXz17VI+OIgtCx4HU3SDoqVDSt4/7oFqJppboTAERkKs80NTa+BeWbla00f7b9uMjc0fYvwyiQh0YHVv3+NmpxxeOyJe3DFxUdgdVU9jaZhUcIynN6iSExKwxHH7IsP3/8aRxy9j3J5iARj1HnPQkPyhkKr6g577YHFs77CuN32w8CiDGVtjURoOBxdYeJuu6J09iyMnrwrBpdktxvG+vBO14+hX8IeN3ky3Ev+RNEWO2Dk0EIJG0j01+LX35cj0b5+Fb+uHI8DFY2Q1KeITkrHkcfviw/e/hqHHrmXtH2N/KPV+Zcff1tjfQ5g3FFb2OKg2Xigbln2+3fwPXEskl4/GckLvzKUyYZVJ5rehlkmkld/Aee3J8D76YlYMvdnNR5ztGy4ItW4EJj7b3MjDP1y5Oqd97jSdBHMM/rzWjH3Nsn7BeaGpu8Q/rdKv96Vi5di6KjhqKitwDa7TkVuCn19zQPCYfWzT7BjkxFDkJuXg6HDBnZgdIPOo1XUY9HAEhQU5WLQsMFITXZEvrcw8NV9QUkJiiSMgUMHIz01KeYwrGDYecVF6Fecj5JNBiIzPRl+EZrOvFE44ciJcLu77o1Lm+TfsJFDkJOXi02GDUCbmX/NTS0497Iz1Gc49FNg46VJlrJ790fJ49sh87fnkeCWPbSn6EKhsUJUa4KnARkrnsbAr7dC+ZuHwxj9vH02nOD95TxpulkMfVOQBZiDzmt6IUkOIw/D4XdL3p9rbmj6Oj6fD2MmTMRX77yL/pmDcfUxJ2BehQ+JcTVm20Ss+ZV/rxK7Nhsckaa5DsKR5ILTYUey6g9gQ0on9gugby+FpU/ilsCJKUTkc3KKWKDfrL+Vvq8Sht2uXu/TX5lWz+TUFDU+MV0fuM/pdEZ934SinBbjVpV2bSrshEQbvFaTxgSR6HBJVeyEK8Wl2rJG+sVLm0oj5eMrny65J5LkcqptjYYELPqlX72KlHNt6Df/bfVGSA3lzEWLXU0kWD5YXswyU1z+KpxP2vD7Dx+qryPZEjaM4F05DVj1vrkRAitU7bfb+2Eepln4866Sgrfif+aGpm9gUY20+mHLHoh/HDgaO++0K7K22RtjB6QpcRcfbWiob1S1WYKvCd9//1e74o9i98uXbsYXPy3BLWcfhNycJFx82lGdKnqbGyROCXYsnf0DFpW34M/vf45Z9LY0NkoFbcf8md/hh69n4cHbHsKiZWV47Lb78eR/p+G7Tz7Dfbc9gu++/BZffjsvJtHrbmqCV4QvRfMPX3yNNkcYf9kQKHbnfPYY3vjwazz+r6NR1ebCjWcc1GHRW1/XIGLXheceewXZuZl47bm3lOjVaLzS8Cn98X2U3jQRxc8dbigQPfqCJh5M4av8e4XNft0Tq17eEWV/fQwf/brC0PnFzN8E/HqhuRECTQhWlkFN76Mg08jTcMy+SMqCiARNn4dCbsoRx+HlD5/FuWfsh4YGi2nDA0TQwvVNftz76I1oafHg5y+n44efFqrJFyLhdbdgxyOvwM5bDcLl909DZZUbdzz6IpoaOzK5wlqaGxpw9k03YkC2DasXzcdd/3kBy+YviEnwNksanXLtVRhW4MKiuUtQVVWO5KJBcLXVY1WlF1N23x5//rkYp5x6AH6a/SeW/70MnMwiGih2D77gIozfROpWdw0ee/A12F1J7RrK/N4WjN7lVBy4xw44+foXkGNrwVUPT5O4xp9u9Q1+3PfkjfB4/fjslReweOlyvP7Wd3BFeKPXSd4dml5A6eJ5SHrzbBSXfm+Mp6uFrqYzMIVvUeNXcPzwf1i5dL6xP4TOL25zbgCalpkbIeSkG53VNH0DdkJhnoajaTnw5/XmhqbXw174Vj3RBHdzM+pq69HUZD3zGc9X4xhEUGJ8HZ6SlgJPixs77D4VyfZWS0HEYALfUfR6vD5TrLXFIXatI8U4uZJTpLJshcdnwyF7bYFX35mlXCjWI3B7IcExjKTkZGkfcspiIKNkGPbYYTheffkDpKalITU1Bb6Garz3yWycc95R8Mq9rE8g8HVh2BxNIlHC9juzsceuW6m0CA9zYG2KUvS6PR60NLWoQS+iErsSB7lkuKiokSRS05LhlgbLzgfsihsvuAuTd9lS7iecSwNDMEa20GwcJNhFlbgy1XBTOuM1nQrLkyjaNkeG1LPhdWbnCt7GRcD8u82NENiz30ocaXov2ZKnVqM2/H2P0XlR06thT3q+XueMaY4oXpWHw5nkQKOI4cREkY1trdJWkgrJ4oFnDL/VhmYPsMWWw9cXf3R34JuFTnxgMiiOxRs6kkAAikr6KxcPG4ltdpmMY46arKzQwRjtAQpaM34hMAzeGSdd6F+UjpdeeB97HXKQVItteG/ah9h8511x6JF7I7HNC48naIQKCYozr9skTBXq+kGrsIlPxOvwzTeFTeJq7FkX+rx1ykQPjFOi/AkXF8k/r8eNzSZMxmEn7oUJ40eJAF9/xA11fthYajQaTQewqOI6V/DOuw1otRhKSLsy9E1YsKzytlWESqSROjS9hozMLPwyawWWLFqIvIJsZOVkRL3kF+bg5x9moXSZD06XE8NGjcXH73yCnNxsY6ICiyUpNRM7Th6nOnEF78/JysQ3039CQXEJlZ4Zw/hR1kqK6MQMLF20WO41fZ3rBRaX04GRE7dHttOL/Y49VIn2wHepaSki6u34+69VGD56DD59fwZycrJVJ7TgxSkNhm2mTgT8Dlx7+xXYcuxwnHnFuTjj3OMxZeq2UiG3YeWiCuy4x3aqccFJHDLS0/Drj/Mxcsxm+PyDLyXcrHXiFbxwDN4JU7ZHmnyGfsfzvvpsJkoGbiL3HP9gYOzAN2TYKHzy3ucqTN5X6LV4nxMnjcfu++yKCTtuq7aDv+d5n3/4FQYPHdmhuGg0Gk202NoCpoGO0rISeH+Y1IZhBojgZAWRhrLS9H5WVQF1YXw3E5zAnvOB5P7mDk1vYmBJMf558/PYcusd8NtP3+DFJ+6Bx11LhWgesS7GC/N1qxQKmozMYhx72kUilMagproCj91zHfr3b0VR/yIloKKFw6DN/f1vuFJG44gTz0J6ZrayzHYcG5YsnIsHb78QO00dqzp/xVI10vr66XtfY//DL8aIMZtLOt0rwm613O8gub91w+EoDTY1EoVFxwoR0jQQc8QDWsS/++onjNpsX0yYtAteeeZBpKdXYdDQgeDoDNFCi/PihUvh8ZTghNMvRpo0YOJNt4SEBFSVl+GpR25HWlolBm8icYklD+X8xQuWwO0uxvESl4ys3HVEb0amC2efeAj2njIOl19xlblX0xdYvlh+u88fjrzVPxvDj/GnETC7xbpOAtvRHEM6sk4C26H7idV3se5vAaZPmYHJO+1o7ljL9BlfYvL0neR5KhvB51mFTay+66z9JJp1Eus5Vudb7ZdqpMI1Di3bvYKSQaJHQ+g8wTv7QmD+veZGCJsUGf6e3YDPb8PMP1Px52IXVpY71JAoWal+DB/gxsRNG5Gb2f7QPT2B2oZEfPt7KuYtd6GqTh6Isq8o14vRg1swYUwjnI7Oyca4oY/eolXmRgjDzga2sHB10fRoKHgvvulZTNxxV1SUlWJ12XI01tetJ+KIIXb5CwsSw3IYJz9Iz8hGQfEApKVnoqWlGUsXzcXcP35BbXVVbK/Y5dD8gmKM3HRL9BuwiQjCzqlXGIeG+losmj8H8/78VeLYYqXp14NVaGJCIgYMGabEblZOAZYtnifC/BdUVZabR8WD4R5RXDJY7ndz5OQXS7jzMf/P2aiuqogt3YTcvEKM2WK8hDekQ+nG63o8HixbNE+lFe8x1rjk5BZg9Nit0H8Q85BuMmvLkxa8fRcteLnDxGq/Frztn2+1v0sEr6cCeH8TUZccRjqEzJRumVGNQve5D3Pw2hfZqK4PX7nbE9uw4xYN+L+DytE/L/KsTt1FebUdD7+Rj89+SlcdZsKRkdKKA3eqwYl7VSDJ2fHsjJvVNUBNmJEZEuXXuddCICnf3KHpLQQE74RJuyq/TPqxcmKDdUStCX1zG0U0ZmTmBFlH5VPEEEVNIhu9sp/iyC2CsqG+Bl6vx6isokHOa26qR2paphJMdgcngejM8s7wG1BfV63Eb5pcJ7rw5Z4SE+BypSpBz9EVAvfn8bpVJRwPtJTTGupMSpYGQ5ZycWBjoaEutnRjOEzzJAmH8XM4nR1PNwnP63ajXuXh+vfY2upXYw7zWuvGUzYYF6cLabynMHHRgrfvogUvd5hY7deCt/3zrfZ3ieD940rgLwtfzSGFQIzjVXaUpWVOXPlofyxcGd3Yj8lJbTj/sDLsu0Otuadn8OmsdNz2fCEamqOzxpSIaL/xtJUYPiDWHuqdRCQr78hLgM1uNjc0vYVgwetxG+UqnDXPZktQQrGpsR5F/fgaf91e+aHVTHAY/CYa+2BCQiJKly9GUnIKsnPy17tGZ8Br1NVUobGhHsUlgyTe7atVdWtyA7yHwH3Gc3+hdFa4PDVwmgrFDKujWMWFZaGutkoJXuZTcBquPc50fQkTFy14+y5a8HKHidV+LXjbP99qfzuCN/iU+PCKSFzwkLkRAn13u1jsLlmVhLPvHBC12CXNbhtuea5IWYN7Ch98l4FrH+8XtdglyyscOPeeAZi7tPMG3I8JDjnHPA/HQikj3hpzQ9ObofgKXSg+09KzlPWV66Hfh7LOd0HrkRaGm5mbjwy5zoYQu4ThJrlccKWkyDVb14tDuEXVtmvWDdb5Pmg9lqWzwmU4a9bVduewzjWC1vnP5/MqC3Dwtdc9Tp5M3NZoNJououOCd9mLImTqzI0Qcrt2GDKPNwGXP9wPlXXxiey7Xy7Az/M4Gnb3Mk8EKwV4GDfJdqlrZBr0R1NLx7M2LnIzzJUQvPXA0hfMDU1fg9a+gMXP+Fxr/esczPApkuQj2LrYmTBcuhAkJbk20H1sBEgeZWXlITMnl5Jd0Gmo0Wi6n46roiXPmishcNpZZ3xjdsbL0+/nYElZ9JbdUFg53/psEfz+7q2gb5Y40Ac5Xsqq7XjojW7yl+WYvFZW3qUWZUXTa6Eo9Hq9qK+rQW11hfLzra2pQnNTZ82yZ4hO+tYy3BYJl36jdbU1yp+1s4QvX8O73S2ol3Draiol7CZ1vSa5bhtrhg0ksPsSzAvmSZ2Uhbo6SbvGeuUeYpQFnX4ajaZ76ZjgbZgPVH1vboSQnWqudA0erw2vz+i4SwLdAr74Jc3c6npmzknB/OWcXLxjvPtNBuqbusnKm22RflU/SJmZZ25oejtsINZWV8LnbUFqahrSM7OQlZOD9PQMNVFFrQhHjwjgeEWpIaCaRDRVSHgJKlyGn5mZjbT0NDV0F0cr8Pt8cV+DQoy+pjXV5XJDPqRKuBmZmcjOyVXX47i29bXVSrR1lrjuazBd2OipkbzgbG9paWkqj7KyjbLAyTxqJQ897vjLgkbT5bCCCywbO+13ZegVdEwRLX3OXAmBVr7kjou2WPjy1zT1Or8z+OBbi9fyXcAH32aaax3D7U3AJz920324nNa+21ZvBDS9CvphUoimZ2UiJTUdNk4FpvbLHxE1dodTRGMOvJ4WtDTHbonl8U2NDWhr9YrIzZXwXCpcuvkYlzBGL8jOzUVTU50S1vFAX9OmhlpkizhLcqWpcDkYmHIn4n3YHSLccmVPGxrq61S8NMHYlGXc3dwgeZEDR1KKSkM+HwNpyJEY2IDw+dy64aDp2bDMcvHLfynEremFLOLGvo0R9RuWdEjOVenBdFmTRr2QDSN4rV5pb0D+XNx51/yjE8OKlc689pxuvA9kWPhCL33eXNH0VihY2As/KytH6kIRNxaTDnB3ekamiN5mZUWNBVoM21p9Skz7JSBVv4qaXkcqyTYnT8jMylbiOJ5KuKmhDpnZ2fAzLAagFLvxjFPIpr/Vj+SUFFDTxyus+yrsfOZuaURGVpbkMdPOSEOm37pp2IrU9AwRvR5lkddoehwsvhS5FHhFm2Hlzg9g4WE/oC1RGtv8biOF979o6k9YPvYetOZuahh7e2l6xC94K2YAjUvMjRCsxM4GZOmqzvMXrm1MRH1T9KMjdCbLyzvvPpaWda0P9TpYlYGmpVJ2ppsbmt5IS3MTUlJSpc6zKX1oZbHjd5yggkKH/pzRWvZ4HH12U9PTTf3J6RessBnCOj0Nzc2x+IrajPtIS1HnR7oGv6FFOyU1TZ2jLZQGTAc2NNLSMow0jJAuKg3loDTJ00Y5R6ehpkfBesYrDTNXGmq2vQLNJ36PAVPPRJLIgFZpeG/MtPp9cNptGDTlXDTt9y2qR10Gv0Oe70yWXiZ84xe8VtbdSK+zY4FPuhaPCCS3FMT2C1xbJ1egxkOwa+nsa8ZcFpnOTG+mu6E04odDlFm5tSyxKDuaHg+FCq2cThen3o2ocdR3PIZj29psHIoqujJFkaw8JOhewGvwT7iLcZNBypJodyr3hFiqAU7e4HAYnVzbvYZasUm8rC3aGxtMFooBToUcXT5xpA0pC7LRyuM0Gx2tfj/a+LYnpIh0GyyG8rhraHNg9Zhj0XbQ03DseRNSUw2DTVurv8dEtbtQP9824w0dG6yOHW9B685PoqzkKDT4ROv1ovZA/IJ31QfmSgicWa0j8GHCGbsWlIqoLmcvMmBRGbBYlkbrCRVSXZ33EOIMbCmdGF608CHfmfcRdVhMV6Yv05npzXRn+jMfOvJwt7LyllmUHU2Ph0KF4tUQgZEwDlCiF63ymRC1WwNfkwesqko78U+oiArAuCj4Gb2QUu4LCsovfqgLcW191lzDOEwLXgOjAWOmjUo++RMpn8yvbOxtqAXvRomnuUG5OPUI+DP2AjWDdkbCMW/BefB/Yd/0YKR3gr2uL5NBm+bIw+Hc7XFgzzdQk7tTr7H2xid42dO+eaW5EQQrtPQOCF63NLWWrDamp1U9HoLwSIquqOSYW7KxfsqOHNh5s4sNLPTCYe+e3Bte0nk+giMHtheW3CPTk+nK9A2G6c98YH4wX+KBQ9OFe/41i5iu/8vc0PQqKEKVWLEQNmGhZU+OD/1NR4DTFKsr8JTAEo5AVGRpk/CjjRWPC8wApvRsIJxw1wnaFzhHYyaXJIdKc5aJQDqGw/xarbLBYHWcpk9jTEoTKDTdBMuePBob4ETVHvfAddz/kDJmT2Sldm1H+95OdkYy0kbsA9f+01C59Z2o94ic5KzzPZj4BO/qz82VEJJE+rP1Hg+sMEtFfHF62kjUNhlCLIQJY9bfFy/bb9ZgrnU923XitSeOaScspiPTMxLMD+ZLPBaZRCkLLBPhKLcoQ5oeDX/dhm6VP0E/9UDp4KexbnzJdcN/U8pRlHUDrcE+H0WReY3AErhIANkOuDL5PB7j1braah8el5joMKZLTjCvE+5s7lLXUB7L8MrvgW4NGhNa7ulKwrxlsljkE/cz/fhq1CeCVx2v2ehgXcB/3QLLoU/+yzJvzBloOmM2cqacC1dajvG9Ji6YfrnbX4Dmg//AX4WnqvTtqRbf+GpuK7HSkRZSZd36VkYrKuTYEL/eof3d2Hxo+69KcjN8uOK4VXj8siW46vhSTNy0EYlBqcB6+IAda82trmff7WvhSFxbUlg1jB/RhMuOLcN/L12Ca/5RiqKc9ptRm/TzYIvhEdKD6cd0jAbmC/MnHqzKRPkX5oqmN2EIzARlpQs8tgw9w5EUuM9YAiXYOIZTzbbGIBTbkJycqiYwSIggjNrkK35L8VRfXweXnGN99LrwuCRXstGBas1Z/AyppWUX3R8Y97r6WriSkg1rtUaRnJwi+VRrtGWsHnAqDaXUJCSqCUNSktkZ0epgjaYTCRQzaW/75THmH7AVSi9pwogTH0LBwJHml5rOoGDAKIw6/FGsOLwR/tyxUB5sfCHGPOghP/c4Ba+FWIl77F1Jjbp2LI3B0MRUv76YO/vg8ohGpKw0Px68aBn2EVE5alAL9tquDv85ezmm3bwAFxy+GvtsV4vrTl6JfnlxvsLvBLLS/bj59JUqLuccUo7/3bQQ916wDPvtUIPRg1uw+zZ1ePjipSjIjtw4OOfQMnPNAqZfDK+YjfyJo9RalQkteHsl/HlRKFJgBgSs4dJrWG6UHZSdk2S/MtCKOHS3tCDR7ohaKPI4h5PlxhBIFLQqxHVOt8lv3bhidWUFkpJSkZgY28gqPN7lSlXnqztg/ELjKNuJsjSI2G31tcHpcplfdD1ej0f9An1erzxMfIZrQDdjdzhgdyajuqpKJV24NOQ+piFnXbMlOOCQc9RxGs2GhD8PEV3ys4Unoz+W7/qgiIRZGJDTjcN1bgQMKkyB/8jZWDrmTnhT+4Mv69YI324mdsFbOxtw8wERAuuveAWvV1JDpUoMuNe3cm66STNO2KvS3FqfK49fhZKC9cVsbqYPh06pxhXy/S7j68293cf2YxtUXI7ctQqFYay5+SJ2rz1ppaW4P2znamw7up0GRJj0iwjzh/kUKywT4eLplnyq/dXc0PQmOJEAzascaoyiU+WvbCvRGzC7SuVGPcxe/E2NjXAlx+bbT8tucgpn7EtAdUUFmps5zi6txHINUdgeT7Oa9re6qlKJVldysrIghoqtSFB0OZ0uOTdNRG8lGhvq1dTIDILX8fo8airj6uoquQ8b0jIyldtE92CMjkGRy9ElfF6fGtu262H6r12Y5pxlL8mVokQvJ5Zwuzl0m5GGRv43oFIaFTY5NjklVbszaDY8/GlI/eMftC3qd7wWVSfOxKDd/k+a0JqugI6MQ6ZcgOrdv0X9ZtfAXzB+zXOhO4ld8FpZ5lwWwiYq4kgFi1dip+xXgSOnsmPbuhwmgpZCsq9Ad4WT9l6/4UGXiPOPWG1uRSCuV4pxnMMyYdUQsvIF1/RoKHRS0tJFgHlRV2t0Ik1I5BQU8p38SRRhw236x9aICEpNy0CiPfauz5ySlmPf8nyPCLxlSxZh9apSlC5fjlUrlomQakJiol0JQVpg6+uq0SSitaWlWYlCjgphdJIxxmQwRFrC2kUUud1hhzMpSU1wQbFbKuGWrVyJMrnOqmXLUFdTo85nWJykollEPu+LkycEwmYhDwhA+aPi3vlwHOB0dQ02HpJcrpgt2uvCeK6NdyA9aLU30sf4ninHe+TIFD65Z3dLk0rr2poqNZVwTXUFqirKUF9bJenSrPavWrEc5ZJ+FavLpHzUwedvFVGcoVxO7N3WYNBsFFDoegCf1EGN212K1qOnIX3va1BU3M/4XtOlFBQPQLo0OFp3exP1Iy+GmpuGL6fjkR+dQOyCt/pHcyWEjvjvyoNNalhzI0o4zqsF5xy6GtefvBKZqXQiASZt3qjcA/oaJ+1TiT22MXxrOQTZ5ceuwuXHrVLb7RIh/cLC/GE+xYOV4K2ZZa5oehsURhQxCYkOZR2tra5UEz+0yFJethKrS1eIMGpASkq6HJNgjL9p2cgKFl7rLhSVFFkiwdQUv5yyOC09AxmZ2XA6nRK2XVkYuZ2WnqncLSiCOWIDBSzdKWilbRSRRnFeXbVaCbSaqnJUlZeJgF4horYSDQ01StSmZ2YpS25igg1F/QciJ78Q2Tn56tppGVlqOmMKQ2X9FWFNKzfFHoVefW2NCOIGNTmFx+1WApHxWIN5T7zfeOCpgXQxtgPhrJtm4RYSEK1sDFCcumWhlby+rkalR2VZqeTdChGqzL/l0rBYJOJ1qawvQ4XkaXVlmTqe+ci0p/BOlYZPdm4hCopK1iyFxSXqO1p/mSecpIRvBTom0DWadqCQEsG7fOw/4DnqTSTucRMcWf2gRxnrXpj+joz+sE+8Be6pb2Jp4fHGF93wgsomlVdsWvuzieFFb0kukNIB/7YVFUBjDENyRXE9t8eG0konBhd33lBfPZElq5KQn+1FSlIMJaipRWoGa/eP9WCDpn+euREjnMyC4/uGkj0e2OV7c0PTExlYUoyLb3oWEybtqiyboRhWVAoprxK1jQ11SkzRZ5dWXdoI7Y4kJZBp/uWQZoZbAEUYnSC4yna3VEOykSD/OJYuhS4tt0l0ORDBpCyPsqxavliJqdz8Ytoe1St+twhPn4i4RIcDSSJIKawYJ04J7Ke/K62xqpprU/GiWOP3vB/6xSo/VLvsaxNRLgK1saFWjfqQLiJX1J0hGuUf4804MPyAFVT5ETNu8slrMD4cdsnvl0XSRPna8tq8PSU8ef921QjgKBEqDDnfsKzS/5kJEophZQ1U1Wqd/+QavA/jepIP6lrq6KB4SBqwt45gTADCNduafVy3Sz6xkcDr093ArvKOaWLEL3CvPNeIAq+gPgJ/1oHXKRfRzPsv6jdQxTEWMjJdOPvEQ7D3lHG4/IqrzL2avsD82T8g9eWj0a/pb8PcxuLDTxLrOuE2aQaW5o1G3tGPwVaytTR8pc4xv+oIS5csRsnTw5HA3wsDDFw/NB7E6rtY90s1O33KDEzeaUdzx1qmz/gSk6fvBNANOfg8q7CJ1XfR7Jf1Vpsdy/efj4EDB5s744c1AY0Q/tKZqPrwVAxsnWcoYtZLQdcMu07CxZEEtuUCFa5xaNnuFZQMGqa+Cib4lOjgGLzhcHZwGtv8TImNqo3bJ02EbhTiOsnZ1ufFLhlU5I5N7BKmH9MxGpgvzJ94sZp5r36uuaLprVD8UTC5XMaYyxRuxSWD0a9kCPJElKalZyvh1CrijJWNM4mv4kXwKZFH1wfDpSA5OQ2JCXbll8ow6euZmZmjvvN63WhpaVKWWlpZ1egK9bVoqq9TFlY3hasIW75qX7ViCcpWLlMWyprKchHNhqBl/FyuFKkQW9Hc2Kisv3zFnpNXgPSMbLleClJT05VFsrDfIORI3DMzs5XluKC4AAOG5KOofw7yC7LUcX7GqbkJXrkvlzxcnUlOJMtnckoyUtNSIZIbtkSnhFWA/gMKMXCTQgwaJuEMLpRr0OrtREoafV8Ni3Sg0UBRWV9biarKMrXQIl1TVYnamso17gM13Cf3plw4mhtUw4A1PtOSecDGAtORItXpTEJaWqa6R1rk6R7CqZ6zcwuQV9BP4jcAuQXFyMrJR2Z2ntxvjnKd4HkBEc6wKZ4D4toQ8HzCcFkfXj8np0CFGavY1WiigkWPL3DZBpfP2v1uRcFlvyB52A7qdxi7sNF0BcyXZJcLqUN2Qv6Jv6Jy7PVGXrJLEauK8FVKpxFbuWhZJRELMzwVBVG8r7sDUDDnZZgbEeCreHnoaDoBpmM0rg3Ml440aHiNcI0ZX4OUqVJzQ9NboSiiOLIn0ipoV9P1UhhxH31wKRqzcvKUuKOoo+WVwle99k5MUK/UGxvrlAglPMbd3KyEE8UyhSpFJjuxMWx6C3NiCtVpSypIiuJ0EcK5eUVKbOcX9kd+UT9lGfZJmI0iiikWm0U0qwktpCzSGkv3g9pquiPQJcNwSzDWjU5yDSKo22yJePKGUzC2yI4t+ydi3EA7Tj7uACSk5ohozMPSH97AhGF2bDfcgYlcZH2bITZ8/ssSeMt/xjYDbdiivw2bFdowJs+GvXeeiMUrK5HYshITh9qw/YhETN0qF/tOLMSUzdMxaXQSKmp90kaUtBQBy/t1yG9PWZdVOrOR4FCjWKiRLyS9KEBpXac2pbCnuM2R9KaATc/KUQKX+UCBzTSk0Ge4zB9lCV4jZA0xG7Akd4SAQNZoOh0WK5+UMSnLqyeehb/PK0XmbpfC5WTdoOkNMJ8ofHN3vBrL91mG2mGnos2RajRiNiCxCd4GC4uclQUvVrLSDFcFKxGWkQwMLOi4uNYYMB2ZnkzXcDAfmB/Ml46irbx9GookClj6uAYLpoCAolhjT36KMY5lW686PZWjqb5eWSBpWTQsjFlKvDIsDr9FEUqrprJ4VqxCfW01vD63CGGnGgOWnec4Zi+tuAGXAwpavsrncGkOEda0alLoJdgSVPxoxU2XhdbiNPN69NtNUWIwRblFKMumxJuuCWXL5mHzPU7GU69/h0de+BzFCctw4H67KmPE6tJFaM0Zjidf+xp3P/Ee7nnqA9zzxPsYO6RARHud6qTx9Js/yv4P8chL0zE4rR7HHrEPsvoNx9PTZuKVT/7GkQfsjKLN9sYrH/6BR178Elnpdvj8dIcwXRJ8PuWawXSk9ZvDerFhQdcMjjJB8Ur3CApM+hVzpAQutEDT/5luGwwnkBeMOO9P3aOWCJreAn9wFESiWtyDJmLlST+j4Kj7MWxAkfpa0zspKSlB5u6PYsXkWWjJNUdz6IC1lw14GkvCEZvgrbdyZ+gkwUv4qn2QiLCibCBbhFZmCpCbLsIsX/blsAu4eaCmU2B6Ml2ZvkxnpjfTnenPfOiIX3YwVmVEC94+BMVUSC1lCiuKNlpa6+uq4G/1ISe/CHmF/ZCZkydizGd2imKHstXyWa7cGJJEfGbn5qvj6LNLyy2Pp0Cm1ZOWYLo0GCM01KjX/BTRFatLldijCGZnKVo/aRll/BrqqlWnukoR0XQX4MgLHI6Mne3oFqGsklLh0s3ALucmuQxLa07REIwZuw1GbDYB19/3HOrmfwrR2VKs5Zj0XGyxzbbYfPyO2HLrnbDNpN3Rr1+RCGm+sUrGhEnjsaV8x3PvefwdtJX/jL9Lm5VLQX5hCdLTUuFMyUT/gZtg0NCRcj26iRiWXYpbWnNdIsbZAZCCnB33aN1V2rXV8Hf2SZoSuoXQcqss4JKuXg/TiB3TjA57tGSzk51hVa83BbHRwS7wkDBcGQJ+uwFhrNF0IyyabnlcpIi43f0GuM/+Fv0Hr++jqem9lAwZCfd+PwJbXYM6m+gRVml8nMQifKWq8rob4W4KP1FWjILXQpx0puAlUuEiQ4QX/UYLRXjlyoPDZTFFraZzYPoynZneTHemP/Ohs7ByibB6a6DpxVAkGYKJfrcNIio5MoDTkaQskkr8NtSK+KyBp6VFWWYpbLNkSc/MVqKO4rO5yRjxgJ8NdbWoE9HGdc6O1tLUpAQaxSytmXyFzk5yDJ+WW4pCdkrj8ewk4RMBzXglp6ara9DazHPpG8wOb36fMYQZ9wUmdmiVeEqwqhJNor+v1MHFJU788e2ncJZsj1T5ibiSXXItF7KyRNpK45BWV3bUa6yjfzGnO08QwSrtxtRkEe1JWDL/ZyYQhgxKVj6uGZlJEhcKWKcI5CRljU7PoM9tlhL2dD+gPy1dOyhEJVHVvdKKocRoguHmQJHMNOdsdwGLOh8UPI4rhtXDEMe8NwpcT0uz2QipUW4cbGiws9nqVctV44PW9EblJ234CbMxYFjQgxYzn+WPui+NplNh8RWxW5/aDzXjjoL7iBfRtstVyOjER5Om5yDVIVrHXYuW7Z9H9YAjUZ8oDZwYrb1S26l/4Yit2DQuNFdCcHSww5qm72Pl0tC4yFzR9F5MK6AsrGg8nhZlTaSllYKKwpPCtUlELjuYGcLLeF3PV/f8nqKqobZGWRxpneRretXZTY6jKKY11yGCme4GdGVIsCeqV/UUgZnZueDkExS5NHvSYqn8gUWMMQyOtUtxRvXKkQzo18tP4xoUm3aJi1cN08X7YIc5XodWVdGaSE3PwvsPnotNMmwYmWXDKWedi7sefRXlq2hZlSI871MMSrNh201smDDMhu1G2JE1KAMZIlrlW+y3wzY4et+tcNxe2+HAQw/BqTe9AVuzPMcpQilKlVWcaWKsr1nkH+PDBoHyjVYuDEnKp5lCmEtyappa2NGMQ4TRXYOi2djOUOKfAp/uIjm5BSqtVAc2OTaFHdhM32jeswpPFt474+OWfGQjg1ZzWs8rylZg5dIFWLlsIcpWLpX7X4HK8lIRy5XSAGlQecQGgxLkLBWMuxLFWgxr4sArWkfEbsOkC+A4+R3YD38SeZvtzPanpivh7zeh6wyOFKUFo3eDffITsO/xNuqHnqXqxmj9e+VppP6FIzbB6601V0Kw6+aWph2sXFE8NeaKprdBIUNh1txUr0RPbXW5cg2gz21zU5MSlBR9SoSx45TqPCVCSwRamrJkZitLJju1Bca/VaMIyDHscEYRxjF3Kdo4mQVfxTfxWjUVappaWoETEjj0GSPDNpVDuUHweipMCS8gAI04ZCoLLz/VdbNykCkLv+NCSykFHkUcO7RxPNqK8lp1rZ2O/Remz67Dhz+U44477sJZe/dHW0YikqTuSxo0Ed/OdeP971ap5Z1vVsInxZqd88j/XXwLjjvhdPz603e44YU5uOT0A9DQuP4Qb7FhiGTJALUEC2XjO0HShM8q5lNgoQClwGfeUDzT0h2Y3INpHUgXjq1Ll4t+JYPRf+BQNYA8l6L+g1FYNEB1EAyIZ6crWTViOA5x6YolxtjG7PgnecXOgnV1tapxo4WvJioobprkI7sIq45+CykH3gHXwHFIS+nAWP+auGnz++Au/cnc6jrSU5OR3G9rpO50D1Zs9RpaXflrR3OIk9iUqs9i2l1dkWnaw0rwcqQGTa+Dr7LpWkAR6nQ6RCRli3jMVf62Rf0HiFAaoCyr7ERFyyz9UGltZaerwKt5CrNgoRa8yB/+V5NFNDXWiiBLkbCLRGiJEJNrcHKD7Nxc1YHN6/NJHFxKyKmwGXJQWMGL/Fl3W6D1uaGhHunpaRKuMcJDTn6Bul5eQaYS3xxLODs3XcR3Ng4+63yUOIH3PpkvIjsRtkQHsnNE3Mv9c8lWfskMnzVzCvY5eFfsf9zp+N+r/8PVR4/G4kr2F42t6t1wrJseaxfDbYIWdsO310grJZolDwOimVZnTixBCzp9pvsPGKjyX03YkZuvxHNaaorKw3oRvhKCCkejWQcWLy6cEV+KyOydbkDCv0pRtM1+a37Tmu4hUTJm+I+Hovzt07BwRZW5t+tgfdN/q0OQcOxq/Fz8L0O1emQJlJkYiK0kWQnecENOaTTBWBURqzKl6bFQ9NDvk5otKydX+Z8SWvkoktqU2IOy4qalpSn/ULUjBngNDhHmSjGG2RJVKWGbYozXkE9aZCm0E2xtqK+vVefEAo+nKwVNBtlyHxzuS12D8Vf3InFW95GGfBFwftF9WTmJ8NW0YblUuMOHDVeWy0R7koh9IDPbbi4OJCcH4sKJMUS4V7dg5JSDsetWA3HcUUciJ7frXhFuSJiGdbVGWcigI3NQPrE8MN9tzKfMHLhcTjm2MuZ80vRx2C5kByUpRJV7Xg/vHW3Y/EhjwhEtdbsZ/lS5+NzI//sxbPJSLirfPU1NsR5bjd4xAuVg3P7XwX1iGyrHXGnEi2UnhojEVp68VhZeXSw17WBVRrTg7XVw2ly73aZ8Pv0UNUQ+lI7hpvnpV2LHjvTMDDUSQtRCxxSiKSJ21QxoQddQwlktxi5+x3gkSM3n9fF9V/Sw4xZ9d+kDS0smg10Tf3UPxkU4PNibD1+KI3Yfi53GFGNEcQJG73oGth8NuL0+NM77BJv374cdRxeoZfzAJLz4/DQkqeEVWwJRRU2FH/9+7nus+ullvPrxHCSZHTndHPO3oXe+6WBZSEw0hpxTDYQAsspZ9dbkk6w7OGuey4WW5mZjp2bjhmVDhK7bngzf4PFoOPp/SN3vaugeQT0QPr6ZMS4gd/5jqHp+Z9TP/0TVXR3wMIgLOrak7HAj6ie8Bm/OVnAzUmtfQkUktqmF38yUAsqexyEMKZLE0GPjaiLglRK5aJW5EYQ9BTgg/BAimu5nvamFRYyy935GZmZAD6p6JpyUDeyn0KVFOC2dnbiig8No0WKovAIEI6zQqmrtVW22NtTV1ku85JwoqjQjTrUSJyl/MOqutfex9nxafavLlqC8slK9Wq2prsLgkVtj2Ca5qK72wF1fhYWLFqxzTY6C0G/gKORlp+D3P+diiy23Uv7GxCGCb+X8X1DvcyoLcVubDdWrF6HenYhBAwcoi3FvQVl3a5hPGeo+SLg0NKC/t+HZVFNTo3ynrfJJTy3cd1kztXD936qgVA2eAkw8BQlbHYWspLW/557CRj+1MAndz0/TKr+q34HI2OofQPFOSEnrwGyscVJV70fbwhfgWvQoUmu+UvtWtg1D47YvYPim26jtYIJvqx3kLv10sAlDFA8YzUaOVRnxsUx1dRtREy+04lFcBssaroXmbvD+NvnHTmc+X3RijkIokcpITjbOV3JJ/jHE4MV4dW7AbdNKGwWG1bgVCXwFL2vGNVSI5pqxcDSH7MJBGD12G4zadDzGbjlBhGw6qipbJC1akZSeg7FbbY/Nx++wZhm37WTk5eWgNSFpHbFLvC0tKBqy+Rp3iNZWCb9gSK8Tu4RWW9j42zXErFUaGt+z3PA7G+zScFhjtddsVLS1+mFrqUZTUhr8+9wA7wlvIWe7o3uk2NVYwKyijcAJFK1+A84P94dv+jnwVMyLdiCFTiMnPRG5WxyH1p2nwbfFtWhqS0aCt0bqm/AxiV7w0rIb7mlC/91onzI9gOp6Oz6cmYGZf6bC7eldP7IWtw0z56So+PM+ehUsI1a+3rrjWu8hoGxMWUPa/xXJESKS17fQhmd9y5+5zY+QhVbGNYfL59pYRUaJr3WuY8hpBXcHLa3m0Gker8cY8ixIwFL0BoZBC17oIsGIBR8bwO8zZj4LQNHb28QuMdLLHACICRohDYPT29hlrGs2LlypGWjc5ky0Hv8asPtVKMzphFk8Nd2HyBC7/LYzFj+LxI8PQd30a7B65VLzy64jPSsPtnHXwL/Ta6gdcBpcyenmN+sSg4U3ghWul7TWWd+edttAXP9kMS64rwR7/3M4bn2uCEtW9ezhThi/W54twr6XDscF9w5Q8T/l1oHykDQP6A2oh92ax+G6BN5ba3o+0mihdS7gj8u/ho1vXbgdyHEKXQo6NRZuVNhEKPrWhKnCksCUZXmdRb4xP3gAq6HAOe0hh6owORoBz6Fsa5W9Nr6aD72O7OP98jgK2cC9a5gelK5GY0aloSRqpDQkHK5Np+DGSb/Bw1Fy4JVIHbuH6Uik6fWwWpcfdGLV70j//Xpkf7gtFn56Oz0zuhSWp7She2Pgzlej38ARxs4Qohe8douWGJ8ytGb0AuoaE7GyYq1LfIvHhre/zsQx1w3GpQ+V4PeFdIzpOfy5yIXLH+6v4vfON5lodq99TKyqcmBFRS/q6e2nxcuinNjDt8Y0PQ+KGk7wwNnNEkTAKCFqfhcM9ytVKXAyglY/R1WIrrphuDYbJ5Zwq+upa7DTowrTtOgGFrk4hRRnNXMmRd9wZZydSS401NeLEGeYQUJtzTXMT3WDNjUjG6s6FReNgkOTsSwwD1QaqkdKcBqaC9NQjqGlPMFmV+uajQ8OXedyJRk/KU3fQhQnp2RwNJVh0F+XwPHSaJRVVJtfdg0sV8lq9svwXR9jqLnlUHYwCkegUuvheH3hf2aM+lezU3H67QNx7l0D8NM8i/vsIn6W6593TwlOvW0QZvyaZpm0DU295MEbKB/hbiTRJaVUt/V7D21qprOmJmNK3oQgNxUKHJXNFD6ym7qQVtna2hokp0TfqKHFkJMh1IoYRZtPCeU22l8pquR7Ja6UYOJ1bEoYuz0euETAxgKnOvaJGOcUu4kU4xIkO1+tuYb5qb6Q69fV1iIlLV32yWY3YMRFrRifPQBO9NHSzJnw3CoN22xh8kkWte73ob6+To2K0XPuQKPRdBr8YUtVmiiP9MTKv5D7Qg7+fP4YlP79k6oTupvYFJOVlVc96Xq+lZeZ0B6zRGyeI6L3/+4YKCJYxGYX5tK3f6Sp654t1//xr1RzrzVOR08oQlEQ6Q2Atu72OuiawBEX6kS8cEIBikEKGmpG6l+KYC5+rxtVFeXmtL/R+5wzLLv8WFNFJFdVVcLjFkEq+7goXSrXMNZp2a1XIoozhUXvMmGQkJggwjoDjY2NaJAwKM4TVdyNa6j7kBWftwXVleVqljhOnNEdME04CQc7DTZKXN0i0jmVb3dDkcupl+tqpSw01Uu+GHljNHZYHgyxyymjq6urkJaWaTQuNBpN30V++8q/1wmMqX0BxR+OR9V7J6G+YqlygesuYqt5LMWJCC9/zxdfjsTo4zh7QTIufag/Drt6E7zwcQ7qN5A1leG+KOEfIde5+P7+6rrRkurq+Y0MBcuGleh1aMHbG7Hb7UgX8eL321BVWYnamio1di4XCtBKEboNDY1qqJqkpCQlemJCjucsXukippqamlFZWSHh1aG5uUG9Qq+R6/G69JSh+LZ6hdUevI+0tCxliayuqERdXY0Kn0tDfa26j6bmFiXqOBVvzPfRSdBqnpzCtz0cdzhFuZVw4o3uR/JJ4pKRkSNloU3ypAo1tVVokfRTjRFJz8qK1WhxuyUvM4186p4k1Gg03QFtHbLkLnoK7ncPhH/OM1LXhhmitAvoHMFLP95e0IPKYY9dIJZWOvDA6/k44LJhuPHpYnzxczqa3B0Tvxxt4evZqSq8Ay4bivsl/OVBvsXRkprcSwQvpya1MpVrC2+vhMKP08vSvYGi1OFwqTYNqwJ7ohOpqRlIFbGrRGKcFj26MnBK4vSMLLlOKhrqakVIuVFdJQK7qkKFy5D9Ph98Hg98Xq+aTII+w4ZbhWFdjLwkwJZIi7JDicgaCZd+vU0i1ivLVymBmSDHMEy/Gk1Bwjcbb6FhbWglR3HO69DKvGaK5s4i5F5UunBhGsti+C3bVLqyA6JKc58XXk+LNA4kvRrr4PG41fc10kior6uTssBzHdJYyERKcpop0jsxzhqNpnfAqlFEb17Tz3B8fgJaPvkHKhd8pYbn70piq30iiROaWno4dAFIcsQXT7fXhve/y8CVj/bDXhcNUz62L3+ajQUr6AcY+UFHK9hCOe4lOf78ewdgr4uH45KHSlR4bm98DwCnvQ2prl4yTIN2aeiTKPElIszudCLJ5UJySqrUazYRjNUifFpVJzLOg76OgFpHVMnvhksYoRg4x+1uRmNDnRJZ3MfrpaamKz9Qh8MBr5dDfPmUMOWn1+tGS0uTYWkWgUwrLV0A6LLAdbXIfk5sUVNNcVujtvnJuCQluZSfboI9EfkF/ZFGsZaaqu6DAo8zi7U0N0qYDMM8V4VXo6zPXhHeAUHM+1r/XjuGCici616TglUtatv8NAUsxam6JxGqLc1NajY8dV+1NdKoWC2CvxTlq1agrHQZSlcsRtnKpagoW4nqytWorhZRW1stad2s7osdADmVdHpmNrLzCpWLSauI4uSUFKmrnMqyy2trNJqNFP78KXdE+BZXv4+Emf9EbXWl+qqriG2mtW8OAErfNTeCyMuAKEnAlUQHLnNnz+SsOwfil/mdOxoDx8jvn+9BboYPacmtkgytaHEnoLElARW1djUyRHuiOFY2H9qMhy7u+vHuYobuDC1uwOMFysPMqFa0F7DD2+aGpqex3kxrEaCQoiBqEoHqEAFEax4FqmHVsyn/U2WZVBZKqfXkJyHyy/hexJBhSTSsiM0iWPnpEhGdlJRsfCdL6fJFytqblZuvvueixshtaVE+uZzi1m53KPFLay8FHS2/dLuiP2miwwmHfE+R1yzCleE6ZR8rQcaFw2qp8DwtyMzOVd5aPCZwfQrFNdtKNAYmuwhYPr2GABfRy/2BOPJc+q7y3o2wjE/DJ5h1gwqkfSgaJbzAxBlqWDB1DRHZ5n5OluFv84vglPhInNaexu+NcYPZQDDug+8beY5xnN2ZpCzJnCiEcaZQpfWb+UiYRiqwNVFmSGvXCfO2YvVK+CXdC/oNCGoARIeeaU3TU9AzrQnt7SfRrJPAtlQJ7uQBqNrtOxQX91NfdQXBUWmflIHmSggB665ZafZkthhqMVtcB+DtLy1z4uf5Kfhydho+/iFDfXK0B+7vbLFLNh/WS+ajpzsD8Vk89FIHmSua3o1NiVSKoZLBw1FQVIKs7Dxp/KXA4UiCS4QoraXs1U+hyFfiFETqUx4mtAbSEssWP622FK5JrmT1Pa2OdTWVqKkqV/soxOpqqlBfU62sjLS62kTs8rzysuVYtXIJKitWKWGn/Eszs2TJVq/WKUSbmgyhm5NbgJy8QqRlZCu3ibSMTKTLsbkFRcjKyVPCmhbfgNhrlR86wzSsxzUS1wrU1VWjQdZpheZoDxSYFHxOEekU6wwzI4vXzlD3boRlU/dMqyqn5mU49IGmRbqxke4BDaa1tQ719TXK4soOa7zn6vIyVFWsNqyslbSy8jwjDvWSTo2ma4Hf61fPFqYVO98xzXltuphkiJDPyStSS7Y0HHLzi1HYbyCKSgYjT9aZb7TUqvyS/AvcP6G8VSJfCXouchUu6moGvDcVhlwnVrGr0Wg2DtpsdqMB3YXEJnjTR5orIXil9UN6gVsDLaN9gS16y30EyoSVs06aRZnS9DIMKyZHVwhYMSn6KPgonCia6G5AAUvRpfaLyKQ7hBJoIowoTukWQTHl93tVeDw2OTVVdRpjOBTPyjosFSXFF6tLdS2nS77PQl5BfxGyhUhPz1axqq4oQ+nyxSKClyrRSIsvj/f6PMq9gQKbgjHg/sCOVuyw5nXTJ5juCQFXCoeKnyHcM5SApgU4Q8QyxaxLxLEjybCOUuTRrcLosFUmInwlqspXKWFruFdQxFYrccrRCxgnikSvuwVuEcHNTQ3KbaLFLd/RN1nSwifxZZi03NKSTXcRin+ma0BgU6CzYxv3c1g3pleWxDFTxKdasiS+WTnKJYRWc94PreFMDxIsYgNLsJCNhbXnazQaTc8gNsGbFn72CnhNUcPWfA+v48YObYlptIaeiF3iP7Y3CF4mc0Dw+iwEb7pFmdL0OigGHSI8RemYe9aKHlpKlRVVRBgtsVUiBGnVZG9+ikcu6Vm0KhrH0CeUrgUUqTUUpbVVysqrLKmeFiXSaAVOSctY489LsclPCk9jMolWFWZ+YX9lseWreZeIwXRaXTNz1izsXJdC1wkJLzHBvkZwU/hyxAmKYmVRrTJ8V6tp2ZW40zrLdVqea2U/O7zVSTwpmulGwcrVYTf8m9n5jp22KFApjpVoFrGcKfFKl0/6v9J3mMfRzcHQmux4R9cJkfWyzXMT5J/ax3/yaZfwKVwNlwVaXv2ytCmxzgYG09qwsq4rQHm88Uej0Wi6iS6ugzrHwusLGg+yh7s1pCb7MXnLBnOrd7LjFg1IT+n57iPrWPw95luAUKzKlKYXIoIqSFQR1UlKFroCGJ28qpV1kiKULgW08tK6Wl62AtUVq1G+eqVyE6BPKYVvbn6h8ZpdhCFfwQesqnQXEGWnLKR0paivrVWjKlBIMyxajZV4FAHr89OnlhZom3IZKCtdrq5VUV4qAppW3TrV+YoCu1XCpAWVFl26IKSlpitBTHeI5OQ0wzLKwSVpW5b7oj8wj+PEGqrTlsSNVlS6BHBRrgxJhl8x04bWXI5XyyHWaFFmmtBtQbksyMLxdSWiIoqTZeH1XEhyJithr4S5Cs+lrOiMAu+LQpfiN5D8ajQJ2ccOdEwbinCKcnbSo3jnwmuzseGWe2be0PJOQWy4QJii2sw7Jbg1Go2mE2F9w/4GXUlsgpc+vJwZKxQ1zqr5oLOy5PUg9p/EwfJ7L/tPqjHXejiBwsyioaxMIbAspWgf3j6F0oGG9ZEVmvJ3Va4EPiUW2SmKAovirr6mSoqIVwlF9uynb2mmiEVaMY1RA0SImq/46XerRkKQde6nFZWilv67dBFoa/MrSyettOnpWUbHNBFzFHYUfRztgdZOWnj5ap+v/e2JdjQ1GNZY5b4gIpF+xT6PVwlu9YpfbomWVUPYJiv3Cgpxw8c1Q01GQcs2rbcUixS07hYO1SXxFCGtrL2yjwKV7gaMW7YIfYr4vMJ+yBURz4XrBcUl6jNL4qfCFSHtVNZhp7L6qk5vFLZMX4pSldimyJX0ZfyNH5vR8U/JVDmW53DMXp5DcUyLL+PIuDVSbEsDg2K4fPUKrFqxxGh8iEBWeSRpzjxcM/oEL6aCNTsZcqEg5mJcMehTo9FowtPq96JFTVzUdcQmeFmRWbk1BKx5/DTqxB7L+FGNKMnr/lmK4qE414ttR3d+x7tOh2Ug0FHNyuqfNkz+6IdjX4Idu5pE7HFIq4rVpUr4eZVV0xCqFLuEIyaIYlIilKMlNIv4aqqnT2sLbCLuDJFHf1i/Er1q+l/ZT19VNWqDnE+hSsGVmZVnCFg5hwWPFl2GS0FIdwJalJVLgQhXtsspQCnAKQJpKaY1l53NWpqaJDybMTKEhOeUhdZeil9DyDaJQDYsphSC9MflfTF+tFQwvhS1qWnpShSnpRuuGtym0OaIB8otgaKVMaWgFgFquCIELfIvICgZR55DCzFFNdOFFl6GR1cO41ocDixLuYzQEq22+Snbyl1EGhHK6pzGjnlGZzS6dXDoMM40lyJxZrw5CxqtyMotReLANOeIG2xQ1FSXqyHKSkUQs1MghypbvWo5KiWPaTFWjQpJHzYy6G8c7D6h0Wg0obDe80td0ZXENiwZ+f5IYPlr5kYQ2elACh84glTscPaEWYCsee7DHDz0Rr651Xs4bf8KnLBX145dFxfspMahyEizFOqqemM9mJJDgAkvmxuankjwsGR8ix6pshB9JuKnUVkClbgTAZjAhUZJHiB/2kQ8UlwmSV0R2KdgwFwPcwFqY9ZSHilGXrdbiU9OVcxJD3iKep0vKxSShsWRQjEQcPvQLcAvwpod4iiIKcoZjoo/RafcuKnP1yxrohkSXxpZW5qN4dtcyS51XqchYbmbvaoRoDrvOWIMXA4PPaO5Se7VXF8XWumNTz4i1MKb5afkL7+kBZr2DTUph1o46Qe/M/IiNY0jPKhTwsLwW5oojs2GsYkelkzTU9DDkgnt7SfRrJPAtvzkGxzFWLXduxg2epz6qiuIXfD+dQvwx9XmRhApLhG9qcY6HzbJSYbfZqPkHt0cuHA/xXCqCOMkUxx3E5wt7ehrhqC8du2QO7HCIYe3dCQiXz4/cEd25RguB2cmJuBnEYLxzsLMcX5fum6RJPW6D4hugcqgSfK2RUStT/KZTy+7KCK7pGeG/BopcgNFq6bRKAehbHodMOpKc0PTEwkI3h2n7op7b7wIdqmszJflYaFADBBatfh9bowZvzvGbDERzz14pbJOxlL7BHxJad1tbGzE6ZfcTTm2nmCKB3aS+/zDNzD/189FSCapejk0cu1VlXSryMwtwZEnX6RS6LnH7kFlZbmkSec0/mktP+H0i1BQ1A8zv/4Us2ZMk7iGcTGLilYkODNw6vnXiugM89uMAPO4ubkZj991sbIqB5IlkD/ESD9ar9XmevBQzsZ28gW3Izk5WY5bm4da8Gp6ClrwCu3tJ9Gsk8B2rxG8Vd8Cn6+fEUrMFucY6xymrMFtCCIrHCKMskQgcwmqKLuST37IwDVPFJtb0eOQ+B7gsmP/ZDtyzai/1OzDs03h3STGOhJwU0YS+NhbLUJxmhz7kQjklhiT/l8nlWKPbcNM3tCVsBFTXS9Pq2ZD9IaDLQE2gNLlV8m8La0Kf+zOM0TFb29uaHoiFLz/vPk5jN9hKuY/NxYnnLaZKK84BWayF5fdPwaFYw7DSf3OQtYmJaKCY/sNKJIS8Mkbf6B2ixkoyksxfEs7SFKyC688dCnuuGC+3F/s03wrkttw6z0N2Pn099TmVw9PxsXnF0p4arNjyG+qadkK3PHb9dh//13wxD034N4LfpKw4zQcyE/zrntXYuJpX8JnWqSjhS4Wf8xZjEnuwzB6u1FrR+mJBakTf/tqLmamvobRIwcq63AALXg1PQUteIX29pNo1klgu5sEb3BUoiN7W8CeZm4EQUFDK67bA5TXRha7hKKYxy0qkzvvniG2dt2mDuNHxOYPO1Eq6oczk3BSylqxSw4X8TuU5q8Q0qQhcEmaU4ldUiDbp6c68EhWEraOwe1jy+HN3S92mV9LJL9qJc2sxC6hiKEgrpD4UiCHO9YuDZ0cKUua3oGUdU+rNGIaRQw2xbvYYUs0fHPrW8J9H+UicWj2cSiu2KuvSNgSRDxKHMNeM8rFjyQzNPkZQNKriWF2zlLfwiHL5DpCgqRjR+Pqa1sb11ih1brRwziEDzuapdGTtM4bAY1Go9mQxF7b2ESk5YWx8BK+tq6oN1R8tFAkr6wyxFEsUERRKPN1eXWDsW419FUELjpqtRrXtj2YUBSqV2ckoYgWzBD4/Tnyfeg3p6U4kEPrdwh5su+6dKc6J0xw68D4XXzkKnMrRnhrdC+gSGU68bNFtmOBebS03Dg/lrxlfoTz3SV5k6Qsxe9OotFoNBqNRhMt8TWvC6aYKyE0GT2w44LCiK++24MiisctWGkI5dU1huWR64vLDGFG62KUDCpy47T9I3cCc9lsuEGE7v6uyAJtuD0BByWvPWaSMxFTkyJbcfeUMK9Jc6prWHHyvpUY0i9GkUp3CTYiFpUCyyRNyqqNdOIn02iB7K9k46QdBcvOZzw/VpEcIDB6Ryj5FmVI03tIlrLNpTONdOkOY0ntQY0h/qYD8erovbokvQJhtdfSjQdpQK8JPwDfPEWuhjoPacSra4fmXzv1oEaj0Wxo4qu+83c2V0KI9Jo7GihUV4kgs4LWXL5S53FWl6IwoyCmAI4yPsfsXoldtw5vieQz6fI0B7Z0RJdUJ6U4lDCmkL1IhGw0jBdhfE2GM+zzb+r4ehy/Z4yjMjANKP7ZiAgMDRYKhWilCOIlqwG3xRBtAbFrNS1wR7BqNGl6CX488PgfuO2B2VhULuWnM0SvS34/Z3yGI8/4Aidc/XPPEL0idl95/jccetoX2P+ETzB/tfwW4r1XEbsfvDEPh5z6OQ488RPMWiz1WGeKXhG7N9/wDY6Q9Nv3pOloYwNd6q3Z3y7FzMUtG170Sv794/RPVP6dHJx/InZfeGYOmhxa9Go0mu4jvqo7axzgNOaqbxdW6HkZQL8cEcqZ8gBpRwTWNQEVteZGEBRntOZGq6np4rC8on0LpsmVx5di61EiqEM4M9UZk68tE5SuD3RVcMbwLNvcnoAzOHpFEPQvZrxiotktIlXuO1qRSos5RW2o6GVjYYWEQ3cGK3h/7JhWJGWB+cv1aHBmSRnaytzQ9DroolNRh4VNSbjk1M0wKFuETRz9ltZFwvR64MnMxktv7YOnb98aaIzdRanTsdsw7etKvPbm3njr9b0xvFDqgnjvVcTnm5+swv+m7YM3JKzxm8jvJd4hW8IhUftlhRcvv70v3nllKmwtkn4SfJY02Nv4BqkTLxUWnxdtGZkq/x4Pzj+pg5Ol8e/c4BHQaDQaa+K0VUjlWTDVXG+HfrlATjqQJpV7dhowIB/oL/s4SoMVVQ3ruiVQvPL1e6zQ0rkiOuuo09GG289cgW2CJnXYRh5Qe3bhq7i95Vo7mOJ6/Mgm3H7WciQ5Y3hIcNzb5XK/MY7+oMTtSjkv2P2AVvJIPtEUt0OKjJE5MlKM/A2st4cqO1TLml5Lkh2ffrEY//z3LFT5pMyyJgnN0lizWMr/l58vxKlHfogb7p5jvP6PFdM1KHiIrA4h4WQl+nDi0R/jsKM/RVNbvOZdYkOeVIVHHfMRjjjyI5RxhvPO/BkkJqBiWZVKvxPP+x5IkTrW14qBmxZhwqCk9oV6R9POacf0L8z8u++vtfnnacVBB24CezyjOWg0Gk0nEX/tPfBocyUCTgfUeLyhpLqAQQXhvwuwSgQXBSstIGUdmEqXfsXsbBUFhuhdjv13qAFl21lRuiR0JqemOnDQ9nX4z9kxil2yihbwGM8JQItwwLJOX99w4+YGyMs0xC3H3Q2FjZr2GBBF2dH0bFp82Gv3obj9um2RlyRCJiEBDbToOaVKoV5KtEn7y5iwIGo8fuy8yyZ47KmpuPrsUQDHtk6KropSkyHIhVt9PjVpBGdS6xTRK/FvTnDiqSd2watPTEYKJ3tgnHifMdOGencCXnx8Kl5+ahcUZkoYbFBTmDJM+grTJ5r7uE7BGIvolwZr8ZBclX5P3baNyiN1Phu07b3wkbTiNM/BaccJJWJC8m8XlX+74ur/G2m8CSC8Py12NRpNNxNPrW1QtLdUzLnmhgWRXAFYGdLS67IQlXxO0p+3UsSXVccn1qe0NHIs30j+YXSHiPIVv8PehkuPLcM9J5SjwBWneIwTm6MVgw5ajouPK1XxiAmK+kgdy9i4YDpZpTfhCA60rkdqIORmADkRRK1THm6RcIpQLpayo+m9UMTmpiPDIw2z+3/DI8/MxUOvL8S7M5bjrsfmoNqfiCeem4NXP12Bfz87zxBx7SJhym/4g/fn44yTP8NR538nZRV46KWFhhiMgN3pws8f/hc//lWKJ249F1kZTtx1w8VwRWpQR4sIP1tzM44/6VMcfern+PKrFXjygzJ88sXyOESvDU54ccQ/PsUxp3yOTz9fgcdeXIh7Hvodz7yzAs889TseeGoe3npvEW556A+8+s4i3PvcguhFb2ICFs1ZjVNP+gyHydLYloj7JbxozmdaMe3akp34z7Xnw9ZSipeefFRNzRw1jgS8+/7fOF3u76yrZ+HWu3+SBnAybrxNPoM682o0Gk13EH2N3eYFmqWS95pWQA4pVXKEsW6JKNJIVoKA6LUSq3ylzo5q4ZDKHYMKDUtjQZbxet3qdTotHDXRWXkDjJlYhYzz/oJzdBh/4g2AY3Sdcb1tYuygRqiNrUQqGwX0naafLdNpYL6xbUU4/+kAnEFNhE5E2tPpA6TM2Mwe5G1SNtxlxqLpPTCPE+24/NwtcdF5W2KbYWmoam7FEWdshmI2EhMSsMMWuVi4sA4D+0sjK1rjXnMiVv95JB6+fQe8ePdELJ9Xi/IaacQFLIUW+Dwt2PaAc7HdZv1x9k2Po7rGjatvewjNMc4gFpY6D558ci88c/ckvPDoFOw4MAkPvTwPf65sNkY/iIV6D267b1e8fO8kPP/IFEwdkYzMwlQsWFaPoYPSsWhFA9LSnGioc2PT4Rn4fXEzsh3t/aCCqHbjmy8Px2N3bI9XH5+CVmnoL65wt5t+hLOtnXPTE7A1tuDaOx8T4dsfJ519AdwtMaRhcwJK5xyNR+7YAQ/cPgGJjU34dPoqzPhV6hR7+3HQaDQ9lD7ygiZyje0uB/78F/Dx5sA0EZPvDQbeEoE6TYTP59vJk8ZCjAZQE1GIaI00WgKFa/+8qCrldWAnqVBrIkVdioVVp1biGuPrflumBynHLELayX/DMaJOacfOhOEx3LR//I3UYxbClhXBQhsJ+jtbdS6jRTZJ0in44Uy3g2h8bYOhtawgio6KkTq5EVc/YNYpwPtDpBy5gHf6G8sbEqev9wGWPi8HxZZPmm6AWdQkv21ZakQUHji5CHdf+wMGj8nH/96Yh9LGVgwbmon6emkox/LDqRKBxnNqPSgZnYdUjpEdRXGg6PVK2TOmyW3rHLEbQO5PxUmWGlmuPXtT3PTwnHYtz2ER0WuE5YHf34bVZY04/8yt8OF7C5BRkIYTjhwKm9xHyYAsXHfZFmhobuf3FIqI3kD6pUsY/dIljlHWe0w7la0ietta/bGJ3QCB/BPRniGN67demI0JE4skg6KLg0aj6UHwZ5tgR/3gI9Dod8jv2NzXS3/O1jX24ieBD0cAc26WyuvPdSvNVqnQqn4Aljxt7rCAPf95mtWwVwEoXGl9jBaO9GD1urJQwgn3gKXo5ggQcWAf0oDU4xci/by/4JpchsSCjj1Meb5rpzKkn/uXCte+SWzW5/Wwsl4znZKkkNolfUP9Gduz1AbDRklBZnSNEk5yYQUnLfnjailbT8lTddm6ZcovabrqQ+CHE4BPtwYa5ptfaHo0vlZsPaEI2S6nCLexmLhVPk45Yhh23qEYRx00CNuNkUZSvP6bIqYPnlqiOj31CPytSC3OxLYi5L/5706GsOsAbRKeLTUFf/2wBAccMBy7bpmDp5//C8M3K8ZWQ1Lld92M0mr5PcXSYAim2YeDdpX0c3dD+kme7S1l4MJztsJZ+0mDlv7YGo2m19Eqz+2asTfBPeV5LM/Zk676fUzw/nULMOtUeVDFMTJCMBQ0HCaLHUo45XAkaHGM1uqYIQ8DKzj6g9XwWHVBIz/EQUK+CNXdSpVQzbhoDlIPWyLCdTUcI+uQWNSChCwPEpL9ouva1GdCpkftpxWXx/H4jAv/VOe7di9FQgeFs4JTNFv57qYzPeVpGc5lhOkUrY8jO6lROEdDJNcRujBEQ82vwBeTgFr51PQYVBFIlSojJWhx2ZBRkoZ+JfQNl985j6FVVooX2lqx+da5cqKUQXV8opRGoyNbWrJsh4YVukjYQ0ZlqM919st5yQwzSstltNhsEv/g64QuEmdHYTLy8hMxdHMR8hx3MOQYR9CsjQ6+xpd7Dru4EmEfmodDx6dj72PHYPyWmRg7qQQnnDgaW0/Mk+8lPDn97GOGG+4AKXakJicy9czQ5TPk2ustKv3ktxuafubSnrt9JJiHqYxjmHDVIvkzYHg6Bo3OQNFQaVyHycNU2bf2fjQaTU+Eb3vaEpOQs/lhyDv4ZTTs/j68yUWiPfilcUxvwSYV17pRLn0H+PagznuYcK70Iql0Oe89LY18vW4FrbBLVxu+u5GgH2qkzlccmYFj8IajvXN7GxybOJyfM0dQoLWbwtbqyWZ1bjCZ0rigz2801l36EXOEh84ibRNglx/kHuT6mm5hYEkxLr75OUzaZSquP3EothyZ2a7XihUJbc2oyzoaW+10JL5+fGcUFhVJWzj2eoZF+/e5pTjt9gVIcdmlquq4BdOV4sJT9/wLuc2vimy3aDC3gz3Bh7/K++Hi/3ygXqjceMqm2HZYg6SXRDgccoxdflc+q7F4g75nNVpaWoqRh32GSTtth+cevgup5Y+g1Raja5KJPbEVvy1OwhUPf4+m+tga3gmJiRKXcrxx65YYOnSAcs2IlcREG/5esAIHXfYTiovz1xkRIiPThbNPPAR7TxmHy6+4ytyr0XQ9S5csRsnTw5HgF00iv0El8II/g7H6Ltb98nOcPmUGJu+0o7ljLdNnfInJ03eSxrdsBJ9nFTax+i6a/bLearNj+f7zMXDgYHMn0NzSgsXvXo7RC++WylN28HgulAkh569ZJ4FtqbIbHMVYtd27GDZ6nPqqK1hX8PrqgPeHieCsMnd0EvSrDQxXFUmAEb4S50QIkRjWTxKtHQG2cJXcT5gnM8eLpf9vX4CjVywsNQpRKBSqaVISmfah7gwBKHYpeq1gPrU3ZnIA5lsME31EzeATgPGPmxuarkYJ3puexcRJuyJR2onxit0ACVI8fBKIPSlRvfiJF9FcaGnwdIrYDUDR29HQ6P3T3GAIyJR0V6dOUkjR2+puhdfrQVKyC23tVIHtwYZDrGI3AEVvUoojYp/k9mAeupu86w1/pgWvpqegBW94wRtg8ZJlcHx1GvrXfSjHysGBOino/LDX6CbBGxwV4O8HIotd1ubxTIVJi2ugEwlfwUd6atLvtL2xXKOpZTkEVzg4iUV7PsW9hep6owCFg0KXQtVK7JJI1jWexg5v0Yhd5mlZtVHgY4Wd6SLFkZ3YmhabG5rugjlLceSR33FHlpbmFvi8XhGr4b+PdmmWuHSm2CXstBXuWrEsAbFLmF5e2ddZi1vCptglbknHcNePZYlX7BKKVOZBuHCjXXh+zGP9ajSaHsPgQQNQfMz7qN33B3jzxhvSjNVyHFKgK1hX8C56xFwJgT6gg/KBocWy9DOGAONr7licwKobpZY2XRUoOCOJXvqMRgo7EE4kskQ0W1mBK+rMlV4M049pGg5adtkwaS9/IpnY2GCIxseXJqzVte27oQQTsLIPl7K0iZQpfloNldYq4S5+wtzQ9GTsdoe0XdatUuLF7uiZbkeODRCvxEQ7Emi+7WQ2RFxjwSgPERqzGo2m18OaK3PAeDTs8jYaN/snGtJGiz6RnT2wLbu2luUYu02yhINj3SYFVZ4UwLTCDi40hEu0HZo4iUTgHR9Fr9X7PtaR9D+1wstz2xFYFLsUveHgLGLsTNeboWi3sqhSUHKWu/awsnRTKLNjYHvjjFJ0c9zeSLOyBZOZsrbMMI7BD0OWJyurfMU35oqmp5Iqjazffp4Jt8fTYaGVnuHCN9M/gtPlUq/OewIUpMkpLnz5+ftIz4xhMoZ2YJh/z/sTNdXVcCZ1TrgUmWmShjM+fVelZXeQlu7Cj99/iTZ5xFD4ajSavk12XjGSd74NnqmvoX7cdVKR97yObWsVTf0ccyWE9sQshQunCS7MNlweIsEbZ6emwMxpngiWXloXKZDCQX9RWhStZmALQMFrZWEo78VWXgpVqyHWlNgVwRpu2t9g6M7QZCFUs9Pbtw6zsVJVL+UmipEvmJeBMhIpXKsGilXZ1PQIsnJceOGpR1E5/0M8ftsZWLx4oRJysUKhlpXrwl03XYXk5j9x04WHo1kapp0lBOOFAr4Vibj2/COR6ZuPW6++AJnZLjUNb0fIyHLhvTdfw7xvn8cbj1+B2b/8qBoOHSFRxKXTlYR/nXccsrEI1118KtJEoHdlwyFb8vCxe2+Hr+xb3POv41BeXg5XcvfmoUaj2fBQLeaUjIFjh3+hdbfXUZE3VfSK7KTM6wHCd22NzTFSw0ERS2GphheLEOOA9c5qSLAAtEpS9PpMsarcGyyEa15WeBGtrLMiZDkcVyTRSwul1TS4PDcasdYTieSSERC87cF7D5edzD/6UUd6FclGCifyaG+EB1rZOb7ygLzo3gJYXdOqbGq6BKczCZxhltPMBi8uWTJE+F33z7MxcVQqjj3+H7jrP7dj5rv34P23/qeEcOg5VgsFMoXaJaccgdOOnoq99zsYDz94J5687TT89ussZVUNd96GXmgpXbjgb9x95RF4+L5/Y7e9DsAlZx6Ji08+SASvQ8U73HntLRT2d95wJfqnrMYZZ56DG264ASt+fR0vPf1fJRjDndPeQrFMcXn9uQfi3tv+hV332A83/+tcXH7qAWiSxm1KavjzOmuhqE1Nd+GKs/+B/aeMxMGHHY3777sT7z59Fb784hNkisAPd976C9vr2iqs0fRW2Ly1FW2H1H1fQdkOz6AqcztDb3Sz6F07SkPNT8Cn26rVdaAI4QxmFC9cp3Bpz5JLMVVeYy1kCcUofXUZFnUOX8GHs0py7NxVYTrS8fW3VOAKCjyrzlUU6RyxIZy/Kq9Nf+T2RnzoSTBtSy06FlKs5kXR0Yw+0Csq1reuMz2KcoCUCIKXeVrbAFS2M0YzO83Roms1bXQodFGpbTKsxqFkjAF2m21uaLqSgQP6Y+oBp2CLrbZb02GK2OQ3w5pj5mf/w9WXnYOUtEy43YabUGZmJj775AN8NnMRttx6x3XOC0eClDvOjPbnzHdw8/VXSRvYD5/PcFnKycnGM089gaW1yRg2Yozq8NZV2B0OLFn4NzJsq3DmmWehsqpa7U9MTER6Wgouv/IabLLlHshIT4c/ys5XyqfVloDvP38DF519HAqKStDcbDS80yWcn374Fq++PxNbb7+b3GvkdAvGbrejtHQFfJV/4JJLL0FVVY3kD4czS0BWZjquu/5G5G2yPXJyc+G3eqvWAXgdznT305fTcMPVFyPBngSPx4h/VlYW3pz2Gn75ux5jNh8v5SFyHqakpuE/N16MU447GFdcqUdp0HQfepQGkU4RRmmIBiovBrfk57dQ+O2BcPHBIRKl0dkPpdu9h2GjtlDHdQVBw5JJtN4uBDxGpb4OGZK6ahIDEwpTCtRIOpGW1zIRvRwVwQqGEzzGK8fHDSeml4k4C/W55SnBbhQUeVaWzUjDb3GyCwr63gDTdHFZeKs2H6T9cg2hGSlf6ArCySHCWWc5KgMt9VbuELwuz6tsxx2EVl0rf9xQ1KQkEicKadVICvMwHnYWsMU95oamKzn44IMxbdo0c2t9Hnn4IYwaPQYFBQVrRJ9DhOLvv/+OX3+ehetvvFnta4+igjzcdPOt2HyLLZCamirFwijjTqcTM2fOxIwvPsWj/31S7etKjj7qcOy51z6YMGHiGqFG0eqR9Z9nzcKtt9yIeX8vUvtj4YH778PIUaPRv3//NelG0Tpv3jz88fvsuIbj2muP3XDEkUdjwsSJa8JkXFnF/yBp+MgjD+K7739U+zcELmci7n/gIYzZdDMlcgN5yPLwyy+/YOZ33+A/d0X/O2a5O/DAA80tjabr0YK344I3mLK6NuT/eAIS5jyL+oQ0LN76TYzdZhfz2w1P0O3KasFUcz0EWlk5tFgAihIK0EjuBBSi7JxEQWslwBgOOz0FXCVa5IESLszCMGHwFFoDA3o90nBnFF9Wr9TpC2vly9rToGi3SnM1qgIts+Z2OJjO7GBGd4RQ2NhIFbFsJXZ5bntilw0O+upGK3YpvumPzXyrlnCt8q9wd3NF09W8/vrrSjCFW1atWoUDDjwIb7zxhhKmAWgB/eabb3DOeRdg9erVYc8NXX797Q9sv8MOeOedd5RACkDL4ezZs3HzrbejpsawWnbV0tDQgHvvexCLFi1eY60kjN/HH32EzcaOxZdffxf23EhLWVkZDj3scLz77rvrpBsF7/Tp03HyKaepY8Kda7W0tLTgmedekJ+QH0uXLlV5QPj5888/K8vuW2+/p4RwuPM7Y1m8dAV232NPJVRDy8MPP/yAy6+8GhUVFWHPDbdosavR9C0KM2xI2OUZVE54AN6UAWv1WxcRJHiFYefQJGBuhMBZtIJ74zOi9IOlSI0Uafa+L8m3FlLs/ESfVCV6ZeEUxKHuB7Qmh+vQRMFUKfEKXD7S0FgFFsNeEVqiA6K7p0JLuZXPMYUm07k914xGOZ+CNdytZqUY6RyOaMQufYcHitiNxleX1ic2oNhIYdhVkocsR+HIGCW/kj3MDU1PorCwUC3JyclKoASgZa+oqAh5eXnIz5fffhTQQjxq1Kg1Qi0Awxo0aBByRbDRVaIroaWZ1x0yZMgaayWh1ZQL48t4xwrP4UIXhuB043ogzWINNykpSZ07dOjQdcIMMGLECBXuhhj+LADLwoABA9YRu4RpV1JSotKSi0aj2bjJ3fpM+A/4AYNGdN2kE2Td2i93BxEtx5obYaDooTgJrk8pXmilo3Bdv541oOVxsFTgVj23KXwopvhMYRjhRDRft4cbJosCma/CafnkOVbCNdKoD4z7qjCuHD0Fxo+i3IrcdOsGRQDmHe8xXPqojmqSPlaCmRZhDilnBa34tOa3J7h5beZtIH+Z78w75qEVW94rCsPCVUXTI6D4o3WSYpUL18OJrvag327g/EBYXKL1j91Q8Prh4hXwM44XpltweFw6SiCugSUQ547GNRYoqkPLQ3CDQaPRaPKzUpGRJbqhC1lfQY65Dkikg4gFdGVYLcIp2JrKhxuHGGuJ4OZAy0KJtO6tZlFT1loRVXxOMrzQMWIppnItLDwUhHzdz89Iois/y1oY0oJKK3ZPg+nBTmpW6Uof5GD/6nBwVAymTzgNwvRinlh1dKPY5flhz2We5lnnaQBVPiR/lRuM5BHD4oxUnLAiUsfGoj0lz7rOv0cTH8uWLcNff/2l/E+5zJkzB3V1Ed4GWEChtGDBAsydO3edsAKd4boLikXGIxAnxm/+/PlKsHaElStXrpNuXK+q6ti07hS8wWEG0jKeBki8LFmyZJ04/Pnnn2hslHpEo9FoupF1BW/tL8D0yVJrWrw6D6A6GIlYoRgK1ivKgucxxKpV/UproFUnMSV6TfcGCjxaAIOhhdbqlTnPoT9wJLcGirtIE1rEMolCV8Epe5mm4eAICJHuh1DERxLytA7TfzccSpRaWJaVv26+0UkuEsrfm28AzHxRVl2z7LTHqg+A749svzxqupVTTz1VdegKWPS4fsopp5jfRg8F72GHHbbG0hsI6+STTzaP6B6OOuooZaVk/Bgnisq99tpLdczqCJ2VbsFMmTJFuRUQhkmhO27cOIwdO1bt6wpOOOGETr8vjUaj6ShrR2momA58vZ8IFIsJDaxg5zRa+EKFKI0fVkONEVr7VlpYLl0ionIlTFpQQgUVfT+XV5gbYaAvKV+vR4KCy0oEUhTTamklArsSCnC6kISD6cvxbZlWVrBhsqjUuvFB0cqxk8NBt4Pl5UZDIhQOB8fZ9yJZ0wNW3UBHNGZzo5QtdoCMlezxwOTPpay1Y8nWaDQajaaT0KM0yKO7E0dp6G6M2/KI8Jx5XOxil1CwstMZx2UN9tNiotHSS+teONFEn9EB+eFfpdM1wsoCSAFMUWsFO2a1B8f/pV9xOBjXFZWRLcVdAQW5ldglvIdIYpewcRAm6ddAv+hwUKSulEZFuHyjCwWHP4skdmnF5bUDYpflgG4w8YhdUj0L+PVCc0Oj0Wg0Go0mNgzBO+c6EaYr1Wrc8LU7O0XRFzbYX4wimNbcUPcEQgvjQBG94dwU+Do91I83AF/jW/mcttd5i1Cr0UIZbsxfQhGvJmYIEvBdiZq4o9bcCAM7mbXnN0vCdfILwKHDrGbFWyENoHD3npNuuKNYaV3mO8tBoLHAdGRDiA2icJb8WFj0XxG+G24MUY1Go9FoNH0XW5u/uQ3v9BdBaiGwKCzpK0prXbRWTwqtLBFHSSGilFZBilt2dgrGL0JpZaUhjIOhsOK4ruGGy2JcOHJB8DmqE1Vu9O4IFGec1CJYoAfDuNIKHcma2dm057LBe6PLRbRxok908KxodBOh2KUvdTgoTsPNdkaLstU0zSRQPlRaysIGC2dOiwbGiXnMYsEwrMTxkJOBrR4xNzTdxUqOrcuGbBSdtugxxSGz8rPD++0vXblS+XlGC69bUlxsbm0YPD4fysrLlc9uNPAe01JTpMoL/8akye1Gc0uLJFd89Uir3488i/TjL2VFaWlMcXVxCDOL8OIh1jzk/Qzo18/c0mh6LtqlQX6vfcilwda28u02fGMxwDenmeUsawH4iptChlbcSDOoBQgMBRZsSWWdr0R0iBhmIvO1d6hIor9o/whjN9IKTOux8veVY2N9pvA+6EtsBcPk9eN7VsUG72WZhd8soUWcAtzKMm0FxShdSyiSKZgjWbYXhvj88hzOaGdlDSaM9xr3BcmLahHM7Vl0+YBm2UiT9GU5CcB7Z0e9cGMOOyUe+0n6aLqNJ599Fqu//hp5efkintrJY4FCbOHSZdjx+OOwx9R1J7Y545RTMKGwEGbJaRcKxuamJtTm5eHKK64w93YubhFjZx56KHbcbCx8rdHFLFHu8ff583G0xGn85pubew1+/PVXPH/LLRgxeLC06y1+1+0hcVokIvw/d99t7ljLcQccgKmbbxF1XJkfi5Ytw04nnojddt7Z3Bs/F194IUY6k2BTgjea+7PB7/VikRx76223mfs0mp6JFrx9TfDOuakNf/zL3AyCQpOdv6ysEhwCTE1RKwsTyAqenplm+N4GB0XXA1r1QoOnAOXIAIFX6hTGQyw6VnUWnG0t0ji8dB+wsoh2FhR6S8qMdA0H04vuH9G4bMQLLd5LgwQlywAn7LByH2HGs3MbxS0XNlZCrfShMCy6RrAhZVW2InW223ux/NhLzA1NV/LFN99gwbvv4uSjjoI3hqHCHMnJuPHee3H4pZdixJAhat/lV16J07fdFoMGDpTsbl84B+AsZx998QVW5efjeIlHZ3PcwYfg2WuvgU9EWaRqLRRHSgouuukmXHb//cg3R29YXVWF2887D7eLEPaJUI8Xu4jJP+fOxRuLFuGKyy4z9wJHizB/QdKR6RdTXCU/bpF4HvLPf67Jj3i44667MTUvF+NE5HtjGOfXYbdj1i+/4Mu6Opx/zjnmXo2m56EFb98SvAlwi8gKB4UJrYJWljq6OVAEDikyXpFbaBeVeBTF9EkNrhTVK/AwPrrskDZYwqSvKIfMimTd7SzYESvPogMXYQcyq6HBOguOyGAldhMlcdn42JBil9D6S99mpjvFNdPeUuwKtOayfFDk0iobSewy7vS95qgQtOxGer1LC7Q8FMNiVV41G5x3HnoIJx99NOpFqLSI4I12qa+pwVXnnot7/2U0rH/4+Wdsm5amhs9qaG5ec5zPfEvg83nVttvjgccTElZDA3afMgXLRPTSVaAzuf766/HYRReqMWObg67ZarrpBOLCYbYC3wWW+upq/EfE6I0XXWQEJlx3wQW4XUQ+v2N48S71IpZHjRiBYVJ//jR7tgr7zrvvxr2nn264S8gSiIffbDx4vR617QkXV8mPy888Ew9ed506Nh5WlJUhZfkyJXbrJb0CYQeEr18Egtqn8tCIS2Dh8eO33BKOxYuxYtUqdbxGo9FsaBJgsxBRfF3Jip5Cj8JGKdcwUMgUmEIm2P0hFLodlImoo0tEICg+4AKvwoPha3SKUI4iwNf4XQGtjhTuVtACbJEEHYZCkTOhhYO6kOnbVcOk0XWB6d7e9ZgWbLBU1K8/+14wFK+qYUShG6FhFIDlgaN0WFr9gpukmq5kcL9+8LXI7zcO/CJsB5m+txTMwwYPVmIsgD3Rjt/+/AXPvvMOfvx7BdIzMuBtrEWNVD/2EH91j4imoQMHorEDVtNwZCQnw5mUZApcA6c07N/6+AM8+/Y7qPUlID0tFctEpLnCjJDil7QZWCSNdZPBsu6XuHYGFIpbjBqF1eXGG5gEiWNOdvYagUto/f7sy09VGi6vbUF6ehpWla+C3ZG03s+O8SopjP/NWY2I5lFDh6p4BaAf79LF89T1P/v1L5WHCZ5GVDR54Qhxo+JkIiM22QR19WH6C2g0Gs0GIAGpQ83VENQEDGY1yVY7O1OpTmsWyoaWQPr80jIYSSxxuDFaMwMuC+EEb3dBYWk15BnvnR3AOhsmZyR3CopPqymZu5M6yUfGO9K0wGxA0FpPl5BIFl1C6zbLGP2BuVi9WUizKK+aDU5Hp4cNCElbQsI6Qo34W/3YbNRmmLN4IbYdkotL/3OHCODf8MrnM7Cy3rNeH02e39GZzkJh7ILFLmH1tMvYAfi7xoG65X/ghqdfw4wfZ+H9b78TMbx+Z9rAsOYkNKyO4vP713RO472H5gcn7Nhh6wn4c9lKpHsqcMVDT2L6z7Pw8TffwJ+wvuGgI/ELm4cSv8GDh+O3RYuw27hRuPHeO/HDL7Pw4qczsKiyUdq+6+YXj+/sPNRoNBorEpC3g7kaAgVpbYO5IbBupJW2kRZfESRWHasodil62dHJqnMUxSP9dGk5NivwHkORiF4r1wG6Zljdd7zQf9jKlYEWc6anVTp2B7x9Cl2OkGGVFOyExtE12ICgO4YVfOCyLLFxRWsxt5m+4UaJIJmbSd5EcD3RdCmJiXblj5nsSlLij4KGVr4Ul2vNdrRQKCa7UpCfnYMPPv0El553Mb748G3c9er7GDogv9N/dtHC62alp6tRJt77aR6O3iIbH/30G76cW4qkOF2MbAmJkm4ONZhNkpNpZ1f3HxDQKUlJKl1jhWGkp6WjMCcbL02fiRsO2h7/+/IHvDrzd2SmdM0bIt5PblY2lv32DXY66DR8+9VH+M8Lb2D0kH4ikLspEzUajUZIQOaWIiQspp2k+4F61R784JJKi2YPvoZXPr4WYo1+mvTvVdY9c18wFDe09DaL0OlJUIBbTdfLCjuaKXFjgSI6HPSRprW5p4nd0ipDpIeDcaavMWeAs5oCmqjyI2UneHIKQquu8vVe13K0hkHHmyua7oZid/7cn/Hr/L/xn6eexeezZmJJtRflpQtxz4sv4/PvvsWC6qaYRK/X70NBTg4mb7s1rvr3jdhsp31x10n74JOfF673SrzLkHrKC4dUCWnYvF8G7v30Txy7z94Yk21Dgy92AZcg6dawbDZuljS76emX8NCb7+PqO2/D/774Ek++9gKeePc93PrM07hBvktNjv3NDq3ATMOdRg/CBU+/g2P22Qv7j+mH5TUt7XoTdQa8frFcf/CYcXjtmXswcLMd8eA5R+Ld737vvjzUaDQawaiBNo3QeYHWN3ZICjdxBF9psZd+4FV0yCsu9R6S/pu09gUPPRVMmQiocJ3XuhOOTmA1DJeVQI2HQLqFQx6wyg0gBsGwwaFLh9VwdPSBHlRo7RLCssF8bpLyFFpWqBtYzmg5tnJxSe4HbHKGuaHpbmjJLV29Gu/O/AWXnnseclGJZbU+1FSuRq00hovTErGgsh6JMbyyZie1o/fcC47MYjx4+RXYb9ep2H/nXTBpkzwRwxaNoA1OK7yOfBy2/SjssvM+uOfCc7HvxK1w7P4Hos0rv9+YsSnXg2223RmJ7iqUFJfIz8KDNlsiXHa7fOdBYdFQDMh0KZtArLi9fhy96y7YdPOJuPeSi3D4jjvgwD32RZY9tpEc4oWd1I7fey94E9Jwz6WX4pgDD8aBO++MqWMGdGMeajQaTUDwFu8PlByqVsNCEbK61hitIFylxZo5YLXjEipaOPwYrX5WIw1YddjqTijgwhHwNe0MrCyl7KjXVZ31YiHYxSUAGwfssMhRLsKJc74BCJQLpl3wU5eHs8FE95ZIZYCiadyDorJSzB2ankCbZODw4my8/eGH+Hz2XPz828+Yt7oZk0bk45NfFyEpjtfyFL30U12yYrnq1EYBFY1QmvH64/j8tzLceMn5avv88y9Rn51DKzw+L+YtW4GK8hVY3eBWna7iJSEhEcsW/YZ+w7bBz79+jwRnMoYV5WNZVR3+cdhROOXA3aWKiT98phmNqQuWLhUB7JNtd1Ri94ZLzlOf555zoWREGe6561G1HSv07f1pzhy1PuuP30TE+7XY1Wg03c7ad0x5k8yVCFDo0QrHXvmh4iUALXe04PFYWoWDj6H1j24OnLUr2He3p/nxEr6St7JKU6R1BlbhcISKnogtKJ+SnMaMbxy6LJw45wOObi+8x1DLP0cACZQlWo2trLoB7GlA1pbmhqZHIL/rVm8TRozZFoPysnDuSRfgkG2HYtcdJ6Gw/6bYfhP5jRuHxQQ7Q7VULcPRN9+F1BSLtwVh2G6fozFpTAEu+NcNavuGG65Wn52DDalSHex55tmYv+AXLKj2rTdyRPRIirT5UYcs7LVZf5x37PG4/ORTUJCTjTOPOg5Onwd1q5ZgaXWTaufFA0dr+Obrj3HHtA+Qwt9plFx49Y3q86abr5PfdwFOPf04tR0rrpRUXH7z1Whu9eKkfz+A5BjyUaPRaDYUhoJpXgb8fpVajQoKGVrluFC4hAoaQqsvO6fx2GBBw0qcU9RuIsKXnZrYMSvScGDdSbLFw6IzxuSlP7CVK0dw7+943mtuKOieQt9sCt1B+cZkIuHgfTGNQjupcLQPWnJLRejybYGVr24o3nrg57PMDU1PwON1Y9ddD8EWhakYNniIGve1X2ERHImJGFZSjHHb7YUpQ2J3ReDUwVlFQ3HIxM3hb68hFIQjKVmuLcI0zXgzk55u8YYmTrxSps8/+kDc9NQr0m6Pf9SUVr8PGYPH4azdt0Z+4QCkSFsxJSUdA4sKpY2dKFWp3LMzHTeddgIam1vMs2KD4wRPnrIPNsl0xtTZL9VMMyPtbBKv+IRqq/zO99l5e1xxxwPYa/Kk9hu0Go1G0wUYgvfv+6RSisM3lRUZhQstdRQy4QQcBRstvmpIsyBoIaHQ5VBmcfZ23qAw3k6LeHGcWD7I+TThAyWah0rgOJ7Dc9Wwb2Hg8G7BBm9/NIF3EfRr5ugbVkKXMJ+DR51g9OnOwM5oHIeZ9x3PLZW+C9T9bm5oegIUvRS0HF6K8FU2RwrgNgVwvK+xvT4vcjKzVHg9BZ/8bvsPGo1Dt98advjlZxz/75KinuZbdvBiOK1tkoZm2pFwQ47FCscqzs/J6ZY05MQhg4dtiaL8LIwfmC/lQAtejUbT/Yi0kgpx6fPmZggUohQ4AwsMH1xa+MKJHdbTFDLldcZwVVwPNS3QvaE3VXwUmokWPoj8jhZMWq/ZCYsL75kLLd7By5r95qJe88u5Vtbd0NENetBDPyoC1hyKXo5osbLSGGYstMFD2Oih+wZnd2MZ43B2HNXDisVPmSua7oC+tZzmNlE+Y17kvISgd/SBfesdJws7dR2/977qM9z3jENgPNrOhELTHnItY7GhzZGCQ7bbGiccfw5GZEu9aAtznMQreFxZrjOenbEErhFMpLyg3+zxe+9tmYah+REPDCdcHHjNQ6buhstOPgmHTt1Vqv4wceB5smg0Gk1XYWtrXNSG94eZmyGU5IrADfP6juKVlt3aJhG7IcI2AOtSnpsuS6Bio/WyJ3bGCgfHCKZoW1lh7giCDwp2wOsIFL2VYcabTaWLR4gPL0VwT7SCh0OVC1r7wwjcACwDFLbpcp/hfCE5XB39xEOhn/nkL8wNTVdzxfHH47RDD5WiK2U3RlxOJ+5//XXc8dRT+JTTAs+ciVHDh6+xDoeSlJRk2TEsScJ69f33cfIttyA3y2IIwTi48ZprcMyWW8JtESclPKUuo8tAOFwOBx557z3c8sgjavvy007DhZJezXGkVyi87t8LF6JpxAjstdtuuFXu/fCRI+GJ0CCOlIbMjwemTcPtTz5p7omNP+bOxUzJy50mTFCCNhyRrs9xhj//9lvscMopGC3lQKPpiSxdshglTw9Hgl/KONublDvBn8FYfRfr/hZg+pQZmLzTjuaOtUyf8SUmT98JoLdR8HlWYROr76LZL+utNjuW7z8fAwcONnf2Xmxt5TPaJHXNzSA4skJ+hiG0rMZUpQ9mVV37oyywsxp9P2kd7qBVoUtghtMaSys1x50NhSKNVsmOYCl4pYEQ6tPMNIvkRtAToBWbI3mEG74uAMsSZ46jVTdSMWDaLDWmUF2H1EHAngvMDU1XM+3NN9HU3LyOFTNa+Oo+LycHe+6+O1paWvDSq6/CKaIrHEkuF15+6SUcceSRcIeZypg/T8bg6COOUNudxV8i4r7/8cewkz5Q7FZWVWHevHmYNGkSvGFELN0HBpaUYPKOxoNq2htvoKKyUp3bGTDdTjjuOKSlpWHJ0qX4fMaMNZNVhMIpkl977TUcdthhYdMwOD/i5elnn4UjQh6+8vLLOFzyKNz1CS3Bxx19dFzlSaPpCrTg7WuCt/rHNnw6wdwMQrkzmNYTVkgUvVYDh9MSSuFLi68VAZFIQdfToWijhZI+iPRPDoXpUJRtbsQJ/ZorJM1CYTpziK9QXEz/HmrlbW9YMaYXhW6mCPlIzzb+0Gg9axIxwTBDyRgD7Dbb3ND0ZR544AGcdVbP6qhYW1uL77//Hrt3QCR2JY888ghOP/10c6vrefDBB3HmmWeaWxpN70ML3r4leBOQYnET9MXka31CtwVa3dyyhPrmEs6wRV9fTjBhJWh53soqQxz3ZHiva/xQzfsPha4ZHcUqDDURRZg0DuRFT4MuDFZil40cjmfMoehotY4kdpnW9G9m+eAMf+FI7f0/OE10eDrBDaCzaW5u7tD4u11Nd6dhT8xDjUaz8ZIAZ66Ikm3MzRCq6wwrZwC6MFCUWAlfWic5Lis7uIUbwzYgnHsyFF6Be7MaJ7cz/JApBq38csP5vzLtIvnFdheNYQQAhS0Frhpz2WJCCsJkZuc9ukPw3niPgc594Sjc01zRaDQajUajiR7DcD3oBPWxHhyNgK+WQ0cUCAhfitdwnSYodil6uXAoq4CPFl9tW41t2xOg0A1Yd3lbVsKrszre0TIeDr7SD0ewGO8psFNiAN5PLi26xcYYy1YuMGxEqclJJH0DFn/eVn2zNLIsrMWJcp0BR5obGo1Go9FoNNFjKJIhpwJZ49TqelBgcbgxipFQsUXhwjFWOfwWxRgtdMFQ+NJvd1g/uUahYfHrsaMNSNwpwgK3wLF2Q25HQWslO/R1BlYzudHiaSVslYXc4rvugH65m4jA5UQizF/66trDCF2WjYDbAu8h0LAg7BVPf2arqZbJptdLunewo6BGo9FoNJqNEkOZ2ESEbvOkbEWwvlKM0NqrfExDCLxup/Dl96HD+tDAqyZUsHi13ROgj2zAWs37sRJfaqQJc72j0Npt1YObojccjBtdLXqQ5lUCN2xDRiK5xporZYNlJFjIc5UNqVVh3iIEk7s9MPwCc0OzMcDpcXsaqakho6f0cOxhRpvoSjprdAqNRqPpDGxtgel9yPJXgB9OEFESQXwQilcOLZUsDyUr4UVRyAqXYqinV3wUZMEWR3aa4liy4eAQa3Rp4H3xlX2b3CjvtT0xT6HHtLLJH4pAuoXwmrSQ0/IZjiK6BVhYxHk9zpPf0xoRLE50hWGjJzhNg2FaUNA3hHlrEEr6SGDHj6Ss9Td3aDYGjjzySEycOFENXRUPLpdLDePVWVC8LVu2DFOnTsW+++5r7u3ZnHHGGTj//PNVZ7toiTR2biwwnGeeeQa33nqruUej6X3oURrkEd2nhiULFrxk1bvAd0eIYIniYUGrHoWvi1bPdYNZB/rwKoEox1v5dXYHyloqgjNYdNHKy6HIItyOJRSfvNfAPVLYKqEbT2CCSwQtfWIjQfHdGaNGdATeI9ONApf3bAW/o9ClVTcasrYEJr0vT898c4dmYyK0aooWjut68cUX44477og7jHD0tvFiP//8c8tJMsKRkpKC2267DZdccgmamiK4F0UBJxQZOnQoRowYYe7RaHofWvDKo71PC15S9R0w81gRJ4vNHe1ACy47L/F1f3vW3IAg5MJju8NCyVtmZ6lwllVOBtGTRpLgsF7tdfRjOrJB0VXClwKX4lUJelkiiQp+RVcGjubQHIPFreRQYPyjcl9hxiTWaNrhqquuwo033mhuaaLlmmuuwXXXXWduaTQbN1rw9i3BG3y7a8mZCOz6EzDoeHNHO1D0cNKJ0mpDMCo/XwshS3FESyCPYQemgN8vxScFFBN7Q0GhFrhuOLFLv92eJHZJlaRne2MXM/0DopL+vTw+2GodL8wruiYwvDX5Jb9GftLnlt9Zid2ANXe1lAlOFRyt2HVmA1s/Dkx4SYtdTdx0pmV3YyJeFxKNRqPp6YQXvIRiY+sngJ2nA5mbmzujgIKRPe5LKw0/WAqjSEbcgACmYOO5ahxWU7jxXApTfk9RFysUfazAGQ7DpFBjWOGehbG8au9qKjllr8S7XeTGKEJ5v7xX3hN9hClW2SmP+5nODIvpwLThOvfxO6a5ygM5NzQfVB5I+OHSLgC/5/ls9NAthBNS0Fc5Gmj5H/IPYI851sPkaTQajUaj0cSBteANkLsDsOssYNtngfTh5s4ooPhh5y8OaUbLL62nFFaRBFOAYMsiz1HWRRFSFHBclCCThfsosCjuAsOjUagFjuN+Cj2GY2XxYTw59mukqXG7CqtRMtgJrNxihIz2CPjWBsQtGxAUsMEimPv4HdNcWdnlehbJtR4BkVvVIPlcFZ9LSP+DpIz9DGz1KODMM3dqNBqNRtN9tLW1wkZ3Bj4Po30m9hXMe7a1+tAWj8GxB9K+4FXYgAFHAbv9DmzzdGwWX0IRRetpeS2wSkQRxSUth9wfD0qQycJMYBgUXVyPRagRij4KSYpkK+hLeoAcM3WmiP7ngDFXS1ocDuRsa4wgkNwPcKRLEkVISn7HY3gsz+HMdgyDYTFMhs1r7PEXkGQh+HhftJxTWHZ34aOAZpopS74pctm4iIUEu6TBESJ0fwQmvgpkbGZ+odFoNBpN95OWloGqoslotLnkuWfu3Big3hCZ0diWjOq8KUhLzzT293LCd1qLhtWfAAseEMHzniSOCKB4YWcrjkbAcTc5U5fqzLYhO7LJ7VJsU4DTehyJzLHAlK8lTinmjnbw8xW+CFKvCEBCkWtPk/NjGL+zYjrw5R5S2CLEjcmTmmx0ErSara2zYGOC1l9aiNlAYNrFWWQUSbmGy8LQsyX+A82dGk3nwpEGOOKAJjauuOIK3HzzzeaWRqMpW10O56KP4PjlYaQt+Qrgi1g+g0NtXHwscl/gM0Cs+1u6udMa7WkUu8U7wT3qDPhK9kBBXt+Y9Cn4dmOjYFdgu2nAXguBMdeKeBlgfhEj9PGk60O1iERObEGL4epawwpMFwWKUuU/ah4fM1Iy1VTIHrmGiNGVZse69sQuLbE7vBW92CUUtkmF0iwcZixcj0XskrzJwNZPSc5EGHifBZNj2DK9ymShuwjdE+K1mBO6TdC1gYKW7iC0JNMPl/nBTmfMD6ZhvGI3f2dgwvPA3suAsSJEtNjVbEC22WYbc00TC7EMY6bRbAwUFuQje8IxwCEvoGK7S1FnS5MfinwR56OwRyP3VYdMVG95FWx7PI+cLY/qM2KXxG/hXQ8JpuxDYKmImpUiFH0bwCd2neHMuMg+fqrxMeX6yrVBVgPuDcp3NQ7rM90VdnhbWnK55o5uoOwj4LvDYk9HpgWtvoE0YtIkyh+uM00CadQmfyhy+dme+I+XtKGG28LA42Q9Bv9vjUaj0Wh6GHzZWbfgWzTMuBv9F71iTDAasE3J4zSi5TTa/d1h4RWh65PtlSVHIm3ChUgftA0cwef0ETpR8AbhbwZK3wGWvQis+kAEVoydmLqTIacAW9wpIjEGy+6GovZX4IcT5fM3c0cvILkYKBGhTp/vbG1l02g0Gk3fgYKJS015GbJf3B62VQuBJNlB41KokCSB7Wj3d6XgJW5Z7bcpqvf6BNl5RWoXb6UvEpwMnUei5AZFz3avA/utMkZ4KDnE8GftqaQOAXb6CNjq4Z4hdknmFsDUH4Ax/5KciuDi0N0w7YafD+z8BbD3UmBzaTBosavRaDSaPkZA1+bkF8J77gL8OfL/RDMArXyZHMcL5W6hVf7zTa/cyF/DLoL3uN+RI2KX99ZXxS7ZMBZeK2jpZaeschFGFV8B1T9KAYmxd39nw6HWRlxsTLJh68GisnkZME+E5OLHAV/Hpv3sMK4CIHcSkCdL4W6ShqPNLzQajUaj2bhYtboCti+uRsHy/8FWX24o4oA5MdiiGthHrPZvKAsvP7kIrakFKC8+DAk73ID83Gxj50ZA1wreUCiAq2eK+P3amM645legaan55QbEkQkU72NYoYv3lR29qE3jqTT8pFe8DlR+IwWYzbQNCC3LGWOArC1NkbsDkKbnx9d0DcsWL8SAwZuYWxqNRtMzYU8Y//Kf4P/iBiQteINGX2X5VQQEZzhRGrp/QwheShyz24572CHAhGtgLx6rBpzYmOhewRsOb43hu1ozG6j7DWhcLCJ4iSzLDIEcD64iIHtrWcaLaJtojBjQk6250eIuA8o+kUbDD7LMkjT7xfCfjgdnFpAyyFjY2YzuFBxvmWLXZjcP0mi6jhP22xEZI7bHuy8/h8//XIFBEWaaPufsc3Hf/feaW/Fz/qETsd25z+OIneQ30Et584k7cOOj0+T55sDX330Bl7k/HC8+8zSOOr5jMxvO/+FDHHryP7HlLofh6buvNvdqNBsntbW1SPzlEdT98T76rf7CEJuUG1aiNHR/ZwteKnFZL82ZgoxR+8K/2anISE83vtvI6HmCNxItpYYF2FsrmVgviznuLRebNKUSkyRjZaEPLkVucn9ZSnq273Bn414NNC83Fk+VCGD59bS6DdcRTvbAtLBzfOBUYz0p3xC5nEpao+lBDB06EgsWzJXf/UrMXCTtr5qfcNy51+OYC27AMeMzcOEtj+K33+fii8/fQVZGLr4SUfz9czfi5c9+w6sffomfpz2I1z76GCsaMvDZm0/juvNOxOtf/oF3vvwBv776H1z/4Cu4+s7Hsd8kY9KTd56+Fzff/wSufPgd7DNe6o1eymabboHf//gV9fNm4NGZzbjo2F1xygkn4ISLb8KOQ1Px5vuf4u1PvsN/H7oLxVlpmPb93xiRVIZzrroLDz39FCrn/4oZ06djQV0yrv/nqZj74ye48YHX8MSTD8NfsQgnn3c1brzrEQwpMIZcPOrA/fDiG2/jkuN3xYl3fIIxBWq3RrPRwlHL6stL4Zg/DenfXQnU1Bjj9wYsviQgSkPFamcJXgpteozm5KNxmxvhGbg/0vOKsDGbr3qX4NVoNBsN37/3LHY79FSM3mE/fP/xq9hyzBj8+Oef2HGzMbjh35di+ooCnDOuBU/MzcTvHzyJ5+64ELtf/DzeuOt07HHC9ZhcXIVTH3kXz513EHY47Dz89+0f8dx/zsaLz7+K2x94Dj99/TYGDt0CSxfOMa8IfPTEjagdcSQOmzTM3NP7uOfKU3Dj05/jn1fdiEvOOAoH77Y9nvnoGxy640Tcc/dNOPJfL+PZcyfi+Xmp+GXaA3jzk48wefK++Par97HLPsfhoHEp8Iw9EavevhFH3/AUTj7yOHz00h0qzDnT38KHn3+BzcdNwOyfvzevaDBm2HD8+fd8c0uj0Sg3h1V/oHb6HSiY/5SxMyB6AyI1VKx2huBl5zlRtuVDTkXm9hcgoWD0Ri10AwQnkUaj0fQMGkuxOm0c6ppa8N+L9sT5t96HtjYbHr7vPpx0waXwN7kxcshgOJI4ooofCYl2rFyyHM31q/HIs2/j2isvQmtiMoqkls/Iy8eSv/7CuInbyrEuHLXvdqhpbMB99z6Iy668Rl0uQGOzPG16NV6cdOWjKF8+H9557+Gq+17DsiVLcNrRR8GZk4/SlVU47NgjsNmk7dDY4EbJgMFwJlbj73l/48jDj0GRbPsTUnDMITthx0k7Ydbnb2D3I/4PeUPG4e5rL8S3P83BUUcegYL8fub1DKZsOQxPfDjT3NJoNIQi01m0KfKOeBI1+76OZpsToNfhhhjNgaKX4Ur4zQnJqNn9beQe+CgcWuyuQQtejUbT80gtxtkiuj75ehZeev19bDdpHwzJd2CrHafgxedeQkqSH/Uihj3uZjS7vahYPg/JQzZF5dKl2Gn8MLz67udwNzepV4u11RUYt9t+eObWq/H6E7fjnLvfRYGtAVN2mYyXXn3NuJ4Jz3F7esvYQuFwYFj/QvlMwLlnn4rqsjLkFfbDCy++iMP22BnZmU7U1TcCLS1we71oqKuTY4tROGQkXnrlFWy76RC0NNejSR6aDXW1KBy2Bb797G05xo//u/BSjBoxAi++9DImTxrHiynOOWQSpp79HxS0lsG7gfvQajS9DXoWUGhlbXkQ6s5zY9Euj8OXJr9Rmn874/dCoWuKXV96PyyZ/BTqz2hC1qb7quv2oi75Gxzt0qDRaHooftx3538wYsIe2GOHLdSeu267FYecch4KHU1oaEtFlsODCrcdLSt+wzJ/IbbI9+Dhlz7BPy84EysW/41+g4dh9fJFyCsZguolv+GFD3/BuacdB3hqcOudj+Kciy9BapD5o6Z8JdqSc5GdxpHkeyerF/yMw084E9kDN8e0Fx6Bp2oxpux5MHY7/p+4+sgJ+HyJF7uOycS7P6xGatnXmFk/CPsOb8FRZ9+Iu5/+H5IrZ2PUjgdg2Q8fImuLPTD3nQdxyd0v4ZMvZ6BhznQceNy5uPCmB3DsvpPU9W69/FwsqmpBVXUdHnnqJeT0kGHMNZqeygppc/Z/d294lnwJp7vBcHOgMuUSi0sDEdHscabBOXQ3rNrtdRTxO01YtODVaDQajUaj6UJo4K2d/SaSf7wdjvIf4PB5DOHrjkLwcugVEbpehwve/G3QMu4SZG1mWHQ11uj00Wg0Go1Go+lC+GIpe/MD4DnwDfj3fAL1JbvCTx+s9kyQHE9X1HJDye7w7/4EvPu+gWwtdqNCp5FGo9FoNBpNF0MBlpWTB9cWx8B+4POoHn+ZErx+b/jOs35Ps7IC125zNez7PQfXZkchMztH++lGiXZp0Gg0Go1Go+lmvF4fsOpHrGwtxqBBg8y9a1myZDH6OyrQVjAODnvwoL6a9gH+H+au0yTM4+fsAAAAAElFTkSuQmCC"}}},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{},"cell_type":"markdown","source":"## Introduction\nThis kernel demonstrates how to use Hugging Face's [transformers](https://github.com/huggingface/transformers) package, more precisely, theier `Tensorflow 2` models, for this competition.\n\n## Motivations\n\n* Give my first contribution on Kaggle.\n* To learn and work on some things new.\n* Provide an easy to read/use training kernel for this competition.\n* (The most important) I want to have your feedbacks, and solutions to some of my questions, so we can improve this kernel!\n\n## Features\nI tried my efforts to make the following things work:\n\n* batch accumulation.\n* Adam optimizer with weight decay and make it work with learning rate decay. (I take [AdamW](https://github.com/tensorflow/addons/blob/master/tensorflow_addons/optimizers/weight_decay_optimizers.py#L340) class from `Tensorflow Addons` with some modification. See `adamw_optimizer.py`)\n* Custom learning rate schedule as close as to the original google Bert code.\n* Easy to choose different model configurations.\n\n## Important Notes\n\n* I didn't use the pre-trained model from starter kernel.\n* I didn't check (yet) the LB scores for the checkpoints I provided.\n* No code for inference/submission in this kernel. (I will work on a separate kernel.)\n* The flag `model_name` is default to `distilbert-base-uncased-distilled-squad`. The `batch_size` and `batch_accumulation_size` is however defaut to `10`. It's your own responsibility to make sure there is no OOM when you change these parameters.\n* In the training loop, if the training is on `Kaggle`, then it stops after `8 hours 45 minutes` so we can still have a saved checkpoint.\n* This kerenl is not optimalized for training. (See next section.)\n* The checkpoints provided in [nq-competition](https://www.kaggle.com/yihdarshieh/nq-competition) (as the value of `NQ_DIR` below) are trained by myself without any effort on choosing the hyperparameters.\n* After you have your new trained checkpoints (or if you create your own .tfrecord file), when you want to use them for resuming training, you have to set your own value for `MY_OWN_NQ_DIR`, which should contain your own checkpoints/.tfrecord file.\n* This kernel is not written for using TPU.\n\n## Checkpoints, Batch size and Training time\n\nIn the following table, the model configurations are:\n\n* `distilled bert`: distilbert-base-uncased-distilled-squad\n* `bert base`: bert-base-uncased\n* `bert large`: bert-large-uncased-whole-word-masking-finetuned-squad\n\nHere is the summary of the checkpoints I provided (no `bert large`, it will eat up my GUP quota. Probably I will provide one after training on GCP).\n\n| model          | batch size | accumulation size | effective batch size | num. updates | training time | epochs | time / epoch |\n|----------------|------------|-------------------|----------------------|--------------|---------------|--------|--------------|\n| distilled bert | 25         | 4                 | 100                  | 9893         | 30823 s       | 2      | 4.28 h       | \n| bert base      | 10         | 10                | 100                  | 4500         | 32226 s       | 0.909  | 9.84 h       |\n| bert large     | 2          | 50                | 100                  | n/a          | n/a           | n/a    | 37.78 h      | \n|                |            |                   |                      |              |               |        |              |\n\n#### Remark:\nFor `distilled bert`, I didn't test with `train_batch_size=10` and `batch_accumulation_size=10` for `epochs=2`. It's possible that this will take more time than the time indicated above.\n\n## A screenshot of training progress\n\nIn training loop, I print the losses and accuracies every `100` effective batch, here `1 effective batch = batch_size * accumulation_size`. It includes those for `start_pos (loss_S, acc_S)`, `end_pos (loss_E, acc_E)` and `answer_type (loss_A, acc_A)`. The following screenshot is however taken from a training where the result is printed every effective batch.\n\n![training.png](attachment:training.png)\n\n#### Remark:\n\nIn the trainin progress, `batch n` means `effective batch n`.\n\n## TODO - Some contribution wanted!\n\n* This kernel is only for training. I will add the prediction/submission parts as soon as possible (In a separate kernel.)\n* The batch accumulation code seems work fine, but not memory/speed efficient. In particular, I can't get rid of the following warning.\n> UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n* Maybe TPU support, not very sure. However, see the 3rd link in the `Reference` section.\n\n## Things tried but not working - Solution wanted!\n\nI tried to use mixed precision for training, but I couldn't make it work.\n\n* Using [tf.keras.mixed_precision.experimental.Policy](https://www.tensorflow.org/api_docs/python/tf/keras/mixed_precision/experimental/Policy) for mixed precision training causes error when loading pretrained models. (` input #1(zero-based) was expected to be a half tensor but is a float tensor [Op:AddV2] name: tf_bert_model_1/bert/embeddings/add/`.)\n* Using [tf.config.optimizer.set_experimental_options](https://www.tensorflow.org/api_docs/python/tf/config/optimizer/set_experimental_options) doesn't make tranining faster. Probably this is designed only for compiled model??\n\n## Disclamation\n* I am not a part of Hugging Face. I choose to use `transformers` package because I found it's easier to use and to extend, so I can focus on other parts of this notebook.\n* I take no responsibility for any (potential) error in this kernel and in the dataset `nq-competition`. (I would appreciate any feedback.)\n* I take no credit of any file (with/without my own modifications) containing in `nq-compeittion`.\n\n## References\n\n* [Transformer model for language understanding (colab tutorial)](https://colab.research.google.com/github/tensorflow/docs/blob/r2.0rc/site/en/r2/tutorials/text/transformer.ipynb#scrollTo=KiVIjF8EJzAx) - Official Tensorflow tutorial on transformer model\n* [淺談神經機器翻譯 & 用 Transformer 與 TensorFlow 2 英翻中](https://leemeng.tw/neural-machine-translation-with-transformer-and-tensorflow2.html) - An annotated chinese tutorial based on the above one\n* [Sequence classification with Transformers & Strategy](https://colab.research.google.com/drive/1yWaLpCWImXZE2fPV0ZYDdWWI8f52__9A#scrollTo=mnhwpzb73KIL) - Hugging Face's tutorial on how to use their TF 2.0 models on TPU\n* [Hugging Face: State-of-the-Art Natural Language Processing in ten lines of TensorFlow 2.0 ](https://blog.tensorflow.org/2019/11/hugging-face-state-of-art-natural.html) - Hugging Face's post on Tensorflow blog","attachments":{"training.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAA50AAAEBCAYAAAD7F8FbAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAAFiUAABYlAUlSJPAAAFH4SURBVHhe7Z0xnty6rqdnL3ODGS/jJe8txEGvxYG9jpP0AiZ24j046iWcyFmPRJEsCgRIUKKqq9pf8P3uaRVFgsCfIODybf+v//N///c7AAAAAAAAwBXQdAIAAAAAAMBl0HQCAAAAAADAZdB0AgAAAAAAwGXQdAIAAAAAAMBl0HQCAAAAAADAZdB0AgAAAAAAwGXQdAIAAAAAAMBl+JvOL9/ef/359/3Pjp/v37/854IxX99f3+SY3++vL4NjFl5ef4sx/76/vX4dGzPLnnvavPLjZzXmz69v42OIvRhD7In95DELxF6OIfajY4g9sd/xobF/Qj8+YFyJfWfMArHfj/nQ2Hfgm04AAAAAAAC4DJpOAAAAAAAAuAyaTgAAAAAAALgMmk4AAAAAAAC4DJpOAAAAAAAAuAyaTgAAAAAAALgMmk4AAAAAAAC4DJpOAAAAAAAAuAyaTgAAAAAAALgMmk4AAAAAAAC4DJrO2Xz59v7rz7/vv378R/8c7s73X/++//n1Tf0MAAAAAACuhaZzNjSdDwdNJwAAAADAx/G0TWdoJN7+eX/58mDN3WdrOr98fX99e+6m7fKm88fP9z9LzDPP6quo3bfXr/rnE3h5/f3cPgIAAACAYdxNZy4WJR9UPE5vOmPB/efP7/fXlxNznm06ZQNz0qbTfrqo6cx60myrfPDz/fuJOJ9qOtV4LGh2P3uDfqDpNPNCRM71ME1nPu8FR216+ef9rTeP0JH0S9Bo8fmOcj5pt6ZDaU/r/CTNLuOu/MMGAAAA+LsZbDrPFf8zmdl0bgXfz/fXsMdHaDr3NiT7jvj+tJ+uaKTKwlnaNnn/K+H9o/Yr9pj8hU3njtDsnDw/H0XS5GjsYoOXz7syT5U74ztdP8t4VGtFvZVnKL7THFNSNMM0nQAAAHAVf33TGfYVC8Rtj4/XdFbF5gCP2HQmm76v/ha2qQ3iSOOnQNPpRDY5owSdPmnTuTCc46x4l5ox8oFnLTlG1bEnN1gazvH+FvZB0wkAAABXMa/pjAXYWrhsY5cCKVC/s/9cKaQiocjK4/ZFUy7AQkGVxgwUjAqbXSeL5suaTvEsrnPbe7Gm8llJVVzGwjWPKeNhxfVoI1sUyWE+rekUz7Z1j8f2YZrOnVZXxJ7C5+uzffyqeGnxVdbcnx9dk3LMyuHmI8RW95U889Uaae/ZR6UftPNwm+uwFgWaHptE+6Rfk0/DHqOtle97uopzZz9Z2pLjNKJPpQ23c3U74+XnAAAAALOY3nTuisD0rCjktoKsmCcVlmWxl+cqxy3Pfsl5iveUtUbZ9uhsMCyMQtSNLEa1fYVn+1hYtgc/tXwSC9Jdwbk8y/bnWCykgjfFbLSRS3PF99Qi/yU1FHF/mn2DBB+M2pqQ8Wgh9rdj3Vf5XItr3OtK9n98VsWjtR9tTPRrNc9OV44GpkVYo+Mra42092DPOub3+9vbOpfYixKPrsY9SP94kPuNPk3fHAabrXnlu4LqPFtxz2vaMVNzQ2mXYw4AAACAM5z+RUK5UNGK2JWySLQKMFFIWg1UiVZobu81GuMOnnW7TGk69z527cnwbbMg9zSPRrF7qNDX4qzNkdYMez8Zj4XzTWeypUCbz/CVRaXXuNYuhnJOT2MY5qk1s/ODiEXAM3eLoMFOvKw1yr2Xuiz3H/+7etezrkZaJ+GMW6b0YZxrs7+2eafz9MyyuXy/eK7lp+1ZI2bBN/Vcu/Nr+RUAAABgEvO/6ZSFWyx6QjFmFMO7gm352dPQaGO6NnbQirphxF6GUZqBVFju5oxFukSu2/RlGRv5WcKIqydGO6JfysI27EvOEW1K66W9V7oaINh69H0lHibWGYgEO9a97JBNZ72W9HX2yYqy1u5zSZxH9b0SoyFC7Dq+stYo916OKX0an6v7Wjh85iJbfAbyR9rvD7Fv2cSlPWRblXdKGpqTsW3+/zGzv8Se5PzSXgAAAIDJ3KnpjAVOKHaUOWJxlIpGT0Ojjena2GF73ygEvYi9DKMVnNK3YYwoEoOfH7fp1GITnpVzWBqK+z3q02CrnNOLFg8Ly/6FYIOYp/KJsZZpf/RLoPCj5xxUvl8pm71irJugp46vrDXKvZdjSp+eta/H6PxxfOVrzzxhv0qMGhpSiWtVZyPbJuKhzR+fXeZXAAAA+Ou5vuksi6tQlCoFkii2PUWz1vR43muxvd8pmntYRaAX4YuAKArVfRq+bTaHnuLYiGtzXkmaIxTBBmEuwx5jb17Mps2DFg8L6wwYz6s4OmKvIt9z2NzS0OHmI7zf8ZUV49LmckzpO8u/sxg9u5Y9Pf+3Yhre9dsQ4lj94UG0S7Mhxrg6fwWH4w8AAABgcG3TKYvYXAwV88RCb/deelYWU+u78hcJiWKra2OH7f1O0dxjtHCVKAVrZZccUxSSct3enrbPxXvL/PlnLa4LQ02nQVhbzBHmFfZuz47H9cObzgXpr+T33b6UtTx73+Yqz5RyziRRM/lsFho63HSEOTq+KhvK8nm593KM9GkYd8JGi7SOoukcK03v0Z7beVHyWYmW2/Jntg0a2llpPTeJ6073KQAAAEDk9C8SysVVLnT3aM3XVhTdUIudaj6lGBfFWVV8e3i5FdsVRxqVKU2ntKXe096Hq2/sdbv+lmuW+05xEL7Q/D9KiJcyh7R3OKaCMN+RWK6o8Vgo7DbPRqnZ1Gwk1vfD3MXetLWkf140vSr+8ZzH3XrrHCebj2Cb3uzUMU1E24Mt8d1W07ni8VMPZQ5r39l2aw0xl5xH6sPMDXEe83MZe03Tmm8Slv1n4w4AAADQwd10djGak7+Os00nTOdU03lPQsOgN20AAAAAAM8KTedsaDofDppOAAAAAICPg6ZzNjSdDwdNJwAAAADAxzGv6QQAAAAAAAAQ0HQCAAAAAADAZdB0gp/GbycFuCv8NXZ4FNAiAABAF5pO8EPTCY8ChT48CmgRAACgC00n+KHphEeBQh8eBbQIAADQxd90PnvDEQuD9R9JHy4OlH9wXZ0j+KgYJ39jqvEPt6v/KHu2t/a5/Q/tL2i/pTX9ZuHl81P/APxZDTyrhmTcrH9kv8VI7HsUWk6YmjZi//L6e/f+jmJ/1ji5nhyn2iN8cEqLEwr9p/mtxgkl7huDZ6rQxJ6f799HdC3z3Q7dpqwT7QyN6COP3dvsyY3D+bPHX6jFZv5YOHS2U0yP5NcV7XxYPs1jjbNT5evG2TC0WM+xofqmOpODZxEA4An4a5rO8pI8UxwE4mWymycWYPlZutTKSy+8179MtqLo5/trsNnp87ieeqEVl9+pQv+sBp5QQ5tuhM3rPkYLRGfsu2hx1vQoPnPFXvlnj8L+O0Vg0Gs5RrMnPCv8GLRwQo/RD2fO8rMV+s0zPkKM8+l5DLYzo2g95cQVoanqnZY+dgW640x5/HbGt3+jFktCrE7k9aSLX/9scT3adEryvHu/du9XmauKd2pNN7QY5nHo82wuBAB4Ev6OpjMm9V8/zhcHAVmcK8V6QF5ejksoFF9xHrXhMegVem+v34KNpy62sxp4Ng1ZcT2CtwDpoc1jFcyjsZd6XQi6ahWBRkx3jahhn6lZD3FOms4DRF2fnkejYWPSxHepKSOWlj7yc+eZ8ugMLZ7gTF6PWgy+S/l2VtO5IOMafu7cr2oslNy4kufXtOjR56wzDQDwBExvOrckvCTthHKRVmO0eUPCLsccv9RvBfD54iAgizZj3rDu8jyPcxZJCetSrGhcXPlSnFFonikuVrzvy9hrRYhHH9Evu3Ejhd3MImgw9iZhHuHD4Nd6/0OxT3sV/glztPav2bNwW3t5rsU9rafY7WLCWXYV+rO06BnTo3HOh5iRCwx2cS8/KzRaaar4bPeOpq3SB+Hzzpny+OysX++lxbjOTUNKLi3OVUZq1jNmhBA/xZZRkl1nbBG08tem1druW71we67quqdFjz49YwAAPglTm86tySoSaLokywSuFRKSsNa5SzxTrjehOFipLiDpm1zUbd8w5WIi2LL66Ear0LEuRYk5rvTjjELToYEmjverveQCSfq7E8f03kiTqVHE7JRuBmNvkv2R4qqcsZXR2Ef75B63eOzt3o0pz1fx3vY8xqz87/WzfA6Loq1818OEs9wr9Kdp0TPGQ9zzqTO8UmjoRrGnoyQtSp+Ksxj8WurV8k94vtfWriGQulIwc2OBZ0yTO2gx+aiMfTqb5bpawyTxjBlCidMhkk5m2RZ9ZsXFjHt8b5+/6nPX1WJ8r0TOkc5C+Pa/GHc6VwAAPCDzmk4rwYfEe3tvS/SdAke8cxhR7JwqDsQFspujtLdcQ64vSZeb8bmrGGqssbsU47hTBWtPAz1671tFtdSWRx/WXEeIc+X4t4pDL53Y90gF54q2x9HYuwvReA7yXMk35T6yv7ZzvjvzpQbOxCi+e+gsR8KeLf9bth3RomeMh+xXSSefOgi+OGujtU/xPBXaWWsph2nPyvlU3zf23siNGc+YHldrcUE9n9L2e+1XEuIyQ9/RNk8espBnpLHP5v2abAnzKGNGtbgS3ylt2s6dyDNhrnN6AgB4ROY1nVbSlRfy7lLwJPzjyXdX7K7PJhQHgbSHdHkk3/wQPor7aBXVlY3VZw2fr4iCznzusKVLTwM9eu9bn0vbnfrY/BfHTSqy8pxnCqNIK/Ymee/xveCz4ud1zGjs4xzec1EVwLLQW20p80H67/JZ8d6h8zjhLDcL/ZladOq1S9zzqTNscXbutEfpT2XeoHurgYo+Cvot86k2v9STRJ4DDc+YHldrMe5di03ly7Cf6EMrR3nGjGCdlVFSjGfYFNmaOl0jW/5V7A77Wd6L8cg5P8XniBYjMudXuXRFmx8A4BNw/6YzP4+JNV5+1oW9XRobQwWRVkRNKA4yZbES5632r9kgMC++zmcB63LSnsdnQz6UnC0ueu9bnzdsd+ljdpEV5zuro258FWTREkj6W+N9IPZq4dOgVcglgp1pzhBXZY34/JAfJ5zlsA+rsAu2zdeiS68WjnxymFJD2uc9ynxYPNf0utNGMbaivFO0+cvPy/dWtHMg8YzxcLUWG5ozfRnP1qY1w0eeMR6sszJKisdALurSODObNoXdlibKnD+qxQK5Zoi7sl/rOQDAM3P3v15b4bz4h5NwvCRanCkSdpdM86Jq7H2htS/1UiyJe6z2EWMh91tyqHDtaaBH732rQIj7acXLpQ9HPFxYfh/kSGGhvlPqL/pKxrtk59843q0Hz3mVRbgR103fB4vdywv9a7V4JPamTTNw7MskacLS5TKvieWD+G7aa/CX9n5G6MhzRied48u1uKDqxXMWPXE9E/uV8P6EvGrp6AyN2OhNZ//cD2uxoIpj0KAY74krAMATMq/pzAVGkUBjAvcUqM1CalYSbhUH8VJxNSVx7M5mWcA49r5devZl32w6Ry/oOP5UwdrTQA/H+9uFXozx7NOpj+EmZ42hVeidLIyasQ9+En5IRJ01tSdpxH7zt98nVXwkyXYRi0rLcdxhPbbOspOwl4ZmLtOiU68Vnlx5hJSrjH1tfmjYO9i8BS00fdi2J6MV7Csjceqt4eEOWtTOffN+SIT3Joxp8XLyXkg4YpLypitunfks/2k5bnvWyJOWFguS7TudJK0XsXfFFQDgCRlsOtfEK9kn2lygRGSBJD/XxuSLpWS0QNNoFQfxUrcvoRLjQshzbHT3pV2Gpp8XSh/EtdyFTryATxWswbYTl6FTQ5WfROxd+lDXahcFKto8B7Toin2iocX95zeaOrBiH/fW0kSlfWXv+zG2PqQPzhTp0wr9wp5E6Y8ZWnTp1UMqUCsGz6Qyj+nHVLiv4zSbHY2CJPhDjD+kjXAOlDPtyY2j+bPFnbRY5yKxd1UfQhueMaMEu47PoZ6PiPRp9pOmNyUvWjlPjgsU+q7j0bk7FC1W+7LOSBWTA/cUAMAT4G86AU4WFwDTmFDoA0wBLQIAAHSh6QQ/NJ3wKFDow6OAFgEAALrQdIKfT9R0Wn+dbceRvwIJ9+ETFfpo8cn5TE1n66+fZviDRwAAGIemEwAAAAAAAC6DphMAAAAAAAAug6YTAAAAAAAALoOmE/zwi4TgUeCXt8CjgBYBAAC6uJrO//6f/1Kfw18GTSc8ChT68CigRQAAgC580wl+aDrhUaDQh0cBLQIAAHS55JvO/E8APMKv+f/y9f31rfx17z/fv38ZLw7kP2vw9vq1Hvfj525Mc608VoyJBcx+noW3f95f5Fx5bKcRLHyg2u3ljk3ny+tve9/3Rsb1qE3Bf8U8Z85HL/aV7kXslc+tfeVYRGRxLT93jzsT2zsW+g+jRSs3HDyT+5x28FwrOrJikv0odW/uayPM5xmT53PkfGW+w1q6oxZvOeQ+edjCOvOJ4XtmZm5cKeK714ahI+1sp7xvnfvWfe/V64g9AABPzvxvOkMSXS7E1zUhH2vwppGKj+IC2wqtMbvCBVteAvGy2V1m4dm+EDDX2hVF4vN4CfUKmDT3a7j8OwVIcTk+RdMZ/fPr9X5NrsVWXAkbVj+MFkXBd0qxcaC46sZe0X1a34x/ekcUO2Gtjvar86Gw+bEsyvT13DjPyWkeSItpz6fOcETGVdW5gzBPoTPzvAS7/xnTvJJTK+QYRfvpvNy0p/hR0bWbe2lxIfj31z9hjzN0MIUQ3xNnI+pjRm5MbDpc4y7i4olVXn/zs5qjFG1WOtOo9Ho/7QAAfDTzv+kMSXVNvB+fTKtCdyXaderCtgobeUkqF9NKtiv7qrav5bet8NjW2uZqXPh5v9/OFypniwsveZ3Nzx9WXClxPoQ1j6GPFq7Ya3Hy7EXaY8RbbVhazWNcW8Zxs79TpFncK788ihZXZuSuFU1307QubAw/x7XiZ641PPYoY1RNSZsaefeQb+O7l2uxOEfdM3dPjDzhworzgdyYCfas8VDi0otVtCd8nmxT/Dxy32e0vd5LOwAAD8D0bzrLZKwm5pWUfJdku6EXnuH9PGb8ArIvhuX5mQtbuYzCWmLObgHUKH68l9C2hu2bmw2bzYeKqkS4zA8WAgOUBVWruHLpI8U7MrT/RtExhBHTZP/RmJixD3ES65W6K8eWSD0aRdRNU9vzVowCWrG1oJ0ZN3cq1mZq0aXXFp4YOlD38bJppspHo0Qb9bzf+ExiaG+HMib4uJfztbnj/g/p6U5a3OXf8r/VcWssI5q/PWO8tGzpYfjuTG685RVl7pFYNfL/bY3bc5kXKzTd3Us7AAAPwNxvOmUCDUlWJOE4Zpew12flpZeSffnu+uxXnfxN4hzlpZWKre+9y6FH2kNp80u6xOO8sdCRl+busmr5p0S59BLbRdcuPkI8FH8ME+Y7WFx4kXZqazr1sflmf6G/vP4csz/GUc4zhNxD3uP27fPRgq8V+13RpvlLEnW304dWJOXnt7mSn0sqXyU/Ri1rsRki2nv4fQ+ztOgZ40GL0QGCNkrNpXha8R4h+MiwUcubGslfjbynjonPyrWDzpYxu5yf4xH1k+xqrdfiHlpcSHsJNip7DcRztnu+PNvZ5hkzQoj5Qd3Id/O+DubGUsNaXFKsS6y4axpLBLvX96OmNJ+WWHON2AMA8OTM/aYzJN6isIqJuUz6WzFcjFFoFdNuYjJPl0BZaG3zt21ose1Bsa8oZtTPpT+kvzTynPq4lq+Cnb0iZYRg/8m49JBriDiuuPQRfX1qvwlZGMwqhFJsR+eLdP0QCyHT5p1eF6yCqHw3+6KhW6sA2/nx+PkLlH7UPp/BJC269Oph57+SAV8KzQXbUtxLncr3PHRyVba/p/d4dpux1caI+PRy/vZstfdknriHFpX8vcvvYYzDv94YjCDPyQgzc6N8xxOXlmbTZ1YTmN9d6ezfo+mVlj0AAE/O1G8660tQXgLiZ4NqniPEtbT/L+Ou0CrfcZAKlapIiZdK2lsuaFp7Dxeu43JpXFjbOsqFJwvI7I8TxVWw42Bx4WTbz94nUg8ufZwtoA1yXEf0k/z2Q/jvZEzM2C8EH6XPUqHZ8Yc6X343scTGodsqRuGd2143+07o0VNQnmSWFj1jXMQ9nzrDC8GeJQ+l/82fefORQZivFZOkpXJNhcM+9eb8lIvTXl9i7j669ztoUb0DZI7Txkg8Y0YJc7Zzi0l6d0JurM6rNy6WT5JONC3Gd5KWt7VvP0s8ms5Y9gAAPDnzvulMBYVKvAhSEp9QdHiwiqCj85sXi7WvWGiH9WWBkD93FDrxEtIu4M0mMa9mz4FLvCLYcbC48JDsDpqR3NZ1xU/z9yzKuGqfS/LZELGOz4/GRI39irb35NuW3ywdC8K6Hf9vZy+de32fpv0evAXlUSZq8Wi+qTipl0TKY3o8HPlIYYt3x7Z0Dlr6enEU3I0xyQ75WRkDdZ8e2yyu1uJCiplG9vlJ3x0mzHnuHJ/Ojdp4b1yiT6q1rJxp5UnrXhj1uWUPAMCTM++bTqvIFwnXU4CdKX5KWsXFcEKPF4palFhzFntPxZBNY7+Wbxe2PYrP4rr6OhuHLrQw78HiwoPQSkb416UPa64ZWMWFRbNIOe5PNfb5ee2f7tmLdja14Snk5H6tWJzZv7egPMpELbr06sHKM6Nofrc06mDbn8OuaH9rjS1Ptn3VGqP6WvhNPQcn9n+5Fhu27fbi0ccsDZWEs3L0HBt7G80NMSe3MONjrZVsq7Ri+NDIGR5N7ziTFwEAHphp33SGy14raGOCzpdKTMy7S2YdI39ex5TzrRfAwV+8Ua5lXgDJLi3ZazYLtnn373Yvm3C5dC6jztpbkeW4oOIFeqrYCLZceBla/pCXv1MfWkyGfpHQuk5VcBiFSI9YFOWCRNFm5qWhxQIz9un9cu74rBX/rl61eRUqvyeflXMf9WMi+u+yQn+mFp167RLnOd0wKL5v5ZEtngta3KOuXTa1NL/y0tdod4yyRqVrzWZ5PkeIa16mxcb8Mm7bz2LssrfyZ8+YIUJM2rmqifR9QyfJdlfe8MQl6knVZCNHaXeLmj97epW07AEAeHLmfNOZkrORKKtknC6VTKO4y2MOXmpyLeuyihdftU5lx57yQsvFWUbZV0lYU7+kbnMo+67GFFiXVdyH+/LTCOueKC46BP8Z8amKYqc+cpESGd6/5uujBUHW2IZpi6XFFW/slXG79Zz72mta93Gle80/2jnyFI4WnoLyBNO16NRrkypvnplL2mPkqnKcjKtpz8rNpjovJvZ2b+PaOdMzxpXzxVlcOayli7W42WrEOJ7j3dmWe9POo2eMl2DDAQ2WCHus3Ji15MkdWlyiv25r1XbLO6Nk9L7v6tVhDwDAZ2He/6cTPj8ziguAGVxd6AN4QYsAAABd5v1/OuHzQ9MJjwKFPjwKaBEAAKAL33SCn0/UdNZ/NUrhzF85g2v5RIU+WnxyPlPTWf11Tw3+4BEAAMah6QQAAAAAAIDLoOkEAAAAAACAy6DpBAAAAAAAgMug6QQAAAAAAIDLoOkEAAAAAACAy6DpBAAAAAAAgMug6QQAAAAAAIDLoOkEAAAAAACAy6DpBAAAAAAAgMug6QQAAAAAAIDLoOkEAAAAAACAy/A3nV++vf/68+/7nx0/379/+c8FY76+v77JMb/fX18Gxyy8vP4WY/59f3v9OjZmlj33tHnlx89qzJ9f38bHEHsxhtgT+8ljFoi9HEPsR8cQe2K/40Nj/4R+fMC4EvvOmAVivx/zobHvwDedAAAAAAAAcBk0nQAAAAAAAHAZNJ0AAAAAAABwGTSdAAAAAAAAcBk0nQAAAAAAAHAZNJ0AAAAAAABwGTSdAAAAAAAAcBk0nQAAAAAAAHAZNJ0AAAAAAABwGTSdAAAAAAAAcBk0nbP58u39159/33/9+I/+Odyd77/+ff/z65v6GQAAAAAAXAtN52xoOh8Omk4AAAAAgI/jaZvO0Ei8/fP+8uXBmrvP1nR++fr++vbcTdvlTeePn+9/lphnntVXUbtvr1/1zyfw8vr7uX0EAAAAAMO4m85cLEo+qHic3nTGgvvPn9/vry8n5jzbdMoG5qRNp/10UdOZ9aTZJnxwtgk61XSq8VjQ7H72Bv1A02nmhYic62GaznzeC87YlHRinbWXf97fWmtp9qyU83lsThqU4/78fP9u6XX5nL+ZAQAAAFcy2HQqhcsHMbPpDHMte3sNe3yEpnNvQ7LviO8fsuksi2dhW6WzWKyfaTzPN51OTfyFTeeOEKuT5+ejSJocjV1+758t9tpZixrOOUFbKz4byhvqPJsGezHcztkSq9etWabpBAAAgCv565vOsK9YtOVC7MGazqpoHeARm85k0/fV36Vthu/Oao+m00n0/1/ZdC4M6yzGO+g1xV6eNUsTUlcH80Zlc1yvGcN17WRnsONEvgIAAABwMK/pLIqdbexSaAXqd/afKwVZJDQLedy+mM2NRCyaNo43JiubXSeL5suaTvEsrnPbe7Gm8llJVZCG+YsxZTysuB5tZONaq61hvnKe4rPdO5pPBniYpnOn1RWh1/D5+mwfvypeWnyVNffnR9ekHLPSbFhahPjpvpJnvloj7T37qPSDdh5ucx3WoqDS4wgp9vL9uAfp++T37AdjXI/KZk/TWRL9PbouAAAAwAjTm85dEagUYluxVcyTCkulcNqPW579kvMU7ylrjbLt8XhzEzhYPGZCEVjYoO0rPNvHwrI9+Knlk1h07orU5Vm2P8diITU2KWajjVyaK75XFcwvRtMZnh+PS/DBqK0JGY8WYn871j2Uz7W4xlisZB/EZ1U8WvvRxkjfauvHuF7RdGasNdLegz3rmN/vb2/rXGIvSjy6Gvcg/TOK5s8V6ZM47u31235f6UyV9Pak2Zzs2M3VyNtSXwAAAAAXcPoXCd3+pN4ousoiUSuS5JjlZ0/zpxWa23uNAquDZ90usXg8XMSl4nuHY0+Gb5sFeSp0RxuYhUOFvhbncg5NQ7mIPh6X803nur5Am8/wlUWl17iW2kSkOa2mrSTMU2tm5wcRi4Bn7hZBg504WWuUey91We4//nf1rmddjbROwhk3lWSnPBOln+N62x47WkmfyzgesDnE3fKPpjkAAACAycz/plMWQaEgjEWNUQzvirHlZ09Do435PE3n3obNLjFnLBYlct2mL8vYyM8SRlw9MdoR/VI2DGFfco5cbCcWX/w42FREgq1HGwolHiadRiLYkfeVKPRqrCV9nfQQUNbafS6J8+i+r2M0xMvZpjO+W44pfRqfq/taOHzmIlt8DuaPZKf0afKJ1HAc3/R1eLe9L5fNrbjGPHLWdwAAAAAt7tR0xmIrFDjKHLEoSoWPLLI1tDFdGzts73eK5h5iL8NojYf0bSwUd0WkUaA2fWm8s8OIqydGJVps1MZHw9KNk2Cr0py50OJhYZ2BhWCDmKfyibGWaX/UQaDwo+ccqL5vNScegp46vrLWKPdejil9eta+HmfmT3YaPq3i4Vkr+LMzxjNPsqGhIZpOAAAAuJLrm85Q1MT3YhFVFTii2PYUzVrT43mvxfZ+p2juEQu8K5rOVFiq+zR822wOXQWrHtfmvJI0x7KWiWnjfu9HMJs2D1o8LKwzYDyv4uiIvYp8z2FzS0OHfR3e7/jK0lxpczmm9J3l31mcObvJtqrpNGz26MozxmNzjKs6JqxxIl8BAAAAOLi26ZRFbBpTzhOLpt176VlZwK3vyl8kJAq8ro0dtvc7RV6PM4XrilJoVnbJMdHPWvHY29P2uXhvmT//bBTNQ02nQVi7NYemgwN8eNO5IP2V/N5rOsN7HU1vc5VnSjlnkqiZfDbjz6tND9t05nEnbLRI6yhay7FqatV+P9l8O1NR1y1Npni0xrTWzGM6Z0jaBgAAAHABp3+RUC6KcqG7RytmtkL6hlpAVvMpxbgopKri20Mq7jRaBZ9FLPIOF3GxCNxT72nvw9U39rpdf8s1y32nOAhfaP4fJcRLjeHNlhnFcLD1SCxX1HgsFHabZ6PUbCr+E+v7Ye4ittpa0scvml4VzXvO4269dY7tncMNXbBNbzqlBvfrLuODLY6mc8Xjpx7KHNa+s+3KGnbsW75W1qriqvjRY7PU2UJ1hpQxmVE/AgAAADhwN51djObkr+Ns0wnTOdV03pOy8dI+BwAAAAB4Qmg6Z0PT+XDQdAIAAAAAfBw0nbOh6Xw4aDoBAAAAAD6OeU0nAAAAAAAAgICmEwAAAAAAAC6DphP8NH47KcBd4a+xw6OAFgEAALrQdIIfmk54FCj04VFAiwAAAF1oOsEPTSc8ChT68CigRQAAgC7+pvPZG45YGKz/APqR4kD+4/bqPyTf+wfglX/c3bQl/TbgPDb+I/qjYxb2tp+I4VkNPKuGZNyO/AP6SuxXVB31KLSc2Omo0sWCZXMxttJiiJeYJ3OL48vrb/OzEjnuVJE+odB/mt9qnFDivjF4pjR9BPT8YTJTH97zIdeU8dN81IrxyXsh8BdqsY7pnkN5LWngSH5dGYl9Hts5O1mX9dnw1ASuumHFeZcDADwzf03TWV6So8VBeLe8CONFVM6zzV9cFLE4al6+yjwBz7ueMelinVXMnNXA2fc/gC2uwuZ1H6M+DbGeUEjEmO7ibukokQoaUczlvb123hdUWlc/3/ssFF+dMzTEX1joq7E/QtTD6XkMjujDdT5C/ihi7slvnTGbLcvn5byj/I1aLDmb13OM/lHz1GGM2G+N4M8l7yk6LNk1gnttBt14agJPzou6vuo8AgA8Cn9H0xmT+q8f54uDQLqM0mVmFB294quaJzzb5vI0k+0xytxnOauBZ9PQTB96imoP2jwePYT3Ct+vP6eCyCqGNI7o04h71YiOENc4c5ZpOk/Oo3FEHyu982GdRalrBTMPB11OuBf+Ri2WnMnrMa7BdynGs5rOBRn78HP08/aZbXd+t6fNFc9doY2ZdaYBAJ6A6U3nlqiXxJpQknA1Rps3FsIlRy/1W3F7vjgIyMsx+EaZt1cQxXmGiq8VzxhnvIY4O6f3fRl7rQjx6CPGezdupLCbWQR5YuYhzCN8GPza0XVr/ehLz7nIhVhrH/KcaTYvuOayuFehH32TOapFz5geswpULe9M4pA+VoJ/Gu8Z8Q4x7Pgk2KTEbdq9cC8txnVuGlJyacpZ5Ti5d8+YEV4m3TUz823Ein3+zLI7+jroqqfNFY/t2hjP3AAAn4SpTedWABQJNF2SVZLtXFBhrXOXeKZcb0JxEEj7SkWCZW/HZ1qBli7J7+Gz1Z8b5dyeMfkyC/97G3Oq0HRooInj/aoQyAVS4SePPtJ7I02mRuG/U7oRcVg5FIvsj2iPdsaqd4oCSvs82tbdn9S9wa2Yj/OF+ZW4h+cHC64JZ7lX6E/TomeMh14cvRQaunEwDiVH9bHSOx/Bh3Us3l6/tc+55ftSk2e1dActpn2UPtn0uV9X9a3AM2YIGZujJF3Osi36zIpLdb4Ldj7y5CmP9pUxrrscAOCTMK/ptBK8KDi3RN9J4OKdw6RLLCX5CcXBSriQSvu0yzIXdmIforiStmxzi4IrvpPGesakgmR3CcYYHS5aexro0Xs/xqeyT2rLow9rriOkYiHRKiy8xD0dnSvHd0HdY9ZfpFXICe2YNPxe2lONUYqtm08ftOm09HNEi54xHqQOM+cbxiqnHeGoPjTk+SjnLmMvc/yK9JOM8ex74ez7C00tLoTP5Rmu9qH4QuIZM0qI1Qx9R9tauapHL/YFmyYVu9Uz3j5jnvOjjdmeiTwT1junJwCAR2Re02klZnkh7y4FY750+cRxR5PvdqkUNk0oDlLxVBWjwuawtx8dnyVfFBejp7jwjKn2HlHf9dLTQI/e+9bncW/Z50597ArdSUVWnvNMYRSxYtQk7z2+F3xW/Ky9s7CtZfjeU+QIfTWJNu3OiCwGV3sdxZzJhLMczoK1n5ladOq1S9xzlXtmcHbus/pQ2J2PFA+ZU2U8FLbC/qaz6tyd1dLVWmzsMeylzEXxLAesHOUZM0KKjZXXvSQNzbApImNfouZETcedPLXN09agNUa9j0fOEgDAE3H/pjM/j4k1Xn7Whb1dGhu9ImWHVkSdLA7SxeG+DDxFdRhz86t6CYnnnjGbrfXa1nMXPQ306L1vfd4oulz6mF1kxfnOFJkrWyzG/KnGL+q6qctWIePZj9BpD4/Owpij8bi60L9Iiy69Wmg5bRYeDbW4Sh9pzmSfof2mT8ox2vj47LCWzr6/0NRiQ3PmGQr6TVoz/OwZ48E6K6OkHDUjRyca+tjpKz3XdBye6f7Z5lhsbpyb1pgQd2W/1nMAgGfm7n+9tqJVDBcMJ+Gw7vJOg6EiIc3nLcoahcIOeaFpF5z0kWeMEa9Tl1lPAz1671sFQnivHS/XvoLfTtifiFo4U2SuHImF+o7nDLX02NtPmn/AVrMYTjx8oX+tFo/EvlVAn8axL5Mr9LGw85Glcc+ZLrUStd5i2AdXa3FB1Yvn3Hvi6hnTIrw/Ia8e0FGXRmy0pjP4eRlvI+/q5VnL/70x4fPOXQ4A8EmY13SmRFkm0Jjwm8nTU0jNSsKt4iBevGoBkz7zrp/23bs847y7vSs+qy5H1xjlAo8X4IcVF473t0u/sw+JUx+bj8QF32L1s1XonSyMNlsOaDHGcKcZR1w3vxp7770/qpve+LS/M+e5dZaduAr9K7To1GtF3PP0pjPlE2NfuRC37J2tjwX1fMj3lDxY4YrZSS3dQYtp72Xsq5yvEd6bMKZFOM8n3k84YpV04cq/nflc/lsJ/hG5M+y5EzPPGEXDbrsAAJ6MwaZzSY4V+2ScC5SILJDk59qYfLGUtBK3l1ZxEC/1Ktmni6u0pSDNJW3W1qj3blws6SLKKM2Ca4y0/eRFFjRwYg6nhqr4i9i79KGupfiohzbPAS1WNreKJkuL1ec3dnrz2Fzpp6C0rVcIquej9vNe+yd1uNI6y060XLRSF/bF5we06NKrBzNmg/5U5jH9WMZXs3mSPtznQ2i/asCVs1GNkZzV0p20WJ9r4UdVH0IbnjGjBLuOz6Gej4j0afaTpg9P7CsfFlhnMsxb+FrV9I1gs2dMmr+KyYF7CgDgCfA3nQAniwuAaUwo9AGmgBYBAAC60HSCH5pOeBQo9OFRQIsAAABdaDrBzydqOq2/zrbjyF+BhPvwiQp9tPjkfKams/XXTzP8wSMAAIxD0wkAAAAAAACXQdMJAAAAAAAAl0HTCQAAAAAAAJdB0wl++EVC8Cjwy1vgUUCLAAAAXVxN53//z3+pz+Evg6YTHgUKfXgU0CIAAEAXvukEPzSd8ChQ6MOjgBYBAAC6XPJNZ/4nAB7h1/x/+fr++lb+uvef79+/jBcH8p81eHv9Wo/78XM3ZrdWLEz2n9/YFSxynrd/3l80m/OcjUaw2P/pouiOTefL6+/23u+JNx49gv+Kec6cj17sK93vNZv9qyH2J8dKHVlz9cad0uMdC/2H0aKZQ46dyX1OO3iuFZ1VMfGMSaSzZvnac4Y853XSvRC4oxZv+79PHrZo5o8F9X5s4YnrCMVZ2cXFOkNCI677fqVld1UPbOzm0ux5hLoJAOAC5n/TGZLociG+rgn3xEU+g1RYFEl8u0zG7AoXbHkpxctkd5mFZ/tCwLWWfC9cYsXP6VISF1Ga+zVc/noBshUGKRYTiiJp21XEuP16vdN6DbIPSxtWP4wWBrE4yTEw4uqhG3tF92n9ZjFonZeO9qvzoeCZZ4jov9Oa7vFAWkx7Hi7oFWQ8VJ07CPMUetHm8Yy5nYd/Ng1qenpxnKEwpphbGzPpXsjcS4sLwXfRRzN0MAXp81E8cR1k09gaUxEXR6zCu45cpeq4JLw3qKkJewcAeFTmf9OZE+39LmKL7VIQST/aderCtooWeVEEXzQuJa05UJD7CD/Hd8yLb107XZzGpTnM2eLCS15n88+HFVfO+HSx5unpQ8EVey1Onr1Ie4x4qw2L1iQknPMMca/8km3/YC2uzMhdK5rupmndYaMcE9cOsUx2SF2cOEPbORH5c+a9cC8tRh+sNnbP3D0xzreLE3E1Cfas8VDiciRWmo0e+8IYoTMHqj4BAD4B07/pLJsvtRFbSUl8Sf4beoIN7+cx4xeQun64CJbnZy7sZH8xh1ZAdy8P58W6zaOPa32WiXs+XRSdKS4GKAuq8r/lOJc+UrwjQ0WlEudDGIVOsv9Qobtgxj7ESS+2zLUGCqtt3ZuuWzEKOOcZ4k6Ffrm31j49WnTptUUvhk7UfUTNnC52o41q3k+0xlhnzoi35wxtOrv5O7wz8164kxZ3+bf8b3Xc5peA5mfPGC8tW3qciKtFeDfEUZn7SKys+77ns6Cp8fPUyjMAAM/M3G86ZULXkm4cs0uq67MygackX767Pvs1kIjjHOWllZL591CEnCiu0h5Km1/SJR7njUVMt8h3XC7bBazbKwsqlWjL6aIo7PFgceFFxk1b06mPzTf7fb+8/hyzPxWjZ/wn95D3+G3bx8GCrxX7XdGm+UuiaSQ8U+YPz29zJT+XHJlniCPF4yiztOgZ4yHu2cwpTqqCOcXBitMIwUcdG1tjkq9kXgzvFLbFcZ4ztMuf+b3b2qfvhXtocSHZGfyi7CMQYiieL8/q89gZM4KMzQgn4qpSaliLS7q7S3p3cHon2SJtzHMJ7UQ/l6iaLwn+uF5LAAAfwdxvOkOSLRKvkkBbDVSiVUy7EQVaWWht8x8oLiLbHhT74mW0XTAd+72Xi1YgFLh8Fec4fZHJAuEK5BoijiuuPUf/di95D7JQmVUIJb2Mzhfp+qEsejprBE3L4ksWW+Wz1vmRmj06T4vSj9rnM5ikRZdePWSfSQZ8KDQXbEtxL3Uq3/OQ81/Dnt6Y9LnU4tEzZGgx/TzlXriHFuM+S+1VZzbpw/LFimfMKPKcjHA0rhryHU9c0juNuFf3ffKh2PM2rqGf4KfCvkSeLzIzNgAAD8TUbzrrS1BeAr6LpJrnCHGt9KeRVaF4cP6tMNnPFxAXShpn7dW1xzRnY9y2TufCj4XX6aIo2HOwuHCiFX7SVy7fnS2gDXJcR/ST/PZD+C9rVGjJSSv2wUfpM6NIygT7DH3IgmiNTfBtuzivYnRwHpM435WF/iwtuvTqIe75qF4SwZ4lL6X/zZ+dicdCmK8Tk+6YdEdIf70cOEPhHTHXFffCHbSonlGZ47QxEs+YUcKcRm7pkd6dkBur8+qNS8MnKd/v7LDOocO3Wk6RbGfk+DkEAHhU5n3TGRPxragsiQk0FRT3aDoXrALn6PzpAqrst/YVigLlEnJcTmlM7/LZbOpc+JYdowSbDhYXHpIfw74lt3Vd8ZMF2UxG/ZnPhoilVbw4MWOv7T35VvHb6HkI63bGb2fPod0D5zDgLSiPMlGLR/NNxUm9JFIek/Nsz48Vu1u827Z5xpg6HT1DL3b+THZI7RyO09VaXEgx08h7j3tu2uEZM0qY82CuHY2rhTbeG5foE7lW9nl13xu2OWze5uz4anTvAABPwrxvOq0iX1xynov9TPFTos5zNKHHRkNtmK05jQt+K3oa+4vzeXzgusRGmySLM8WFB8Nf0r8ufVhzzWC46Wz9ocRxf1qxt/yjnr3oJ/d5iLFo7t3ab4lnnhZn3+8xUYsuvXoQax9G050nZgbb/tp2ecYEzKZz4AxFP1k+V+NxxrdXa7ERm92Z9uxhloZKwlk5mMdG4toi5uQWZny0tdJ8xnkIfpefWTmjwFP/XK4nAIAPYto3neEi15JpTKA5QcfEvEvY6xj58zqmnG+9nA7+4o1yLbPhS3ZpF51ms2Cbd/+uulacy7z00yXsLFK3AqpzOccL9PQlFmwfKARGCXYq+04+EcVVTx9aTIZ+kdC6jtS0tKUc30LGQNFm5qWhxQIz9un9cu74TOrOPA8a2rwKmt93OOdpEv13WWE2U4tOvXaJ85xuGBQNt/LIFs8FLV5R102bPGMSrfPlOUPp/ZamlfeGzoEkzneZFhvzy7htP4uxi9/Knz1jhnhZz3M7VzXxxDWSbFf1IfHEJdgu1tKeSSqbG7qNqH6XOOYBAHhW5nzTmRKlkaSrCz1dKplGcZfHHLzU5FpWMo+XSLVOZcee8gLJxVmm3levuMmXqkbyb7oUNdKYyscFRy+0sO6J4qJD8I1h2+aXYm2nPqQ/hwt2zdetYqRF1tiGaYulxRVP7I1x1XpxTMsne03rPq50r/jHM88QnoLyBNO1OCOfmWf6yFzSHiMnleNkXFs5JtnkGbPM1cp7uxh3zpArf65Iu84U+RdrcduzEWPtDAsfqfnKM8ZLsOGABks6cU3kPOKJlxaX6K/bWsLu6lzsaWlR+rDSomaznGOhlY8BAJ6Zef+fTvj8zCguAGZwdaEP4AUtAgAAdJn3/+mEzw9NJzwKFPrwKKBFAACALnzTCX4+UdNZ/ZVQjTN/5Qyu5RMV+mjxyflMTWf11081+INHAAAYh6YTAAAAAAAALoOmEwAAAAAAAC6DphMAAAAAAAAug6YTAAAAAAAALoOmEwAAAAAAAC6DphMAAAAAAAAug6YTAAAAAAAALoOmEwAAAAAAAC6DphMAAAAAAAAug6YTAAAAAAAALoOmEwAAAAAAAC7D33R++fb+68+/7392/Hz//uU/F4z5+v76Jsf8fn99GRyz8PL6W4z59/3t9evYmFn23NPmlR8/qzF/fn0bH0PsxRhiT+wnj1kg9nIMsR8dQ+yJ/Y4Pjf0T+vEB40rsO2MWiP1+zIfGvgPfdAIAAAAAAMBl0HQCAAAAAADAZdB0AgAAAAAAwGXQdAIAAAAAAMBl0HQCAAAAAADAZdB0AgAAAAAAwGXQdAIAAAAAAMBl0HQCAAAAAADAZdB0AgAAAAAAwGXQdAIAAAAAAMBl0HTO5su3919//n3/9eM/+udwd77/+vf9z69v6mcAAAAAAHAtNJ2zoel8OGg6AQAAAAA+jqdtOkMj8fbP+8uXB2vuPlvT+eXr++vbczdtlzedP36+/1linnlWX0Xtvr1+1T+fwMvr7+f2EQAAAAAM4246c7Eo+aDicXrTGQvuP39+v7++nJjzbNMpG5iTNp3208Sm09JQ5auXf97fyjEn1z7VdKrxWNB8+uwN+oGm08wLETnXwzSd+bwXHLXJo1ehI9XH0iapMa/N0p4/P9+/7+aJOt2NUcYBAAAATGKw6XycomRm0xnmWvb2Gvb4CE3n3oZk3xHfP1zT2bMlFszZf6nQPrH++abTqYm/sOncEWJ38vx8FEd15tBrlTvjOzs/V/NELbXOi2ZzfNacJz47HGcAAACAQf76pjPsKxZt2x4fr+msCtIBnqrptNYaafwUaDqdRO3+lU3nwnCO8+jVyAdyLVWjjnPvsllqmKYTAAAA7sy8prMoZLaxSxEVqN/Zf64UW5FQiOVx+2I2F2mhoEpjBgpGhc2uk0XzZU2neBbXue29WFP5rKQqNmNxm8eU8bDieqCRDe83m07dd0kHR4vkh2k6d1pdEXoNn6/P9vGr9q3FV1lzf350TcoxK0f93Go65Zmv1kh7zz4q/aCdh9tcM/7waaWrT4lHr9HWyvelrizdxPlb8XDZHH16yw80nQAAAHBfpjeduyIwPSuKoq0gK+ZJhWVZOOW5ynHLs19ynuI9Za1Rtj06GwwLoxB1Uxaj68/avsKzfSws24OfWj6JBemuAF2eyQI1+DoVxSlmg43cZmOcK7Lz04toWuLab6/fNhsG10sEHxx8t4pHi+Qrba11b+VzLa4xFju/xGdVPFr70cYE3yrz7HTVb3KayPhpWGukvQd71jG/39/e1rnEXpR4dDXuQfrHg9xv9OlOr9a85btWTPN8RjycNle5Ia23+jvTyO0AAAAAJzn9i4RyQaQVsStlkWgVSaKQ9DR/WqG5vXe8ePKs22VK07n3sWtPhm+bBbmneTQK4imFftxr1lCpg9KPVlHu5HzTubwv0eYbtLPSa1xrF0M5p9W0lYR5as3s/FD6Oo3xzN0iaLBzfqw1yr2Xuiz3H/+7etezrkZaJ+GMW6b0YZyr0mv67/KspGeFzVru2Z6J/Y7aHHzTHxe0ccSHAAAAAA7mf9Mpi5tY9IRizCiGdwXb8rOnodHGdG3soBV+w4i9DKM0A6n43M0Zi3SJXLfpyzI28rOEEVdPjDzs5gn2LHv/IZqIaMPRZiis0SvOLZR4mFhnIBLsEPGqm856LenrpIeAstbuc0mcJ4yR8Yva/bimM75bjil9Gp+r+1o4fOYiW3wG8odXr2kP2VblnQUZt/SNaSseTZuzvxx7Oht7AAAAgAZ3ajpjcRUKS2WOWPCkolEW2RramK6NHbb394XgMGIvw2iNh/RtGCMKxODnJ206U8ysIvlkQRzWUJozF0YjqGKdgYVtn1qTUezVWMu0P+ogUMTCcw7CGBm/s41H0FPHV9Ya5d7LMaVPz9rXY3T+OP6QXsN+2zFK87TPp7FWts2r3Tj+6DkBAAAAaHB901kWV1aTI4ptT9GsNT2e91ps7zuLNAtPodhC+CIQfZsKS3Wfhm81P2WsgrXkyqZTzt3U0PG4mE2bh5G1LfuN51UcHbFXke85bG5pqLlWi/B+x1eW5kqbyzGl7yz/zmL07Fr29PzvielCiFHvjGk2J7t6sSiJsT+ctwAAAAAaXNt0yiI2F0PFPLFo2r2XnpUF1/qu/EVCf0nTWdklx0Q/r994yXV7e9o+F+8t8+eftbguzGg6wxzStrC3wh5NH4N8eNO5IP2V/N5rOjcftTW9zVWeKeWcSaJm8tksNNRrhkzCHB1fxXhWa5R7L8dIn0Z9HLbRIq2jaDrHStP7qF7T552zo54NiWGz690Sp00AAAAARzn9i4RycZUL3T2yCVrZiqIbagFZzacU46JIqopvDy+3YrviSKMSCzht3y5iEbun3tPeh6tv7HW7/pZrlvuWRX9E838PaYfpX2HP2QYjrHsklitqPBaKvZtno9RsKuwT6/th7iK22lrSx6peFc17zuNuvXWO7Z3D/g626c1OFfvdusv4YEt8N/oq2KHpz+OnHsoc1r6z7dYaYi45j9SHmhtkXDW9emzWfJNI9kstLhzOVwAAAAAO3E1nF6M5+es423TCdE41nfckNAx60wYAAAAA8KzQdM6GpvPhoOkEAAAAAPg4aDpnQ9P5cNB0AgAAAAB8HPOaTgAAAAAAAAABTScAAAAAAABcBk0n+Gn8dlKAu8JfY4dHAS0CAAB0oekEPzSd8ChQ6MOjgBYBAAC60HSCH5pOeBQo9OFRQIsAAABd/E3nszccsTA4+g+hy3/cXv2H5Kt/mF35R/vLcdY/Nl/Y2h4Xf2NwHivWqz63/wF8F2c18KwaknG14tGi0sbGoXhIfSyomi7i39L8Xtt2fF5ef29jyv2HmJbvl9zmyu8qnx1iQqH/NL/VOKHE/ZAvlbywYeQrC2fsE1kDls89+dObY1d6OT9/jhZHqc/znkN5rXcv9tDOh+XTXuyltuU83nzu0au0++j+AQAenL+m6SwvydHiILxbXgTxItnNE57t/bMV88Ulky6XX/9sRZ92ucTLLs+dCkQ5No4zL/f0XnlZ9t7pEd4/oYGz738Am26Ezes+RgvEoI/Bol4jamgXQ0WP2e5XRauJrEfHXsrCyFEUbevb+1X9OkK0Z/Qslzxr03n4/CZibjg9j0EV+5x3/rH15smfnjEFmx3r57VO0nuvZ3W48jdqseRsXs95qHEvHiHPu/drN/ZRrzme2jyefO7Ra7WWcd8DAHwC/o6mMyf288VBQGno1KKhvHTiO2HtxsWizlNdTNs+mkWjFi/F7iHOauDs+/fmrL9KPEWKB20eqYd1TNJWGK9ofnBvQZfLnN/XQq1XEHn06RnTIr5/5iyrZ+2ROeuzRIz96Xk0pI3h55QDt880n3fzp3NMJuSaVR+1TkIzGuc5/YcfK3+jFkvO5PWoxeC7lJMmNlzyD0C6sbfyotRZ+Lmdzw9rOmvXnhsA4BmZ3nRuiXxJpAnlIq3GaPOG5FyOOZ6EQ2IPF9n54iCgXI63NW5zywsvY12u1oUnCznHhadeXHKeUcKcfQ2YeN+XsdeKEI8+UpFboujRZGYR5ImZhzCP8KEW60T0U/XZSCyL+YOmO/4wdV9y9ixOOMuuQn+WFj1jepw9v4mo6yuazmbs03lUfO7JnyM51pvzt/dP5LSVe2kx+S+j2J1yVjlOatYzZoSXk/dCYma+jYT4GvOpsTdiGeKzPB+5g7t6Tfvt3fcAAJ+EqU3nlpiLRJwuyTLxhmTduaDCWucu8Uy53oTiIJD2VV4W0ea8/1hkqheHdbmal9D2PM2VLtLwrVNYc6N5Uaa5OxdlE4cGmjjerwoBzW6PPixfjlI0C6d0U8yTOFRUZH9Ee7QzVhLXrWwPzxefCrsqm4QfW0VcQDsbClpBNsSEsxxsaNg5TYueMR7ing/ppqTQ0I0TeSHRi33r8+ijbEfU5W6vnjEr4bkv51cxPsIdtJj2Xu51s32/rudcnT57kmDbSR+uJF3Osi36bCj2ci/RprfXb7s8mLRXUukwrm/qVeTWTF5TzAcA8OTMazqtBF8WAMvPW6LvFDjincPIpD6hOFgJl7ZmX1ovXDQN+9M45XLVLsJUXKRLaFtfXErxQtP9H21qFTUeehro0Xs/xse6vPPePPqw5jpCnGuaH1fino7OlTSx0tyjoYv8frl+tKnW1c3X4b1WUdiITWlzN349ri70Z2rRM8aD1GHmfMNo5rQRevtM9ps+d+TP3pjBnL9p8uS+r9biQvhcnrtqr+JnDc+YUcKZOOnDlWTbmaZTnpHGPtXYlxou49rzW8wL1efpvWBP7SPNhpQnp9xdAAAPxLymMyRrpfiRF/LuUjDm2yXq45f5lrwLmyYUB+aFIC6dNE69pDqXa343kv6UNa3pKkDSOHGBmj730NNAj9771udxb9nnTn3s/GgVC4PkOc8URpFtrsGGIe89vhd8Vvwsx4dzWfvIWnunraiZUuvhPWvvigZNot2HC6sJZ7lZ6Af7JmnRqdcuSjymcXZuT+xTDtLGJB3Hz/I5K8c6xlS67uhkG38ip61crUWpuYLqPMbzHrDOqWfMCCEuJ324kjQ0w6bIdgfquVGNfdrLD7GnRgwSlfbCXMv6LU2XzyPyvgcA+Czcv+nMz32F2HZppGQ8kIS1IupkcWBdGmbBZRT8w5ersHvXGBTjds/D2iJeZy/1ngZ69N63Pm9c+C59zC6yrLgOohY9HarCZiXqQy1aDVvVecRzbUx4ZvlQ01wDywYXJ8/yStCO5rOVi7To0quFltNm0dKQB0/srTU8+dM1RvFPRyebBsfOYEVnDQ9NLTY0Z57HoN9lzoBxxjxjPFhnZZQU4xk5OqFpIqLGPmlU+qMxT2I3n0ev5fOSCXoCAHhE7v7Xayus5CwIl/LIZRSTe4uhpJ7m0+y0LiTLJ4OXa1VYBFvkpbj343YB1oXEsB9Lwn5OFBe990f9WODaV0+LXqIWzhYFR2KhvtM6Q5atRixu88c5l3dNSjsGNb1iFsweJhRmYa9W3rlYi0dib9o0A8e+TLyxj/ZXPvf42jMm5egGcn9q4zFKtO2Q7yJNLS6oemmd+4Qnrp4xLcL7E/LqgRzSpREbNfaWTx13xy5GHr2WzwtO5UUAgAdmXtOZknXZ6MTE27wUreRc4rlcPbSKg3gZqBdL+qxXFIh3t2fKnyAPXK7avJpfqwtUszk+O1y0hvcbGujheL/ar8dXTn1sPlLiYbH6Wa47ELsWmy0HtBgL610MW02w9Zm2j9Y8EbMgcry7Y3S8pHWWnQStjZzpWVp06rXCkyuPkPKJsa/NDw17vbFU8lZCy3Pbs9t59Yyp6OikyptHuIMWk4/L2LtsD+9NGNMi5KuTPlxxnK+UN5tnMNGZz/Sf1HNDt4lkV6mBI3rV3gEA+CwMNp1rQpTsE+iWNG/IAkl+ro3JF0vJaIGm0SoO4kVTJfx0cZW2FNSXTMneN+q+Inke6Wdr3+kizCgXmRKzUwVrmO/EhejUUOUn4QOXPtS17MveRJvngBYrm1tFk6XF6vMbO01X2igo16203Y9t2Ie0vVcsqmfoQCxKZhX6O5s26sK++PyAFl169WDGdfBMKvOYfixjp9nci/2C5Wdpdy9/esfs0HTyYuWhhRNxuVqLtd1i76o+hDY8Y0YJdh2fQz0fEenT7CdNb0perO47b+zFXN0axdB/V6/SniP6AwB4EvxNJ8DJ4gJgGhMKfYApoEUAAIAuNJ3gh6YTHgUKfXgU0CIAAEAXmk7w84maTuuvs+3grzo9Lp+o0EeLT85najrlX/dU4Q8eAQBgHJpOAAAAAAAAuAyaTgAAAAAAALgMmk4AAAAAAAC4DJpO8MMvEoJHgV/eAo8CWgQAAOjiajr/+3/+S30Ofxk0nfAoUOjDo4AWAQAAuvBNJ/ih6YRHgUIfHgW0CAAA0OWSbzrzPwHwCL/m/8vX99e38te9/3z//mW8OJD/rMHb69d63I+fuzHqWmJMNU8sYMoxgbd/3l/SXNoY6etq3wvlHEe4Y9P58vp7js0zkHE9alPwXzHPmfORNWDEw6N7aY91NnqanTnGyx0L/YfRopUbDp7JfU47eK6VPFPFxDOmJOukkatbYzza94zxckct3s7sx/7hXz4TBsNne2ZuXCnOyi4u1hkSZ9tz37tqAs++ZmoRAOCBmf9NZ0jqy4X4uhYFH5w8UzIvEv12UYzZFS7Y8lKKBc/uMgvP9oWAXGu7qIu144W0u6yOFDDpIm1d1MkXZwrnYO8dip1o66/XO63XYIuZsGH1w2hRFGOd4+qJmUHS1atm20qcu9RVpXupM0sfUtdxHzvNzhozwpFzcoQH0qIW16MEPRSxVnXuIMxTaFibxzMmk3S47NPM060x6bNivVr7jjEj3EuLC8F3v/4J9s/QwRTCWT5xNmIumJEbE5vG1piKuDhiFd7t3PeeMa59zdYiAMADM/+bzpB814R5v4vYYrt4ZFGy2XXqwrYuCnlJloW24Y/KxoN+U/cqkYX/KGeLCy95nc3PH1ZcKXE+hDXPgXiEOMd5tpjX74fnsnn06F7aY7yz09qsMcVzFwfPyTCPosUVTww9aLqbpnWHjY0xWRPBRl0brTGqpsR6njFDxHcv12KM0Wpj2IM84x/FmXthYm7MBHvWeChxORIrz9mQY5z7mq5FAIAHZvo3nWXzpTZiKykhL4l1QyTdyPYnfonxC8huBJfnZy7sZH8xR1hLzLm7UPJFKNaUl+vBAsZVhChF2hBhDwcLgQHKvbT25dJHindk6CJX4nwII6bJ/qPFxaavet+aFs0iqCT6Ktupxbs4u2HcrDHpuZc7FfoztejSa4tJxai6jxCj1a4T+WEl2tjUmTWm3J+Vqzpjgo87Od8zZvdZj2jT1VrcnaPyv9Vxy14SWiw8Y7y0bOlh+C6dlSNaD++GOCpzH4mV5x6QY5z7mq5FAIAHZu43nTLRaoVDHLNLqOuzMvHmgrR4d332ayAJxznKSysVW99DsX6iuEp7KG1+SZd4nDdeHHn9+Hl12YXnxYWd5i7pXT7W3CVx3iOXeEbaegUybtqaTn1sTdneLy+vP8fsTwVAz78t5B7yHr9t+zhY8G37q+NxK7qU5421qvnk+Y0aSt8ghBjNGrM+HyHPsd/jVGZp0TPGw4wzvFDpIMUn/G+tpyGCjzo2GmN2upWa8YyR8Vqoc75nzOD+76HFhWRn2Luyj0DwiXi+PNvZ5hkzQojnQd3Id/O+DubGsLc4nxaX+Czl9EDyaTlPSXqnZYsc49nXFVoEAHhg5n7TGRJ+kShD4t0n/VA0dJJpVfweIV4CKaGXhdY2//GEvu1BsS9eIttlJj5Pn5UXXB7f2GseI+yVl6d2Ie7sWehdrj3kRXoF1WW9j+OKSx9Re7vC6igeX7ewCqEUn9H5IqYfYlGpFZrmWtFf5ee7c1LGpYjJrDE7WzyUftQ+n0Fp6/qzYq9Hi54xLuL6WYeZgVwmNBdsS3mh1Kl8z4OVqzxjgq+LeMq7xDNGxEfP+Z4xxZoe7qHF6LdSe8H23X0S9WGd8RXPmFHkORmh1FzpxyO5Ub7jiYtDs+Z9X1CN8ewrPp+qRQCAB2bqN531JSgvAd9FUs1zhLhW+pPFqlA8OP92GeznC4SLV14aYq/5gkssl9IPx4Ud525dntul176kNps6a7UIdpx434F22Uo9uPRxtoA2yHEd0U/ym4x11qjQkpNmPMP+V01EFh0Gv2lnLxWiUj+pqE//mz4ri6hZY9KaXs6862SWFl169RD3fFQviaSDSg8yPoOE+ToxUcdo90KlFf+Yds73jBnc/x20qN4BYf/F+dfGSDxjRglzHsy16d0JubE6r964NHyyzdm2Qx3j2dcVWgQAeGDmfdMZE3wocCviRRCTrFr4Fswq0qwi6Oj86XKp7Lf2FYqCzqXnKfTipdi8gD0FqdP/JukiPVJceEj2rT6uuK3rip8syGbiiWtJPhsizp6YNdj06NxjLnDEWtk2ZZ6ou8rX8XnY/6wx6ZmXaPehdz1M1OKsfHZWLwmrkN6ed3KRQcq1LdvMMdpZlXnRM2YhrSF1UcbAM6Z83uVqLS7ku0ch+9NznjxjRglzOvOQJOefk7lRG++NS/SJXMu87wvMMc59TdciAMADM++bTq0oWBGXnCeZnil+StR5Ri+zRNifcrmsWHOKvVdYjYDE8m2J54L1rmcR9nOwuPBg+Uv416WPnu/PELXgnjs1MOofShz35+YH5/thLXkWUmNlzGHoeuf/WWOK5y48ej/DRC2e2meJ4cdhNN1ZGnWw7a9tV2tMKrxtfr7/P8eY1b+qrz0xO+Pbq7XYiM3uPvXsYZaGSs7cC7NyY8zJLcz4aGul+VrnoTXGua/pWgQAeGCmfdMZkqfWTMYEmpNvLOZ2yXgdI39ex5TzrUn84C/eKNfaChylAEx2aRedZrNgm3f/rrnWirZHDcfa+YLrzNW0x0OwZaAQGCVcyIp9cn9OfWgxGfpFQus60qdOX1fEAiUXPoo2My8NLRZsBUs/Htu4uujS/COp1oi2lQXRrDFDRP9dVujP1KJTr13iPKeLUUXDLS1tOlnQtBp13bTJM0Zi+b9EG6OcqyrvecaMEOe7rum055dx234WYxc/lT97xgzxcvJeiPrI6yvxSSTbXfnXE5dgu1hLeybxjPHsS3l2SosAAA/MnG86UxFjJGDz0s8oCTbNmcccvNTkWtZlFS+Iap3Kjj3lhZaLs8x+X/nCjKiXYbrMMsq+s603qoKummehdUF6CHOeKC46BP8Z8ZHFlVcf0ufDBftMP4q4mbZYWlzR7EkkuzyaVzSUEeM9up01xk3c46k5GkzX4ox8VuXNM3NJe4witxwndW/asxJt8owp50wEfXYKb2uMR/+eMV7iXFdpcdun4auXLR/scok821q+8ozxEmw4oMESYY+VG/Md64mXFpfor9tawu7qXOwJc3nGpPk8+5qpRQCAB2be/6cTPj8ziguAGVxd6AN4QYsAAABd5v1/OuHzQ9MJjwKFPjwKaBEAAKAL33SCn0/UdNZ/FVrhzF85g2v5RIU+WnxyPlPTWf31Uw3+4BEAAMah6QQAAAAAAIDLoOkEAAAAAACAy6DpBAAAAAAAgMug6QQAAAAAAIDLoOkEAAAAAACAy6DpBAAAAAAAgMug6QQAAAAAAIDLoOkEAAAAAACAy6DpBAAAAAAAgMug6QQAAAAAAIDLoOkEAAAAAACAy/A3nV++vf/68+/7nx0/379/+c8FY76+v77JMb/fX18Gxyy8vP4WY/59f3v9OjZmlj33tHnlx89qzJ9f38bHEHsxhtgT+8ljFoi9HEPsR8cQe2K/40Nj/4R+fMC4EvvOmAVivx/zobHvwDedAAAAAAAAcBk0nQAAAAAAAHAZNJ0AAAAAAABwGTSdAAAAAAAAcBk0nQAAAAAAAHAZNJ0AAAAAAABwGTSdAAAAAAAAcBk0nQAAAAAAAHAZNJ0AAAAAAABwGTSdAAAAAAAAcBk0nQAAAAAAAHAZNJ0AAAAAAABwGTSdAAAAAAAAcBk0nQAAAAAAAHAR//v9/wOi7ug0FIs8bQAAAABJRU5ErkJggg=="}}},{"metadata":{"trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport os\nimport sys\nimport json\nimport tensorflow as tf # Yes, we are going to play with Tensorflow 2!\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport absl # For using flags without tf.compat.v1.flags.Flag\nimport datetime","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Input data files are available in the \"../input/\" directory.\nIS_KAGGLE = True\nINPUT_DIR = \"/kaggle/input/\"\n\n# The original Bert Joint Baseline data.\nBERT_JOINT_BASE_DIR = os.path.join(INPUT_DIR, \"bertjointbaseline\")\n\n# This nq dir contains all files for publicly use.\nNQ_DIR = os.path.join(INPUT_DIR, \"nq-competition\")\n\n# If you want to use your own .tfrecord or new trained checkpoints, you can put them under you own nq dir (`MY_OWN_NQ_DIR`)\n# Default to NQ_DIR. You have to change it to the dir containing your own working files.\nMY_OWN_NQ_DIR = NQ_DIR\n\n# For local usage.\nif not os.path.isdir(INPUT_DIR):\n    IS_KAGGLE = False\n    INPUT_DIR = \"./\"\n    NQ_DIR = \"./\"\n    MY_OWN_NQ_DIR = \"./\"\n\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nfor dirname, _, filenames in os.walk(INPUT_DIR):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\n\nINPUT_DIR","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# NQ_DIR contains some packages / modules\nsys.path.append(NQ_DIR)\nsys.path.append(os.path.join(NQ_DIR, \"transformers\"))\n\nfrom nq_flags import DEFAULT_FLAGS as FLAGS\nfrom nq_flags import del_all_flags\nfrom nq_dataset_utils import *\n\nimport sacremoses as sm\nimport transformers\nfrom adamw_optimizer import AdamW","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Hugging Face pretrained Bert model names\n\n* Model names taken from: https://github.com/huggingface/transformers/blob/master/transformers/\n* To use the models, we have to get configuration, vocab, and model files. These files have to be renamed to `config.json`, `vocab.txt` and `tf_model.h5`.\n* Under the above transformers web page, you can find links to\n    * Models: see links in modeling_bert.py / modeling_distilbert.py\n    * Configurations, see configuration_bert.py / configuration_distilbert.py\n    * Vocabs, see tokenization_bert.py / tokenization_distilbert.py\n\n#### Important\n\nWe are using a particular dataset `nq-train.tfrecords-00000-of-00001`, which is created with a customized bert vocabulary `vocab-nq` and it is uncased. If you want to use cased bert model, you have to regenerate the training dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"PRETRAINED_MODELS = {\n    \"BERT\": [\n        'bert-base-uncased',\n        'bert-large-uncased',\n        'bert-base-cased',\n        'bert-large-cased',\n        'bert-base-multilingual-uncased',\n        'bert-base-multilingual-cased',\n        'bert-base-chinese',\n        'bert-base-german-cased',\n        'bert-large-uncased-whole-word-masking',\n        'bert-large-cased-whole-word-masking',\n        'bert-large-uncased-whole-word-masking-finetuned-squad',\n        'bert-large-cased-whole-word-masking-finetuned-squad',\n        'bert-base-cased-finetuned-mrpc'\n    ],\n    \"DISTILBERT\": [\n        'distilbert-base-uncased',\n        'distilbert-base-uncased-distilled-squad'\n    ]\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Abseil Flags - Datasets\n\n`Tensorflow 2` no longer supports `flags`, and it is in favor of the now open-source `absl-py`.\n\nI keep using (Abseil) flags in this kernel for making definitions clear, but my way of using it is not for command-line use. At some point later, `FLAGS.mark_as_parsed()` is called.\n\nThis section contains flags about the characteristics of the datasets."},{"metadata":{"trusted":true},"cell_type":"code","source":"flags = absl.flags\ndel_all_flags(flags.FLAGS)\n\nflags.DEFINE_bool(\n    \"do_lower_case\", True,\n    \"Whether to lower case the input text. Should be True for uncased \"\n    \"models and False for cased models.\")\n\nvocab_file = os.path.join(NQ_DIR, \"vocab-nq.txt\")\n\nflags.DEFINE_string(\"vocab_file\", vocab_file,\n                    \"The vocabulary file that the BERT model was trained on.\")\n\nflags.DEFINE_integer(\n    \"max_seq_length_for_training\", 512,\n    \"The maximum total input sequence length after WordPiece tokenization for training examples. \"\n    \"Sequences longer than this will be truncated, and sequences shorter \"\n    \"than this will be padded.\")\n\nflags.DEFINE_integer(\n    \"max_seq_length\", 512,\n    \"The maximum total input sequence length after WordPiece tokenization. \"\n    \"Sequences longer than this will be truncated, and sequences shorter \"\n    \"than this will be padded.\")\n\nflags.DEFINE_integer(\n    \"doc_stride\", 128,\n    \"When splitting up a long document into chunks, how much stride to \"\n    \"take between chunks.\")\n\nflags.DEFINE_float(\n    \"include_unknowns_for_training\", 0.02,\n    \"If positive, for converting training dataset, probability of including answers of type `UNKNOWN`.\")\n\nflags.DEFINE_float(\n    \"include_unknowns\", -1.0,\n    \"If positive, probability of including answers of type `UNKNOWN`.\")\n\nflags.DEFINE_boolean(\n    \"skip_nested_contexts\", True,\n    \"Completely ignore context that are not top level nodes in the page.\")\n\nflags.DEFINE_integer(\"max_contexts\", 48,\n                     \"Maximum number of contexts to output for an example.\")\n\nflags.DEFINE_integer(\n    \"max_position\", 50,\n    \"Maximum context position for which to generate special tokens.\")\n\nflags.DEFINE_integer(\n    \"max_query_length\", 64,\n    \"The maximum number of tokens for the question. Questions longer than \"\n    \"this will be truncated to this length.\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Abseil Flags - Hyperparameters and file paths"},{"metadata":{"trusted":true},"cell_type":"code","source":"if os.path.isfile(os.path.join(MY_OWN_NQ_DIR, \"nq_train.tfrecord\")):\n    TRAIN_TF_RECORD = os.path.join(MY_OWN_NQ_DIR, \"nq_train.tfrecord\")\nelif os.path.isfile(os.path.join(MY_OWN_NQ_DIR, \"nq-train.tfrecords-00000-of-00001\")):\n    TRAIN_TF_RECORD = os.path.join(MY_OWN_NQ_DIR, \"nq-train.tfrecords-00000-of-00001\")\nelse:\n    TRAIN_TF_RECORD = os.path.join(BERT_JOINT_BASE_DIR, \"nq-train.tfrecords-00000-of-00001\")\n    \nflags.DEFINE_string(\"train_tf_record\", TRAIN_TF_RECORD,\n                    \"Precomputed tf records for training dataset.\")\n\nflags.DEFINE_string(\"valid_tf_record\", os.path.join(NQ_DIR, \"nq_valid.tfrecord\"),\n                    \"Precomputed tf records for validation dataset.\")\n\nflags.DEFINE_string(\"valid_small_tf_record\", os.path.join(NQ_DIR, \"nq_valid_small.tfrecord\"),\n                    \"Precomputed tf records for a smaller validation dataset.\")\n\nflags.DEFINE_string(\"valid_tf_record_with_labels\", \"nq_valid_with_labels.tfrecord\",\n                    \"Precomputed tf records for validation dataset with labels.\")\n\nflags.DEFINE_string(\"valid_small_tf_record_with_labels\", \"nq_valid_small_with_labels.tfrecord\",\n                    \"Precomputed tf records for a smaller validation dataset with labels.\")\n\n# This file should be generated when the kernel is running using the provided test dataset!\nflags.DEFINE_string(\"test_tf_record\", \"nq_test.tfrecord\",\n                    \"Precomputed tf records for test dataset.\")\n\nflags.DEFINE_bool(\"do_train\", False, \"Whether to run training dataset.\")\n\nflags.DEFINE_bool(\"do_valid\", False, \"Whether to run validation dataset.\")\n\nflags.DEFINE_bool(\"smaller_valid_dataset\", True, \"Whether to use the smaller validation dataset\")\n\nflags.DEFINE_bool(\"do_predict\", True, \"Whether to run test dataset.\")\n\nflags.DEFINE_string(\n    \"validation_prediction_output_file\", \"validatioin_predictions.json\",\n    \"Where to print predictions for validation dataset in NQ prediction format, to be passed to natural_questions.nq_eval.\")\n\nflags.DEFINE_string(\n    \"validation_small_prediction_output_file\", \"validatioin_small_predictions.json\",\n    \"Where to print predictions for validation dataset in NQ prediction format, to be passed to natural_questions.nq_eval.\")\n\nflags.DEFINE_string(\n    \"prediction_output_file\", \"predictions.json\",\n    \"Where to print predictions for test dataset in NQ prediction format, to be passed to natural_questions.nq_eval.\")\n\nflags.DEFINE_string(\n    \"input_checkpoint_dir\", os.path.join(MY_OWN_NQ_DIR, \"checkpoints\"),\n    \"The root directory that contains checkpoints to be loaded of all trained models.\")\n\nflags.DEFINE_string(\n    \"output_checkpoint_dir\", \"checkpoints\",\n    \"The output directory where the model checkpoints will be written to.\")\n\n# If you want to use other Hugging Face's models, change this to `MY_OWN_NQ_DIR` and put the downloaded models at the right place.\nflags.DEFINE_string(\"model_dir\", NQ_DIR, \"Root dir of all Hugging Face's models\")\n\nflags.DEFINE_string(\"model_name\", \"distilbert-base-uncased-distilled-squad\", \"Name of Hugging Face's model to use.\")\n# flags.DEFINE_string(\"model_name\", \"bert-base-uncased\", \"Name of Hugging Face's model to use.\")\n# flags.DEFINE_string(\"model_name\", \"bert-large-uncased-whole-word-masking-finetuned-squad\", \"Name of Hugging Face's model to use.\")\n\nflags.DEFINE_integer(\"epochs\", 1, \"Total epochs for training.\")\n\nflags.DEFINE_integer(\"train_batch_size\", 10, \"Batch size for training.\")\n\nflags.DEFINE_integer(\"shuffle_buffer_size\", 100000, \"Shuffle buffer size for training.\")\n\nflags.DEFINE_integer(\"batch_accumulation_size\", 50, \"Number of batches to accumulate gradient before applying optimization.\")\n\nflags.DEFINE_float(\"init_learning_rate\", 5e-5, \"The initial learning rate for AdamW optimizer.\")\n\nflags.DEFINE_bool(\"cyclic_learning_rate\", True, \"If to use cyclic learning rate.\")\n\nflags.DEFINE_float(\"init_weight_decay_rate\", 0.01, \"The initial weight decay rate for AdamW optimizer.\")\n\nflags.DEFINE_integer(\"num_warmup_steps\", 0, \"Number of training steps to perform linear learning rate warmup.\")\n\nflags.DEFINE_integer(\"num_train_examples\", None, \"Number of precomputed training steps in 1 epoch.\")\n\nflags.DEFINE_integer(\"predict_batch_size\", 25, \"Batch size for predictions.\")\n\n# ----------------------------------------------------------------------------------------\nflags.DEFINE_integer(\n    \"n_best_size\", 10,\n    \"The total number of n-best predictions to generate in the \"\n    \"nbest_predictions.json output file.\")\n\nflags.DEFINE_integer(\n    \"max_answer_length\", 30,\n    \"The maximum length of an answer that can be generated. This is needed \"\n    \"because the start and end predictions are not conditioned on one another.\")\n\nflags.DEFINE_string(\n    \"validation_predict_file\", os.path.join(NQ_DIR, \"simplified-nq-dev.jsonl\"),\n    \"\")\n\nflags.DEFINE_string(\n    \"validation_predict_file_small\", os.path.join(NQ_DIR, \"simplified-nq-dev-small.jsonl\"),\n    \"\")\n\n# ----------------------------------------------------------------------------------------\n## Special flags - do not change\n\nif IS_KAGGLE: \n    flags.DEFINE_string(\n        \"predict_file\", \"/kaggle/input/tensorflow2-question-answering/simplified-nq-test.jsonl\",\n        \"NQ json for predictions. E.g., dev-v1.1.jsonl.gz or test-v1.1.jsonl.gz\")\nelse:\n    flags.DEFINE_string(\n        \"predict_file\", os.path.join(NQ_DIR, \"simplified-nq-test.jsonl\"),\n        \"NQ json for predictions. E.g., dev-v1.1.jsonl.gz or test-v1.1.jsonl.gz\")\n    \nif IS_KAGGLE: \n    flags.DEFINE_string(\n        \"sample_submission_csv\", \"/kaggle/input/tensorflow2-question-answering/sample_submission.csv\",\n        \"path to sample submission csv file.\")\nelse:\n    flags.DEFINE_string(\n        \"sample_submission_csv\", os.path.join(NQ_DIR, \"sample_submission.csv\"),\n        \"path to sample submission csv file.\")    \n    \nflags.DEFINE_boolean(\"logtostderr\", True, \"Logs to stderr\")\nflags.DEFINE_boolean(\"undefok\", True, \"it's okay to be undefined\")\nflags.DEFINE_string('f', '', 'kernel')\nflags.DEFINE_string('HistoryManager.hist_file', '', 'kernel')\n\n# Make the default flags as parsed flags\nFLAGS.mark_as_parsed()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IS_SUBMITTING = False\n\ntest_answers_df = pd.read_csv(FLAGS.sample_submission_csv)\n\nif IS_KAGGLE and len(test_answers_df) != 692:\n    IS_SUBMITTING = True\n    FLAGS.do_train = False\n    FLAGS.do_valid = False\n    FLAGS.do_predict = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(test_answers_df))\nprint(IS_SUBMITTING)\nprint(FLAGS.do_train)\nprint(FLAGS.do_valid)\nprint(FLAGS.do_predict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NB_ANSWER_TYPES = 5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get Datasets from TF Record files\n\nA method to get datasets from tf_record_file. See next section for the usage.[](http://)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataset(tf_record_file, seq_length, batch_size=1, shuffle_buffer_size=0, is_training=False):\n\n    if is_training:\n        features = {\n            \"unique_ids\": tf.io.FixedLenFeature([], tf.int64),\n            \"input_ids\": tf.io.FixedLenFeature([seq_length], tf.int64),\n            \"input_mask\": tf.io.FixedLenFeature([seq_length], tf.int64),\n            \"segment_ids\": tf.io.FixedLenFeature([seq_length], tf.int64),\n            \"start_positions\": tf.io.FixedLenFeature([], tf.int64),\n            \"end_positions\": tf.io.FixedLenFeature([], tf.int64),\n            \"answer_types\": tf.io.FixedLenFeature([], tf.int64)\n        }\n    else:\n        features = {\n            \"unique_ids\": tf.io.FixedLenFeature([], tf.int64),\n            \"input_ids\": tf.io.FixedLenFeature([seq_length], tf.int64),\n            \"input_mask\": tf.io.FixedLenFeature([seq_length], tf.int64),\n            \"segment_ids\": tf.io.FixedLenFeature([seq_length], tf.int64)\n        }        \n\n    # Taken from the TensorFlow models repository: https://github.com/tensorflow/models/blob/befbe0f9fe02d6bc1efb1c462689d069dae23af1/official/nlp/bert/input_pipeline.py#L24\n    def decode_record(record, features):\n        \"\"\"Decodes a record to a TensorFlow example.\"\"\"\n        example = tf.io.parse_single_example(record, features)\n\n        # tf.Example only supports tf.int64, but the TPU only supports tf.int32.\n        # So cast all int64 to int32.\n        for name in list(example.keys()):\n            t = example[name]\n            if t.dtype == tf.int64:\n                t = tf.cast(t, tf.int32)\n            example[name] = t\n        return example\n\n    def select_data_from_record(record):\n        \n        x = {\n            'unique_ids': record['unique_ids'],\n            'input_ids': record['input_ids'],\n            'input_mask': record['input_mask'],\n            'segment_ids': record['segment_ids']\n        }\n\n        if is_training:\n            y = {\n                'start_positions': record['start_positions'],\n                'end_positions': record['end_positions'],\n                'answer_types': record['answer_types']\n            }\n\n            return (x, y)\n        \n        return x\n\n    dataset = tf.data.TFRecordDataset(tf_record_file)\n    \n    dataset = dataset.map(lambda record: decode_record(record, features))\n    dataset = dataset.map(select_data_from_record)\n    \n    if shuffle_buffer_size > 0:\n        dataset = dataset.shuffle(shuffle_buffer_size)\n    \n    dataset = dataset.batch(batch_size)\n    \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Set num_train_examples if FLAGS.num_train_examples is None"},{"metadata":{"trusted":true},"cell_type":"code","source":"if FLAGS.num_train_examples is None:\n    FLAGS.num_train_examples = 494670","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check a small batch in training / validation dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_tf_record = FLAGS.valid_tf_record\nif FLAGS.smaller_valid_dataset:\n    valid_tf_record = FLAGS.valid_small_tf_record\n    \nvalid_tf_record_with_labels = FLAGS.valid_tf_record_with_labels\nif FLAGS.smaller_valid_dataset:\n    valid_tf_record_with_labels = FLAGS.valid_small_tf_record_with_labels\n\ntrain_dataset = get_dataset(FLAGS.train_tf_record,\n                    seq_length=FLAGS.max_seq_length_for_training,\n                    batch_size=2,\n                    shuffle_buffer_size=FLAGS.shuffle_buffer_size,\n                    is_training=True\n                )    \n\nvalidation_dataset = get_dataset(os.path.join(NQ_DIR, valid_tf_record),\n                         seq_length=FLAGS.max_seq_length,\n                         batch_size=2,\n                         is_training=False\n                     )\n\nvalidation_dataset_with_labels = get_dataset(os.path.join(NQ_DIR, valid_tf_record_with_labels),\n                         seq_length=FLAGS.max_seq_length,\n                         batch_size=2,\n                         is_training=True\n                     )\n\ntest_dataset = get_dataset(FLAGS.test_tf_record,\n                         seq_length=FLAGS.max_seq_length,\n                         batch_size=2,\n                         is_training=False\n                     )\n\n# Can't use next(train_dataset)!\nfeatures, targets = next(iter(train_dataset))\nprint(features)\nprint(targets)\n\nfeatures = next(iter(validation_dataset))\nprint(features)\n\nfeatures, labels = next(iter(validation_dataset_with_labels))\nprint(features)\n\n# features = next(iter(test_dataset))\n# print(features)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Demo of using Hugging Face Tensorflow 2.0 models"},{"metadata":{"trusted":true},"cell_type":"code","source":"from transformers import TFBertModel\nfrom transformers import TFBertMainLayer, TFBertPreTrainedModel\nfrom transformers.modeling_tf_utils import get_initializer\n\nclass TFBertForNQDemo(TFBertPreTrainedModel):\n    \n    def __init__(self, config, *inputs, **kwargs):\n        \n        super(TFBertForNQDemo, self).__init__(config, *inputs, **kwargs)\n\n        self.bert = TFBertMainLayer(config, name='bert')\n        \n        self.seq_output_dropout = tf.keras.layers.Dropout(kwargs.get('seq_output_dropout_prob', 0.05))\n        self.pooled_output_dropout = tf.keras.layers.Dropout(kwargs.get('pooled_output_dropout_prob', 0.05))\n        \n        self.pos_classifier = tf.keras.layers.Dense(2,\n                                        kernel_initializer=get_initializer(config.initializer_range),\n                                        name='pos_classifier')\n        \n        self.answer_type_classifier = tf.keras.layers.Dense(NB_ANSWER_TYPES,\n                                        kernel_initializer=get_initializer(config.initializer_range),\n                                        name='answer_type_classifier')         \n        \n    def call(self, inputs, **kwargs):\n        \n        bert_outputs = self.bert(inputs, **kwargs)\n\n        # sequence / [CLS] outputs from original bert\n        sequence_output, pooled_output = bert_outputs[0], bert_outputs[1]  # shape = (batch_size, seq_len, hidden_dim) / (batch_size, hidden_dim)\n\n        # dropout\n        sequence_output = self.seq_output_dropout(sequence_output, training=kwargs.get('training', False))\n        pooled_output = self.pooled_output_dropout(pooled_output, training=kwargs.get('training', False))\n        \n        pos_logits = self.pos_classifier(sequence_output)  # shape = (batch_size, seq_len, 2)\n        start_pos_logits = pos_logits[:, :, 0]  # shape = (batch_size, seq_len)\n        end_pos_logits = pos_logits[:, :, 1]  # shape = (batch_size, seq_len)\n        \n        answer_type_logits = self.answer_type_classifier(pooled_output)  # shape = (batch_size, NB_ANSWER_TYPES)\n\n        outputs = (start_pos_logits, end_pos_logits, answer_type_logits)\n\n        return outputs  # logits","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Try the demo model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from transformers import BertTokenizer\n\npretrained_weights = os.path.join(NQ_DIR, \"bert-base-uncased\")\n\nbert_tokenizer = BertTokenizer.from_pretrained(pretrained_weights)\n\nbert_barebone = TFBertModel.from_pretrained(pretrained_weights)\nbert_for_nq = TFBertForNQDemo.from_pretrained(pretrained_weights)\n\ninput_ids = tf.constant(bert_tokenizer.encode(\"My cat is so cute!\"))[None, :]  # Batch size 1\ninput_masks = tf.constant(0, shape=input_ids.shape)\nsegment_ids = tf.constant(0, shape=input_ids.shape)\n\n# Actual inputs to model's `call()` method.\ninputs = (input_ids, input_masks, segment_ids)\n\n# Outputs from `bert_barebone`\noutputs = bert_barebone(inputs)\nlast_hidden_states = outputs[0]\nprint(last_hidden_states.shape)\n\n# Outputs from `bert_for_nq`\noutputs = bert_for_nq(inputs)\n(start_pos_logits, end_pos_logits, answer_type_logits) = outputs\nprint(start_pos_logits.shape)\nprint(end_pos_logits.shape)\nprint(answer_type_logits.shape)\n\nlen(bert_for_nq.trainable_variables)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## An interface  for NQ models"},{"metadata":{"trusted":true},"cell_type":"code","source":"from transformers import BertTokenizer\nfrom transformers import TFBertModel, TFDistilBertModel\nfrom transformers import TFBertMainLayer, TFDistilBertMainLayer, TFBertPreTrainedModel, TFDistilBertPreTrainedModel\nfrom transformers.modeling_tf_utils import get_initializer\n\nclass TFNQModel:\n    \n    def __init__(self, config, *inputs, **kwargs):\n        \"\"\"\n        \n        Subclasses of this class are different in self.backend,\n        which should be a model that outputs a tensor of shape (batch_size, hidden_dim), and the\n        `backend_call()` method.\n        \n        We will use Hugging Face Bert/DistilBert as backend in this notebook.\n        \"\"\"\n\n        self.backend = None\n        \n        self.seq_output_dropout = tf.keras.layers.Dropout(kwargs.get('seq_output_dropout_prob', 0.05))\n        self.pooled_output_dropout = tf.keras.layers.Dropout(kwargs.get('pooled_output_dropout_prob', 0.05))\n        \n        self.pos_classifier = tf.keras.layers.Dense(2,\n                                        kernel_initializer=get_initializer(config.initializer_range),\n                                        name='pos_classifier')       \n\n        self.answer_type_classifier = tf.keras.layers.Dense(NB_ANSWER_TYPES,\n                                        kernel_initializer=get_initializer(config.initializer_range),\n                                        name='answer_type_classifier')         \n                \n    def backend_call(self, inputs, **kwargs):\n        \"\"\"This method should be implemented by subclasses.\n           \n           The implementation should take into account the (somehow) different input formats of Hugging Face's\n           models.\n           \n           For example, the `TFDistilBert` model, unlike `Bert` model, doesn't have segment_id as input.\n           \n           Then it calls `self.backend_call()` to get the outputs from Bert's model, which is used in self.call().\n        \"\"\"\n        \n        raise NotImplementedError\n\n    \n    def call(self, inputs, **kwargs):\n        \n        # sequence / [CLS] outputs from original bert\n        sequence_output, pooled_output = self.backend_call(inputs, **kwargs)  # shape = (batch_size, seq_len, hidden_dim) / (batch_size, hidden_dim)\n        \n        # dropout\n        sequence_output = self.seq_output_dropout(sequence_output, training=kwargs.get('training', False))\n        pooled_output = self.pooled_output_dropout(pooled_output, training=kwargs.get('training', False))\n        \n        pos_logits = self.pos_classifier(sequence_output)  # shape = (batch_size, seq_len, 2)\n        start_pos_logits = pos_logits[:, :, 0]  # shape = (batch_size, seq_len)\n        end_pos_logits = pos_logits[:, :, 1]  # shape = (batch_size, seq_len)\n        \n        answer_type_logits = self.answer_type_classifier(pooled_output)  # shape = (batch_size, NB_ANSWER_TYPES)\n\n        outputs = (start_pos_logits, end_pos_logits, answer_type_logits)\n\n        return outputs  # logits\n    \n    \nclass TFBertForNQ(TFNQModel, TFBertPreTrainedModel):\n    \n    def __init__(self, config, *inputs, **kwargs):\n        \n        TFBertPreTrainedModel.__init__(self, config, *inputs, **kwargs)  # explicit calls without super\n        TFNQModel.__init__(self, config)\n\n        self.bert = TFBertMainLayer(config, name='bert')\n        \n    def backend_call(self, inputs, **kwargs):\n        \n        outputs = self.bert(inputs, **kwargs)\n        sequence_output, pooled_output = outputs[0], outputs[1]  # shape = (batch_size, seq_len, hidden_dim) / (batch_size, hidden_dim)\n        \n        return sequence_output, pooled_output\n        \nclass TFDistilBertForNQ(TFNQModel, TFDistilBertPreTrainedModel):\n    \n    def __init__(self, config, *inputs, **kwargs):\n        \n        TFDistilBertPreTrainedModel.__init__(self, config, *inputs, **kwargs)  # explicit calls without super\n        TFNQModel.__init__(self, config)\n\n        self.backend = TFDistilBertMainLayer(config, name=\"distilbert\")\n        \n    def backend_call(self, inputs, **kwargs):\n        \n        if isinstance(inputs, tuple):\n            # Distil bert has no segment_id (i.e. `token_type_ids`)\n            inputs = inputs[:2]\n        else:\n            inputs = inputs\n        \n        outputs = self.backend(inputs, **kwargs)\n        \n        # TFDistilBertModel's output[0] is of shape (batch_size, sequence_length, hidden_size)\n        # We take only for the [CLS].\n        \n        sequence_output = outputs[0]  # shape = (batch_size, seq_len, hidden_dim)\n        pooled_output = sequence_output[:, 0, :]  # shape = (batch_size, hidden_dim)\n        \n        return sequence_output, pooled_output\n    \n    \nmodel_mapping = {\n    \"bert\": TFBertForNQ,\n    \"distilbert\": TFDistilBertForNQ\n}\n\n\ndef get_pretrained_model(model_name):\n    \n    pretrained_path = os.path.join(FLAGS.model_dir, model_name)\n    \n    tokenizer = BertTokenizer.from_pretrained(pretrained_path)\n    \n    model_type = model_name.split(\"-\")[0]\n    if model_type not in model_mapping:\n        raise ValueError(\"Model definition not found.\")\n    \n    model_class = model_mapping[model_type]\n    model = model_class.from_pretrained(pretrained_path)\n    \n    return tokenizer, model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Try Bert / DistillBert models for NQ"},{"metadata":{"trusted":true},"cell_type":"code","source":"bert_tokenizer, bert_for_nq = get_pretrained_model('bert-base-uncased')\n_, distil_bert_for_nq = get_pretrained_model('distilbert-base-uncased-distilled-squad')\n\ninput_ids = tf.constant(bert_tokenizer.encode(\"Hello, my dog is cute\"))[None, :]  # Batch size 1\ninput_masks = tf.constant(0, shape=input_ids.shape)\nsegment_ids = tf.constant(0, shape=input_ids.shape)\n\n# Actual inputs to model\ninputs = (input_ids, input_masks, segment_ids)\n\n# Outputs from bert_for_nq using backend_call()\noutputs = bert_for_nq(inputs)\n(start_pos_logits, end_pos_logits, answer_type_logits) = outputs\nprint(start_pos_logits.shape)\nprint(end_pos_logits.shape)\nprint(answer_type_logits.shape)\n# print(start_pos_binary_probs.shape)\n# print(end_pos_binary_probs.shape)\n\nlen(bert_for_nq.trainable_variables)\n\n# Outputs from distil_bert_for_nq using backend_call()\noutputs = distil_bert_for_nq(inputs)\n(start_pos_logits, end_pos_logits, answer_type_logits) = outputs\nprint(start_pos_logits.shape)\nprint(end_pos_logits.shape)\nprint(answer_type_logits.shape)\n# print(start_pos_binary_probs.shape)\n# print(end_pos_binary_probs.shape)\n\nlen(distil_bert_for_nq.trainable_variables)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Choose the model to use\n\nGet a new model, determined by `FLAGS.model_name` (for example, `bert-base-uncased`), and print its trainable variables."},{"metadata":{"trusted":true},"cell_type":"code","source":"bert_tokenizer, bert_nq = get_pretrained_model(FLAGS.model_name)\n\nif not IS_KAGGLE:\n    bert_nq.trainable_variables","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Metrics"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_metrics(name):\n\n    loss = tf.keras.metrics.Mean(name=f'{name}_loss')\n    loss_start_pos = tf.keras.metrics.Mean(name=f'{name}_loss_start_pos')\n    loss_end_pos = tf.keras.metrics.Mean(name=f'{name}_loss_end_pos')\n    loss_ans_type = tf.keras.metrics.Mean(name=f'{name}_loss_ans_type')\n    \n    acc = tf.keras.metrics.SparseCategoricalAccuracy(name=f'{name}_acc')\n    acc_start_pos = tf.keras.metrics.SparseCategoricalAccuracy(name=f'{name}_acc_start_pos')\n    acc_end_pos = tf.keras.metrics.SparseCategoricalAccuracy(name=f'{name}_acc_end_pos')\n    acc_ans_type = tf.keras.metrics.SparseCategoricalAccuracy(name=f'{name}_acc_ans_type')\n    \n    return loss, loss_start_pos, loss_end_pos, loss_ans_type, acc, acc_start_pos, acc_end_pos, acc_ans_type\n\ntrain_loss, train_loss_start_pos, train_loss_end_pos, train_loss_ans_type, train_acc, train_acc_start_pos, train_acc_end_pos, train_acc_ans_type = get_metrics(\"train\")\nvalid_loss, valid_loss_start_pos, valid_loss_end_pos, valid_loss_ans_type, valid_acc, valid_acc_start_pos, valid_acc_end_pos, valid_acc_ans_type = get_metrics(\"valid\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loss function\n\nThe loss is not averaged over a batch of training examples. The caller of `loss_function` have to take care of this. The reason of not using average here is that I need the sum of losses to do `batch accumulation`."},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='none')\n\ndef loss_function(nq_labels, nq_logits):\n    \n    (start_pos_labels, end_pos_labels, answer_type_labels) = nq_labels\n    (start_pos_logits, end_pos_logits, answer_type_logits) = nq_logits\n    \n    loss_start_pos = loss_object(start_pos_labels, start_pos_logits)\n    loss_end_pos = loss_object(end_pos_labels, end_pos_logits)\n    loss_ans_type = loss_object(answer_type_labels, answer_type_logits)\n    \n    loss_start_pos = tf.math.reduce_sum(loss_start_pos)\n    loss_end_pos = tf.math.reduce_sum(loss_end_pos)\n    loss_ans_type = tf.math.reduce_sum(loss_ans_type)\n    \n    loss = (loss_start_pos + loss_end_pos + loss_ans_type) / 3.0\n    \n    return loss, loss_start_pos, loss_end_pos, loss_ans_type","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Custom Learning Schedule\n\nA learning rate sheduler which is close to the original Bert's one: https://github.com/google-research/bert/blob/master/optimization.py#L32.\n\nIt supports the following\n* warmup stesp (used in the original Bert's optimizer)\n* cyclic learning rate"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomSchedule(tf.keras.optimizers.schedules.PolynomialDecay):\n    \n    def __init__(self,\n      initial_learning_rate,\n      decay_steps,\n      end_learning_rate=0.0001,\n      power=1.0,\n      cycle=False,\n      name=None,\n      num_warmup_steps=1000):\n        \n        # Since we have a custom __call__() method, we pass cycle=False when calling `super().__init__()` and\n        # in self.__call__(), we simply do `step = step % self.decay_steps` to have cyclic behavior.\n        super(CustomSchedule, self).__init__(initial_learning_rate, decay_steps, end_learning_rate, power, cycle=False, name=name)\n        \n        self.num_warmup_steps = num_warmup_steps\n        \n        self.cycle = tf.constant(cycle, dtype=tf.bool)\n        \n    def __call__(self, step):\n        \"\"\" `step` is actually the step index, starting at 0.\n        \"\"\"\n        \n        # For cyclic behavior\n        step = tf.cond(self.cycle and step >= self.decay_steps, lambda: step % self.decay_steps, lambda: step)\n        \n        learning_rate = super(CustomSchedule, self).__call__(step)\n\n        # Copy (including the comments) from original bert optimizer with minor change.\n        # Ref: https://github.com/google-research/bert/blob/master/optimization.py#L25\n        \n        # Implements linear warmup: if global_step < num_warmup_steps, the\n        # learning rate will be `global_step / num_warmup_steps * init_lr`.\n        if self.num_warmup_steps > 0:\n            \n            steps_int = tf.cast(step, tf.int32)\n            warmup_steps_int = tf.constant(self.num_warmup_steps, dtype=tf.int32)\n\n            steps_float = tf.cast(steps_int, tf.float32)\n            warmup_steps_float = tf.cast(warmup_steps_int, tf.float32)\n\n            # The first training step has index (`step`) 0.\n            # The original code use `steps_float / warmup_steps_float`, which gives `warmup_percent_done` being 0,\n            # and causing `learning_rate` = 0, which is undesired.\n            # For this reason, we use `(steps_float + 1) / warmup_steps_float`.\n            # At `step = warmup_steps_float - 1`, i.e , at the `warmup_steps_float`-th step, \n            #`learning_rate` is `self.initial_learning_rate`.\n            warmup_percent_done = (steps_float + 1) / warmup_steps_float\n            \n            warmup_learning_rate = self.initial_learning_rate * warmup_percent_done\n\n            is_warmup = tf.cast(steps_int < warmup_steps_int, tf.float32)\n            learning_rate = ((1.0 - is_warmup) * learning_rate + is_warmup * warmup_learning_rate)\n                        \n        return learning_rate\n    \nnum_train_steps = int(FLAGS.epochs * FLAGS.num_train_examples / FLAGS.train_batch_size / FLAGS.batch_accumulation_size)\n\nlearning_rate = CustomSchedule(\n    initial_learning_rate=FLAGS.init_learning_rate,\n    decay_steps=num_train_steps,\n    end_learning_rate=FLAGS.init_learning_rate,\n    power=1.0,\n    cycle=FLAGS.cyclic_learning_rate,    \n    num_warmup_steps=FLAGS.num_warmup_steps\n)\n\n# Normally we want to use this way to plot since it's much fast.\n# However, we get the error`The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()`.\n# This is due to some bool operations on `step`. So we use the second way (which is slow) to plot.\n# (This error also occurs even we use `tf.keras.optimizers.schedules.PolynomialDecay` with `cycle=True`.)\n\n# Get error here.\n# plt.plot(learning_rate(tf.range(num_train_steps)))\n\n# This is a workaround to plot, but it's very slow. So we plot the learning rate only when `num_train_steps` is small.\nif num_train_steps <= 5000:\n    xs = tf.range(2 * num_train_steps)\n    ys = [learning_rate(x) for x in xs]\n    plt.plot(xs, ys)\n    \n    plt.ylabel(\"Learning Rate\")\n    plt.xlabel(\"Train Step\")\n    plt.show()\n\nprint(f'num_train_steps: {num_train_steps}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get variables to apply weight decay in AdamW optimizer\n\nIn the original Bert's optimizer [AdamWeightDecayOptimizer](https://github.com/google-research/bert/blob/master/optimization.py#L87), it accepts an arguement `exclude_from_weight_decay`, and the list `[\"LayerNorm\", \"layer_norm\", \"bias\"]` is passed to it, see [here](https://github.com/google-research/bert/blob/master/optimization.py#L59).\n\nHere we use [tfa.optimizers.AdamW](https://github.com/tensorflow/addons/blob/master/tensorflow_addons/optimizers/weight_decay_optimizers.py#L340), which accepts an arguement `decay_var_list`."},{"metadata":{"trusted":true},"cell_type":"code","source":"decay_var_list = []\n\nfor i in range(len(bert_nq.trainable_variables)):\n    name = bert_nq.trainable_variables[i].name\n    if any(x in name for x in [\"LayerNorm\", \"layer_norm\", \"bias\"]):\n        decay_var_list.append(name)\n        \ndecay_var_list","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## AdamW Optimizer (Weight decay)\n\nThe `Adam` optimizer from [tf.kears.optimizers](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/keras/optimizer_v2/adam.py#L32) doesn't support weight decay. But the original Bert's [optimizer](https://github.com/google-research/bert/blob/master/optimization.py#L59) uses weight decay.\n\nThere is a `AdamW` class from Tensorflow addons, [tfa.optimizers.AdamW](https://github.com/tensorflow/addons/blob/master/tensorflow_addons/optimizers/weight_decay_optimizers.py#L340). However, it doesn't support [update_with_lr ](https://github.com/google-research/bert/blob/master/optimization.py#L149) as in the original Bert.\n\nThere is a [note](https://github.com/tensorflow/addons/blob/master/tensorflow_addons/optimizers/weight_decay_optimizers.py#L362) about how to integrate learing rate decay in `tfa.optimizers.AdamW`:\n\n> when applying a decay to the learning rate, be sure to manually apply the decay to the `weight_decay` as well ...\n\nbut I didn't make it work with schedules.\n\nTherefore, we use a modified version. The difference is: Unlike in [DecoupledWeightDecayExtension._decay_weights_op()](https://github.com/tensorflow/addons/blob/master/tensorflow_addons/optimizers/weight_decay_optimizers.py#L154), we compute `weight_decay` as `init_weight_decay * learning_rate`.\n\nThis modified version is included in `adamw_optimizer.py` in [nq-competition](https://www.kaggle.com/yihdarshieh/nq-competition) dataset.\n\n#### Note\n\nFor performance, we shouldn't re-compute `learning_rate` in modified `DecoupledWeightDecayExtension._decay_weights_op()`.\nInstead, we should prepare states to use multiple times as in [apply_state](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/keras/optimizer_v2/optimizer_v2.py#L440) in `tf.keras.optimizers.Optimizer`.\nHowever, it inovles modification of tensorflow code, which I tried to avoid."},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.optimizers import Adam\nfrom adamw_optimizer import AdamW\n\n# The hyperparameters are copied from AdamWeightDecayOptimizer in original bert code.\n# (https://github.com/google-research/bert/blob/master/optimization.py#L25)\noptimizer = AdamW(weight_decay=FLAGS.init_weight_decay_rate, learning_rate=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-6, decay_var_list=decay_var_list)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Try to load the latest checkpoint"},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint_path = os.path.join(FLAGS.input_checkpoint_dir, FLAGS.model_name)\nckpt = tf.train.Checkpoint(model=bert_nq)\nckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=10000)\n\n# if a checkpoint exists, restore the latest checkpoint.\nif ckpt_manager.latest_checkpoint:\n    ckpt.restore(ckpt_manager.latest_checkpoint)\n    last_epoch = int(ckpt_manager.latest_checkpoint.split(\"-\")[-1])\n    print (f'Latest BertNQ checkpoint restored -- Model trained for {last_epoch} epochs')\nelse:\n    print('Checkpoint not found. Train BertNQ from scratch')\n    last_epoch = 0\n    \n    \n# Reset saving path, because the FLAGS.input_checkpoint_dir is not writable on Kaggle\nprint(ckpt_manager._directory)\nckpt_manager._directory = os.path.join(FLAGS.output_checkpoint_dir, FLAGS.model_name)\nckpt_manager._checkpoint_prefix = os.path.join(ckpt_manager._directory, \"ckpt\")\nprint(ckpt_manager._directory)\n\nfrom tensorflow.python.lib.io.file_io import recursive_create_dir\nrecursive_create_dir(ckpt_manager._directory)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## tf.function and input_signature"},{"metadata":{"trusted":true},"cell_type":"code","source":"# The @tf.function trace-compiles train_step into a TF graph for faster\n# execution. The function specializes to the precise shape of the argument\n# tensors. To avoid re-tracing due to the variable sequence lengths or variable\n# batch sizes (the last batch is smaller), use input_signature to specify\n# more generic shapes.\n\ninput_signature = [\n        tf.TensorSpec(shape=(None, None), dtype=tf.int32),\n        tf.TensorSpec(shape=(None, None), dtype=tf.int32),\n        tf.TensorSpec(shape=(None, None), dtype=tf.int32),\n        tf.TensorSpec(shape=(None,), dtype=tf.int32),\n        tf.TensorSpec(shape=(None,), dtype=tf.int32),\n        tf.TensorSpec(shape=(None,), dtype=tf.int32)\n]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Simple Training Step\n\nNo batch accumulation here."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_loss_and_gradients(input_ids, input_masks, segment_ids, start_pos_labels, end_pos_labels, answer_type_labels):\n    \n    nq_inputs = (input_ids, input_masks, segment_ids)\n    nq_labels = (start_pos_labels, end_pos_labels, answer_type_labels)\n\n    with tf.GradientTape() as tape:\n\n        nq_logits = bert_nq(nq_inputs, training=True)\n        loss, loss_start_pos, loss_end_pos, loss_ans_type = loss_function(nq_labels, nq_logits)\n                \n    gradients = tape.gradient(loss, bert_nq.trainable_variables)        \n        \n    (start_pos_logits, end_pos_logits, answer_type_logits) = nq_logits\n        \n    train_acc.update_state(start_pos_labels, start_pos_logits)\n    train_acc.update_state(end_pos_labels, end_pos_logits)\n    train_acc.update_state(answer_type_labels, answer_type_logits)\n\n    train_acc_start_pos.update_state(start_pos_labels, start_pos_logits)\n    train_acc_end_pos.update_state(end_pos_labels, end_pos_logits)\n    train_acc_ans_type.update_state(answer_type_labels, answer_type_logits)\n    \n    return loss, gradients, loss_start_pos, loss_end_pos, loss_ans_type\n\n\n@tf.function(input_signature=input_signature)\ndef train_step_simple(input_ids, input_masks, segment_ids, start_pos_labels, end_pos_labels, answer_type_labels):\n\n    nb_examples = tf.math.reduce_sum(tf.cast(tf.math.not_equal(start_pos_labels, -2), tf.int32))\n    \n    loss, gradients, loss_start_pos, loss_end_pos, loss_ans_type = get_loss_and_gradients(input_ids, input_masks, segment_ids, start_pos_labels, end_pos_labels, answer_type_labels)\n    \n    average_loss = tf.math.divide(loss, tf.cast(nb_examples, tf.float32))\n    \n    # For this simple training step, it's better to use tf.math.reduce_mean() in loss_function() instead of tf.math.reduce_sum(), and not using the following line\n    # to average gradients manually.\n    \n    # Using this line causing `UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape.`.\n    average_gradients = [tf.divide(x, tf.cast(nb_examples, tf.float32)) for x in gradients]\n    \n    optimizer.apply_gradients(zip(gradients, bert_nq.trainable_variables))\n\n    average_loss_start_pos = tf.math.divide(loss_start_pos, tf.cast(nb_examples, tf.float32))\n    average_loss_end_pos = tf.math.divide(loss_end_pos, tf.cast(nb_examples, tf.float32))\n    average_loss_ans_type = tf.math.divide(loss_ans_type, tf.cast(nb_examples, tf.float32))\n    \n    train_loss(average_loss)\n    train_loss_start_pos(average_loss_start_pos)\n    train_loss_end_pos(average_loss_end_pos)\n    train_loss_ans_type(average_loss_ans_type)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training Step - Batch Accumulation\n\nMy effort to use batch accumulation.\n\nBecause of the line `[x + y for x, y in zip(total_gradients_over_batch, gradients)]`, I got \n>`UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape.`\n\nI tried avoid this warning by making sparse gradient operations as possible as I can. These are the 4 lines starting with `### `, but it doesn't work and gives errors, so they are not used.\n\n## *** Question ***\n\nI would like to see a better implementation of batch accumulation! I think the code here uses more memory and maybe also run slowly."},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function(input_signature=input_signature)\ndef train_step_with_batch_accumulation(input_ids, input_masks, segment_ids, start_pos_labels, end_pos_labels, answer_type_labels):\n\n    # This gets None! (probably due to input_signature)\n    # batch_size = input_ids.shape[0]\n    \n    # Try this.\n    nb_examples = tf.math.reduce_sum(tf.cast(tf.math.not_equal(start_pos_labels, -2), tf.int32))\n\n    total_loss = 0.0\n    total_loss_start_pos = 0.0\n    total_loss_end_pos = 0.0\n    total_loss_ans_type = 0.0\n    \n    total_gradients = [tf.constant(0, shape=x.shape, dtype=tf.float32) for x in bert_nq.trainable_variables]        \n    ### total_gradients_sparse = [tf.IndexedSlices(values=tf.constant(0.0, shape=[1] + x.shape.as_list()[1:]), indices=tf.constant([0], dtype=tf.int32), dense_shape=x.shape.as_list()) for x in bert_nq.trainable_variables]        \n\n    for idx in tf.range(FLAGS.batch_accumulation_size):    \n                \n        start_idx = FLAGS.train_batch_size * idx\n        end_idx = FLAGS.train_batch_size * (idx + 1)\n        \n        if start_idx >= nb_examples:\n            break\n\n        (input_ids_mini, input_masks_mini, segment_ids_mini) = (input_ids[start_idx:end_idx], input_masks[start_idx:end_idx], segment_ids[start_idx:end_idx])\n        (start_pos_labels_mini, end_pos_labels_mini, answer_type_labels_mini) = (start_pos_labels[start_idx:end_idx], end_pos_labels[start_idx:end_idx], answer_type_labels[start_idx:end_idx])\n        \n        loss, gradients, loss_start_pos, loss_end_pos, loss_ans_type = get_loss_and_gradients(input_ids_mini, input_masks_mini, segment_ids_mini, start_pos_labels_mini, end_pos_labels_mini, answer_type_labels_mini)\n        \n        total_loss += loss\n        total_loss_start_pos += loss_start_pos\n        total_loss_end_pos += loss_end_pos\n        total_loss_ans_type += loss_ans_type\n        \n        total_gradients = [x + y for x, y in zip(total_gradients, gradients)]        \n        ### total_gradients_sparse = [_add_grads_for_var(x, y) for x, y in zip(total_gradients_sparse, gradients)]\n\n    average_loss = tf.math.divide(total_loss, tf.cast(nb_examples, tf.float32))        \n    average_gradients = [tf.divide(x, tf.cast(nb_examples, tf.float32)) for x in total_gradients]\n    ### average_gradients_sparse = [tf.scalar_mul(tf.divide(1.0, tf.cast(nb_examples, tf.float32)), x) for x in total_gradients_sparse]\n    \n    optimizer.apply_gradients(zip(average_gradients, bert_nq.trainable_variables))\n    ### optimizer.apply_gradients(zip(average_gradients_sparse, bert_nq.trainable_variables))\n\n    average_loss_start_pos = tf.math.divide(total_loss_start_pos, tf.cast(nb_examples, tf.float32))\n    average_loss_end_pos = tf.math.divide(total_loss_end_pos, tf.cast(nb_examples, tf.float32))\n    average_loss_ans_type = tf.math.divide(total_loss_ans_type, tf.cast(nb_examples, tf.float32))    \n    \n    train_loss(average_loss)\n    train_loss_start_pos(average_loss_start_pos)\n    train_loss_end_pos(average_loss_end_pos)\n    train_loss_ans_type(average_loss_ans_type)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training\n\nFinally, we can sit here and look the progress!"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_step = train_step_simple\nif FLAGS.batch_accumulation_size > 1:\n    train_step = train_step_with_batch_accumulation\n\n# Because the previous checkpoint manager (for loading model) is created with a path which is read-only, ckpt_manager.save() fails here.\n# So we create another checkpoint manager for saving checkpoints.\n# checkpoint_path = os.path.join(FLAGS.output_checkpoint_dir, FLAGS.model_name)\n# ckpt = tf.train.Checkpoint(model=bert_nq, optimizer=optimizer)\n# ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=3)\n\ntrain_start_time = datetime.datetime.now()\n\nepochs = FLAGS.epochs\nfor epoch in range(epochs):\n\n    train_dataset = get_dataset(\n        FLAGS.train_tf_record,\n        FLAGS.max_seq_length_for_training,\n        FLAGS.batch_accumulation_size * FLAGS.train_batch_size,\n        FLAGS.shuffle_buffer_size,\n        is_training=True\n    )     \n    \n    train_loss.reset_states()\n    train_loss_start_pos.reset_states()\n    train_loss_end_pos.reset_states()\n    train_loss_ans_type.reset_states()    \n    \n    train_acc.reset_states()\n    train_acc_start_pos.reset_states()\n    train_acc_end_pos.reset_states()\n    train_acc_ans_type.reset_states()\n    \n    epoch_start_time = datetime.datetime.now()\n    \n    for (batch_idx, (features, targets)) in enumerate(train_dataset):\n        \n        # If the training is on kaggle, we stop the training after 8h45m, so we can get the checkpoints!\n        if IS_KAGGLE and (datetime.datetime.now() - train_start_time).total_seconds() > 31500:\n            break\n        \n        (input_ids, input_masks, segment_ids) = (features['input_ids'], features['input_mask'], features['segment_ids'])\n        (start_pos_labels, end_pos_labels, answer_type_labels) = (targets['start_positions'], targets['end_positions'], targets['answer_types'])\n    \n        batch_start_time = datetime.datetime.now()\n        \n        train_step(input_ids, input_masks, segment_ids, start_pos_labels, end_pos_labels, answer_type_labels)\n\n        batch_end_time = datetime.datetime.now()\n        batch_elapsed_time = (batch_end_time - batch_start_time).total_seconds()\n        \n        if (batch_idx + 1) % 100 == 0:\n            print('Epoch {} | Batch {} | Elapsed Time {}'.format(\n                epoch + 1,\n                batch_idx + 1,\n                batch_elapsed_time\n            ))\n            print('Loss {:.6f} | Loss_S {:.6f} | Loss_E {:.6f} | Loss_T {:.6f}'.format(\n                train_loss.result(),\n                train_loss_start_pos.result(),\n                train_loss_end_pos.result(),\n                train_loss_ans_type.result()\n            ))\n            print(' Acc {:.6f} |  Acc_S {:.6f} |  Acc_E {:.6f} |  Acc_T {:.6f}'.format(\n                train_acc.result(),\n                train_acc_start_pos.result(),\n                train_acc_end_pos.result(),\n                train_acc_ans_type.result()\n            ))\n            print(\"-\" * 100)\n       \n    epoch_end_time = datetime.datetime.now()\n    epoch_elapsed_time = (epoch_end_time - epoch_start_time).total_seconds()\n            \n    if (epoch + 1) % 1 == 0:\n        \n        ckpt_save_path = ckpt_manager.save()\n        print ('\\nSaving checkpoint for epoch {} at {}'.format(epoch+1, ckpt_save_path))\n        \n        print('\\nEpoch {}'.format(epoch + 1))\n        print('Loss {:.6f} | Loss_S {:.6f} | Loss_E {:.6f} | Loss_T {:.6f}'.format(\n            train_loss.result(),\n            train_loss_start_pos.result(),\n            train_loss_end_pos.result(),\n            train_loss_ans_type.result()\n        ))\n        print(' Acc {:.6f} |  Acc_S {:.6f} |  Acc_E {:.6f} |  Acc_T {:.6f}'.format(\n            train_acc.result(),\n            train_acc_start_pos.result(),\n            train_acc_end_pos.result(),\n            train_acc_ans_type.result()\n        ))\n\n    print('\\nTime taken for 1 epoch: {} secs\\n'.format(epoch_elapsed_time))\n    print(\"-\" * 80 + \"\\n\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check saved checkpoints"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"{ckpt_manager._directory}\")\nos.system(f\"ls -l {ckpt_manager._directory} > results.txt\")\nwith open(\"results.txt\", \"r\", encoding=\"UTF-8\") as fp:\n    for line in fp:\n        print(line.strip())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(f\"{ckpt_manager._directory}/checkpoint\", \"r\", encoding=\"UTF-8\") as fp:\n    for line in fp:\n        print(line.strip())","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}