{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Use Hugging Face's Tensorflow 2 transformer models for NQ - Inference\n\n![logo.png](attachment:logo.png)","attachments":{"logo.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAArwAAACvCAYAAAAfZw1fAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAAFiUAABYlAUlSJPAAAKK+SURBVHhe7Z0HgBPF/se/uUtyud4bHE06ioqooKKI2Hvv9Vn/9vbsPnt56rN3n72XJ/bewI5iwYKA9HIc1/ul3f1/39kNhJDNJbnjGvOBvexudmdnZyaz3/ntb2ZsbQI0Go1Go9FoNJo+SoL5qdFoNBqNRqPR9Em04NVoNBqNRqPpAXj8sribza118bhb4G01NzQxowWvRqPRaDQaTTciOheoW4rmz/+FOb/9rPaFMvf3X9D8xfWoK18GrXtjRwtejUbTLn//8hXuuu9hte5pqEZFVZ1a12g0Gk38sBNVVVU1Gn96AW1vHo3ML25AVYPH+DKEyrpmZHx+DdxvHoum2S+jqrpWC98Y0IJXo9FE5KOnb8FZt7+KCWP6oWDIVlg2+3O8+PoX5rcajUajiQfK2orvnkDSG/vC9cmpsC39GnCIMEtINA4IwZZoB5xAfuUMOD8+GUlv7YOqmc+ocDTto0dp0Gg0ERm/+RaYNftXtb5o/lxUL/oZf3s2wY/TbsbcylY0uQbhuasOxuHn/RtV9V789uWL2G7KIagsr8D3f81Bdvi6W6PRaDZaqmrqkf31ufDOfg7OVp9hfrTJ4gamT5mByTvtqI4LZvqMLzF5+k6ASzao3GTxJDrg3Px4rNr6PhTlJKvjNOHRFl6NRhORxNQMcw0YMnyk1LVetDQ344Jr78BV/7wAi/6chTm/zEL+8K3w0vOPo/T3H9DozMPzL7+ATC12NRqNZg1VfuCvt29DzsMZsP36FJxtInbt8kVA8EYDj+NCg2+bF/jlcRQ9nYK579+FOu3jYIm28EZLawvQMB+onyvLPKBxEeCrNxav+Rm8TRzpUiCDluDt1CFA+ghZRgJpw6Wws8mm0fQ8dhq/OWbMmq3Wb7jgFGw3dS80OYtw6yWn4ZZHXsB1V/4Tn33yEX77YQYOPfhIvPXLQiRV/I3TjjsUJ93xJo7aScq4RqPRbMQ0iy5N/P42OD+/VHSC7HDKEjA5BpseRWq0a+GlIZfKLXBeQMXJNVpkffn4OzFotwvg0CbNddCCNxxNS4HyL4Canwxx2yAil/s2VFLZpKmWMlCErwgDiuCscUD+FGOfRtPNLP3pA2x/2D9x4M6bYXZtPp47bxK+qsjFHZeegv0OPxTX3/48Pn3hKtzy4ky0rJiP+++5HkeddROGpbfgkoffwqQReWZIGo1Gs3HhF9ngWfUH/NOvR9q8VwyRyjdflBNcD3wGiFfwBtZNC2/j8KPg2/pqpBSP1sLXRAte4i4zBO7qz+TzcxG4C80vupm0TQzhy6VAlqRC8wuNpqvxYdGyCgwZUGRuG5RV1qEw13B5qC5bjozCElWXextr0GRLQ2YK39VpNBrNxgU7krUs/RWJf7+F1F/vAWoq5RkuOwNuCxtK8BJu+4GKxALYtz4PicP2h7P/ZuryGzMbr+Ct/QVY+jyw6gOgbo65s4eTMRoo3AMYeIxhBdZoNBqNRtNjoIG1ZvlcuOa9hJafnkVO3QI18oISpKGilNuhYrWzBC+3GRkfUJ0+As6xx8Iz9Chk9RsWtatwX2PjErwtK0XkviDLcyJ4fzd39lIyNgUGHSfi92jA1c/cqdFoNBqNpjugVbftjzfgmX4D0st+MlwXKDy5WInS0P2dKXgJ17mI+G3I3xr27a9B24h9VRAbG8FJ1DfxNxuW3K/2At4bDPx2We8Xu6TuD+NeeE9f7WncI+9Vo+lkHnvwQXNtfZ7473/NtfV5/PEnzTWNRqPpu1DoLp49HY3PHgjnu8cjvULELq26PWWUGsbDDqRV/Yikj4+B+5VDsOSPrza6SSv6roXXVwf8/YAs9wDuCnNnB7FLqXFKKXaan4nSXkiwGZ/seJYQtE38UpxaJXlb5ZPJHNjmp8crCyfNlk+fmlSw4yTlAsPOA4aeJT+2THOnRtMxJm+9Nab/+KO5Bcyf/T0WVduw++Rtsd8+e+Gmm2/A8LFbI1mK/apFf2JBTSJ2GDcSu0zZHZ99/pF5lkaj0fQ9yhv8yH//SPj/eE1ZEG38o1b4rUCFxe3AZwCr/RvCwhu8Lgs/GM8FOUcg45CXkJ+qvu3zBCdR38BTCfxxNfDeEOMzXrGbJII2Ow3olwMMKhAh2Q/YpAgoEVFZkAVkSQlJl1KX6gJcTuN4hwjhgNglXOc+fsdjeCzP4bkMg2ExTIbNa/BavCaPjwc37/1fwPu896skLTpJ6Gs2apKT1778mv/lK7jknmmY8cLNePnrpWha9AvenzEL48dPgrd8Lo69+E589vRNeOP7ucjN1o0ujUbT9wiYqBZ/9wbyb7ADv76GRBG4NlpSKXQDYrenYcZNiXJRvUMrXkb+vTYs+el9JYI7yfTWY+k7grelFJh9sSH2/roF8NaaX0SJUwothWixiM6hxYYAzZcHdpo87ClAabndUDBsXoPX4jWVwJY4UAAzToxbLHjrJA1ulbTYRNLkIsN3WaPpBIbveDi2LPTiu9kLULlqJQZvvRsuPed0nLDLWNx+27/RWLMav/1divfefg9Jzo29T7BGo+lLKFHYXIdlXzyFpocnYPA7BwEpsjMw+sIGlAmdSiCulBYS/0Gf7o2FD26Hld8+C7+7oc+6OvR+wdvqBuZcB3wwDJh/N+BrMr+IAorMPBGYtLIOLjSsrrTABltpuwu6R1AAM06MG+NIMRyL9ZdpMf8eSZvhwJ/XGmml0cSIz7+23X/nxUdg5D7n4cGbL0JjUzPKli1R+xcsX4msjExceffTeOWdj3HJ/x2BmjpzAhaNRqPp5XhEBdbOnQHf68dj8GcnIWXlTKwzeURvhcJX7mOo+zsM+P54eN4/HjULvoK3D6re3p1Vq94HPh4rYu4GaXZFKeYoZuk2QCsqlxxZp29uT4dxDI4316MV5kybOTcaabXqPXOnRhMd40YWYupue2Cnybthh8POwrWn7I0bHn0PuZkOTNxyKLafsDVcY/bAmVffhX+ftBNGjhyNVkc6Nt10tBmCRqPR9E7YIa184Wy0fXw5El49BElz3jSUE21PvcWi2x68D96TLMkLp8H+wSHAN1diyfzf1KRwfYXe2WmteRnw64XAimnmjihIcYlITDX8aPsSjS1ATaPxGS39DwS2uEtK9gBzh0aj0Wg0mmAqlsxBxs93ofnv6cjkrKtUS3QD4GeovSmwL/S7WPdvyE5rJLAdaT8REbzaMQLZw6egbtQFyO3f+6eID77dnk+bH5h3O/DRZtGLXboFDMo3Ooj1NbFLeE/95d54j7zXaFjxhqThpsDc24w01Wg0Go1Go6D5yP3rK8j4375w/vgYMutE7AasoBsD5n0WeObB8dsjcH6wL1b+9L9eb+3tPdnXvJzNHuC3ywFfo7kzAvTF5at/dvxKoqNNH4f3GBhRIiMK4Uv/3t+vMNKUaavRaDQazUYMzT8Lf5mO5ge2R9K0I+CsXmhYdAOjL2xMBAS+3Ht6498o/vZQND63AxbM/qrXCt/eIXjpd/rpeKDyG3NHBGjlZCcvjrYQ7/BevRnec5Hc+xBJg2gsvkzTT7aSNH7X3KHRaDQazcYD3+JXly7G8runYJPndkb2ym+hhiqgQtrYhG44RPTaJJEyq7/B0I93xIqndkN95fJeN5pDzxa8fN3+26XANwcYY8xGgkN38dU+rZyxDuPVF3FIGjAtmCbtpYenStL4QEnrSyTN+5KLukaj0Wg04XHL485dX4nV3zyH7LuHYNDqL4xhxmgr6z3vv7sOWrpdwKC6T5D+9AAsnfkyPI1V8PQS2dBzs5Qd06bvDMz7j4iwgBd1GNj6ykmXHCjsmz66HYVpwrTJlTSK1FJlGs+700jzpqXmTo0mOqTJhAN22MHY0Gg0mh4MLZO+ujLUfXYHKh7eE4VvHmcMMbYRvhSOGeoI2tAkvQZ/dyQqXtoLFd/ciea68jX93XoqPVPwVn0PfLo1UPmtucMCjrxAMZeXoV87RIJpkytpRFcPplkkKr8z0r5KPjWaKFjk8+FGmw1vfvMNTs7KQoxTvmg0Gk2XwfF0a35+C22vHYv86f9E/+ofDbGrLbqxYQrffs0zkffzRfB+dCzq5r7bo/17e14Wc2zdL3eL7MLAhOYkDBx5QbsvRA/dHJhmTLtIDQS6OHy5ux6zV9Muv1RX4QmHA3eqLRser63FFQkJ0HP7aTSankatG0h851wkv3k8HPM+MRSQlhDxQx2RKDLMAWSUfQTXjGNR8f4FqOPgxT2QniV4lz4HfHuQMYKAFZyAoSTfmHhBEx9MuwGShg465FjAPPj2YMmTZ80dGs26fLl4ET7PycUNaivQgrLhgbY23G+zYV5jFKOpaDQazQamqhlY9uG9SHtoUyTOvA/JnlpD6EYy/GhiQ9RkkrcGOX/fjbTXN8PyL+9DRQ97BPSciSfm32l0UIsUHY46UJQtCatLaafQKmm9qhpokNrAChEu2OxWYMRF5g7NxgSrh/LyciRwqmsTp9OJz2bNwrJddsE5ak+432Mbrpa/e/76KzYdOBA+n+4MqYkev9+PwsJCc0ujiQ/WOst+noGij05CUvVCw8IXELqUGoFqLZp1EtgO3U+svot1f0+YeCKW/SR0XRb6SbekDEPFhGdQMnq7dU7tLnqG4KXQZec0K1g48zK1VXdDUd0AVEiLN1JJGHEhMPY2c0OzsTBq1CjMnTvX3AqPUWzWit4FsmeYua7RxMvdd9+N8847z9zSaKLD1FtoWDEHmHY6MhZ+afjoBl5oWom09tZJYDt0P7H6Ltb9fUHwBo6n6pWltnBnJO78KFLyh6snRTgTSVfQ/YL3t3+K2L3L3AgDLYwcXkuPwLBh4dTEpVWG1deK4fLw2TxCw0TT5xhYUoxLbn0B2+04BR729jBhheVwJuDdn2bi/IMnICOoCvtFarqfvluFsQWF8PtaVb2n0URLekYCzjruMOwxeQtcceVV5l6Npn3cUtkkLJuFRV88if6/P4TUNqmzgocYY2UU7zoJbIfuJ1bfxbq/rwhewnU+GvzSAGlLRGn//8OQHU6GL2dLuCJ4VG4ogqPf9cy7I7LYTZToDcjTYrcrYBqXSFoHvbpej/n3SH7dbm5oNhbYJm6V54bf71uz+GRplUqsTb5gnRYMt7nf729TxwWfpxe9tLewrLXJP9o6NJpoaaytRs2n98Fx39YYMfsBpCZKQaJlV9O98HdsB9LsfgxfcT/sr45DxfcPobmxzvi+C+k+wcvOUL9dZm6EgZ3TKHZdusR2GUzrge10ZuPUzkueNjc0GwMUr0qEUNwGLdzXKn84Trspc9VC4wP3t4oiDj1HL3ppf5FS1AM87TS9g9raOjT+8Bw8Lx2Pwo/ONVRNkizda87ThGIKX7qWlPx6JryfnoCGP19CZU3X9WzrniLB4a5mnWpuhIFDjVF4cawLTdfCtOcIDpGGe/vpdD0VsUZVYCmiTm6W1QdkuU+Wh2S5X5YUZZ7TJjqNRrNhYIe02u+fg+OFA2F//WRk//2O4aerhW7Phvkj+ZSx8g0kfnUSkr44CHV/vESvhw1O1/vwVn0LzNgd8FuMDBAQXHRn0HQf/lZgWTks5wxMTAZ2/BDI3d7coemL0If34puexYRJu8Lv88HusMNPVwapNVSDXX6uTVIU2oK0baJ8l9REFwjD5kvty+M8bo+y/BKbLQHOJOPtTSyymOHx2h6P21jpZBIT7eoeY5XqjIlffjM+rzEApU1u2uFM6pTX8q2tbfDyfgWG65RwGcF40o3hdFaV35G48A/zMFxcMjJdOPvEQ7D3lHG4/Artw6tZnzp5LDU+fQwK/n4ViT6vIaK4sCCySAXLh+DtjqyTwHbofmL1Xaz7+5oPb6RzmF/ySPDbnSgvOAbpez+B1AgvmDtK1wrepsXAp9saExuEg6/SKXbpzqDpfryiWCh6fRZtL2c2sMtMIHWIuUPT1wgI3u0n74qVy1biyQcuRlFx6jp9Gyl8QgmuVmy2Nqxc0YRzL38EyclJqp5r89vw6nMPoLG+Juz5VrS2taKweDAOOPwk+HydJ96I3eFE6bJleG/aUxLHNqmLo4sXY0D3jW222w3jt9sRXhG9CQl2vPHS46gqXwlbJL/4SEjADHf46K0wZc/9RUzLg10aCq8//yhqqlZJukUfbpukW3ZuMQ455nQR5p4OpxvzzCb3+PoLEpfK+OJy8NGnyf1514uLFrwaK1Y3i4z46GYMnnUTWpuajC4ngaIX+GRxCi6OwdsdWSeB7dD9xOq7WPdvTIKX6/yUhc+UhOR0LBl0DZK3vggFqTyoc+k6wdsmlfUXklFVP5g7QmDJVW4MEV6la7oeWngpemnxDUf21sCUL6UkafeTvggF7z9vfg7b7TQVt119Iu554gq43RZlwQLq2cTENpx/yp24+NrHlMXypssOw+U3noys7BzZjj68xMRELJw/H88+9jUuuPoBEYGG5bOj0LK7YtlSvPnyZbjs+gslTtwbfdVot9vxv+dfk5/JOOx76JFyfyfhjAv2Rr8BA0RgxvuyzgaHhPvNjK/w00wP/nH2Zbjp0hNwzqUHorBfv5jSjeMor1q5Eg/c/iYuvfEpeD3yVI0bmzQOknDrlcfj3EsPiiMuiRKX5bj/39NwmTSmvN5146IFryYU/oJKv3wB+W8djyS3bAX76AZEU6iQCmD1XazrJLAdup9YfRfr/o1N8BKu077AT5EcHmcSqrZ+EYVbHMRvozQ9tE9wVDYsHGvXUuzK7ehpgnsmzJN+kjdWk31wHvLZl5gbmr4IxdLq0goMH52PlhY/GhuaYloa6pvAeSf6D0hGU2MzZs74BEf/Y1f0KxkkYTtFLLqiXmzSsNpi620wZmw6VixZjAQRwJ0BBe+bLz+EW+67TtZ5raT1rh1pEcmLU887HYsXzsCfs//E9jsPwqjNNpewkuB0psS5JMOW4MA+Bx8Ah2MFvv/yK+yy1ygMHTkm5nTj8cPkvB2nDsP8P39X9xsvbHTM/eNXTN5tWJxxcUhcNsXk3UfiLwmnI3HR9F2ofZpbWtD413S03LMjSl47BkltInbZS5Y/+85SQZqeAfOTitQhsqPNjaLvD0bla7uhZtE3cLvdqjx0lK4RvCvfBObfa26EwJssztGjMfRkkiVvmEdWFczf90kev2FuaPoarGgaG+tEXCZEfBUeyTWBp/Fbd0sTli9fiOL+/eFxG6+zY108LR7kF+ajrHS5hGl9zVjwiiJPsHnUUGu0Voa7bntLS7MbGekuzPvjFwwcPEgqacN1oKMLw83JzcQfs2ehZNAA5Qsd7rj2Fo/Hg6LiApSWLouYV+3Bc5cunIcBgwbFHxc5r7h/P6xctqhDcdH0Terld9j2+xuofO402B6bitTFX+mRFzYmmM/SqMmr+gSuT3ZG1cf/Byx/Fw1e4+t42fDFp2kJMOtkcyMM+Vl6nN3eAPOIeWXFrFMMH21Nn4Oikh2yIjWx0zPS0dLUjDT5jCB7RVD6laCkL+562BKQnJKMlNRk+YxcJ9AP1N/q65RWvyguNRyWuscwOF0uFacUiZu9nc60fgnH42XnvHVj5kxKMsKQpb0w1kfSX8JlhzjRiuvhCA7b7P9Av+Fw12mTeDEPOpJuFKzGeLkh6SX5xzRiPJKT180/R+iIO1JIVDlQgzl3Si5q+gj1NRVwv301Ep48CCW/PYsUm5QR2sN0u2jjgvkt1VlyohfFK5+E7eN94fn+OtXvI142rOBt8wHfHwV4LCKYngxkbQDPZM2GgXnFPAsH85h5zTzX9D0iiJL0zAxce9pZuO2Ge3DovmciKdV6CnAVjPoT8vQSsdTWVI27brwXj9//FO687cl2RC8VU+cIJcakTUnA9cOj2H3n6efw8D1P4Z6b78WcRRURBKthb6YYD747it0PXnwJD9/9JO69+T78Pm9VTKI3EBbFaigUu19Mm4YH73oC9916P2b9uhhOEZcNq5fj1zkrw18nXGOjo0j+2ZqqcMcNRv7de+fTSEgy8o9i94uPvlxf9JLOyUJNH6BBluYfX4L9mYOQN/1Go+DTqhv8Y9JsfDD/WXVItZXz17VI+OIgtCx4HU3SDoqVDSt4/7oFqJppboTAERkKs80NTa+BeWbla00f7b9uMjc0fYvwyiQh0YHVv3+NmpxxeOyJe3DFxUdgdVU9jaZhUcIynN6iSExKwxHH7IsP3/8aRxy9j3J5iARj1HnPQkPyhkKr6g577YHFs77CuN32w8CiDGVtjURoOBxdYeJuu6J09iyMnrwrBpdktxvG+vBO14+hX8IeN3ky3Ev+RNEWO2Dk0EIJG0j01+LX35cj0b5+Fb+uHI8DFY2Q1KeITkrHkcfviw/e/hqHHrmXtH2N/KPV+Zcff1tjfQ5g3FFb2OKg2Xigbln2+3fwPXEskl4/GckLvzKUyYZVJ5rehlkmkld/Aee3J8D76YlYMvdnNR5ztGy4ItW4EJj7b3MjDP1y5Oqd97jSdBHMM/rzWjH3Nsn7BeaGpu8Q/rdKv96Vi5di6KjhqKitwDa7TkVuCn19zQPCYfWzT7BjkxFDkJuXg6HDBnZgdIPOo1XUY9HAEhQU5WLQsMFITXZEvrcw8NV9QUkJiiSMgUMHIz01KeYwrGDYecVF6Fecj5JNBiIzPRl+EZrOvFE44ciJcLu77o1Lm+TfsJFDkJOXi02GDUCbmX/NTS0497Iz1Gc49FNg46VJlrJ790fJ49sh87fnkeCWPbSn6EKhsUJUa4KnARkrnsbAr7dC+ZuHwxj9vH02nOD95TxpulkMfVOQBZiDzmt6IUkOIw/D4XdL3p9rbmj6Oj6fD2MmTMRX77yL/pmDcfUxJ2BehQ+JcTVm20Ss+ZV/rxK7Nhsckaa5DsKR5ILTYUey6g9gQ0on9gugby+FpU/ilsCJKUTkc3KKWKDfrL+Vvq8Sht2uXu/TX5lWz+TUFDU+MV0fuM/pdEZ934SinBbjVpV2bSrshEQbvFaTxgSR6HBJVeyEK8Wl2rJG+sVLm0oj5eMrny65J5LkcqptjYYELPqlX72KlHNt6Df/bfVGSA3lzEWLXU0kWD5YXswyU1z+KpxP2vD7Dx+qryPZEjaM4F05DVj1vrkRAitU7bfb+2Eepln4866Sgrfif+aGpm9gUY20+mHLHoh/HDgaO++0K7K22RtjB6QpcRcfbWiob1S1WYKvCd9//1e74o9i98uXbsYXPy3BLWcfhNycJFx82lGdKnqbGyROCXYsnf0DFpW34M/vf45Z9LY0NkoFbcf8md/hh69n4cHbHsKiZWV47Lb78eR/p+G7Tz7Dfbc9gu++/BZffjsvJtHrbmqCV4QvRfMPX3yNNkcYf9kQKHbnfPYY3vjwazz+r6NR1ebCjWcc1GHRW1/XIGLXheceewXZuZl47bm3lOjVaLzS8Cn98X2U3jQRxc8dbigQPfqCJh5M4av8e4XNft0Tq17eEWV/fQwf/brC0PnFzN8E/HqhuRECTQhWlkFN76Mg08jTcMy+SMqCiARNn4dCbsoRx+HlD5/FuWfsh4YGi2nDA0TQwvVNftz76I1oafHg5y+n44efFqrJFyLhdbdgxyOvwM5bDcLl909DZZUbdzz6IpoaOzK5wlqaGxpw9k03YkC2DasXzcdd/3kBy+YviEnwNksanXLtVRhW4MKiuUtQVVWO5KJBcLXVY1WlF1N23x5//rkYp5x6AH6a/SeW/70MnMwiGih2D77gIozfROpWdw0ee/A12F1J7RrK/N4WjN7lVBy4xw44+foXkGNrwVUPT5O4xp9u9Q1+3PfkjfB4/fjslReweOlyvP7Wd3BFeKPXSd4dml5A6eJ5SHrzbBSXfm+Mp6uFrqYzMIVvUeNXcPzwf1i5dL6xP4TOL25zbgCalpkbIeSkG53VNH0DdkJhnoajaTnw5/XmhqbXw174Vj3RBHdzM+pq69HUZD3zGc9X4xhEUGJ8HZ6SlgJPixs77D4VyfZWS0HEYALfUfR6vD5TrLXFIXatI8U4uZJTpLJshcdnwyF7bYFX35mlXCjWI3B7IcExjKTkZGkfcspiIKNkGPbYYTheffkDpKalITU1Bb6Garz3yWycc95R8Mq9rE8g8HVh2BxNIlHC9juzsceuW6m0CA9zYG2KUvS6PR60NLWoQS+iErsSB7lkuKiokSRS05LhlgbLzgfsihsvuAuTd9lS7iecSwNDMEa20GwcJNhFlbgy1XBTOuM1nQrLkyjaNkeG1LPhdWbnCt7GRcD8u82NENiz30ocaXov2ZKnVqM2/H2P0XlR06thT3q+XueMaY4oXpWHw5nkQKOI4cREkY1trdJWkgrJ4oFnDL/VhmYPsMWWw9cXf3R34JuFTnxgMiiOxRs6kkAAikr6KxcPG4ltdpmMY46arKzQwRjtAQpaM34hMAzeGSdd6F+UjpdeeB97HXKQVItteG/ah9h8511x6JF7I7HNC48naIQKCYozr9skTBXq+kGrsIlPxOvwzTeFTeJq7FkX+rx1ykQPjFOi/AkXF8k/r8eNzSZMxmEn7oUJ40eJAF9/xA11fthYajQaTQewqOI6V/DOuw1otRhKSLsy9E1YsKzytlWESqSROjS9hozMLPwyawWWLFqIvIJsZOVkRL3kF+bg5x9moXSZD06XE8NGjcXH73yCnNxsY6ICiyUpNRM7Th6nOnEF78/JysQ3039CQXEJlZ4Zw/hR1kqK6MQMLF20WO41fZ3rBRaX04GRE7dHttOL/Y49VIn2wHepaSki6u34+69VGD56DD59fwZycrJVJ7TgxSkNhm2mTgT8Dlx7+xXYcuxwnHnFuTjj3OMxZeq2UiG3YeWiCuy4x3aqccFJHDLS0/Drj/Mxcsxm+PyDLyXcrHXiFbxwDN4JU7ZHmnyGfsfzvvpsJkoGbiL3HP9gYOzAN2TYKHzy3ucqTN5X6LV4nxMnjcfu++yKCTtuq7aDv+d5n3/4FQYPHdmhuGg0Gk202NoCpoGO0rISeH+Y1IZhBojgZAWRhrLS9H5WVQF1YXw3E5zAnvOB5P7mDk1vYmBJMf558/PYcusd8NtP3+DFJ+6Bx11LhWgesS7GC/N1qxQKmozMYhx72kUilMagproCj91zHfr3b0VR/yIloKKFw6DN/f1vuFJG44gTz0J6ZrayzHYcG5YsnIsHb78QO00dqzp/xVI10vr66XtfY//DL8aIMZtLOt0rwm613O8gub91w+EoDTY1EoVFxwoR0jQQc8QDWsS/++onjNpsX0yYtAteeeZBpKdXYdDQgeDoDNFCi/PihUvh8ZTghNMvRpo0YOJNt4SEBFSVl+GpR25HWlolBm8icYklD+X8xQuWwO0uxvESl4ys3HVEb0amC2efeAj2njIOl19xlblX0xdYvlh+u88fjrzVPxvDj/GnETC7xbpOAtvRHEM6sk4C26H7idV3se5vAaZPmYHJO+1o7ljL9BlfYvL0neR5KhvB51mFTay+66z9JJp1Eus5Vudb7ZdqpMI1Di3bvYKSQaJHQ+g8wTv7QmD+veZGCJsUGf6e3YDPb8PMP1Px52IXVpY71JAoWal+DB/gxsRNG5Gb2f7QPT2B2oZEfPt7KuYtd6GqTh6Isq8o14vRg1swYUwjnI7Oyca4oY/eolXmRgjDzga2sHB10fRoKHgvvulZTNxxV1SUlWJ12XI01tetJ+KIIXb5CwsSw3IYJz9Iz8hGQfEApKVnoqWlGUsXzcXcP35BbXVVbK/Y5dD8gmKM3HRL9BuwiQjCzqlXGIeG+losmj8H8/78VeLYYqXp14NVaGJCIgYMGabEblZOAZYtnifC/BdUVZabR8WD4R5RXDJY7ndz5OQXS7jzMf/P2aiuqogt3YTcvEKM2WK8hDekQ+nG63o8HixbNE+lFe8x1rjk5BZg9Nit0H8Q85BuMmvLkxa8fRcteLnDxGq/Frztn2+1v0sEr6cCeH8TUZccRjqEzJRumVGNQve5D3Pw2hfZqK4PX7nbE9uw4xYN+L+DytE/L/KsTt1FebUdD7+Rj89+SlcdZsKRkdKKA3eqwYl7VSDJ2fHsjJvVNUBNmJEZEuXXuddCICnf3KHpLQQE74RJuyq/TPqxcmKDdUStCX1zG0U0ZmTmBFlH5VPEEEVNIhu9sp/iyC2CsqG+Bl6vx6isokHOa26qR2paphJMdgcngejM8s7wG1BfV63Eb5pcJ7rw5Z4SE+BypSpBz9EVAvfn8bpVJRwPtJTTGupMSpYGQ5ZycWBjoaEutnRjOEzzJAmH8XM4nR1PNwnP63ajXuXh+vfY2upXYw7zWuvGUzYYF6cLabynMHHRgrfvogUvd5hY7deCt/3zrfZ3ieD940rgLwtfzSGFQIzjVXaUpWVOXPlofyxcGd3Yj8lJbTj/sDLsu0Otuadn8OmsdNz2fCEamqOzxpSIaL/xtJUYPiDWHuqdRCQr78hLgM1uNjc0vYVgwetxG+UqnDXPZktQQrGpsR5F/fgaf91e+aHVTHAY/CYa+2BCQiJKly9GUnIKsnPy17tGZ8Br1NVUobGhHsUlgyTe7atVdWtyA7yHwH3Gc3+hdFa4PDVwmgrFDKujWMWFZaGutkoJXuZTcBquPc50fQkTFy14+y5a8HKHidV+LXjbP99qfzuCN/iU+PCKSFzwkLkRAn13u1jsLlmVhLPvHBC12CXNbhtuea5IWYN7Ch98l4FrH+8XtdglyyscOPeeAZi7tPMG3I8JDjnHPA/HQikj3hpzQ9ObofgKXSg+09KzlPWV66Hfh7LOd0HrkRaGm5mbjwy5zoYQu4ThJrlccKWkyDVb14tDuEXVtmvWDdb5Pmg9lqWzwmU4a9bVduewzjWC1vnP5/MqC3Dwtdc9Tp5M3NZoNJououOCd9mLImTqzI0Qcrt2GDKPNwGXP9wPlXXxiey7Xy7Az/M4Gnb3Mk8EKwV4GDfJdqlrZBr0R1NLx7M2LnIzzJUQvPXA0hfMDU1fg9a+gMXP+Fxr/esczPApkuQj2LrYmTBcuhAkJbk20H1sBEgeZWXlITMnl5Jd0Gmo0Wi6n46roiXPmishcNpZZ3xjdsbL0+/nYElZ9JbdUFg53/psEfz+7q2gb5Y40Ac5Xsqq7XjojW7yl+WYvFZW3qUWZUXTa6Eo9Hq9qK+rQW11hfLzra2pQnNTZ82yZ4hO+tYy3BYJl36jdbU1yp+1s4QvX8O73S2ol3Draiol7CZ1vSa5bhtrhg0ksPsSzAvmSZ2Uhbo6SbvGeuUeYpQFnX4ajaZ76ZjgbZgPVH1vboSQnWqudA0erw2vz+i4SwLdAr74Jc3c6npmzknB/OWcXLxjvPtNBuqbusnKm22RflU/SJmZZ25oejtsINZWV8LnbUFqahrSM7OQlZOD9PQMNVFFrQhHjwjgeEWpIaCaRDRVSHgJKlyGn5mZjbT0NDV0F0cr8Pt8cV+DQoy+pjXV5XJDPqRKuBmZmcjOyVXX47i29bXVSrR1lrjuazBd2OipkbzgbG9paWkqj7KyjbLAyTxqJQ897vjLgkbT5bCCCywbO+13ZegVdEwRLX3OXAmBVr7kjou2WPjy1zT1Or8z+OBbi9fyXcAH32aaax3D7U3AJz920324nNa+21ZvBDS9CvphUoimZ2UiJTUdNk4FpvbLHxE1dodTRGMOvJ4WtDTHbonl8U2NDWhr9YrIzZXwXCpcuvkYlzBGL8jOzUVTU50S1vFAX9OmhlpkizhLcqWpcDkYmHIn4n3YHSLccmVPGxrq61S8NMHYlGXc3dwgeZEDR1KKSkM+HwNpyJEY2IDw+dy64aDp2bDMcvHLfynEremFLOLGvo0R9RuWdEjOVenBdFmTRr2QDSN4rV5pb0D+XNx51/yjE8OKlc689pxuvA9kWPhCL33eXNH0VihY2As/KytH6kIRNxaTDnB3ekamiN5mZUWNBVoM21p9Skz7JSBVv4qaXkcqyTYnT8jMylbiOJ5KuKmhDpnZ2fAzLAagFLvxjFPIpr/Vj+SUFFDTxyus+yrsfOZuaURGVpbkMdPOSEOm37pp2IrU9AwRvR5lkddoehwsvhS5FHhFm2Hlzg9g4WE/oC1RGtv8biOF979o6k9YPvYetOZuahh7e2l6xC94K2YAjUvMjRCsxM4GZOmqzvMXrm1MRH1T9KMjdCbLyzvvPpaWda0P9TpYlYGmpVJ2ppsbmt5IS3MTUlJSpc6zKX1oZbHjd5yggkKH/pzRWvZ4HH12U9PTTf3J6RessBnCOj0Nzc2x+IrajPtIS1HnR7oGv6FFOyU1TZ2jLZQGTAc2NNLSMow0jJAuKg3loDTJ00Y5R6ehpkfBesYrDTNXGmq2vQLNJ36PAVPPRJLIgFZpeG/MtPp9cNptGDTlXDTt9y2qR10Gv0Oe70yWXiZ84xe8VtbdSK+zY4FPuhaPCCS3FMT2C1xbJ1egxkOwa+nsa8ZcFpnOTG+mu6E04odDlFm5tSyxKDuaHg+FCq2cThen3o2ocdR3PIZj29psHIoqujJFkaw8JOhewGvwT7iLcZNBypJodyr3hFiqAU7e4HAYnVzbvYZasUm8rC3aGxtMFooBToUcXT5xpA0pC7LRyuM0Gx2tfj/a+LYnpIh0GyyG8rhraHNg9Zhj0XbQ03DseRNSUw2DTVurv8dEtbtQP9824w0dG6yOHW9B685PoqzkKDT4ROv1ovZA/IJ31QfmSgicWa0j8GHCGbsWlIqoLmcvMmBRGbBYlkbrCRVSXZ33EOIMbCmdGF608CHfmfcRdVhMV6Yv05npzXRn+jMfOvJwt7LyllmUHU2Ph0KF4tUQgZEwDlCiF63ymRC1WwNfkwesqko78U+oiArAuCj4Gb2QUu4LCsovfqgLcW191lzDOEwLXgOjAWOmjUo++RMpn8yvbOxtqAXvRomnuUG5OPUI+DP2AjWDdkbCMW/BefB/Yd/0YKR3gr2uL5NBm+bIw+Hc7XFgzzdQk7tTr7H2xid42dO+eaW5EQQrtPQOCF63NLWWrDamp1U9HoLwSIquqOSYW7KxfsqOHNh5s4sNLPTCYe+e3Bte0nk+giMHtheW3CPTk+nK9A2G6c98YH4wX+KBQ9OFe/41i5iu/8vc0PQqKEKVWLEQNmGhZU+OD/1NR4DTFKsr8JTAEo5AVGRpk/CjjRWPC8wApvRsIJxw1wnaFzhHYyaXJIdKc5aJQDqGw/xarbLBYHWcpk9jTEoTKDTdBMuePBob4ETVHvfAddz/kDJmT2Sldm1H+95OdkYy0kbsA9f+01C59Z2o94ic5KzzPZj4BO/qz82VEJJE+rP1Hg+sMEtFfHF62kjUNhlCLIQJY9bfFy/bb9ZgrnU923XitSeOaScspiPTMxLMD+ZLPBaZRCkLLBPhKLcoQ5oeDX/dhm6VP0E/9UDp4KexbnzJdcN/U8pRlHUDrcE+H0WReY3AErhIANkOuDL5PB7j1braah8el5joMKZLTjCvE+5s7lLXUB7L8MrvgW4NGhNa7ulKwrxlsljkE/cz/fhq1CeCVx2v2ehgXcB/3QLLoU/+yzJvzBloOmM2cqacC1dajvG9Ji6YfrnbX4Dmg//AX4WnqvTtqRbf+GpuK7HSkRZSZd36VkYrKuTYEL/eof3d2Hxo+69KcjN8uOK4VXj8siW46vhSTNy0EYlBqcB6+IAda82trmff7WvhSFxbUlg1jB/RhMuOLcN/L12Ca/5RiqKc9ptRm/TzYIvhEdKD6cd0jAbmC/MnHqzKRPkX5oqmN2EIzARlpQs8tgw9w5EUuM9YAiXYOIZTzbbGIBTbkJycqiYwSIggjNrkK35L8VRfXweXnGN99LrwuCRXstGBas1Z/AyppWUX3R8Y97r6WriSkg1rtUaRnJwi+VRrtGWsHnAqDaXUJCSqCUNSktkZ0epgjaYTCRQzaW/75THmH7AVSi9pwogTH0LBwJHml5rOoGDAKIw6/FGsOLwR/tyxUB5sfCHGPOghP/c4Ba+FWIl77F1Jjbp2LI3B0MRUv76YO/vg8ohGpKw0Px68aBn2EVE5alAL9tquDv85ezmm3bwAFxy+GvtsV4vrTl6JfnlxvsLvBLLS/bj59JUqLuccUo7/3bQQ916wDPvtUIPRg1uw+zZ1ePjipSjIjtw4OOfQMnPNAqZfDK+YjfyJo9RalQkteHsl/HlRKFJgBgSs4dJrWG6UHZSdk2S/MtCKOHS3tCDR7ohaKPI4h5PlxhBIFLQqxHVOt8lv3bhidWUFkpJSkZgY28gqPN7lSlXnqztg/ELjKNuJsjSI2G31tcHpcplfdD1ej0f9An1erzxMfIZrQDdjdzhgdyajuqpKJV24NOQ+piFnXbMlOOCQc9RxGs2GhD8PEV3ys4Unoz+W7/qgiIRZGJDTjcN1bgQMKkyB/8jZWDrmTnhT+4Mv69YI324mdsFbOxtw8wERAuuveAWvV1JDpUoMuNe3cm66STNO2KvS3FqfK49fhZKC9cVsbqYPh06pxhXy/S7j68293cf2YxtUXI7ctQqFYay5+SJ2rz1ppaW4P2znamw7up0GRJj0iwjzh/kUKywT4eLplnyq/dXc0PQmOJEAzascaoyiU+WvbCvRGzC7SuVGPcxe/E2NjXAlx+bbT8tucgpn7EtAdUUFmps5zi6txHINUdgeT7Oa9re6qlKJVldysrIghoqtSFB0OZ0uOTdNRG8lGhvq1dTIDILX8fo8airj6uoquQ8b0jIyldtE92CMjkGRy9ElfF6fGtu262H6r12Y5pxlL8mVokQvJ5Zwuzl0m5GGRv43oFIaFTY5NjklVbszaDY8/GlI/eMftC3qd7wWVSfOxKDd/k+a0JqugI6MQ6ZcgOrdv0X9ZtfAXzB+zXOhO4ld8FpZ5lwWwiYq4kgFi1dip+xXgSOnsmPbuhwmgpZCsq9Ad4WT9l6/4UGXiPOPWG1uRSCuV4pxnMMyYdUQsvIF1/RoKHRS0tJFgHlRV2t0Ik1I5BQU8p38SRRhw236x9aICEpNy0CiPfauz5ySlmPf8nyPCLxlSxZh9apSlC5fjlUrlomQakJiol0JQVpg6+uq0SSitaWlWYlCjgphdJIxxmQwRFrC2kUUud1hhzMpSU1wQbFbKuGWrVyJMrnOqmXLUFdTo85nWJykollEPu+LkycEwmYhDwhA+aPi3vlwHOB0dQ02HpJcrpgt2uvCeK6NdyA9aLU30sf4ninHe+TIFD65Z3dLk0rr2poqNZVwTXUFqirKUF9bJenSrPavWrEc5ZJ+FavLpHzUwedvFVGcoVxO7N3WYNBsFFDoegCf1EGN212K1qOnIX3va1BU3M/4XtOlFBQPQLo0OFp3exP1Iy+GmpuGL6fjkR+dQOyCt/pHcyWEjvjvyoNNalhzI0o4zqsF5xy6GtefvBKZqXQiASZt3qjcA/oaJ+1TiT22MXxrOQTZ5ceuwuXHrVLb7RIh/cLC/GE+xYOV4K2ZZa5oehsURhQxCYkOZR2tra5UEz+0yFJethKrS1eIMGpASkq6HJNgjL9p2cgKFl7rLhSVFFkiwdQUv5yyOC09AxmZ2XA6nRK2XVkYuZ2WnqncLSiCOWIDBSzdKWilbRSRRnFeXbVaCbSaqnJUlZeJgF4horYSDQ01StSmZ2YpS25igg1F/QciJ78Q2Tn56tppGVlqOmMKQ2X9FWFNKzfFHoVefW2NCOIGNTmFx+1WApHxWIN5T7zfeOCpgXQxtgPhrJtm4RYSEK1sDFCcumWhlby+rkalR2VZqeTdChGqzL/l0rBYJOJ1qawvQ4XkaXVlmTqe+ci0p/BOlYZPdm4hCopK1iyFxSXqO1p/mSecpIRvBTom0DWadqCQEsG7fOw/4DnqTSTucRMcWf2gRxnrXpj+joz+sE+8Be6pb2Jp4fHGF93wgsomlVdsWvuzieFFb0kukNIB/7YVFUBjDENyRXE9t8eG0konBhd33lBfPZElq5KQn+1FSlIMJaipRWoGa/eP9WCDpn+euREjnMyC4/uGkj0e2OV7c0PTExlYUoyLb3oWEybtqiyboRhWVAoprxK1jQ11SkzRZ5dWXdoI7Y4kJZBp/uWQZoZbAEUYnSC4yna3VEOykSD/OJYuhS4tt0l0ORDBpCyPsqxavliJqdz8Ytoe1St+twhPn4i4RIcDSSJIKawYJ04J7Ke/K62xqpprU/GiWOP3vB/6xSo/VLvsaxNRLgK1saFWjfqQLiJX1J0hGuUf4804MPyAFVT5ETNu8slrMD4cdsnvl0XSRPna8tq8PSU8ef921QjgKBEqDDnfsKzS/5kJEophZQ1U1Wqd/+QavA/jepIP6lrq6KB4SBqwt45gTADCNduafVy3Sz6xkcDr093ArvKOaWLEL3CvPNeIAq+gPgJ/1oHXKRfRzPsv6jdQxTEWMjJdOPvEQ7D3lHG4/IqrzL2avsD82T8g9eWj0a/pb8PcxuLDTxLrOuE2aQaW5o1G3tGPwVaytTR8pc4xv+oIS5csRsnTw5HA3wsDDFw/NB7E6rtY90s1O33KDEzeaUdzx1qmz/gSk6fvBNANOfg8q7CJ1XfR7Jf1Vpsdy/efj4EDB5s744c1AY0Q/tKZqPrwVAxsnWcoYtZLQdcMu07CxZEEtuUCFa5xaNnuFZQMGqa+Cib4lOjgGLzhcHZwGtv8TImNqo3bJ02EbhTiOsnZ1ufFLhlU5I5N7BKmH9MxGpgvzJ94sZp5r36uuaLprVD8UTC5XMaYyxRuxSWD0a9kCPJElKalZyvh1CrijJWNM4mv4kXwKZFH1wfDpSA5OQ2JCXbll8ow6euZmZmjvvN63WhpaVKWWlpZ1egK9bVoqq9TFlY3hasIW75qX7ViCcpWLlMWyprKchHNhqBl/FyuFKkQW9Hc2Kisv3zFnpNXgPSMbLleClJT05VFsrDfIORI3DMzs5XluKC4AAOG5KOofw7yC7LUcX7GqbkJXrkvlzxcnUlOJMtnckoyUtNSIZIbtkSnhFWA/gMKMXCTQgwaJuEMLpRr0OrtREoafV8Ni3Sg0UBRWV9biarKMrXQIl1TVYnamso17gM13Cf3plw4mhtUw4A1PtOSecDGAtORItXpTEJaWqa6R1rk6R7CqZ6zcwuQV9BP4jcAuQXFyMrJR2Z2ntxvjnKd4HkBEc6wKZ4D4toQ8HzCcFkfXj8np0CFGavY1WiigkWPL3DZBpfP2v1uRcFlvyB52A7qdxi7sNF0BcyXZJcLqUN2Qv6Jv6Jy7PVGXrJLEauK8FVKpxFbuWhZJRELMzwVBVG8r7sDUDDnZZgbEeCreHnoaDoBpmM0rg3Ml440aHiNcI0ZX4OUqVJzQ9NboSiiOLIn0ipoV9P1UhhxH31wKRqzcvKUuKOoo+WVwle99k5MUK/UGxvrlAglPMbd3KyEE8UyhSpFJjuxMWx6C3NiCtVpSypIiuJ0EcK5eUVKbOcX9kd+UT9lGfZJmI0iiikWm0U0qwktpCzSGkv3g9pquiPQJcNwSzDWjU5yDSKo22yJePKGUzC2yI4t+ydi3EA7Tj7uACSk5ohozMPSH97AhGF2bDfcgYlcZH2bITZ8/ssSeMt/xjYDbdiivw2bFdowJs+GvXeeiMUrK5HYshITh9qw/YhETN0qF/tOLMSUzdMxaXQSKmp90kaUtBQBy/t1yG9PWZdVOrOR4FCjWKiRLyS9KEBpXac2pbCnuM2R9KaATc/KUQKX+UCBzTSk0Ge4zB9lCV4jZA0xG7Akd4SAQNZoOh0WK5+UMSnLqyeehb/PK0XmbpfC5WTdoOkNMJ8ofHN3vBrL91mG2mGnos2RajRiNiCxCd4GC4uclQUvVrLSDFcFKxGWkQwMLOi4uNYYMB2ZnkzXcDAfmB/Ml46irbx9GookClj6uAYLpoCAolhjT36KMY5lW686PZWjqb5eWSBpWTQsjFlKvDIsDr9FEUqrprJ4VqxCfW01vD63CGGnGgOWnec4Zi+tuAGXAwpavsrncGkOEda0alLoJdgSVPxoxU2XhdbiNPN69NtNUWIwRblFKMumxJuuCWXL5mHzPU7GU69/h0de+BzFCctw4H67KmPE6tJFaM0Zjidf+xp3P/Ee7nnqA9zzxPsYO6RARHud6qTx9Js/yv4P8chL0zE4rR7HHrEPsvoNx9PTZuKVT/7GkQfsjKLN9sYrH/6BR178Elnpdvj8dIcwXRJ8PuWawXSk9ZvDerFhQdcMjjJB8Ur3CApM+hVzpAQutEDT/5luGwwnkBeMOO9P3aOWCJreAn9wFESiWtyDJmLlST+j4Kj7MWxAkfpa0zspKSlB5u6PYsXkWWjJNUdz6IC1lw14GkvCEZvgrbdyZ+gkwUv4qn2QiLCibCBbhFZmCpCbLsIsX/blsAu4eaCmU2B6Ml2ZvkxnpjfTnenPfOiIX3YwVmVEC94+BMVUSC1lCiuKNlpa6+uq4G/1ISe/CHmF/ZCZkydizGd2imKHstXyWa7cGJJEfGbn5qvj6LNLyy2Pp0Cm1ZOWYLo0GCM01KjX/BTRFatLldijCGZnKVo/aRll/BrqqlWnukoR0XQX4MgLHI6Mne3oFqGsklLh0s3ALucmuQxLa07REIwZuw1GbDYB19/3HOrmfwrR2VKs5Zj0XGyxzbbYfPyO2HLrnbDNpN3Rr1+RCGm+sUrGhEnjsaV8x3PvefwdtJX/jL9Lm5VLQX5hCdLTUuFMyUT/gZtg0NCRcj26iRiWXYpbWnNdIsbZAZCCnB33aN1V2rXV8Hf2SZoSuoXQcqss4JKuXg/TiB3TjA57tGSzk51hVa83BbHRwS7wkDBcGQJ+uwFhrNF0IyyabnlcpIi43f0GuM/+Fv0Hr++jqem9lAwZCfd+PwJbXYM6m+gRVml8nMQifKWq8rob4W4KP1FWjILXQpx0puAlUuEiQ4QX/UYLRXjlyoPDZTFFraZzYPoynZneTHemP/Ohs7ByibB6a6DpxVAkGYKJfrcNIio5MoDTkaQskkr8NtSK+KyBp6VFWWYpbLNkSc/MVqKO4rO5yRjxgJ8NdbWoE9HGdc6O1tLUpAQaxSytmXyFzk5yDJ+WW4pCdkrj8ewk4RMBzXglp6ara9DazHPpG8wOb36fMYQZ9wUmdmiVeEqwqhJNor+v1MHFJU788e2ncJZsj1T5ibiSXXItF7KyRNpK45BWV3bUa6yjfzGnO08QwSrtxtRkEe1JWDL/ZyYQhgxKVj6uGZlJEhcKWKcI5CRljU7PoM9tlhL2dD+gPy1dOyhEJVHVvdKKocRoguHmQJHMNOdsdwGLOh8UPI4rhtXDEMe8NwpcT0uz2QipUW4cbGiws9nqVctV44PW9EblJ234CbMxYFjQgxYzn+WPui+NplNh8RWxW5/aDzXjjoL7iBfRtstVyOjER5Om5yDVIVrHXYuW7Z9H9YAjUZ8oDZwYrb1S26l/4Yit2DQuNFdCcHSww5qm72Pl0tC4yFzR9F5MK6AsrGg8nhZlTaSllYKKwpPCtUlELjuYGcLLeF3PV/f8nqKqobZGWRxpneRretXZTY6jKKY11yGCme4GdGVIsCeqV/UUgZnZueDkExS5NHvSYqn8gUWMMQyOtUtxRvXKkQzo18tP4xoUm3aJi1cN08X7YIc5XodWVdGaSE3PwvsPnotNMmwYmWXDKWedi7sefRXlq2hZlSI871MMSrNh201smDDMhu1G2JE1KAMZIlrlW+y3wzY4et+tcNxe2+HAQw/BqTe9AVuzPMcpQilKlVWcaWKsr1nkH+PDBoHyjVYuDEnKp5lCmEtyappa2NGMQ4TRXYOi2djOUOKfAp/uIjm5BSqtVAc2OTaFHdhM32jeswpPFt474+OWfGQjg1ZzWs8rylZg5dIFWLlsIcpWLpX7X4HK8lIRy5XSAGlQecQGgxLkLBWMuxLFWgxr4sArWkfEbsOkC+A4+R3YD38SeZvtzPanpivh7zeh6wyOFKUFo3eDffITsO/xNuqHnqXqxmj9e+VppP6FIzbB6601V0Kw6+aWph2sXFE8NeaKprdBIUNh1txUr0RPbXW5cg2gz21zU5MSlBR9SoSx45TqPCVCSwRamrJkZitLJju1Bca/VaMIyDHscEYRxjF3Kdo4mQVfxTfxWjUVappaWoETEjj0GSPDNpVDuUHweipMCS8gAI04ZCoLLz/VdbNykCkLv+NCSykFHkUcO7RxPNqK8lp1rZ2O/Remz67Dhz+U44477sJZe/dHW0YikqTuSxo0Ed/OdeP971ap5Z1vVsInxZqd88j/XXwLjjvhdPz603e44YU5uOT0A9DQuP4Qb7FhiGTJALUEC2XjO0HShM8q5lNgoQClwGfeUDzT0h2Y3INpHUgXjq1Ll4t+JYPRf+BQNYA8l6L+g1FYNEB1EAyIZ6crWTViOA5x6YolxtjG7PgnecXOgnV1tapxo4WvJioobprkI7sIq45+CykH3gHXwHFIS+nAWP+auGnz++Au/cnc6jrSU5OR3G9rpO50D1Zs9RpaXflrR3OIk9iUqs9i2l1dkWnaw0rwcqQGTa+Dr7LpWkAR6nQ6RCRli3jMVf62Rf0HiFAaoCyr7ERFyyz9UGltZaerwKt5CrNgoRa8yB/+V5NFNDXWiiBLkbCLRGiJEJNrcHKD7Nxc1YHN6/NJHFxKyKmwGXJQWMGL/Fl3W6D1uaGhHunpaRKuMcJDTn6Bul5eQaYS3xxLODs3XcR3Ng4+63yUOIH3PpkvIjsRtkQHsnNE3Mv9c8lWfskMnzVzCvY5eFfsf9zp+N+r/8PVR4/G4kr2F42t6t1wrJseaxfDbYIWdsO310grJZolDwOimVZnTixBCzp9pvsPGKjyX03YkZuvxHNaaorKw3oRvhKCCkejWQcWLy6cEV+KyOydbkDCv0pRtM1+a37Tmu4hUTJm+I+Hovzt07BwRZW5t+tgfdN/q0OQcOxq/Fz8L0O1emQJlJkYiK0kWQnecENOaTTBWBURqzKl6bFQ9NDvk5otKydX+Z8SWvkoktqU2IOy4qalpSn/ULUjBngNDhHmSjGG2RJVKWGbYozXkE9aZCm0E2xtqK+vVefEAo+nKwVNBtlyHxzuS12D8Vf3InFW95GGfBFwftF9WTmJ8NW0YblUuMOHDVeWy0R7koh9IDPbbi4OJCcH4sKJMUS4V7dg5JSDsetWA3HcUUciJ7frXhFuSJiGdbVGWcigI3NQPrE8MN9tzKfMHLhcTjm2MuZ80vRx2C5kByUpRJV7Xg/vHW3Y/EhjwhEtdbsZ/lS5+NzI//sxbPJSLirfPU1NsR5bjd4xAuVg3P7XwX1iGyrHXGnEi2UnhojEVp68VhZeXSw17WBVRrTg7XVw2ly73aZ8Pv0UNUQ+lI7hpvnpV2LHjvTMDDUSQtRCxxSiKSJ21QxoQddQwlktxi5+x3gkSM3n9fF9V/Sw4xZ9d+kDS0smg10Tf3UPxkU4PNibD1+KI3Yfi53GFGNEcQJG73oGth8NuL0+NM77BJv374cdRxeoZfzAJLz4/DQkqeEVWwJRRU2FH/9+7nus+ullvPrxHCSZHTndHPO3oXe+6WBZSEw0hpxTDYQAsspZ9dbkk6w7OGuey4WW5mZjp2bjhmVDhK7bngzf4PFoOPp/SN3vaugeQT0QPr6ZMS4gd/5jqHp+Z9TP/0TVXR3wMIgLOrak7HAj6ie8Bm/OVnAzUmtfQkUktqmF38yUAsqexyEMKZLE0GPjaiLglRK5aJW5EYQ9BTgg/BAimu5nvamFRYyy935GZmZAD6p6JpyUDeyn0KVFOC2dnbiig8No0WKovAIEI6zQqmrtVW22NtTV1ku85JwoqjQjTrUSJyl/MOqutfex9nxafavLlqC8slK9Wq2prsLgkVtj2Ca5qK72wF1fhYWLFqxzTY6C0G/gKORlp+D3P+diiy23Uv7GxCGCb+X8X1DvcyoLcVubDdWrF6HenYhBAwcoi3FvQVl3a5hPGeo+SLg0NKC/t+HZVFNTo3ynrfJJTy3cd1kztXD936qgVA2eAkw8BQlbHYWspLW/557CRj+1MAndz0/TKr+q34HI2OofQPFOSEnrwGyscVJV70fbwhfgWvQoUmu+UvtWtg1D47YvYPim26jtYIJvqx3kLv10sAlDFA8YzUaOVRnxsUx1dRtREy+04lFcBssaroXmbvD+NvnHTmc+X3RijkIokcpITjbOV3JJ/jHE4MV4dW7AbdNKGwWG1bgVCXwFL2vGNVSI5pqxcDSH7MJBGD12G4zadDzGbjlBhGw6qipbJC1akZSeg7FbbY/Nx++wZhm37WTk5eWgNSFpHbFLvC0tKBqy+Rp3iNZWCb9gSK8Tu4RWW9j42zXErFUaGt+z3PA7G+zScFhjtddsVLS1+mFrqUZTUhr8+9wA7wlvIWe7o3uk2NVYwKyijcAJFK1+A84P94dv+jnwVMyLdiCFTiMnPRG5WxyH1p2nwbfFtWhqS0aCt0bqm/AxiV7w0rIb7mlC/91onzI9gOp6Oz6cmYGZf6bC7eldP7IWtw0z56So+PM+ehUsI1a+3rrjWu8hoGxMWUPa/xXJESKS17fQhmd9y5+5zY+QhVbGNYfL59pYRUaJr3WuY8hpBXcHLa3m0Gker8cY8ixIwFL0BoZBC17oIsGIBR8bwO8zZj4LQNHb28QuMdLLHACICRohDYPT29hlrGs2LlypGWjc5ky0Hv8asPtVKMzphFk8Nd2HyBC7/LYzFj+LxI8PQd30a7B65VLzy64jPSsPtnHXwL/Ta6gdcBpcyenmN+sSg4U3ghWul7TWWd+edttAXP9kMS64rwR7/3M4bn2uCEtW9ezhThi/W54twr6XDscF9w5Q8T/l1oHykDQP6A2oh92ax+G6BN5ba3o+0mihdS7gj8u/ho1vXbgdyHEKXQo6NRZuVNhEKPrWhKnCksCUZXmdRb4xP3gAq6HAOe0hh6owORoBz6Fsa5W9Nr6aD72O7OP98jgK2cC9a5gelK5GY0aloSRqpDQkHK5Np+DGSb/Bw1Fy4JVIHbuH6Uik6fWwWpcfdGLV70j//Xpkf7gtFn56Oz0zuhSWp7She2Pgzlej38ARxs4Qohe8douWGJ8ytGb0AuoaE7GyYq1LfIvHhre/zsQx1w3GpQ+V4PeFdIzpOfy5yIXLH+6v4vfON5lodq99TKyqcmBFRS/q6e2nxcuinNjDt8Y0PQ+KGk7wwNnNEkTAKCFqfhcM9ytVKXAyglY/R1WIrrphuDYbJ5Zwq+upa7DTowrTtOgGFrk4hRRnNXMmRd9wZZydSS401NeLEGeYQUJtzTXMT3WDNjUjG6s6FReNgkOTsSwwD1QaqkdKcBqaC9NQjqGlPMFmV+uajQ8OXedyJRk/KU3fQhQnp2RwNJVh0F+XwPHSaJRVVJtfdg0sV8lq9svwXR9jqLnlUHYwCkegUuvheH3hf2aM+lezU3H67QNx7l0D8NM8i/vsIn6W6593TwlOvW0QZvyaZpm0DU295MEbKB/hbiTRJaVUt/V7D21qprOmJmNK3oQgNxUKHJXNFD6ym7qQVtna2hokp0TfqKHFkJMh1IoYRZtPCeU22l8pquR7Ja6UYOJ1bEoYuz0euETAxgKnOvaJGOcUu4kU4xIkO1+tuYb5qb6Q69fV1iIlLV32yWY3YMRFrRifPQBO9NHSzJnw3CoN22xh8kkWte73ob6+To2K0XPuQKPRdBr8YUtVmiiP9MTKv5D7Qg7+fP4YlP79k6oTupvYFJOVlVc96Xq+lZeZ0B6zRGyeI6L3/+4YKCJYxGYX5tK3f6Sp654t1//xr1RzrzVOR08oQlEQ6Q2Atu72OuiawBEX6kS8cEIBikEKGmpG6l+KYC5+rxtVFeXmtL/R+5wzLLv8WFNFJFdVVcLjFkEq+7goXSrXMNZp2a1XIoozhUXvMmGQkJggwjoDjY2NaJAwKM4TVdyNa6j7kBWftwXVleVqljhOnNEdME04CQc7DTZKXN0i0jmVb3dDkcupl+tqpSw01Uu+GHljNHZYHgyxyymjq6urkJaWaTQuNBpN30V++8q/1wmMqX0BxR+OR9V7J6G+YqlygesuYqt5LMWJCC9/zxdfjsTo4zh7QTIufag/Drt6E7zwcQ7qN5A1leG+KOEfIde5+P7+6rrRkurq+Y0MBcuGleh1aMHbG7Hb7UgX8eL321BVWYnamio1di4XCtBKEboNDY1qqJqkpCQlemJCjucsXukippqamlFZWSHh1aG5uUG9Qq+R6/G69JSh+LZ6hdUevI+0tCxliayuqERdXY0Kn0tDfa26j6bmFiXqOBVvzPfRSdBqnpzCtz0cdzhFuZVw4o3uR/JJ4pKRkSNloU3ypAo1tVVokfRTjRFJz8qK1WhxuyUvM4186p4k1Gg03QFtHbLkLnoK7ncPhH/OM1LXhhmitAvoHMFLP95e0IPKYY9dIJZWOvDA6/k44LJhuPHpYnzxczqa3B0Tvxxt4evZqSq8Ay4bivsl/OVBvsXRkprcSwQvpya1MpVrC2+vhMKP08vSvYGi1OFwqTYNqwJ7ohOpqRlIFbGrRGKcFj26MnBK4vSMLLlOKhrqakVIuVFdJQK7qkKFy5D9Ph98Hg98Xq+aTII+w4ZbhWFdjLwkwJZIi7JDicgaCZd+vU0i1ivLVymBmSDHMEy/Gk1Bwjcbb6FhbWglR3HO69DKvGaK5s4i5F5UunBhGsti+C3bVLqyA6JKc58XXk+LNA4kvRrr4PG41fc10kior6uTssBzHdJYyERKcpop0jsxzhqNpnfAqlFEb17Tz3B8fgJaPvkHKhd8pYbn70piq30iiROaWno4dAFIcsQXT7fXhve/y8CVj/bDXhcNUz62L3+ajQUr6AcY+UFHK9hCOe4lOf78ewdgr4uH45KHSlR4bm98DwCnvQ2prl4yTIN2aeiTKPElIszudCLJ5UJySqrUazYRjNUifFpVJzLOg76OgFpHVMnvhksYoRg4x+1uRmNDnRJZ3MfrpaamKz9Qh8MBr5dDfPmUMOWn1+tGS0uTYWkWgUwrLV0A6LLAdbXIfk5sUVNNcVujtvnJuCQluZSfboI9EfkF/ZFGsZaaqu6DAo8zi7U0N0qYDMM8V4VXo6zPXhHeAUHM+1r/XjuGCici616TglUtatv8NAUsxam6JxGqLc1NajY8dV+1NdKoWC2CvxTlq1agrHQZSlcsRtnKpagoW4nqytWorhZRW1stad2s7osdADmVdHpmNrLzCpWLSauI4uSUFKmrnMqyy2trNJqNFP78KXdE+BZXv4+Emf9EbXWl+qqriG2mtW8OAErfNTeCyMuAKEnAlUQHLnNnz+SsOwfil/mdOxoDx8jvn+9BboYPacmtkgytaHEnoLElARW1djUyRHuiOFY2H9qMhy7u+vHuYobuDC1uwOMFysPMqFa0F7DD2+aGpqex3kxrEaCQoiBqEoHqEAFEax4FqmHVsyn/U2WZVBZKqfXkJyHyy/hexJBhSTSsiM0iWPnpEhGdlJRsfCdL6fJFytqblZuvvueixshtaVE+uZzi1m53KPFLay8FHS2/dLuiP2miwwmHfE+R1yzCleE6ZR8rQcaFw2qp8DwtyMzOVd5aPCZwfQrFNdtKNAYmuwhYPr2GABfRy/2BOPJc+q7y3o2wjE/DJ5h1gwqkfSgaJbzAxBlqWDB1DRHZ5n5OluFv84vglPhInNaexu+NcYPZQDDug+8beY5xnN2ZpCzJnCiEcaZQpfWb+UiYRiqwNVFmSGvXCfO2YvVK+CXdC/oNCGoARIeeaU3TU9AzrQnt7SfRrJPAtlQJ7uQBqNrtOxQX91NfdQXBUWmflIHmSggB665ZafZkthhqMVtcB+DtLy1z4uf5Kfhydho+/iFDfXK0B+7vbLFLNh/WS+ajpzsD8Vk89FIHmSua3o1NiVSKoZLBw1FQVIKs7Dxp/KXA4UiCS4QoraXs1U+hyFfiFETqUx4mtAbSEssWP622FK5JrmT1Pa2OdTWVqKkqV/soxOpqqlBfU62sjLS62kTs8rzysuVYtXIJKitWKWGn/Eszs2TJVq/WKUSbmgyhm5NbgJy8QqRlZCu3ibSMTKTLsbkFRcjKyVPCmhbfgNhrlR86wzSsxzUS1wrU1VWjQdZpheZoDxSYFHxOEekU6wwzI4vXzlD3boRlU/dMqyqn5mU49IGmRbqxke4BDaa1tQ719TXK4soOa7zn6vIyVFWsNqyslbSy8jwjDvWSTo2ma4Hf61fPFqYVO98xzXltuphkiJDPyStSS7Y0HHLzi1HYbyCKSgYjT9aZb7TUqvyS/AvcP6G8VSJfCXouchUu6moGvDcVhlwnVrGr0Wg2DtpsdqMB3YXEJnjTR5orIXil9UN6gVsDLaN9gS16y30EyoSVs06aRZnS9DIMKyZHVwhYMSn6KPgonCia6G5AAUvRpfaLyKQ7hBJoIowoTukWQTHl93tVeDw2OTVVdRpjOBTPyjosFSXFF6tLdS2nS77PQl5BfxGyhUhPz1axqq4oQ+nyxSKClyrRSIsvj/f6PMq9gQKbgjHg/sCOVuyw5nXTJ5juCQFXCoeKnyHcM5SApgU4Q8QyxaxLxLEjybCOUuTRrcLosFUmInwlqspXKWFruFdQxFYrccrRCxgnikSvuwVuEcHNTQ3KbaLFLd/RN1nSwifxZZi03NKSTXcRin+ma0BgU6CzYxv3c1g3pleWxDFTxKdasiS+WTnKJYRWc94PreFMDxIsYgNLsJCNhbXnazQaTc8gNsGbFn72CnhNUcPWfA+v48YObYlptIaeiF3iP7Y3CF4mc0Dw+iwEb7pFmdL0OigGHSI8RemYe9aKHlpKlRVVRBgtsVUiBGnVZG9+ikcu6Vm0KhrH0CeUrgUUqTUUpbVVysqrLKmeFiXSaAVOSctY489LsclPCk9jMolWFWZ+YX9lseWreZeIwXRaXTNz1izsXJdC1wkJLzHBvkZwU/hyxAmKYmVRrTJ8V6tp2ZW40zrLdVqea2U/O7zVSTwpmulGwcrVYTf8m9n5jp22KFApjpVoFrGcKfFKl0/6v9J3mMfRzcHQmux4R9cJkfWyzXMT5J/ax3/yaZfwKVwNlwVaXv2ytCmxzgYG09qwsq4rQHm88Uej0Wi6iS6ugzrHwusLGg+yh7s1pCb7MXnLBnOrd7LjFg1IT+n57iPrWPw95luAUKzKlKYXIoIqSFQR1UlKFroCGJ28qpV1kiKULgW08tK6Wl62AtUVq1G+eqVyE6BPKYVvbn6h8ZpdhCFfwQesqnQXEGWnLKR0paivrVWjKlBIMyxajZV4FAHr89OnlhZom3IZKCtdrq5VUV4qAppW3TrV+YoCu1XCpAWVFl26IKSlpitBTHeI5OQ0wzLKwSVpW5b7oj8wj+PEGqrTlsSNVlS6BHBRrgxJhl8x04bWXI5XyyHWaFFmmtBtQbksyMLxdSWiIoqTZeH1XEhyJithr4S5Cs+lrOiMAu+LQpfiN5D8ajQJ2ccOdEwbinCKcnbSo3jnwmuzseGWe2be0PJOQWy4QJii2sw7Jbg1Go2mE2F9w/4GXUlsgpc+vJwZKxQ1zqr5oLOy5PUg9p/EwfJ7L/tPqjHXejiBwsyioaxMIbAspWgf3j6F0oGG9ZEVmvJ3Va4EPiUW2SmKAovirr6mSoqIVwlF9uynb2mmiEVaMY1RA0SImq/46XerRkKQde6nFZWilv67dBFoa/MrSyettOnpWUbHNBFzFHYUfRztgdZOWnj5ap+v/e2JdjQ1GNZY5b4gIpF+xT6PVwlu9YpfbomWVUPYJiv3Cgpxw8c1Q01GQcs2rbcUixS07hYO1SXxFCGtrL2yjwKV7gaMW7YIfYr4vMJ+yBURz4XrBcUl6jNL4qfCFSHtVNZhp7L6qk5vFLZMX4pSldimyJX0ZfyNH5vR8U/JVDmW53DMXp5DcUyLL+PIuDVSbEsDg2K4fPUKrFqxxGh8iEBWeSRpzjxcM/oEL6aCNTsZcqEg5mJcMehTo9FowtPq96JFTVzUdcQmeFmRWbk1BKx5/DTqxB7L+FGNKMnr/lmK4qE414ttR3d+x7tOh2Ug0FHNyuqfNkz+6IdjX4Idu5pE7HFIq4rVpUr4eZVV0xCqFLuEIyaIYlIilKMlNIv4aqqnT2sLbCLuDJFHf1i/Er1q+l/ZT19VNWqDnE+hSsGVmZVnCFg5hwWPFl2GS0FIdwJalJVLgQhXtsspQCnAKQJpKaY1l53NWpqaJDybMTKEhOeUhdZeil9DyDaJQDYsphSC9MflfTF+tFQwvhS1qWnpShSnpRuuGtym0OaIB8otgaKVMaWgFgFquCIELfIvICgZR55DCzFFNdOFFl6GR1cO41ocDixLuYzQEq22+Snbyl1EGhHK6pzGjnlGZzS6dXDoMM40lyJxZrw5CxqtyMotReLANOeIG2xQ1FSXqyHKSkUQs1MghypbvWo5KiWPaTFWjQpJHzYy6G8c7D6h0Wg0obDe80td0ZXENiwZ+f5IYPlr5kYQ2elACh84glTscPaEWYCsee7DHDz0Rr651Xs4bf8KnLBX145dFxfspMahyEizFOqqemM9mJJDgAkvmxuankjwsGR8ix6pshB9JuKnUVkClbgTAZjAhUZJHiB/2kQ8UlwmSV0R2KdgwFwPcwFqY9ZSHilGXrdbiU9OVcxJD3iKep0vKxSShsWRQjEQcPvQLcAvwpod4iiIKcoZjoo/RafcuKnP1yxrohkSXxpZW5qN4dtcyS51XqchYbmbvaoRoDrvOWIMXA4PPaO5Se7VXF8XWumNTz4i1MKb5afkL7+kBZr2DTUph1o46Qe/M/IiNY0jPKhTwsLwW5oojs2GsYkelkzTU9DDkgnt7SfRrJPAtvzkGxzFWLXduxg2epz6qiuIXfD+dQvwx9XmRhApLhG9qcY6HzbJSYbfZqPkHt0cuHA/xXCqCOMkUxx3E5wt7ehrhqC8du2QO7HCIYe3dCQiXz4/cEd25RguB2cmJuBnEYLxzsLMcX5fum6RJPW6D4hugcqgSfK2RUStT/KZTy+7KCK7pGeG/BopcgNFq6bRKAehbHodMOpKc0PTEwkI3h2n7op7b7wIdqmszJflYaFADBBatfh9bowZvzvGbDERzz14pbJOxlL7BHxJad1tbGzE6ZfcTTm2nmCKB3aS+/zDNzD/189FSCapejk0cu1VlXSryMwtwZEnX6RS6LnH7kFlZbmkSec0/mktP+H0i1BQ1A8zv/4Us2ZMk7iGcTGLilYkODNw6vnXiugM89uMAPO4ubkZj991sbIqB5IlkD/ESD9ar9XmevBQzsZ28gW3Izk5WY5bm4da8Gp6ClrwCu3tJ9Gsk8B2rxG8Vd8Cn6+fEUrMFucY6xymrMFtCCIrHCKMskQgcwmqKLuST37IwDVPFJtb0eOQ+B7gsmP/ZDtyzai/1OzDs03h3STGOhJwU0YS+NhbLUJxmhz7kQjklhiT/l8nlWKPbcNM3tCVsBFTXS9Pq2ZD9IaDLQE2gNLlV8m8La0Kf+zOM0TFb29uaHoiFLz/vPk5jN9hKuY/NxYnnLaZKK84BWayF5fdPwaFYw7DSf3OQtYmJaKCY/sNKJIS8Mkbf6B2ixkoyksxfEs7SFKyC688dCnuuGC+3F/s03wrkttw6z0N2Pn099TmVw9PxsXnF0p4arNjyG+qadkK3PHb9dh//13wxD034N4LfpKw4zQcyE/zrntXYuJpX8JnWqSjhS4Wf8xZjEnuwzB6u1FrR+mJBakTf/tqLmamvobRIwcq63AALXg1PQUteIX29pNo1klgu5sEb3BUoiN7W8CeZm4EQUFDK67bA5TXRha7hKKYxy0qkzvvniG2dt2mDuNHxOYPO1Eq6oczk3BSylqxSw4X8TuU5q8Q0qQhcEmaU4ldUiDbp6c68EhWEraOwe1jy+HN3S92mV9LJL9qJc2sxC6hiKEgrpD4UiCHO9YuDZ0cKUua3oGUdU+rNGIaRQw2xbvYYUs0fHPrW8J9H+UicWj2cSiu2KuvSNgSRDxKHMNeM8rFjyQzNPkZQNKriWF2zlLfwiHL5DpCgqRjR+Pqa1sb11ih1brRwziEDzuapdGTtM4bAY1Go9mQxF7b2ESk5YWx8BK+tq6oN1R8tFAkr6wyxFEsUERRKPN1eXWDsW419FUELjpqtRrXtj2YUBSqV2ckoYgWzBD4/Tnyfeg3p6U4kEPrdwh5su+6dKc6J0xw68D4XXzkKnMrRnhrdC+gSGU68bNFtmOBebS03Dg/lrxlfoTz3SV5k6Qsxe9OotFoNBqNRhMt8TWvC6aYKyE0GT2w44LCiK++24MiisctWGkI5dU1huWR64vLDGFG62KUDCpy47T9I3cCc9lsuEGE7v6uyAJtuD0BByWvPWaSMxFTkyJbcfeUMK9Jc6prWHHyvpUY0i9GkUp3CTYiFpUCyyRNyqqNdOIn02iB7K9k46QdBcvOZzw/VpEcIDB6Ryj5FmVI03tIlrLNpTONdOkOY0ntQY0h/qYD8erovbokvQJhtdfSjQdpQK8JPwDfPEWuhjoPacSra4fmXzv1oEaj0Wxo4qu+83c2V0KI9Jo7GihUV4kgs4LWXL5S53FWl6IwoyCmAI4yPsfsXoldtw5vieQz6fI0B7Z0RJdUJ6U4lDCmkL1IhGw0jBdhfE2GM+zzb+r4ehy/Z4yjMjANKP7ZiAgMDRYKhWilCOIlqwG3xRBtAbFrNS1wR7BqNGl6CX488PgfuO2B2VhULuWnM0SvS34/Z3yGI8/4Aidc/XPPEL0idl95/jccetoX2P+ETzB/tfwW4r1XEbsfvDEPh5z6OQ488RPMWiz1WGeKXhG7N9/wDY6Q9Nv3pOloYwNd6q3Z3y7FzMUtG170Sv794/RPVP6dHJx/InZfeGYOmhxa9Go0mu4jvqo7axzgNOaqbxdW6HkZQL8cEcqZ8gBpRwTWNQEVteZGEBRntOZGq6np4rC8on0LpsmVx5di61EiqEM4M9UZk68tE5SuD3RVcMbwLNvcnoAzOHpFEPQvZrxiotktIlXuO1qRSos5RW2o6GVjYYWEQ3cGK3h/7JhWJGWB+cv1aHBmSRnaytzQ9DroolNRh4VNSbjk1M0wKFuETRz9ltZFwvR64MnMxktv7YOnb98aaIzdRanTsdsw7etKvPbm3njr9b0xvFDqgnjvVcTnm5+swv+m7YM3JKzxm8jvJd4hW8IhUftlhRcvv70v3nllKmwtkn4SfJY02Nv4BqkTLxUWnxdtGZkq/x4Pzj+pg5Ol8e/c4BHQaDQaa+K0VUjlWTDVXG+HfrlATjqQJpV7dhowIB/oL/s4SoMVVQ3ruiVQvPL1e6zQ0rkiOuuo09GG289cgW2CJnXYRh5Qe3bhq7i95Vo7mOJ6/Mgm3H7WciQ5Y3hIcNzb5XK/MY7+oMTtSjkv2P2AVvJIPtEUt0OKjJE5MlKM/A2st4cqO1TLml5Lkh2ffrEY//z3LFT5pMyyJgnN0lizWMr/l58vxKlHfogb7p5jvP6PFdM1KHiIrA4h4WQl+nDi0R/jsKM/RVNbvOZdYkOeVIVHHfMRjjjyI5RxhvPO/BkkJqBiWZVKvxPP+x5IkTrW14qBmxZhwqCk9oV6R9POacf0L8z8u++vtfnnacVBB24CezyjOWg0Gk0nEX/tPfBocyUCTgfUeLyhpLqAQQXhvwuwSgQXBSstIGUdmEqXfsXsbBUFhuhdjv13qAFl21lRuiR0JqemOnDQ9nX4z9kxil2yihbwGM8JQItwwLJOX99w4+YGyMs0xC3H3Q2FjZr2GBBF2dH0bFp82Gv3obj9um2RlyRCJiEBDbToOaVKoV5KtEn7y5iwIGo8fuy8yyZ47KmpuPrsUQDHtk6KropSkyHIhVt9PjVpBGdS6xTRK/FvTnDiqSd2watPTEYKJ3tgnHifMdOGencCXnx8Kl5+ahcUZkoYbFBTmDJM+grTJ5r7uE7BGIvolwZr8ZBclX5P3baNyiN1Phu07b3wkbTiNM/BaccJJWJC8m8XlX+74ur/G2m8CSC8Py12NRpNNxNPrW1QtLdUzLnmhgWRXAFYGdLS67IQlXxO0p+3UsSXVccn1qe0NHIs30j+YXSHiPIVv8PehkuPLcM9J5SjwBWneIwTm6MVgw5ajouPK1XxiAmK+kgdy9i4YDpZpTfhCA60rkdqIORmADkRRK1THm6RcIpQLpayo+m9UMTmpiPDIw2z+3/DI8/MxUOvL8S7M5bjrsfmoNqfiCeem4NXP12Bfz87zxBx7SJhym/4g/fn44yTP8NR538nZRV46KWFhhiMgN3pws8f/hc//lWKJ249F1kZTtx1w8VwRWpQR4sIP1tzM44/6VMcfern+PKrFXjygzJ88sXyOESvDU54ccQ/PsUxp3yOTz9fgcdeXIh7Hvodz7yzAs889TseeGoe3npvEW556A+8+s4i3PvcguhFb2ICFs1ZjVNP+gyHydLYloj7JbxozmdaMe3akp34z7Xnw9ZSipeefFRNzRw1jgS8+/7fOF3u76yrZ+HWu3+SBnAybrxNPoM682o0Gk13EH2N3eYFmqWS95pWQA4pVXKEsW6JKNJIVoKA6LUSq3ylzo5q4ZDKHYMKDUtjQZbxet3qdTotHDXRWXkDjJlYhYzz/oJzdBh/4g2AY3Sdcb1tYuygRqiNrUQqGwX0naafLdNpYL6xbUU4/+kAnEFNhE5E2tPpA6TM2Mwe5G1SNtxlxqLpPTCPE+24/NwtcdF5W2KbYWmoam7FEWdshmI2EhMSsMMWuVi4sA4D+0sjK1rjXnMiVv95JB6+fQe8ePdELJ9Xi/IaacQFLIUW+Dwt2PaAc7HdZv1x9k2Po7rGjatvewjNMc4gFpY6D558ci88c/ckvPDoFOw4MAkPvTwPf65sNkY/iIV6D267b1e8fO8kPP/IFEwdkYzMwlQsWFaPoYPSsWhFA9LSnGioc2PT4Rn4fXEzsh3t/aCCqHbjmy8Px2N3bI9XH5+CVmnoL65wt5t+hLOtnXPTE7A1tuDaOx8T4dsfJ519AdwtMaRhcwJK5xyNR+7YAQ/cPgGJjU34dPoqzPhV6hR7+3HQaDQ9lD7ygiZyje0uB/78F/Dx5sA0EZPvDQbeEoE6TYTP59vJk8ZCjAZQE1GIaI00WgKFa/+8qCrldWAnqVBrIkVdioVVp1biGuPrflumBynHLELayX/DMaJOacfOhOEx3LR//I3UYxbClhXBQhsJ+jtbdS6jRTZJ0in44Uy3g2h8bYOhtawgio6KkTq5EVc/YNYpwPtDpBy5gHf6G8sbEqev9wGWPi8HxZZPmm6AWdQkv21ZakQUHji5CHdf+wMGj8nH/96Yh9LGVgwbmon6emkox/LDqRKBxnNqPSgZnYdUjpEdRXGg6PVK2TOmyW3rHLEbQO5PxUmWGlmuPXtT3PTwnHYtz2ER0WuE5YHf34bVZY04/8yt8OF7C5BRkIYTjhwKm9xHyYAsXHfZFmhobuf3FIqI3kD6pUsY/dIljlHWe0w7la0ietta/bGJ3QCB/BPRniGN67demI0JE4skg6KLg0aj6UHwZ5tgR/3gI9Dod8jv2NzXS3/O1jX24ieBD0cAc26WyuvPdSvNVqnQqn4Aljxt7rCAPf95mtWwVwEoXGl9jBaO9GD1urJQwgn3gKXo5ggQcWAf0oDU4xci/by/4JpchsSCjj1Meb5rpzKkn/uXCte+SWzW5/Wwsl4znZKkkNolfUP9Gduz1AbDRklBZnSNEk5yYQUnLfnjailbT8lTddm6ZcovabrqQ+CHE4BPtwYa5ptfaHo0vlZsPaEI2S6nCLexmLhVPk45Yhh23qEYRx00CNuNkUZSvP6bIqYPnlqiOj31CPytSC3OxLYi5L/5706GsOsAbRKeLTUFf/2wBAccMBy7bpmDp5//C8M3K8ZWQ1Lld92M0mr5PcXSYAim2YeDdpX0c3dD+kme7S1l4MJztsJZ+0mDlv7YGo2m19Eqz+2asTfBPeV5LM/Zk676fUzw/nULMOtUeVDFMTJCMBQ0HCaLHUo45XAkaHGM1uqYIQ8DKzj6g9XwWHVBIz/EQUK+CNXdSpVQzbhoDlIPWyLCdTUcI+uQWNSChCwPEpL9ouva1GdCpkftpxWXx/H4jAv/VOe7di9FQgeFs4JTNFv57qYzPeVpGc5lhOkUrY8jO6lROEdDJNcRujBEQ82vwBeTgFr51PQYVBFIlSojJWhx2ZBRkoZ+JfQNl985j6FVVooX2lqx+da5cqKUQXV8opRGoyNbWrJsh4YVukjYQ0ZlqM919st5yQwzSstltNhsEv/g64QuEmdHYTLy8hMxdHMR8hx3MOQYR9CsjQ6+xpd7Dru4EmEfmodDx6dj72PHYPyWmRg7qQQnnDgaW0/Mk+8lPDn97GOGG+4AKXakJicy9czQ5TPk2ustKv3ktxuafubSnrt9JJiHqYxjmHDVIvkzYHg6Bo3OQNFQaVyHycNU2bf2fjQaTU+Eb3vaEpOQs/lhyDv4ZTTs/j68yUWiPfilcUxvwSYV17pRLn0H+PagznuYcK70Iql0Oe89LY18vW4FrbBLVxu+u5GgH2qkzlccmYFj8IajvXN7GxybOJyfM0dQoLWbwtbqyWZ1bjCZ0rigz2801l36EXOEh84ibRNglx/kHuT6mm5hYEkxLr75OUzaZSquP3EothyZ2a7XihUJbc2oyzoaW+10JL5+fGcUFhVJWzj2eoZF+/e5pTjt9gVIcdmlquq4BdOV4sJT9/wLuc2vimy3aDC3gz3Bh7/K++Hi/3ygXqjceMqm2HZYg6SXRDgccoxdflc+q7F4g75nNVpaWoqRh32GSTtth+cevgup5Y+g1Raja5KJPbEVvy1OwhUPf4+m+tga3gmJiRKXcrxx65YYOnSAcs2IlcREG/5esAIHXfYTiovz1xkRIiPThbNPPAR7TxmHy6+4ytyr0XQ9S5csRsnTw5HgF00iv0El8II/g7H6Ltb98nOcPmUGJu+0o7ljLdNnfInJ03eSxrdsBJ9nFTax+i6a/bLearNj+f7zMXDgYHMn0NzSgsXvXo7RC++WylN28HgulAkh569ZJ4FtqbIbHMVYtd27GDZ6nPqqK1hX8PrqgPeHieCsMnd0EvSrDQxXFUmAEb4S50QIkRjWTxKtHQG2cJXcT5gnM8eLpf9vX4CjVywsNQpRKBSqaVISmfah7gwBKHYpeq1gPrU3ZnIA5lsME31EzeATgPGPmxuarkYJ3puexcRJuyJR2onxit0ACVI8fBKIPSlRvfiJF9FcaGnwdIrYDUDR29HQ6P3T3GAIyJR0V6dOUkjR2+puhdfrQVKyC23tVIHtwYZDrGI3AEVvUoojYp/k9mAeupu86w1/pgWvpqegBW94wRtg8ZJlcHx1GvrXfSjHysGBOino/LDX6CbBGxwV4O8HIotd1ubxTIVJi2ugEwlfwUd6atLvtL2xXKOpZTkEVzg4iUV7PsW9hep6owCFg0KXQtVK7JJI1jWexg5v0Yhd5mlZtVHgY4Wd6SLFkZ3YmhabG5rugjlLceSR33FHlpbmFvi8XhGr4b+PdmmWuHSm2CXstBXuWrEsAbFLmF5e2ddZi1vCptglbknHcNePZYlX7BKKVOZBuHCjXXh+zGP9ajSaHsPgQQNQfMz7qN33B3jzxhvSjNVyHFKgK1hX8C56xFwJgT6gg/KBocWy9DOGAONr7licwKobpZY2XRUoOCOJXvqMRgo7EE4kskQ0W1mBK+rMlV4M049pGg5adtkwaS9/IpnY2GCIxseXJqzVte27oQQTsLIPl7K0iZQpfloNldYq4S5+wtzQ9GTsdoe0XdatUuLF7uiZbkeODRCvxEQ7Emi+7WQ2RFxjwSgPERqzGo2m18OaK3PAeDTs8jYaN/snGtJGiz6RnT2wLbu2luUYu02yhINj3SYFVZ4UwLTCDi40hEu0HZo4iUTgHR9Fr9X7PtaR9D+1wstz2xFYFLsUveHgLGLsTNeboWi3sqhSUHKWu/awsnRTKLNjYHvjjFJ0c9zeSLOyBZOZsrbMMI7BD0OWJyurfMU35oqmp5Iqjazffp4Jt8fTYaGVnuHCN9M/gtPlUq/OewIUpMkpLnz5+ftIz4xhMoZ2YJh/z/sTNdXVcCZ1TrgUmWmShjM+fVelZXeQlu7Cj99/iTZ5xFD4ajSavk12XjGSd74NnqmvoX7cdVKR97yObWsVTf0ccyWE9sQshQunCS7MNlweIsEbZ6emwMxpngiWXloXKZDCQX9RWhStZmALQMFrZWEo78VWXgpVqyHWlNgVwRpu2t9g6M7QZCFUs9Pbtw6zsVJVL+UmipEvmJeBMhIpXKsGilXZ1PQIsnJceOGpR1E5/0M8ftsZWLx4oRJysUKhlpXrwl03XYXk5j9x04WHo1kapp0lBOOFAr4Vibj2/COR6ZuPW6++AJnZLjUNb0fIyHLhvTdfw7xvn8cbj1+B2b/8qBoOHSFRxKXTlYR/nXccsrEI1118KtJEoHdlwyFb8vCxe2+Hr+xb3POv41BeXg5XcvfmoUaj2fBQLeaUjIFjh3+hdbfXUZE3VfSK7KTM6wHCd22NzTFSw0ERS2GphheLEOOA9c5qSLAAtEpS9PpMsarcGyyEa15WeBGtrLMiZDkcVyTRSwul1TS4PDcasdYTieSSERC87cF7D5edzD/6UUd6FclGCifyaG+EB1rZOb7ygLzo3gJYXdOqbGq6BKczCZxhltPMBi8uWTJE+F33z7MxcVQqjj3+H7jrP7dj5rv34P23/qeEcOg5VgsFMoXaJaccgdOOnoq99zsYDz94J5687TT89ussZVUNd96GXmgpXbjgb9x95RF4+L5/Y7e9DsAlZx6Ji08+SASvQ8U73HntLRT2d95wJfqnrMYZZ56DG264ASt+fR0vPf1fJRjDndPeQrFMcXn9uQfi3tv+hV332A83/+tcXH7qAWiSxm1KavjzOmuhqE1Nd+GKs/+B/aeMxMGHHY3777sT7z59Fb784hNkisAPd976C9vr2iqs0fRW2Ly1FW2H1H1fQdkOz6AqcztDb3Sz6F07SkPNT8Cn26rVdaAI4QxmFC9cp3Bpz5JLMVVeYy1kCcUofXUZFnUOX8GHs0py7NxVYTrS8fW3VOAKCjyrzlUU6RyxIZy/Kq9Nf+T2RnzoSTBtSy06FlKs5kXR0Yw+0Csq1reuMz2KcoCUCIKXeVrbAFS2M0YzO83Roms1bXQodFGpbTKsxqFkjAF2m21uaLqSgQP6Y+oBp2CLrbZb02GK2OQ3w5pj5mf/w9WXnYOUtEy43YabUGZmJj775AN8NnMRttx6x3XOC0eClDvOjPbnzHdw8/VXSRvYD5/PcFnKycnGM089gaW1yRg2Yozq8NZV2B0OLFn4NzJsq3DmmWehsqpa7U9MTER6Wgouv/IabLLlHshIT4c/ys5XyqfVloDvP38DF519HAqKStDcbDS80yWcn374Fq++PxNbb7+b3GvkdAvGbrejtHQFfJV/4JJLL0FVVY3kD4czS0BWZjquu/5G5G2yPXJyc+G3eqvWAXgdznT305fTcMPVFyPBngSPx4h/VlYW3pz2Gn75ux5jNh8v5SFyHqakpuE/N16MU447GFdcqUdp0HQfepQGkU4RRmmIBiovBrfk57dQ+O2BcPHBIRKl0dkPpdu9h2GjtlDHdQVBw5JJtN4uBDxGpb4OGZK6ahIDEwpTCtRIOpGW1zIRvRwVwQqGEzzGK8fHDSeml4k4C/W55SnBbhQUeVaWzUjDb3GyCwr63gDTdHFZeKs2H6T9cg2hGSlf6ArCySHCWWc5KgMt9VbuELwuz6tsxx2EVl0rf9xQ1KQkEicKadVICvMwHnYWsMU95oamKzn44IMxbdo0c2t9Hnn4IYwaPQYFBQVrRJ9DhOLvv/+OX3+ehetvvFnta4+igjzcdPOt2HyLLZCamirFwijjTqcTM2fOxIwvPsWj/31S7etKjj7qcOy51z6YMGHiGqFG0eqR9Z9nzcKtt9yIeX8vUvtj4YH778PIUaPRv3//NelG0Tpv3jz88fvsuIbj2muP3XDEkUdjwsSJa8JkXFnF/yBp+MgjD+K7739U+zcELmci7n/gIYzZdDMlcgN5yPLwyy+/YOZ33+A/d0X/O2a5O/DAA80tjabr0YK344I3mLK6NuT/eAIS5jyL+oQ0LN76TYzdZhfz2w1P0O3KasFUcz0EWlk5tFgAihIK0EjuBBSi7JxEQWslwBgOOz0FXCVa5IESLszCMGHwFFoDA3o90nBnFF9Wr9TpC2vly9rToGi3SnM1qgIts+Z2OJjO7GBGd4RQ2NhIFbFsJXZ5bntilw0O+upGK3YpvumPzXyrlnCt8q9wd3NF09W8/vrrSjCFW1atWoUDDjwIb7zxhhKmAWgB/eabb3DOeRdg9erVYc8NXX797Q9sv8MOeOedd5RACkDL4ezZs3HzrbejpsawWnbV0tDQgHvvexCLFi1eY60kjN/HH32EzcaOxZdffxf23EhLWVkZDj3scLz77rvrpBsF7/Tp03HyKaepY8Kda7W0tLTgmedekJ+QH0uXLlV5QPj5888/K8vuW2+/p4RwuPM7Y1m8dAV232NPJVRDy8MPP/yAy6+8GhUVFWHPDbdosavR9C0KM2xI2OUZVE54AN6UAWv1WxcRJHiFYefQJGBuhMBZtIJ74zOi9IOlSI0Uafa+L8m3FlLs/ESfVCV6ZeEUxKHuB7Qmh+vQRMFUKfEKXD7S0FgFFsNeEVqiA6K7p0JLuZXPMYUm07k914xGOZ+CNdytZqUY6RyOaMQufYcHitiNxleX1ic2oNhIYdhVkocsR+HIGCW/kj3MDU1PorCwUC3JyclKoASgZa+oqAh5eXnIz5fffhTQQjxq1Kg1Qi0Awxo0aBByRbDRVaIroaWZ1x0yZMgaayWh1ZQL48t4xwrP4UIXhuB043ogzWINNykpSZ07dOjQdcIMMGLECBXuhhj+LADLwoABA9YRu4RpV1JSotKSi0aj2bjJ3fpM+A/4AYNGdN2kE2Td2i93BxEtx5obYaDooTgJrk8pXmilo3Bdv541oOVxsFTgVj23KXwopvhMYRjhRDRft4cbJosCma/CafnkOVbCNdKoD4z7qjCuHD0Fxo+i3IrcdOsGRQDmHe8xXPqojmqSPlaCmRZhDilnBa34tOa3J7h5beZtIH+Z78w75qEVW94rCsPCVUXTI6D4o3WSYpUL18OJrvag327g/EBYXKL1j91Q8Prh4hXwM44XpltweFw6SiCugSUQ547GNRYoqkPLQ3CDQaPRaPKzUpGRJbqhC1lfQY65Dkikg4gFdGVYLcIp2JrKhxuHGGuJ4OZAy0KJtO6tZlFT1loRVXxOMrzQMWIppnItLDwUhHzdz89Iois/y1oY0oJKK3ZPg+nBTmpW6Uof5GD/6nBwVAymTzgNwvRinlh1dKPY5flhz2We5lnnaQBVPiR/lRuM5BHD4oxUnLAiUsfGoj0lz7rOv0cTH8uWLcNff/2l/E+5zJkzB3V1Ed4GWEChtGDBAsydO3edsAKd4boLikXGIxAnxm/+/PlKsHaElStXrpNuXK+q6ti07hS8wWEG0jKeBki8LFmyZJ04/Pnnn2hslHpEo9FoupF1BW/tL8D0yVJrWrw6D6A6GIlYoRgK1ivKgucxxKpV/UproFUnMSV6TfcGCjxaAIOhhdbqlTnPoT9wJLcGirtIE1rEMolCV8Epe5mm4eAICJHuh1DERxLytA7TfzccSpRaWJaVv26+0UkuEsrfm28AzHxRVl2z7LTHqg+A749svzxqupVTTz1VdegKWPS4fsopp5jfRg8F72GHHbbG0hsI6+STTzaP6B6OOuooZaVk/Bgnisq99tpLdczqCJ2VbsFMmTJFuRUQhkmhO27cOIwdO1bt6wpOOOGETr8vjUaj6ShrR2momA58vZ8IFIsJDaxg5zRa+EKFKI0fVkONEVr7VlpYLl0ionIlTFpQQgUVfT+XV5gbYaAvKV+vR4KCy0oEUhTTamklArsSCnC6kISD6cvxbZlWVrBhsqjUuvFB0cqxk8NBt4Pl5UZDIhQOB8fZ9yJZ0wNW3UBHNGZzo5QtdoCMlezxwOTPpay1Y8nWaDQajaaT0KM0yKO7E0dp6G6M2/KI8Jx5XOxil1CwstMZx2UN9tNiotHSS+teONFEn9EB+eFfpdM1wsoCSAFMUWsFO2a1B8f/pV9xOBjXFZWRLcVdAQW5ldglvIdIYpewcRAm6ddAv+hwUKSulEZFuHyjCwWHP4skdmnF5bUDYpflgG4w8YhdUj0L+PVCc0Oj0Wg0Go0mNgzBO+c6EaYr1Wrc8LU7O0XRFzbYX4wimNbcUPcEQgvjQBG94dwU+Do91I83AF/jW/mcttd5i1Cr0UIZbsxfQhGvJmYIEvBdiZq4o9bcCAM7mbXnN0vCdfILwKHDrGbFWyENoHD3npNuuKNYaV3mO8tBoLHAdGRDiA2icJb8WFj0XxG+G24MUY1Go9FoNH0XW5u/uQ3v9BdBaiGwKCzpK0prXbRWTwqtLBFHSSGilFZBilt2dgrGL0JpZaUhjIOhsOK4ruGGy2JcOHJB8DmqE1Vu9O4IFGec1CJYoAfDuNIKHcma2dm057LBe6PLRbRxok908KxodBOh2KUvdTgoTsPNdkaLstU0zSRQPlRaysIGC2dOiwbGiXnMYsEwrMTxkJOBrR4xNzTdxUqOrcuGbBSdtugxxSGz8rPD++0vXblS+XlGC69bUlxsbm0YPD4fysrLlc9uNPAe01JTpMoL/8akye1Gc0uLJFd89Uir3488i/TjL2VFaWlMcXVxCDOL8OIh1jzk/Qzo18/c0mh6LtqlQX6vfcilwda28u02fGMxwDenmeUsawH4iptChlbcSDOoBQgMBRZsSWWdr0R0iBhmIvO1d6hIor9o/whjN9IKTOux8veVY2N9pvA+6EtsBcPk9eN7VsUG72WZhd8soUWcAtzKMm0FxShdSyiSKZgjWbYXhvj88hzOaGdlDSaM9xr3BcmLahHM7Vl0+YBm2UiT9GU5CcB7Z0e9cGMOOyUe+0n6aLqNJ599Fqu//hp5efkintrJY4FCbOHSZdjx+OOwx9R1J7Y545RTMKGwEGbJaRcKxuamJtTm5eHKK64w93YubhFjZx56KHbcbCx8rdHFLFHu8ff583G0xGn85pubew1+/PVXPH/LLRgxeLC06y1+1+0hcVokIvw/d99t7ljLcQccgKmbbxF1XJkfi5Ytw04nnojddt7Z3Bs/F194IUY6k2BTgjea+7PB7/VikRx76223mfs0mp6JFrx9TfDOuakNf/zL3AyCQpOdv6ysEhwCTE1RKwsTyAqenplm+N4GB0XXA1r1QoOnAOXIAIFX6hTGQyw6VnUWnG0t0ji8dB+wsoh2FhR6S8qMdA0H04vuH9G4bMQLLd5LgwQlywAn7LByH2HGs3MbxS0XNlZCrfShMCy6RrAhZVW2InW223ux/NhLzA1NV/LFN99gwbvv4uSjjoI3hqHCHMnJuPHee3H4pZdixJAhat/lV16J07fdFoMGDpTsbl84B+AsZx998QVW5efjeIlHZ3PcwYfg2WuvgU9EWaRqLRRHSgouuukmXHb//cg3R29YXVWF2887D7eLEPaJUI8Xu4jJP+fOxRuLFuGKyy4z9wJHizB/QdKR6RdTXCU/bpF4HvLPf67Jj3i44667MTUvF+NE5HtjGOfXYbdj1i+/4Mu6Opx/zjnmXo2m56EFb98SvAlwi8gKB4UJrYJWljq6OVAEDikyXpFbaBeVeBTF9EkNrhTVK/AwPrrskDZYwqSvKIfMimTd7SzYESvPogMXYQcyq6HBOguOyGAldhMlcdn42JBil9D6S99mpjvFNdPeUuwKtOayfFDk0iobSewy7vS95qgQtOxGer1LC7Q8FMNiVV41G5x3HnoIJx99NOpFqLSI4I12qa+pwVXnnot7/2U0rH/4+Wdsm5amhs9qaG5ec5zPfEvg83nVttvjgccTElZDA3afMgXLRPTSVaAzuf766/HYRReqMWObg67ZarrpBOLCYbYC3wWW+upq/EfE6I0XXWQEJlx3wQW4XUQ+v2N48S71IpZHjRiBYVJ//jR7tgr7zrvvxr2nn264S8gSiIffbDx4vR617QkXV8mPy888Ew9ed506Nh5WlJUhZfkyJXbrJb0CYQeEr18Egtqn8tCIS2Dh8eO33BKOxYuxYtUqdbxGo9FsaBJgsxBRfF3Jip5Cj8JGKdcwUMgUmEIm2P0hFLodlImoo0tEICg+4AKvwoPha3SKUI4iwNf4XQGtjhTuVtACbJEEHYZCkTOhhYO6kOnbVcOk0XWB6d7e9ZgWbLBU1K8/+14wFK+qYUShG6FhFIDlgaN0WFr9gpukmq5kcL9+8LXI7zcO/CJsB5m+txTMwwYPVmIsgD3Rjt/+/AXPvvMOfvx7BdIzMuBtrEWNVD/2EH91j4imoQMHorEDVtNwZCQnw5mUZApcA6c07N/6+AM8+/Y7qPUlID0tFctEpLnCjJDil7QZWCSNdZPBsu6XuHYGFIpbjBqF1eXGG5gEiWNOdvYagUto/f7sy09VGi6vbUF6ehpWla+C3ZG03s+O8SopjP/NWY2I5lFDh6p4BaAf79LF89T1P/v1L5WHCZ5GVDR54Qhxo+JkIiM22QR19WH6C2g0Gs0GIAGpQ83VENQEDGY1yVY7O1OpTmsWyoaWQPr80jIYSSxxuDFaMwMuC+EEb3dBYWk15BnvnR3AOhsmZyR3CopPqymZu5M6yUfGO9K0wGxA0FpPl5BIFl1C6zbLGP2BuVi9WUizKK+aDU5Hp4cNCElbQsI6Qo34W/3YbNRmmLN4IbYdkotL/3OHCODf8MrnM7Cy3rNeH02e39GZzkJh7ILFLmH1tMvYAfi7xoG65X/ghqdfw4wfZ+H9b78TMbx+Z9rAsOYkNKyO4vP713RO472H5gcn7Nhh6wn4c9lKpHsqcMVDT2L6z7Pw8TffwJ+wvuGgI/ELm4cSv8GDh+O3RYuw27hRuPHeO/HDL7Pw4qczsKiyUdq+6+YXj+/sPNRoNBorEpC3g7kaAgVpbYO5IbBupJW2kRZfESRWHasodil62dHJqnMUxSP9dGk5NivwHkORiF4r1wG6Zljdd7zQf9jKlYEWc6anVTp2B7x9Cl2OkGGVFOyExtE12ICgO4YVfOCyLLFxRWsxt5m+4UaJIJmbSd5EcD3RdCmJiXblj5nsSlLij4KGVr4Ul2vNdrRQKCa7UpCfnYMPPv0El553Mb748G3c9er7GDogv9N/dtHC62alp6tRJt77aR6O3iIbH/30G76cW4qkOF2MbAmJkm4ONZhNkpNpZ1f3HxDQKUlJKl1jhWGkp6WjMCcbL02fiRsO2h7/+/IHvDrzd2SmdM0bIt5PblY2lv32DXY66DR8+9VH+M8Lb2D0kH4ikLspEzUajUZIQOaWIiQspp2k+4F61R784JJKi2YPvoZXPr4WYo1+mvTvVdY9c18wFDe09DaL0OlJUIBbTdfLCjuaKXFjgSI6HPSRprW5p4nd0ipDpIeDcaavMWeAs5oCmqjyI2UneHIKQquu8vVe13K0hkHHmyua7oZid/7cn/Hr/L/xn6eexeezZmJJtRflpQtxz4sv4/PvvsWC6qaYRK/X70NBTg4mb7s1rvr3jdhsp31x10n74JOfF673SrzLkHrKC4dUCWnYvF8G7v30Txy7z94Yk21Dgy92AZcg6dawbDZuljS76emX8NCb7+PqO2/D/774Ek++9gKeePc93PrM07hBvktNjv3NDq3ATMOdRg/CBU+/g2P22Qv7j+mH5TUt7XoTdQa8frFcf/CYcXjtmXswcLMd8eA5R+Ld737vvjzUaDQawaiBNo3QeYHWN3ZICjdxBF9psZd+4FV0yCsu9R6S/pu09gUPPRVMmQiocJ3XuhOOTmA1DJeVQI2HQLqFQx6wyg0gBsGwwaFLh9VwdPSBHlRo7RLCssF8bpLyFFpWqBtYzmg5tnJxSe4HbHKGuaHpbmjJLV29Gu/O/AWXnnseclGJZbU+1FSuRq00hovTErGgsh6JMbyyZie1o/fcC47MYjx4+RXYb9ep2H/nXTBpkzwRwxaNoA1OK7yOfBy2/SjssvM+uOfCc7HvxK1w7P4Hos0rv9+YsSnXg2223RmJ7iqUFJfIz8KDNlsiXHa7fOdBYdFQDMh0KZtArLi9fhy96y7YdPOJuPeSi3D4jjvgwD32RZY9tpEc4oWd1I7fey94E9Jwz6WX4pgDD8aBO++MqWMGdGMeajQaTUDwFu8PlByqVsNCEbK61hitIFylxZo5YLXjEipaOPwYrX5WIw1YddjqTijgwhHwNe0MrCyl7KjXVZ31YiHYxSUAGwfssMhRLsKJc74BCJQLpl3wU5eHs8FE95ZIZYCiadyDorJSzB2ankCbZODw4my8/eGH+Hz2XPz828+Yt7oZk0bk45NfFyEpjtfyFL30U12yYrnq1EYBFY1QmvH64/j8tzLceMn5avv88y9Rn51DKzw+L+YtW4GK8hVY3eBWna7iJSEhEcsW/YZ+w7bBz79+jwRnMoYV5WNZVR3+cdhROOXA3aWKiT98phmNqQuWLhUB7JNtd1Ri94ZLzlOf555zoWREGe6561G1HSv07f1pzhy1PuuP30TE+7XY1Wg03c7ad0x5k8yVCFDo0QrHXvmh4iUALXe04PFYWoWDj6H1j24OnLUr2He3p/nxEr6St7JKU6R1BlbhcISKnogtKJ+SnMaMbxy6LJw45wOObi+8x1DLP0cACZQlWo2trLoB7GlA1pbmhqZHIL/rVm8TRozZFoPysnDuSRfgkG2HYtcdJ6Gw/6bYfhP5jRuHxQQ7Q7VULcPRN9+F1BSLtwVh2G6fozFpTAEu+NcNavuGG65Wn52DDalSHex55tmYv+AXLKj2rTdyRPRIirT5UYcs7LVZf5x37PG4/ORTUJCTjTOPOg5Onwd1q5ZgaXWTaufFA0dr+Obrj3HHtA+Qwt9plFx49Y3q86abr5PfdwFOPf04tR0rrpRUXH7z1Whu9eKkfz+A5BjyUaPRaDYUhoJpXgb8fpVajQoKGVrluFC4hAoaQqsvO6fx2GBBw0qcU9RuIsKXnZrYMSvScGDdSbLFw6IzxuSlP7CVK0dw7+943mtuKOieQt9sCt1B+cZkIuHgfTGNQjupcLQPWnJLRejybYGVr24o3nrg57PMDU1PwON1Y9ddD8EWhakYNniIGve1X2ERHImJGFZSjHHb7YUpQ2J3ReDUwVlFQ3HIxM3hb68hFIQjKVmuLcI0zXgzk55u8YYmTrxSps8/+kDc9NQr0m6Pf9SUVr8PGYPH4azdt0Z+4QCkSFsxJSUdA4sKpY2dKFWp3LMzHTeddgIam1vMs2KD4wRPnrIPNsl0xtTZL9VMMyPtbBKv+IRqq/zO99l5e1xxxwPYa/Kk9hu0Go1G0wUYgvfv+6RSisM3lRUZhQstdRQy4QQcBRstvmpIsyBoIaHQ5VBmcfZ23qAw3k6LeHGcWD7I+TThAyWah0rgOJ7Dc9Wwb2Hg8G7BBm9/NIF3EfRr5ugbVkKXMJ+DR51g9OnOwM5oHIeZ9x3PLZW+C9T9bm5oegIUvRS0HF6K8FU2RwrgNgVwvK+xvT4vcjKzVHg9BZ/8bvsPGo1Dt98advjlZxz/75KinuZbdvBiOK1tkoZm2pFwQ47FCscqzs/J6ZY05MQhg4dtiaL8LIwfmC/lQAtejUbT/Yi0kgpx6fPmZggUohQ4AwsMH1xa+MKJHdbTFDLldcZwVVwPNS3QvaE3VXwUmokWPoj8jhZMWq/ZCYsL75kLLd7By5r95qJe88u5Vtbd0NENetBDPyoC1hyKXo5osbLSGGYstMFD2Oih+wZnd2MZ43B2HNXDisVPmSua7oC+tZzmNlE+Y17kvISgd/SBfesdJws7dR2/977qM9z3jENgPNrOhELTHnItY7GhzZGCQ7bbGiccfw5GZEu9aAtznMQreFxZrjOenbEErhFMpLyg3+zxe+9tmYah+REPDCdcHHjNQ6buhstOPgmHTt1Vqv4wceB5smg0Gk1XYWtrXNSG94eZmyGU5IrADfP6juKVlt3aJhG7IcI2AOtSnpsuS6Bio/WyJ3bGCgfHCKZoW1lh7giCDwp2wOsIFL2VYcabTaWLR4gPL0VwT7SCh0OVC1r7wwjcACwDFLbpcp/hfCE5XB39xEOhn/nkL8wNTVdzxfHH47RDD5WiK2U3RlxOJ+5//XXc8dRT+JTTAs+ciVHDh6+xDoeSlJRk2TEsScJ69f33cfIttyA3y2IIwTi48ZprcMyWW8JtESclPKUuo8tAOFwOBx557z3c8sgjavvy007DhZJezXGkVyi87t8LF6JpxAjstdtuuFXu/fCRI+GJ0CCOlIbMjwemTcPtTz5p7omNP+bOxUzJy50mTFCCNhyRrs9xhj//9lvscMopGC3lQKPpiSxdshglTw9Hgl/KONublDvBn8FYfRfr/hZg+pQZmLzTjuaOtUyf8SUmT98JoLdR8HlWYROr76LZL+utNjuW7z8fAwcONnf2Xmxt5TPaJHXNzSA4skJ+hiG0rMZUpQ9mVV37oyywsxp9P2kd7qBVoUtghtMaSys1x50NhSKNVsmOYCl4pYEQ6tPMNIvkRtAToBWbI3mEG74uAMsSZ46jVTdSMWDaLDWmUF2H1EHAngvMDU1XM+3NN9HU3LyOFTNa+Oo+LycHe+6+O1paWvDSq6/CKaIrHEkuF15+6SUcceSRcIeZypg/T8bg6COOUNudxV8i4r7/8cewkz5Q7FZWVWHevHmYNGkSvGFELN0HBpaUYPKOxoNq2htvoKKyUp3bGTDdTjjuOKSlpWHJ0qX4fMaMNZNVhMIpkl977TUcdthhYdMwOD/i5elnn4UjQh6+8vLLOFzyKNz1CS3Bxx19dFzlSaPpCrTg7WuCt/rHNnw6wdwMQrkzmNYTVkgUvVYDh9MSSuFLi68VAZFIQdfToWijhZI+iPRPDoXpUJRtbsQJ/ZorJM1CYTpziK9QXEz/HmrlbW9YMaYXhW6mCPlIzzb+0Gg9axIxwTBDyRgD7Dbb3ND0ZR544AGcdVbP6qhYW1uL77//Hrt3QCR2JY888ghOP/10c6vrefDBB3HmmWeaWxpN70ML3r4leBOQYnET9MXka31CtwVa3dyyhPrmEs6wRV9fTjBhJWh53soqQxz3ZHiva/xQzfsPha4ZHcUqDDURRZg0DuRFT4MuDFZil40cjmfMoehotY4kdpnW9G9m+eAMf+FI7f0/OE10eDrBDaCzaW5u7tD4u11Nd6dhT8xDjUaz8ZIAZ66Ikm3MzRCq6wwrZwC6MFCUWAlfWic5Lis7uIUbwzYgnHsyFF6Be7MaJ7cz/JApBq38csP5vzLtIvnFdheNYQQAhS0Frhpz2WJCCsJkZuc9ukPw3niPgc594Sjc01zRaDQajUajiR7DcD3oBPWxHhyNgK+WQ0cUCAhfitdwnSYodil6uXAoq4CPFl9tW41t2xOg0A1Yd3lbVsKrszre0TIeDr7SD0ewGO8psFNiAN5PLi26xcYYy1YuMGxEqclJJH0DFn/eVn2zNLIsrMWJcp0BR5obGo1Go9FoNNFjKJIhpwJZ49TqelBgcbgxipFQsUXhwjFWOfwWxRgtdMFQ+NJvd1g/uUahYfHrsaMNSNwpwgK3wLF2Q25HQWslO/R1BlYzudHiaSVslYXc4rvugH65m4jA5UQizF/66trDCF2WjYDbAu8h0LAg7BVPf2arqZbJptdLunewo6BGo9FoNJqNEkOZ2ESEbvOkbEWwvlKM0NqrfExDCLxup/Dl96HD+tDAqyZUsHi13ROgj2zAWs37sRJfaqQJc72j0Npt1YObojccjBtdLXqQ5lUCN2xDRiK5xporZYNlJFjIc5UNqVVh3iIEk7s9MPwCc0OzMcDpcXsaqakho6f0cOxhRpvoSjprdAqNRqPpDGxtgel9yPJXgB9OEFESQXwQilcOLZUsDyUr4UVRyAqXYqinV3wUZMEWR3aa4liy4eAQa3Rp4H3xlX2b3CjvtT0xT6HHtLLJH4pAuoXwmrSQ0/IZjiK6BVhYxHk9zpPf0xoRLE50hWGjJzhNg2FaUNA3hHlrEEr6SGDHj6Ss9Td3aDYGjjzySEycOFENXRUPLpdLDePVWVC8LVu2DFOnTsW+++5r7u3ZnHHGGTj//PNVZ7toiTR2biwwnGeeeQa33nqruUej6X3oURrkEd2nhiULFrxk1bvAd0eIYIniYUGrHoWvi1bPdYNZB/rwKoEox1v5dXYHyloqgjNYdNHKy6HIItyOJRSfvNfAPVLYKqEbT2CCSwQtfWIjQfHdGaNGdATeI9ONApf3bAW/o9ClVTcasrYEJr0vT898c4dmYyK0aooWjut68cUX44477og7jHD0tvFiP//8c8tJMsKRkpKC2267DZdccgmamiK4F0UBJxQZOnQoRowYYe7RaHofWvDKo71PC15S9R0w81gRJ4vNHe1ACy47L/F1f3vW3IAg5MJju8NCyVtmZ6lwllVOBtGTRpLgsF7tdfRjOrJB0VXClwKX4lUJelkiiQp+RVcGjubQHIPFreRQYPyjcl9hxiTWaNrhqquuwo033mhuaaLlmmuuwXXXXWduaTQbN1rw9i3BG3y7a8mZCOz6EzDoeHNHO1D0cNKJ0mpDMCo/XwshS3FESyCPYQemgN8vxScFFBN7Q0GhFrhuOLFLv92eJHZJlaRne2MXM/0DopL+vTw+2GodL8wruiYwvDX5Jb9GftLnlt9Zid2ANXe1lAlOFRyt2HVmA1s/Dkx4SYtdTdx0pmV3YyJeFxKNRqPp6YQXvIRiY+sngJ2nA5mbmzujgIKRPe5LKw0/WAqjSEbcgACmYOO5ahxWU7jxXApTfk9RFysUfazAGQ7DpFBjWOGehbG8au9qKjllr8S7XeTGKEJ5v7xX3hN9hClW2SmP+5nODIvpwLThOvfxO6a5ygM5NzQfVB5I+OHSLgC/5/ls9NAthBNS0Fc5Gmj5H/IPYI851sPkaTQajUaj0cSBteANkLsDsOssYNtngfTh5s4ooPhh5y8OaUbLL62nFFaRBFOAYMsiz1HWRRFSFHBclCCThfsosCjuAsOjUagFjuN+Cj2GY2XxYTw59mukqXG7CqtRMtgJrNxihIz2CPjWBsQtGxAUsMEimPv4HdNcWdnlehbJtR4BkVvVIPlcFZ9LSP+DpIz9DGz1KODMM3dqNBqNRtN9tLW1wkZ3Bj4Po30m9hXMe7a1+tAWj8GxB9K+4FXYgAFHAbv9DmzzdGwWX0IRRetpeS2wSkQRxSUth9wfD0qQycJMYBgUXVyPRagRij4KSYpkK+hLeoAcM3WmiP7ngDFXS1ocDuRsa4wgkNwPcKRLEkVISn7HY3gsz+HMdgyDYTFMhs1r7PEXkGQh+HhftJxTWHZ34aOAZpopS74pctm4iIUEu6TBESJ0fwQmvgpkbGZ+odFoNBpN95OWloGqoslotLnkuWfu3Big3hCZ0diWjOq8KUhLzzT293LCd1qLhtWfAAseEMHzniSOCKB4YWcrjkbAcTc5U5fqzLYhO7LJ7VJsU4DTehyJzLHAlK8lTinmjnbw8xW+CFKvCEBCkWtPk/NjGL+zYjrw5R5S2CLEjcmTmmx0ErSara2zYGOC1l9aiNlAYNrFWWQUSbmGy8LQsyX+A82dGk3nwpEGOOKAJjauuOIK3HzzzeaWRqMpW10O56KP4PjlYaQt+Qrgi1g+g0NtXHwscl/gM0Cs+1u6udMa7WkUu8U7wT3qDPhK9kBBXt+Y9Cn4dmOjYFdgu2nAXguBMdeKeBlgfhEj9PGk60O1iERObEGL4epawwpMFwWKUuU/ah4fM1Iy1VTIHrmGiNGVZse69sQuLbE7vBW92CUUtkmF0iwcZixcj0XskrzJwNZPSc5EGHifBZNj2DK9ymShuwjdE+K1mBO6TdC1gYKW7iC0JNMPl/nBTmfMD6ZhvGI3f2dgwvPA3suAsSJEtNjVbEC22WYbc00TC7EMY6bRbAwUFuQje8IxwCEvoGK7S1FnS5MfinwR56OwRyP3VYdMVG95FWx7PI+cLY/qM2KXxG/hXQ8JpuxDYKmImpUiFH0bwCd2neHMuMg+fqrxMeX6yrVBVgPuDcp3NQ7rM90VdnhbWnK55o5uoOwj4LvDYk9HpgWtvoE0YtIkyh+uM00CadQmfyhy+dme+I+XtKGG28LA42Q9Bv9vjUaj0Wh6GHzZWbfgWzTMuBv9F71iTDAasE3J4zSi5TTa/d1h4RWh65PtlSVHIm3ChUgftA0cwef0ETpR8AbhbwZK3wGWvQis+kAEVoydmLqTIacAW9wpIjEGy+6GovZX4IcT5fM3c0cvILkYKBGhTp/vbG1l02g0Gk3fgYKJS015GbJf3B62VQuBJNlB41KokCSB7Wj3d6XgJW5Z7bcpqvf6BNl5RWoXb6UvEpwMnUei5AZFz3avA/utMkZ4KDnE8GftqaQOAXb6CNjq4Z4hdknmFsDUH4Ax/5KciuDi0N0w7YafD+z8BbD3UmBzaTBosavRaDSaPkZA1+bkF8J77gL8OfL/RDMArXyZHMcL5W6hVf7zTa/cyF/DLoL3uN+RI2KX99ZXxS7ZMBZeK2jpZaeschFGFV8B1T9KAYmxd39nw6HWRlxsTLJh68GisnkZME+E5OLHAV/Hpv3sMK4CIHcSkCdL4W6ShqPNLzQajUaj2bhYtboCti+uRsHy/8FWX24o4oA5MdiiGthHrPZvKAsvP7kIrakFKC8+DAk73ID83Gxj50ZA1wreUCiAq2eK+P3amM645legaan55QbEkQkU72NYoYv3lR29qE3jqTT8pFe8DlR+IwWYzbQNCC3LGWOArC1NkbsDkKbnx9d0DcsWL8SAwZuYWxqNRtMzYU8Y//Kf4P/iBiQteINGX2X5VQQEZzhRGrp/QwheShyz24572CHAhGtgLx6rBpzYmOhewRsOb43hu1ozG6j7DWhcLCJ4iSzLDIEcD64iIHtrWcaLaJtojBjQk6250eIuA8o+kUbDD7LMkjT7xfCfjgdnFpAyyFjY2YzuFBxvmWLXZjcP0mi6jhP22xEZI7bHuy8/h8//XIFBEWaaPufsc3Hf/feaW/Fz/qETsd25z+OIneQ30Et584k7cOOj0+T55sDX330Bl7k/HC8+8zSOOr5jMxvO/+FDHHryP7HlLofh6buvNvdqNBsntbW1SPzlEdT98T76rf7CEJuUG1aiNHR/ZwteKnFZL82ZgoxR+8K/2anISE83vtvI6HmCNxItpYYF2FsrmVgviznuLRebNKUSkyRjZaEPLkVucn9ZSnq273Bn414NNC83Fk+VCGD59bS6DdcRTvbAtLBzfOBUYz0p3xC5nEpao+lBDB06EgsWzJXf/UrMXCTtr5qfcNy51+OYC27AMeMzcOEtj+K33+fii8/fQVZGLr4SUfz9czfi5c9+w6sffomfpz2I1z76GCsaMvDZm0/juvNOxOtf/oF3vvwBv776H1z/4Cu4+s7Hsd8kY9KTd56+Fzff/wSufPgd7DNe6o1eymabboHf//gV9fNm4NGZzbjo2F1xygkn4ISLb8KOQ1Px5vuf4u1PvsN/H7oLxVlpmPb93xiRVIZzrroLDz39FCrn/4oZ06djQV0yrv/nqZj74ye48YHX8MSTD8NfsQgnn3c1brzrEQwpMIZcPOrA/fDiG2/jkuN3xYl3fIIxBWq3RrPRwlHL6stL4Zg/DenfXQnU1Bjj9wYsviQgSkPFamcJXgpteozm5KNxmxvhGbg/0vOKsDGbr3qX4NVoNBsN37/3LHY79FSM3mE/fP/xq9hyzBj8+Oef2HGzMbjh35di+ooCnDOuBU/MzcTvHzyJ5+64ELtf/DzeuOt07HHC9ZhcXIVTH3kXz513EHY47Dz89+0f8dx/zsaLz7+K2x94Dj99/TYGDt0CSxfOMa8IfPTEjagdcSQOmzTM3NP7uOfKU3Dj05/jn1fdiEvOOAoH77Y9nvnoGxy640Tcc/dNOPJfL+PZcyfi+Xmp+GXaA3jzk48wefK++Par97HLPsfhoHEp8Iw9EavevhFH3/AUTj7yOHz00h0qzDnT38KHn3+BzcdNwOyfvzevaDBm2HD8+fd8c0uj0Sg3h1V/oHb6HSiY/5SxMyB6AyI1VKx2huBl5zlRtuVDTkXm9hcgoWD0Ri10AwQnkUaj0fQMGkuxOm0c6ppa8N+L9sT5t96HtjYbHr7vPpx0waXwN7kxcshgOJI4ooofCYl2rFyyHM31q/HIs2/j2isvQmtiMoqkls/Iy8eSv/7CuInbyrEuHLXvdqhpbMB99z6Iy668Rl0uQGOzPG16NV6cdOWjKF8+H9557+Gq+17DsiVLcNrRR8GZk4/SlVU47NgjsNmk7dDY4EbJgMFwJlbj73l/48jDj0GRbPsTUnDMITthx0k7Ydbnb2D3I/4PeUPG4e5rL8S3P83BUUcegYL8fub1DKZsOQxPfDjT3NJoNIQi01m0KfKOeBI1+76OZpsToNfhhhjNgaKX4Ur4zQnJqNn9beQe+CgcWuyuQQtejUbT80gtxtkiuj75ehZeev19bDdpHwzJd2CrHafgxedeQkqSH/Uihj3uZjS7vahYPg/JQzZF5dKl2Gn8MLz67udwNzepV4u11RUYt9t+eObWq/H6E7fjnLvfRYGtAVN2mYyXXn3NuJ4Jz3F7esvYQuFwYFj/QvlMwLlnn4rqsjLkFfbDCy++iMP22BnZmU7U1TcCLS1we71oqKuTY4tROGQkXnrlFWy76RC0NNejSR6aDXW1KBy2Bb797G05xo//u/BSjBoxAi++9DImTxrHiynOOWQSpp79HxS0lsG7gfvQajS9DXoWUGhlbXkQ6s5zY9Euj8OXJr9Rmn874/dCoWuKXV96PyyZ/BTqz2hC1qb7quv2oi75Gxzt0qDRaHooftx3538wYsIe2GOHLdSeu267FYecch4KHU1oaEtFlsODCrcdLSt+wzJ/IbbI9+Dhlz7BPy84EysW/41+g4dh9fJFyCsZguolv+GFD3/BuacdB3hqcOudj+Kciy9BapD5o6Z8JdqSc5GdxpHkeyerF/yMw084E9kDN8e0Fx6Bp2oxpux5MHY7/p+4+sgJ+HyJF7uOycS7P6xGatnXmFk/CPsOb8FRZ9+Iu5/+H5IrZ2PUjgdg2Q8fImuLPTD3nQdxyd0v4ZMvZ6BhznQceNy5uPCmB3DsvpPU9W69/FwsqmpBVXUdHnnqJeT0kGHMNZqeygppc/Z/d294lnwJp7vBcHOgMuUSi0sDEdHscabBOXQ3rNrtdRTxO01YtODVaDQajUaj6UJo4K2d/SaSf7wdjvIf4PB5DOHrjkLwcugVEbpehwve/G3QMu4SZG1mWHQ11uj00Wg0Go1Go+lC+GIpe/MD4DnwDfj3fAL1JbvCTx+s9kyQHE9X1HJDye7w7/4EvPu+gWwtdqNCp5FGo9FoNBpNF0MBlpWTB9cWx8B+4POoHn+ZErx+b/jOs35Ps7IC125zNez7PQfXZkchMztH++lGiXZp0Gg0Go1Go+lmvF4fsOpHrGwtxqBBg8y9a1myZDH6OyrQVjAODnvwoL6a9gH+H+au0yTM4+fsAAAAAElFTkSuQmCC"}}},{"metadata":{},"cell_type":"markdown","source":"## Introduction\nThis inference kernel is the continuation of my TF2 training kerenl [Use Hugging Face's Tensorflow 2 transformer models for NQ](https://www.kaggle.com/yihdarshieh/use-hugging-face-s-tensorflow-2-transformer-models). These demonstrate how to use Hugging Face's [transformers](https://github.com/huggingface/transformers) package, more precisely, theier `Tensorflow 2` models, for this competition.\n\n\n## Disclamation\n* I am not a part of Hugging Face. I choose to use `transformers` package because I found it's easier to use and to extend, so I can focus on other parts of this notebook.\n* I take no responsibility for any (potential) error in this kernel and in the dataset `nq-competition`. (I would appreciate any feedback.)\n* I take no credit of any file (with/without my own modifications) containing in `nq-compeittion`."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport os\nimport sys\nimport json\nimport tensorflow as tf # Yes, we are going to play with Tensorflow 2!\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport absl # For using flags without tf.compat.v1.flags.Flag\nimport datetime","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.extend(['../input/bert-joint-baseline/'])\n\nimport modeling \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Input data files are available in the \"../input/\" directory.\nIS_KAGGLE = True\nINPUT_DIR = \"/kaggle/input/\"\n\n# The original Bert Joint Baseline data.\nBERT_JOINT_BASE_DIR = os.path.join(INPUT_DIR, \"bertjointbaseline\")\n\n# This nq dir contains all files for publicly use.\nNQ_DIR = os.path.join(INPUT_DIR, \"nq-competition\")\n\n# If you want to use your own .tfrecord or new trained checkpoints, you can put them under you own nq dir (`MY_OWN_NQ_DIR`)\n# Default to NQ_DIR. You have to change it to the dir containing your own working files.\nMY_OWN_NQ_DIR = NQ_DIR\n\n# For local usage.\nif not os.path.isdir(INPUT_DIR):\n    IS_KAGGLE = False\n    INPUT_DIR = \"./\"\n    NQ_DIR = \"./\"\n    MY_OWN_NQ_DIR = \"./\"\n\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nfor dirname, _, filenames in os.walk(INPUT_DIR):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\n\nINPUT_DIR","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# NQ_DIR contains some packages / modules\nsys.path.append(NQ_DIR)\nsys.path.append(os.path.join(NQ_DIR, \"transformers\"))\n\nfrom nq_flags import DEFAULT_FLAGS as FLAGS\nfrom nq_flags import del_all_flags\nfrom nq_dataset_utils import *\n\nimport sacremoses as sm\nimport transformers\nfrom adamw_optimizer import AdamW","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Hugging Face pretrained Bert model names"},{"metadata":{"trusted":true},"cell_type":"code","source":"PRETRAINED_MODELS = {\n    \"BERT\": [\n        'bert-base-uncased',\n        'bert-large-uncased',\n        'bert-base-cased',\n        'bert-large-cased',\n        'bert-base-multilingual-uncased',\n        'bert-base-multilingual-cased',\n        'bert-base-chinese',\n        'bert-base-german-cased',\n        'bert-large-uncased-whole-word-masking',\n        'bert-large-cased-whole-word-masking',\n        'bert-large-uncased-whole-word-masking-finetuned-squad',\n        'bert-large-cased-whole-word-masking-finetuned-squad',\n        'bert-base-cased-finetuned-mrpc'\n    ],\n    \"DISTILBERT\": [\n        'distilbert-base-uncased',\n        'distilbert-base-uncased-distilled-squad'\n    ]\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Abseil Flags - Datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"flags = absl.flags\ndel_all_flags(flags.FLAGS)\n\nflags.DEFINE_bool(\n    \"do_lower_case\", True,\n    \"Whether to lower case the input text. Should be True for uncased \"\n    \"models and False for cased models.\")\n\nvocab_file = os.path.join(NQ_DIR, \"vocab-nq.txt\")\n\nflags.DEFINE_string(\"vocab_file\", vocab_file,\n                    \"The vocabulary file that the BERT model was trained on.\")\n\nflags.DEFINE_integer(\n    \"max_seq_length_for_training\", 512,\n    \"The maximum total input sequence length after WordPiece tokenization for training examples. \"\n    \"Sequences longer than this will be truncated, and sequences shorter \"\n    \"than this will be padded.\")\n\nflags.DEFINE_integer(\n    \"max_seq_length\", 512,\n    \"The maximum total input sequence length after WordPiece tokenization. \"\n    \"Sequences longer than this will be truncated, and sequences shorter \"\n    \"than this will be padded.\")\n\nflags.DEFINE_integer(\n    \"doc_stride\", 128,\n    \"When splitting up a long document into chunks, how much stride to \"\n    \"take between chunks.\")\n\nflags.DEFINE_float(\n    \"include_unknowns_for_training\", 0.02,\n    \"If positive, for converting training dataset, probability of including answers of type `UNKNOWN`.\")\n\nflags.DEFINE_float(\n    \"include_unknowns\", -1.0,\n    \"If positive, probability of including answers of type `UNKNOWN`.\")\n\nflags.DEFINE_boolean(\n    \"skip_nested_contexts\", True,\n    \"Completely ignore context that are not top level nodes in the page.\")\n\nflags.DEFINE_integer(\"max_contexts\", 48,\n                     \"Maximum number of contexts to output for an example.\")\n\nflags.DEFINE_integer(\n    \"max_position\", 50,\n    \"Maximum context position for which to generate special tokens.\")\n\nflags.DEFINE_integer(\n    \"max_query_length\", 64,\n    \"The maximum number of tokens for the question. Questions longer than \"\n    \"this will be truncated to this length.\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Abseil Flags - Hyperparameters and file paths"},{"metadata":{"trusted":true},"cell_type":"code","source":"if os.path.isfile(os.path.join(MY_OWN_NQ_DIR, \"nq_train.tfrecord\")):\n    TRAIN_TF_RECORD = os.path.join(MY_OWN_NQ_DIR, \"nq_train.tfrecord\")\nelif os.path.isfile(os.path.join(MY_OWN_NQ_DIR, \"nq-train.tfrecords-00000-of-00001\")):\n    TRAIN_TF_RECORD = os.path.join(MY_OWN_NQ_DIR, \"nq-train.tfrecords-00000-of-00001\")\nelse:\n    TRAIN_TF_RECORD = os.path.join(BERT_JOINT_BASE_DIR, \"nq-train.tfrecords-00000-of-00001\")\n    \nflags.DEFINE_string(\"train_tf_record\", TRAIN_TF_RECORD,\n                    \"Precomputed tf records for training dataset.\")\n\nflags.DEFINE_string(\"valid_tf_record\", os.path.join(NQ_DIR, \"nq_valid.tfrecord\"),\n                    \"Precomputed tf records for validation dataset.\")\n\nflags.DEFINE_string(\"valid_small_tf_record\", os.path.join(NQ_DIR, \"nq_valid_small.tfrecord\"),\n                    \"Precomputed tf records for a smaller validation dataset.\")\n\nflags.DEFINE_string(\"valid_tf_record_with_labels\", \"nq_valid_with_labels.tfrecord\",\n                    \"Precomputed tf records for validation dataset with labels.\")\n\nflags.DEFINE_string(\"valid_small_tf_record_with_labels\", \"nq_valid_small_with_labels.tfrecord\",\n                    \"Precomputed tf records for a smaller validation dataset with labels.\")\n\n# This file should be generated when the kernel is running using the provided test dataset!\nflags.DEFINE_string(\"test_tf_record\", \"nq_test.tfrecord\",\n                    \"Precomputed tf records for test dataset.\")\n\nflags.DEFINE_bool(\"do_train\", False, \"Whether to run training dataset.\")\n\nflags.DEFINE_bool(\"do_valid\", False, \"Whether to run validation dataset.\")\n\nflags.DEFINE_bool(\"smaller_valid_dataset\", True, \"Whether to use the smaller validation dataset\")\n\nflags.DEFINE_bool(\"do_predict\", True, \"Whether to run test dataset.\")\n\nflags.DEFINE_string(\n    \"validation_prediction_output_file\", \"validatioin_predictions.json\",\n    \"Where to print predictions for validation dataset in NQ prediction format, to be passed to natural_questions.nq_eval.\")\n\nflags.DEFINE_string(\n    \"validation_small_prediction_output_file\", \"validatioin_small_predictions.json\",\n    \"Where to print predictions for validation dataset in NQ prediction format, to be passed to natural_questions.nq_eval.\")\n\nflags.DEFINE_string(\n    \"prediction_output_file\", \"predictions.json\",\n    \"Where to print predictions for test dataset in NQ prediction format, to be passed to natural_questions.nq_eval.\")\n\nflags.DEFINE_string(\n    \"input_checkpoint_dir\", os.path.join(MY_OWN_NQ_DIR, \"checkpoints\"),\n    \"The root directory that contains checkpoints to be loaded of all trained models.\")\n\nflags.DEFINE_string(\n    \"output_checkpoint_dir\", \"checkpoints\",\n    \"The output directory where the model checkpoints will be written to.\")\n\n# If you want to use other Hugging Face's models, change this to `MY_OWN_NQ_DIR` and put the downloaded models at the right place.\nflags.DEFINE_string(\"model_dir\", NQ_DIR, \"Root dir of all Hugging Face's models\")\n\n# flags.DEFINE_string(\"model_name\", \"distilbert-base-uncased-distilled-squad\", \"Name of Hugging Face's model to use.\")\nflags.DEFINE_string(\"model_name\", \"bert-base-uncased\", \"Name of Hugging Face's model to use.\")\n# flags.DEFINE_string(\"model_name\", \"bert-large-uncased-whole-word-masking-finetuned-squad\", \"Name of Hugging Face's model to use.\")\n\nflags.DEFINE_integer(\"epochs\", 1, \"Total epochs for training.\")\n\nflags.DEFINE_integer(\"train_batch_size\", 5, \"Batch size for training.\")\n\nflags.DEFINE_integer(\"shuffle_buffer_size\", 10000, \"Shuffle buffer size for training.\")\n\nflags.DEFINE_integer(\"batch_accumulation_size\", 100, \"Number of batches to accumulate gradient before applying optimization.\")\n\nflags.DEFINE_float(\"init_learning_rate\", 5e-5, \"The initial learning rate for AdamW optimizer.\")\n\nflags.DEFINE_bool(\"cyclic_learning_rate\", True, \"If to use cyclic learning rate.\")\n\nflags.DEFINE_float(\"init_weight_decay_rate\", 0.01, \"The initial weight decay rate for AdamW optimizer.\")\n\nflags.DEFINE_integer(\"num_warmup_steps\", 0, \"Number of training steps to perform linear learning rate warmup.\")\n\nflags.DEFINE_integer(\"num_train_examples\", None, \"Number of precomputed training steps in 1 epoch.\")\n\nflags.DEFINE_integer(\"predict_batch_size\", 25, \"Batch size for predictions.\")\n\n# ----------------------------------------------------------------------------------------\nflags.DEFINE_integer(\n    \"n_best_size\", 20,\n    \"The total number of n-best predictions to generate in the \"\n    \"nbest_predictions.json output file.\")\n\nflags.DEFINE_integer(\n    \"max_answer_length\", 30,\n    \"The maximum length of an answer that can be generated. This is needed \"\n    \"because the start and end predictions are not conditioned on one another.\")\n\nflags.DEFINE_string(\n    \"validation_predict_file\", os.path.join(NQ_DIR, \"simplified-nq-dev.jsonl\"),\n    \"\")\n\nflags.DEFINE_string(\n    \"validation_predict_file_small\", os.path.join(NQ_DIR, \"simplified-nq-dev-small.jsonl\"),\n    \"\")\n\n# ----------------------------------------------------------------------------------------\n## Special flags - do not change\n\nif IS_KAGGLE: \n    flags.DEFINE_string(\n        \"predict_file\", \"/kaggle/input/tensorflow2-question-answering/simplified-nq-test.jsonl\",\n        \"NQ json for predictions. E.g., dev-v1.1.jsonl.gz or test-v1.1.jsonl.gz\")\nelse:\n    flags.DEFINE_string(\n        \"predict_file\", os.path.join(NQ_DIR, \"simplified-nq-test.jsonl\"),\n        \"NQ json for predictions. E.g., dev-v1.1.jsonl.gz or test-v1.1.jsonl.gz\")\n    \nif IS_KAGGLE: \n    flags.DEFINE_string(\n        \"sample_submission_csv\", \"/kaggle/input/tensorflow2-question-answering/sample_submission.csv\",\n        \"path to sample submission csv file.\")\nelse:\n    flags.DEFINE_string(\n        \"sample_submission_csv\", os.path.join(NQ_DIR, \"sample_submission.csv\"),\n        \"path to sample submission csv file.\")    \n    \nflags.DEFINE_boolean(\"logtostderr\", True, \"Logs to stderr\")\nflags.DEFINE_boolean(\"undefok\", True, \"it's okay to be undefined\")\nflags.DEFINE_string('f', '', 'kernel')\nflags.DEFINE_string('HistoryManager.hist_file', '', 'kernel')\n\n# Make the default flags as parsed flags\nFLAGS.mark_as_parsed()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IS_SUBMITTING = False\n\ntest_answers_df = pd.read_csv(FLAGS.sample_submission_csv)\n\nif IS_KAGGLE and len(test_answers_df) != 692:\n    IS_SUBMITTING = True\n    FLAGS.do_train = False\n    FLAGS.do_valid = False\n    FLAGS.do_predict = True\n# FLAGS.do_valid = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(test_answers_df))\nprint(IS_SUBMITTING)\nprint(FLAGS.do_train)\nprint(FLAGS.do_valid)\nprint(FLAGS.do_predict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NB_ANSWER_TYPES = 5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make TF record file for test dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"def jsonl_iterator(jsonl_files, to_json=False):\n\n    for file_path in jsonl_files:\n        with open(file_path, \"r\", encoding=\"UTF-8\") as fp:\n            for jsonl in fp:\n                raw_example = jsonl\n                if to_json:\n                    raw_example = json.loads(jsonl)\n                yield raw_example\n\n                \n# Convert test examples to tf records.\ncreator = TFExampleCreator(is_training=False)\nnq_lines = jsonl_iterator([FLAGS.predict_file])\ncreator.process_nq_lines(nq_lines=nq_lines, output_tfrecord=FLAGS.test_tf_record, max_examples=0, collect_nq_features=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get Datasets from TF Record files"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataset(tf_record_file, seq_length, batch_size=1, shuffle_buffer_size=0, is_training=False):\n\n    if is_training:\n        features = {\n            \"unique_ids\": tf.io.FixedLenFeature([], tf.int64),\n            \"input_ids\": tf.io.FixedLenFeature([seq_length], tf.int64),\n            \"input_mask\": tf.io.FixedLenFeature([seq_length], tf.int64),\n            \"segment_ids\": tf.io.FixedLenFeature([seq_length], tf.int64),\n            \"start_positions\": tf.io.FixedLenFeature([], tf.int64),\n            \"end_positions\": tf.io.FixedLenFeature([], tf.int64),\n            \"answer_types\": tf.io.FixedLenFeature([], tf.int64)\n        }\n    else:\n        features = {\n            \"unique_ids\": tf.io.FixedLenFeature([], tf.int64),\n            \"input_ids\": tf.io.FixedLenFeature([seq_length], tf.int64),\n            \"input_mask\": tf.io.FixedLenFeature([seq_length], tf.int64),\n            \"segment_ids\": tf.io.FixedLenFeature([seq_length], tf.int64),\n            \"token_map\": tf.io.FixedLenFeature([seq_length], tf.int64)\n        }        \n\n    # Taken from the TensorFlow models repository: https://github.com/tensorflow/models/blob/befbe0f9fe02d6bc1efb1c462689d069dae23af1/official/nlp/bert/input_pipeline.py#L24\n    def decode_record(record, features):\n        \"\"\"Decodes a record to a TensorFlow example.\"\"\"\n        example = tf.io.parse_single_example(record, features)\n    \n        # tf.Example only supports tf.int64, but the TPU only supports tf.int32.\n        # So cast all int64 to int32.\n        for name in list(example.keys()):\n                        \n            t = example[name]\n            if t.dtype == tf.int64:\n                t = tf.cast(t, tf.int32)\n            example[name] = t\n        return example\n\n    def select_data_from_record(record):\n        \n        x = {\n            'unique_ids': record['unique_ids'],\n            'input_ids': record['input_ids'],\n            'input_mask': record['input_mask'],\n            'segment_ids': record['segment_ids']\n        }\n        \n        if not is_training:\n            x['token_map'] = record['token_map']\n\n        if is_training:\n            y = {\n                'start_positions': record['start_positions'],\n                'end_positions': record['end_positions'],\n                'answer_types': record['answer_types']\n            }\n\n            return (x, y)\n        \n        return x\n\n    dataset = tf.data.TFRecordDataset(tf_record_file)\n    \n    dataset = dataset.map(lambda record: decode_record(record, features))\n    dataset = dataset.map(select_data_from_record)\n    \n    if shuffle_buffer_size > 0:\n        dataset = dataset.shuffle(shuffle_buffer_size)\n    \n    dataset = dataset.batch(batch_size)\n    \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check a small batch in validation / test datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_tf_record = FLAGS.valid_tf_record\nif FLAGS.smaller_valid_dataset:\n    valid_tf_record = FLAGS.valid_small_tf_record\n    \nvalid_tf_record_with_labels = FLAGS.valid_tf_record_with_labels\nif FLAGS.smaller_valid_dataset:\n    valid_tf_record_with_labels = FLAGS.valid_small_tf_record_with_labels\n\ntrain_dataset = get_dataset(FLAGS.train_tf_record,\n                    seq_length=FLAGS.max_seq_length_for_training,\n                    batch_size=2,\n                    shuffle_buffer_size=FLAGS.shuffle_buffer_size,\n                    is_training=True\n                )    \n\nvalidation_dataset = get_dataset(os.path.join(NQ_DIR, valid_tf_record),\n                         seq_length=FLAGS.max_seq_length,\n                         batch_size=2,\n                         is_training=False\n                     )\n\nvalidation_dataset_with_labels = get_dataset(os.path.join(NQ_DIR, valid_tf_record_with_labels),\n                         seq_length=FLAGS.max_seq_length,\n                         batch_size=2,\n                         is_training=True\n                     )\n\ntest_dataset = get_dataset(FLAGS.test_tf_record,\n                         seq_length=FLAGS.max_seq_length,\n                         batch_size=2,\n                         is_training=False\n                     )\n\n# Can't use next(train_dataset)!\nfeatures, targets = next(iter(train_dataset))\nprint(features)\nprint(targets)\n\nfeatures = next(iter(validation_dataset))\nprint(features)\n\nfeatures, labels = next(iter(validation_dataset_with_labels))\nprint(features)\n\nfeatures = next(iter(test_dataset))\nprint(features)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get the actual validation / test datasets from TF Records"},{"metadata":{"trusted":true},"cell_type":"code","source":"validation_dataset = get_dataset(os.path.join(NQ_DIR, valid_tf_record),\n                         seq_length=FLAGS.max_seq_length,\n                         batch_size=FLAGS.predict_batch_size,\n                         is_training=False\n                     )\n\nvalidation_dataset_with_labels = get_dataset(os.path.join(NQ_DIR, valid_tf_record_with_labels),\n                         seq_length=FLAGS.max_seq_length,\n                         batch_size=FLAGS.predict_batch_size,\n                         is_training=True\n                     )\n\ntest_dataset = get_dataset(FLAGS.test_tf_record,\n                         seq_length=FLAGS.max_seq_length,\n                         batch_size=FLAGS.predict_batch_size,\n                         is_training=False\n                     )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get validation / test features"},{"metadata":{"trusted":true},"cell_type":"code","source":"# r is a string Tensor, we use r.numpy() to get underlying byte string\nvalidation_features = (tf.train.Example.FromString(r.numpy()) for r in tf.data.TFRecordDataset(valid_tf_record))\ntest_features = (tf.train.Example.FromString(r.numpy()) for r in tf.data.TFRecordDataset(FLAGS.test_tf_record))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## An interface  for NQ models"},{"metadata":{"trusted":true},"cell_type":"code","source":"from transformers import BertTokenizer\nfrom transformers import TFBertModel, TFDistilBertModel\nfrom transformers import TFBertMainLayer, TFDistilBertMainLayer, TFBertPreTrainedModel, TFDistilBertPreTrainedModel\nfrom transformers.modeling_tf_utils import get_initializer\n\nclass TFNQModel:\n    \n    def __init__(self, config, *inputs, **kwargs):\n        \"\"\"\n        \n        Subclasses of this class are different in self.backend,\n        which should be a model that outputs a tensor of shape (batch_size, hidden_dim), and the\n        `backend_call()` method.\n        \n        We will use Hugging Face Bert/DistilBert as backend in this notebook.\n        \"\"\"\n\n        self.backend = None\n        \n        self.seq_output_dropout = tf.keras.layers.Dropout(kwargs.get('seq_output_dropout_prob', 0.05))\n        self.pooled_output_dropout = tf.keras.layers.Dropout(kwargs.get('pooled_output_dropout_prob', 0.05))\n        \n        self.pos_classifier = tf.keras.layers.Dense(2,\n                                        kernel_initializer=get_initializer(config.initializer_range),\n                                        name='pos_classifier')       \n\n        self.answer_type_classifier = tf.keras.layers.Dense(NB_ANSWER_TYPES,\n                                        kernel_initializer=get_initializer(config.initializer_range),\n                                        name='answer_type_classifier')         \n                \n    def backend_call(self, inputs, **kwargs):\n        \"\"\"This method should be implemented by subclasses.\n           \n           The implementation should take into account the (somehow) different input formats of Hugging Face's\n           models.\n           \n           For example, the `TFDistilBert` model, unlike `Bert` model, doesn't have segment_id as input.\n           \n           Then it calls `self.backend_call()` to get the outputs from Bert's model, which is used in self.call().\n        \"\"\"\n        \n        raise NotImplementedError\n\n    \n    def call(self, inputs, **kwargs):\n        \n        # sequence / [CLS] outputs from original bert\n        sequence_output, pooled_output = self.backend_call(inputs, **kwargs)  # shape = (batch_size, seq_len, hidden_dim) / (batch_size, hidden_dim)\n        \n        # dropout\n        sequence_output = self.seq_output_dropout(sequence_output, training=kwargs.get('training', False))\n        pooled_output = self.pooled_output_dropout(pooled_output, training=kwargs.get('training', False))\n        \n        pos_logits = self.pos_classifier(sequence_output)  # shape = (batch_size, seq_len, 2)\n        start_pos_logits = pos_logits[:, :, 0]  # shape = (batch_size, seq_len)\n        end_pos_logits = pos_logits[:, :, 1]  # shape = (batch_size, seq_len)\n        \n        answer_type_logits = self.answer_type_classifier(pooled_output)  # shape = (batch_size, NB_ANSWER_TYPES)\n\n        outputs = (start_pos_logits, end_pos_logits, answer_type_logits)\n\n        return outputs  # logits\n    \n    \nclass TFBertForNQ(TFNQModel, TFBertPreTrainedModel):\n    \n    def __init__(self, config, *inputs, **kwargs):\n        \n        TFBertPreTrainedModel.__init__(self, config, *inputs, **kwargs)  # explicit calls without super\n        TFNQModel.__init__(self, config)\n\n        self.bert = TFBertMainLayer(config, name='bert')\n        \n    def backend_call(self, inputs, **kwargs):\n        \n        outputs = self.bert(inputs, **kwargs)\n        sequence_output, pooled_output = outputs[0], outputs[1]  # shape = (batch_size, seq_len, hidden_dim) / (batch_size, hidden_dim)\n        \n        return sequence_output, pooled_output\n        \nclass TFDistilBertForNQ(TFNQModel, TFDistilBertPreTrainedModel):\n    \n    def __init__(self, config, *inputs, **kwargs):\n        \n        TFDistilBertPreTrainedModel.__init__(self, config, *inputs, **kwargs)  # explicit calls without super\n        TFNQModel.__init__(self, config)\n\n        self.backend = TFDistilBertMainLayer(config, name=\"distilbert\")\n        \n    def backend_call(self, inputs, **kwargs):\n        \n        if isinstance(inputs, tuple):\n            # Distil bert has no segment_id (i.e. `token_type_ids`)\n            inputs = inputs[:2]\n        else:\n            inputs = inputs\n        \n        outputs = self.backend(inputs, **kwargs)\n        \n        # TFDistilBertModel's output[0] is of shape (batch_size, sequence_length, hidden_size)\n        # We take only for the [CLS].\n        \n        sequence_output = outputs[0]  # shape = (batch_size, seq_len, hidden_dim)\n        pooled_output = sequence_output[:, 0, :]  # shape = (batch_size, hidden_dim)\n        \n        return sequence_output, pooled_output\n    \n    \nmodel_mapping = {\n    \"bert\": TFBertForNQ,\n    \"distilbert\": TFDistilBertForNQ\n}\n\n\ndef get_pretrained_model(model_name):\n    \n    pretrained_path = os.path.join(FLAGS.model_dir, model_name)\n    \n    tokenizer = BertTokenizer.from_pretrained(pretrained_path)\n    \n    model_type = model_name.split(\"-\")[0]\n    if model_type not in model_mapping:\n        raise ValueError(\"Model definition not found.\")\n    \n    model_class = model_mapping[model_type]\n    model = model_class.from_pretrained(pretrained_path)\n    \n    return tokenizer, model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Try Bert / DistillBert models for NQ"},{"metadata":{"trusted":true},"cell_type":"code","source":"bert_tokenizer, bert_for_nq = get_pretrained_model('bert-base-uncased')\n_, distil_bert_for_nq = get_pretrained_model('distilbert-base-uncased-distilled-squad')\n\ninput_ids = tf.constant(bert_tokenizer.encode(\"Hello, my dog is cute\"))[None, :]  # Batch size 1\ninput_masks = tf.constant(0, shape=input_ids.shape)\nsegment_ids = tf.constant(0, shape=input_ids.shape)\n\n# Actual inputs to model\ninputs = (input_ids, input_masks, segment_ids)\n\n# Outputs from bert_for_nq using backend_call()\noutputs = bert_for_nq(inputs)\n(start_pos_logits, end_pos_logits, answer_type_logits) = outputs\nprint(start_pos_logits.shape)\nprint(end_pos_logits.shape)\nprint(answer_type_logits.shape)\n\nlen(bert_for_nq.trainable_variables)\n\n# Outputs from distil_bert_for_nq using backend_call()\noutputs = distil_bert_for_nq(inputs)\n(start_pos_logits, end_pos_logits, answer_type_logits) = outputs\nprint(start_pos_logits.shape)\nprint(end_pos_logits.shape)\nprint(answer_type_logits.shape)\n\nlen(distil_bert_for_nq.trainable_variables)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Bert Joint Baseline"},{"metadata":{"trusted":true},"cell_type":"code","source":"class TDense(tf.keras.layers.Layer):\n    def __init__(self,\n                 output_size,\n                 kernel_initializer=None,\n                 bias_initializer=\"zeros\",\n                **kwargs):\n        super().__init__(**kwargs)\n        self.output_size = output_size\n        self.kernel_initializer = kernel_initializer\n        self.bias_initializer = bias_initializer\n    def build(self,input_shape):\n        dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())\n        if not (dtype.is_floating or dtype.is_complex):\n            raise TypeError(\"Unable to build `TDense` layer with \"\n                          \"non-floating point (and non-complex) \"\n                          \"dtype %s\" % (dtype,))\n        input_shape = tf.TensorShape(input_shape)\n        if tf.compat.dimension_value(input_shape[-1]) is None:\n            raise ValueError(\"The last dimension of the inputs to \"\n                           \"`TDense` should be defined. \"\n                           \"Found `None`.\")\n        last_dim = tf.compat.dimension_value(input_shape[-1])\n        self.input_spec = tf.keras.layers.InputSpec(min_ndim=3, axes={-1: last_dim})\n        self.kernel = self.add_weight(\n            \"kernel\",\n            shape=[self.output_size,last_dim],\n            initializer=self.kernel_initializer,\n            dtype=self.dtype,\n            trainable=True)\n        self.bias = self.add_weight(\n            \"bias\",\n            shape=[self.output_size],\n            initializer=self.bias_initializer,\n            dtype=self.dtype,\n            trainable=True)\n        super(TDense, self).build(input_shape)\n    def call(self,x):\n        return tf.matmul(x,self.kernel,transpose_b=True)+self.bias\n    \ndef mk_model(config):\n    seq_len = config['max_position_embeddings']\n    # unique_id  = tf.keras.Input(shape=(1,),dtype=tf.int64,name='unique_id')\n    input_ids   = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_ids')\n    input_mask  = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_mask')\n    segment_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='segment_ids')\n    BERT = modeling.BertModel(config=config,name='bert')\n    pooled_output, sequence_output = BERT(input_word_ids=input_ids,\n                                          input_mask=input_mask,\n                                          input_type_ids=segment_ids)\n    \n    logits = TDense(2,name='logits')(sequence_output)\n    start_logits,end_logits = tf.split(logits,axis=-1,num_or_size_splits= 2,name='split')\n    start_logits = tf.squeeze(start_logits,axis=-1,name='start_squeeze')\n    end_logits   = tf.squeeze(end_logits,  axis=-1,name='end_squeeze')\n    \n    ans_type      = TDense(5,name='ans_type')(pooled_output)\n    return tf.keras.Model([input_ for input_ in [input_ids,input_mask,segment_ids] \n                           if input_ is not None],\n                          [start_logits,end_logits,ans_type],\n                          name='bert-baseline')    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Choose the model to use"},{"metadata":{"trusted":true},"cell_type":"code","source":"bert_tokenizer, bert_nq = get_pretrained_model(FLAGS.model_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# bert_v2_tokenizer, bert_v2_nq = get_pretrained_model('distilbert-base-uncased-distilled-squad')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# base_bert_tokenizer, base_bert_nq = get_pretrained_model('bert-large-uncased-whole-word-masking-finetuned-squad')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('../input/bert-joint-baseline/bert_config.json','r') as f:\n    config = json.load(f)\nprint(json.dumps(config,indent=4))\n\nbase_bert_nq= mk_model(config)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Try to load the latest checkpoint"},{"metadata":{"trusted":true},"cell_type":"code","source":"# # checkpoint_path = os.path.join(FLAGS.input_checkpoint_dir, FLAGS.model_name)\n# checkpoint_path = os.path.join('../input/bertjointfinetuned/bert-large-uncased-whole-word-masking-finetuned-squad',\n#                                'bert-large-uncased-whole-word-masking-finetuned-squad')\n# # checkpoint_path = os.path.join('../input/bertjointfinetuned/bert-base-uncased-finetuned',\n# #                                FLAGS.model_name)\n# ckpt = tf.train.Checkpoint(model=bert_nq)\n# ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n\n# # if a checkpoint exists, restore the latest checkpoint.\n# if ckpt_manager.latest_checkpoint:\n#     ckpt.restore(ckpt_manager.latest_checkpoint)\n#     last_epoch = int(ckpt_manager.latest_checkpoint.split(\"-\")[-1])\n#     print (f'Latest BertNQ checkpoint restored -- Model trained for {last_epoch} epochs')\n# else:\n#     print('Checkpoint not found. Train BertNQ from scratch')\n#     last_epoch = 0\n    \n    \n# # Reset saving path, because the FLAGS.input_checkpoint_dir is not writable on Kaggle\n# print(ckpt_manager._directory)\n# ckpt_manager._directory = os.path.join(FLAGS.output_checkpoint_dir, FLAGS.model_name)\n# ckpt_manager._checkpoint_prefix = os.path.join(ckpt_manager._directory, \"ckpt\")\n# print(ckpt_manager._directory)\n\n# from tensorflow.python.lib.io.file_io import recursive_create_dir\n# recursive_create_dir(ckpt_manager._directory)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# checkpoint_path = os.path.join(FLAGS.input_checkpoint_dir, 'distilbert-base-uncased-distilled-squad')\n\n# ckpt = tf.train.Checkpoint(model=bert_v2_nq)\n# ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n\n# # if a checkpoint exists, restore the latest checkpoint.\n# if ckpt_manager.latest_checkpoint:\n#     ckpt.restore(ckpt_manager.latest_checkpoint)\n#     last_epoch = int(ckpt_manager.latest_checkpoint.split(\"-\")[-1])\n#     print (f'Latest BertNQ checkpoint restored -- Model trained for {last_epoch} epochs')\n# else:\n#     print('Checkpoint not found. Train BertNQ from scratch')\n#     last_epoch = 0\n    \n    \n# # Reset saving path, because the FLAGS.input_checkpoint_dir is not writable on Kaggle\n# print(ckpt_manager._directory)\n# ckpt_manager._directory = os.path.join(FLAGS.output_checkpoint_dir, FLAGS.model_name)\n# ckpt_manager._checkpoint_prefix = os.path.join(ckpt_manager._directory, \"ckpt\")\n# print(ckpt_manager._directory)\n\n# from tensorflow.python.lib.io.file_io import recursive_create_dir\n# recursive_create_dir(ckpt_manager._directory)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # checkpoint_path = os.path.join(FLAGS.input_checkpoint_dir, 'distilbert-base-uncased-distilled-squad')\n# checkpoint_path = os.path.join('../input/bertjointfinetuned/bert-large-uncased-whole-word-masking-finetuned-squad',\n#                                'bert-large-uncased-whole-word-masking-finetuned-squad')\n\n# ckpt = tf.train.Checkpoint(model=base_bert_nq)\n# ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n\n# # if a checkpoint exists, restore the latest checkpoint.\n# if ckpt_manager.latest_checkpoint:\n#     ckpt.restore(ckpt_manager.latest_checkpoint)\n#     last_epoch = int(ckpt_manager.latest_checkpoint.split(\"-\")[-1])\n#     print (f'Latest BertNQ checkpoint restored -- Model trained for {last_epoch} epochs')\n# else:\n#     print('Checkpoint not found. Train BertNQ from scratch')\n#     last_epoch = 0\n    \n    \n# # Reset saving path, because the FLAGS.input_checkpoint_dir is not writable on Kaggle\n# print(ckpt_manager._directory)\n# ckpt_manager._directory = os.path.join(FLAGS.output_checkpoint_dir, FLAGS.model_name)\n# ckpt_manager._checkpoint_prefix = os.path.join(ckpt_manager._directory, \"ckpt\")\n# print(ckpt_manager._directory)\n\n# from tensorflow.python.lib.io.file_io import recursive_create_dir\n# recursive_create_dir(ckpt_manager._directory)\n# # cpkt = tf.train.Checkpoint(model=bert_nq)\n# # cpkt.restore('../input/bertjointbaseline/bert_joint.ckpt').assert_consumed()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bert_nq.load_weights('../input/bertjointfinetuned/bert-base-e4.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cpkt = tf.train.Checkpoint(model=base_bert_nq)\ncpkt.restore('../input/bert-joint-baseline/model_cpkt-1').assert_consumed()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Some methods for evaluation"},{"metadata":{"trusted":true},"cell_type":"code","source":"Span = collections.namedtuple(\"Span\", [\"start_token_idx\", \"end_token_idx\", \"score\"])\n\n\nclass EvalExample(object):\n    \"\"\"Eval data available for a single example.\"\"\"\n\n    def __init__(self, example_id, candidates):\n        self.example_id = example_id\n        self.candidates = candidates\n        self.results = {}\n        self.features = {}\n\n\nclass ScoreSummary(object):\n\n    def __init__(self):\n        self.predicted_label = None\n        self.short_span_score = None\n        self.cls_token_score = None\n        self.answer_type_logits = None\n        self.start_prob = None\n        self.end_prob = None\n        self.answer_type_prob_dist = None\n\n        \ndef read_candidates_from_one_split(input_path):\n    \"\"\"Read candidates from a single jsonl file.\"\"\"\n    candidates_dict = {}\n    if input_path.endswith(\".gz\"):\n        with gzip.GzipFile(fileobj=tf.io.gfile.GFile(input_path, \"rb\")) as input_file:\n            print(\"Reading examples from: {}\".format(input_path))\n            for index, line in enumerate(input_file):\n                e = json.loads(line)\n                candidates_dict[e[\"example_id\"]] = e[\"long_answer_candidates\"]\n                # if index > 100:\n                #     break\n    else:\n        with tf.io.gfile.GFile(input_path, \"r\") as input_file:\n            print(\"Reading examples from: {}\".format(input_path))\n            for index, line in enumerate(input_file):\n                e = json.loads(line)\n                candidates_dict[e[\"example_id\"]] = e[\"long_answer_candidates\"]\n                # if index > 100:\n                #     break\n\n\n    return candidates_dict\n\n\ndef read_candidates(input_pattern):\n    \"\"\"Read candidates with real multiple processes.\"\"\"\n    input_paths = tf.io.gfile.glob(input_pattern)\n    final_dict = {}\n    for input_path in input_paths:\n        final_dict.update(read_candidates_from_one_split(input_path))\n    return final_dict\n\n\ndef get_best_indexes(logits, n_best_size, token_map=None):\n    # Return a sorted list of (idx, logit)\n    index_and_score = sorted(enumerate(logits[1:], 1), key=lambda x: x[1], reverse=True)\n        \n    best_indexes = []\n    for i in range(len(index_and_score)):\n        \n        idx = index_and_score[i][0]\n\n        if token_map is not None and token_map[idx] == -1:\n            continue\n\n        best_indexes.append(idx)\n\n        if len(best_indexes) >= n_best_size:\n            break    \n    \n    return best_indexes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_duplicates(span):\n    start_end = []\n    for s in span:\n        cont = 0\n        if not start_end:\n            start_end.append(Span(s[0], s[1], s[2]))\n            cont += 1\n        else:\n            for i in range(len(start_end)):\n                if start_end[i][0] == s[0] and start_end[i][1] == s[1]:\n                    cont += 1\n        if cont == 0:\n            start_end.append(Span(s[0], s[1], s[2]))\n            \n    return start_end\n\ndef get_short_long_span(predictions, example):\n    \n    sorted_predictions = sorted(predictions, key=lambda x: (x[0], x[2], x[3]), reverse=True)\n    short_span = []\n    long_span = []\n    for prediction in sorted_predictions:\n        score, summary, start_span, end_span = prediction\n        # get scores > zero\n        if score > 0:\n            short_span.append(Span(int(start_span), int(end_span), float(score)))\n\n    short_span = remove_duplicates(short_span)\n\n    for s in range(len(short_span)):\n        for c in example.candidates:\n            start = short_span[s].start_token_idx\n            end = short_span[s].end_token_idx\n\n            if c[\"top_level\"] and c[\"start_token\"] <= start and c[\"end_token\"] >= end:\n                long_span.append(Span(int(c[\"start_token\"]), int(c[\"end_token\"]), float(short_span[s].score)))\n                break\n    long_span = remove_duplicates(long_span)\n    \n    if not long_span:\n        long_span = [Span(-1, -1, -10000.0)]\n    if not short_span:\n        short_span = [Span(-1, -1, -10000.0)]\n        \n    \n    return short_span, long_span","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def compute_predictions(example):\n    \"\"\"Converts an example into an NQEval object for evaluation.\n    \n       Unlike the starter kernel, this returns a list of `ScoreSummary`, sorted by score.\n    \"\"\"\n    \n    predictions = []\n    max_answer_length = FLAGS.max_answer_length\n    null_entry_threshold = -0.3\n    score_null = 1000000\n\n\n    for unique_id, result in example.results.items():\n        \n        if unique_id not in example.features:\n            raise ValueError(\"No feature found with unique_id:\", unique_id)\n        token_map = example.features[unique_id][\"token_map\"].int64_list.value\n        \n        # feature_null_score = result[\"cls_start_logit\"] + result[\"cls_end_logit\"]\n        feature_null_score = result[\"start_logits\"][0] + result[\"end_logits\"][0]\n        score_null = min(score_null, feature_null_score)\n        \n        for start_index, start_logit, start_prob in zip(result[\"start_indexes\"], result[\"start_logits\"], result[\"start_pos_prob_dist\"]):\n\n            if token_map[start_index] == -1:\n                continue            \n            \n            for end_index, end_logit, end_prob in zip(result[\"end_indexes\"], result[\"end_logits\"], result[\"end_pos_prob_dist\"]):\n\n                if token_map[end_index] == -1:\n                    continue\n\n                if end_index < start_index:\n                    continue                    \n                    \n                length = end_index - start_index + 1\n                if length > max_answer_length:\n                    continue\n\n                summary = ScoreSummary()\n                \n                summary.instance_id = unique_id\n                \n                summary.short_span_score = start_logit + end_logit\n                summary.cls_token_score = result[\"cls_start_logit\"] + result[\"cls_end_logit\"]\n                summary.answer_type_logits = result[\"answer_type_logits\"]\n                \n                summary.start_indexes = result[\"start_indexes\"]\n                summary.end_indexes = result[\"end_indexes\"]\n\n                summary.start_logits = result[\"start_logits\"]\n                summary.end_logits = result[\"end_logits\"]                \n                \n                summary.start_pos_prob_dist = result[\"start_pos_prob_dist\"]\n                summary.end_pos_prob_dist = result[\"end_pos_prob_dist\"]                \n                           \n                summary.start_index = start_index\n                summary.end_index = end_index\n                \n                summary.start_logit = start_logit\n                summary.end_logit = end_logit\n                \n                answer_type_prob_dist = result[\"answer_type_prob_dist\"]\n                summary.start_prob = start_prob\n                summary.end_prob = end_prob\n                summary.answer_type_prob_dist = {\n                    \"unknown\": answer_type_prob_dist[0],\n                    \"yes\": answer_type_prob_dist[1],\n                    \"no\": answer_type_prob_dist[2],\n                    \"short\": answer_type_prob_dist[3],\n                    \"long\": answer_type_prob_dist[4]\n                }\n                \n                start_span = token_map[start_index]\n                end_span = token_map[end_index] + 1\n\n                # Span logits minus the cls logits seems to be close to the best.\n                score = summary.short_span_score - summary.cls_token_score\n                predictions.append((score, summary, start_span, end_span))\n                \n    all_summaries = []      \n\n    short_span = [Span(-1, -1, 0)]\n    long_span = [Span(-1, -1, 0)]\n    score = 0\n    summary = ScoreSummary() \n        \n    if predictions:\n        short_span, long_span = get_short_long_span(predictions, example)\n\n    if len(all_summaries) == 0:\n       \n        summary.predicted_label = {\n                \"example_id\": example.example_id,\n                \"instance_id\": None,\n                \"long_answer\": {\n                        \"tokens_and_score\": long_span,\n                        \"start_byte\": -1,\n                        \"end_byte\": -1\n                },\n                \"long_answer_score\": score,\n                \"short_answers\": [{\n                        \"tokens_and_score\": short_span,\n                        \"start_byte\": -1,\n                        \"end_byte\": -1\n                }],\n                \"short_answers_score\": score,\n                \"yes_no_answer\": \"NONE\",\n                \"no_answer\": True\n        }        \n        \n        all_summaries.append(summary)\n            \n    all_summaries = all_summaries[:min(FLAGS.n_best_size, len(all_summaries))]        \n    \n    return all_summaries\n\n\ndef compute_pred_dict(candidates_dict, dev_features, raw_results):\n    \"\"\"Computes official answer key from raw logits.\n    \n       Unlike the starter kernel, each nq_pred_dict[example_id] is a list of `predicted_label`\n       that is defined in `compute_predictions`.\n    \"\"\"\n\n    raw_results_by_id = [(int(res[\"unique_id\"]), 1, res, None) for res in raw_results]\n\n    examples_by_id = [(int(tf.cast(int(k), dtype=tf.int32)), 0, v, k) for k, v in candidates_dict.items()]\n    \n    features_by_id = [(int(tf.cast(f.features.feature[\"unique_ids\"].int64_list.value[0], dtype=tf.int32)), 2, f.features.feature, None) for f in dev_features]\n    \n    print('merging examples...')\n    merged = sorted(examples_by_id + raw_results_by_id + features_by_id)\n    print('done.')\n    \n    examples = []\n    for idx, type_, datum, orig_example_id in merged:\n        if type_ == 0: # Here, datum the list `long_answer_candidates`\n            examples.append(EvalExample(orig_example_id, datum))\n        elif type_ == 2: # Here, datum is a feature with `token_map`\n            examples[-1].features[idx] = datum\n        else: # Here, datum is a raw_result given by the model\n            examples[-1].results[idx] = datum    \n    \n    # Construct prediction objects.\n    summary_dict = {}\n    nq_pred_dict = {}\n    for e in examples:\n        \n        all_summaries = compute_predictions(e)\n        summary_dict[e.example_id] = all_summaries\n        nq_pred_dict[e.example_id] = [summary.predicted_label for summary in all_summaries]\n        if len(nq_pred_dict) % 100 == 0:\n            print(\"Examples processed: %d\" % len(nq_pred_dict))\n\n    return nq_pred_dict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get logits from model - Save to a json file"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_prediction_json(mode, max_nb_pos_logits=-1):\n    \n    if mode == 'valid':\n        dataset = validation_dataset\n        if FLAGS.smaller_valid_dataset:\n            predict_file = FLAGS.validation_predict_file_small\n            prediction_output_file = FLAGS.validation_small_prediction_output_file\n        else:\n            predict_file = FLAGS.validation_predict_file\n            prediction_output_file = FLAGS.validation_prediction_output_file\n        eval_features = validation_features\n    else:\n        dataset = test_dataset\n        predict_file = FLAGS.predict_file\n        eval_features = test_features\n        prediction_output_file = FLAGS.prediction_output_file\n    \n    print(predict_file)\n    print(prediction_output_file)\n    \n    all_results = []\n\n    prediction_start_time = datetime.datetime.now()\n\n    for (batch_idx, features) in enumerate(dataset):\n\n        batch_start_time = datetime.datetime.now()\n\n        unique_ids = features['unique_ids']\n        token_maps = features['token_map']       \n        \n        (input_ids, input_masks, segment_ids) = (features['input_ids'], features['input_mask'], features['segment_ids'])\n        \n        nq_inputs = (input_ids, input_masks, segment_ids)\n        nq_logits = bert_nq(nq_inputs, training=False)\n        \n        base_nq_logits = base_bert_nq(nq_inputs, training=False)\n        #distill_nq_logits = bert_v2_nq(nq_inputs, training=False)\n\n        (start_pos_logits, end_pos_logits, answer_type_logits) = nq_logits\n        (base_start_pos_logits, base_end_pos_logits, baseanswer_type_logits) = base_nq_logits\n        #(distill_start_pos_logits, distill_end_pos_logits, distill_answer_type_logits) = distill_nq_logits\n        \n        start_pos_logits = (0.2 * start_pos_logits + 0.8 * base_start_pos_logits)\n        end_pos_logits = (0.2 * end_pos_logits + 0.8 * base_end_pos_logits)\n        answer_type_logits = (0.2 * answer_type_logits + 0.8 * baseanswer_type_logits)\n\n        # start_pos_logits = (0.12 * start_pos_logits + 0.8 * base_start_pos_logits + 0.08 * distill_start_pos_logits)\n        # end_pos_logits = (0.12 * end_pos_logits + 0.8 * base_end_pos_logits + 0.08 * distill_end_pos_logits)\n        # answer_type_logits = (0.12 * answer_type_logits + 0.8 * baseanswer_type_logits + 0.08 * distill_answer_type_logits)\n        \n\n        unique_ids = unique_ids.numpy().tolist()\n        \n        token_maps = token_maps.numpy().tolist()\n        \n        start_pos_prob_dist = tf.nn.softmax(start_pos_logits, axis=-1).numpy().tolist()\n        end_pos_prob_dist = tf.nn.softmax(end_pos_logits, axis=-1).numpy().tolist()\n        answer_type_prob_dist = tf.nn.softmax(answer_type_logits, axis=-1).numpy().tolist()\n        \n        start_pos_logits = start_pos_logits.numpy().tolist()\n        end_pos_logits = end_pos_logits.numpy().tolist()\n        answer_type_logits = answer_type_logits.numpy().tolist()\n\n        for uid, token_map, s, e, a, sp, ep, ap in zip(unique_ids, token_maps, start_pos_logits, end_pos_logits, answer_type_logits, start_pos_prob_dist, end_pos_prob_dist, answer_type_prob_dist):\n\n            if max_nb_pos_logits < 0:\n                max_nb_pos_logits = len(start_pos_logits)\n            \n            # full_start_logits = s\n            # full_end_logits = e\n            \n            cls_start_logit = s[0]\n            cls_end_logit = e[0]\n            \n            start_indexes = get_best_indexes(s, max_nb_pos_logits, token_map)\n            end_indexes = get_best_indexes(e, max_nb_pos_logits, token_map)            \n            \n            s = [s[idx] for idx in start_indexes]\n            e = [e[idx] for idx in end_indexes]\n            sp = [sp[idx] for idx in start_indexes]\n            ep = [ep[idx] for idx in end_indexes]            \n            \n            raw_result = {\n                \"unique_id\": uid,\n                \"start_indexes\": start_indexes,\n                \"end_indexes\": end_indexes,\n                \"start_logits\": s,\n                \"end_logits\": e,\n                \"answer_type_logits\": a,\n                \"start_pos_prob_dist\": sp,\n                \"end_pos_prob_dist\": ep,\n                \"answer_type_prob_dist\": ap,\n                \"cls_start_logit\": cls_start_logit,\n                \"cls_end_logit\": cls_end_logit\n                # \"full_start_logits\": full_start_logits,\n                # \"full_end_logits\": full_end_logits\n            }\n            all_results.append(raw_result)\n\n        batch_end_time = datetime.datetime.now()\n        batch_elapsed_time = (batch_end_time - batch_start_time).total_seconds()\n\n        if (batch_idx + 1) % 100 == 0:\n            print('Batch {} | Elapsed Time {}'.format(\n                batch_idx + 1,\n                batch_elapsed_time\n            ))\n      \n    prediction_end_time = datetime.datetime.now()\n    prediction_elapsed_time = (prediction_end_time - prediction_start_time).total_seconds()\n\n    print('\\nTime taken for prediction: {} secs\\n'.format(prediction_elapsed_time))\n    print(\"-\" * 80 + \"\\n\")\n\n    print(\"Going to candidates file\")\n    candidates_dict = read_candidates(predict_file)\n\n    print (\"setting up eval features\")\n    # eval_features = ...\n\n    print (\"compute_pred_dict\")\n    nq_pred_dict = compute_pred_dict(candidates_dict, eval_features, all_results)\n    \n    predictions_json = {\"predictions\": list(nq_pred_dict.values())}\n\n    print (\"writing json\")\n    with tf.io.gfile.GFile(prediction_output_file, \"w\") as f:\n        json.dump(predictions_json, f, indent=4)\n        \n    return predictions_json","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Code for metrics"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import f1_score\n\ndef create_answer_from_token_indices(answer):\n    \n    if answer[\"start_token\"] == -1 or answer[\"end_token\"] == -1:\n        return \"\"\n    else:\n        return str(answer[\"start_token\"]) + \":\" + str(answer[\"end_token\"])\n    \ndef create_long_answer_from_1_pred(pred):\n    \"\"\"\n    Args:\n        pred: A `predicted_label` as defined in `compute_predictions`.\n    \n    Returns:\n        A string. It's either an empty string \"\" or a string of the form \"start_token:end_token\",\n        where start_token and end_token are string forms of integers.\n    \"\"\"\n    if pred[\"long_answer_score\"] < 1.5 or pred['answer_type'] == 0:\n        return \"\"\n    long_answer = create_answer_from_token_indices(pred[\"long_answer\"])\n    \n    return long_answer\n    \n    \ndef create_short_answers_from_1_pred(pred):\n    \"\"\"\n    Args:\n        pred: A `predicted_label` as defined in `compute_predictions`.\n    \n    Returns:\n        A list of strings. Each element can be [\"\"], [\"YES\"], [\"NO\"] or an list of strings with\n        the form \"start_token:end_token\" as describe in `create_long_answer_from_1_pred`.\n    \"\"\"\n    \n    short_answers = []\n    \n    for predicted_short_answer in pred[\"short_answers\"]:\n        \n        short_answer = create_answer_from_token_indices(predicted_short_answer)\n\n        # Custom\n        if \"answer_type_prob_dist\" in pred:\n\n            if pred[\"short_answers_score\"] < 1.5 or pred['answer_type'] == 0:\n                if short_answer not in [\"YES\", \"NO\"]:\n                    short_answer = \"\"\n    \n        short_answers.append(short_answer)\n    \n    return short_answers\n\n\ndef is_pred_ok(pred, annotations):\n    \"\"\"\n    Args:\n        pred: A `predicted_label` as defined in `compute_predictions`.\n        annotations: A list of annotations. See `simplified-nq-dev.jsonl` for the format.\n        \n    Returns:\n        has_long_label: bool\n        has_short_label: bool\n        has_long_pred: bool\n        has_short_pred: bool\n        is_long_pred_correct: bool\n        is_short_pred_correct: bool\n    \"\"\"    \n        \n    long_labels = []\n    \n    for annotation in annotations:\n        \n        long_label = create_answer_from_token_indices(annotation[\"long_answer\"])\n        long_labels.append(long_label)\n        \n    non_null_long_labels = [x for x in long_labels if x != \"\"]\n    has_long_label = len(non_null_long_labels) > 1\n    \n    long_pred = create_long_answer_from_1_pred(pred)\n    has_long_pred = (long_pred != \"\")\n    \n    short_label_lists = []\n    \n    for annotation in annotations:\n        \n        if len(annotation[\"short_answers\"]) == 0:\n            if annotation[\"yes_no_answer\"] == \"YES\":\n                short_label_lists.append([\"YES\"])\n            elif annotation[\"yes_no_answer\"] == \"NO\":\n                short_label_lists.append([\"NO\"])\n            else:\n                short_label_lists.append([\"\"])\n        else:\n            \n            short_labels = []\n            for anno_short_answer in annotation[\"short_answers\"]:\n                short_label = create_answer_from_token_indices(anno_short_answer)\n                short_labels.append(short_label)\n            \n            short_label_lists.append(short_labels)        \n    \n    non_null_short_label_lists = [x for x in short_label_lists if x != [\"\"]]\n    has_short_label = len(non_null_short_label_lists) > 1\n    \n    # It can be [\"\"], [\"YES\"], [\"NO\"] or [\"start_token:end_token\"].\n    short_preds = create_short_answers_from_1_pred(pred)\n    has_short_pred = (short_preds != [\"\"])\n    \n    is_long_pred_correct = False\n    is_short_pred_correct = False\n    \n    for long_label in long_labels:\n        \n        if has_long_label and long_label == \"\":\n            continue\n        if not has_long_label and long_label != \"\":\n            continue\n            \n        if long_pred == long_label:\n            is_long_pred_correct = True\n            break\n\n    for short_labels in short_label_lists:\n\n        if has_short_label and short_labels == [\"\"]:\n            continue\n        if not has_short_label and short_labels != [\"\"]:\n            continue        \n            \n        if has_short_label:\n            \n            if short_labels == [\"YES\"] or short_labels == [\"NO\"]:\n\n                if short_preds == short_labels:\n                    is_short_pred_correct = True\n                    break\n\n            else:\n                if short_preds[0] in short_labels:\n\n                    is_short_pred_correct = True\n                    break\n                        \n        else:\n            \n            if short_preds == short_labels:\n                is_short_pred_correct = True\n                break\n\n     \n    return has_long_label, has_short_label, has_long_pred, has_short_pred, is_long_pred_correct, is_short_pred_correct\n\n\ndef compute_f1_scores(predictions_json, gold_jsonl_file):\n    \n    predictions = predictions_json[\"predictions\"]\n    \n    golden_nq_lines = jsonl_iterator([gold_jsonl_file])\n    golden_dict = dict()\n    for nq_line in golden_nq_lines:\n        nq_data = json.loads(nq_line)\n        golden = dict()\n        golden[\"example_id\"] = nq_data[\"example_id\"]\n        golden[\"annotations\"] = nq_data[\"annotations\"]\n        golden_dict[golden[\"example_id\"]] = golden\n        \n    long_labels = []\n    long_preds = []\n    short_labels = []\n    short_preds = []\n\n    for preds in predictions:\n        \n        # Let's take only the 1st pred for now. We can play with multiple preds if we want.\n        pred = preds[0]\n        \n        example_id = pred[\"example_id\"]\n        assert example_id in golden_dict\n        golden = golden_dict[example_id]\n        assert example_id == golden[\"example_id\"]\n        \n        has_long_label, has_short_label, has_long_pred, has_short_pred, is_long_correct, is_short_correct = is_pred_ok(pred, golden[\"annotations\"])\n        \n        if has_long_label or has_long_pred:\n            if is_long_correct:\n                long_labels.append(1)\n                long_preds.append(1)\n            else:\n                long_labels.append(1)\n                long_preds.append(0)            \n        \n        if has_short_label or has_short_pred:        \n            if is_short_correct:\n                short_labels.append(1)\n                short_preds.append(1)\n            else:\n                short_labels.append(1)\n                short_preds.append(0)\n            \n    f1 = f1_score(long_labels + short_labels, long_preds + short_preds)\n    long_f1 = f1_score(long_labels, long_preds)\n    short_f1 = f1_score(short_labels, short_preds)\n\n    return f1, long_f1, short_f1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get metrics for test dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"def df_long_index_score(df):\n    #print(df)\n    answers = []\n    cont = 0\n    for e in df[0]['long_answer']['tokens_and_score']:\n        # if score > 2\n        if e[2] >= 1.5: \n            index = {}\n            index['start'] = e[0]\n            index['end'] = e[1]\n            index['score'] = e[2]\n            answers.append(str(index['start']) + \":\" + str(index['end']))\n            cont += 1\n        # number of answers\n        if cont == 1:\n            break\n    if len(answers) == 0:\n        return \"\"\n    return answers[0]\n\ndef df_short_index_score(df):\n    answers = []\n    cont = 0\n    for e in df[0]['short_answers'][0]['tokens_and_score']:\n        # if score > 2\n        if e[2] >= 3.0:\n            index = {}\n            index['start'] = e[0]\n            index['end'] = e[1]\n            index['score'] = e[2]\n            answers.append(str(index['start']) + \":\" + str(index['end']))\n            cont += 1\n        # number of answers\n        if cont == 1:\n            break\n            \n    if len(answers) == 0:\n        return \"\"\n    return answers[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Run on test dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"if FLAGS.do_predict:\n    test_features = (tf.train.Example.FromString(r.numpy()) for r in tf.data.TFRecordDataset(FLAGS.test_tf_record))\n    predictions_json = get_prediction_json(mode='test', max_nb_pos_logits=FLAGS.n_best_size)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission files\n\n\nThe Bert model produces a confidence score, which the Kaggle metric does not use. You, however, can use that score to determine which answers get submitted. See the limits commented out in create_short_answer and create_long_answer below for an example.\n\nValues for confidence will range between 1.0 and 2.0."},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_long_answer(preds):\n    \"\"\"\n    Args:\n        preds: A list of `predicted_label` as defined in `compute_predictions`.\n    \n    Returns:\n        A string represented a long answer.\n    \"\"\"\n    \n    # Currently, return the long answer from the 1st pred in preds.\n    return create_long_answer_from_1_pred(preds[0])  \n    \n    \ndef create_short_answer(preds):\n    \"\"\"\n    Args:\n        pred: A list of `predicted_label` as defined in `compute_predictions`.\n    \n    Returns:\n        A string represented a short answer.\n    \"\"\"\n    \n    # Currently, return the short answer from the 1st pred in preds.\n    return create_short_answers_from_1_pred(preds[0])[0]\n\n\nif FLAGS.do_predict:\n\n    test_answers_df = pd.read_json(FLAGS.prediction_output_file)\n\n    test_answers_df[\"long_answer_score\"] = test_answers_df[\"predictions\"].apply(lambda q: q[0][\"long_answer_score\"])\n    test_answers_df[\"short_answer_score\"] = test_answers_df[\"predictions\"].apply(lambda q: q[0][\"short_answers_score\"])\n\n    test_answers_df[\"long_answer_score\"].describe()\n    \n    sample_submission = pd.read_csv(FLAGS.sample_submission_csv)\n    sample_submission.to_csv(\"submission.csv\", index=False)\n\n    # We re-format the JSON answers to match the requirements for submission.\n    # test_answers_df[\"long_answer\"] = test_answers_df[\"predictions\"].apply(create_long_answer)\n    # test_answers_df[\"short_answer\"] = test_answers_df[\"predictions\"].apply(create_short_answer)\n    test_answers_df[\"long_answer\"] = test_answers_df[\"predictions\"].apply(df_long_index_score)\n    test_answers_df[\"short_answer\"] = test_answers_df[\"predictions\"].apply(df_short_index_score)\n    \n    test_answers_df[\"example_id\"] = test_answers_df[\"predictions\"].apply(lambda q: str(q[0][\"example_id\"]))\n\n    long_answers = dict(zip(test_answers_df[\"example_id\"], test_answers_df[\"long_answer\"]))\n    short_answers = dict(zip(test_answers_df[\"example_id\"], test_answers_df[\"short_answer\"]))\n\n    # Then we add them to our sample submission. Recall that each sample has both a _long and _short entry in the sample submission, one for each type of answer.\n\n    long_prediction_strings = sample_submission[sample_submission[\"example_id\"].str.contains(\"_long\")].apply(lambda q: long_answers[q[\"example_id\"].replace(\"_long\", \"\")], axis=1)\n    short_prediction_strings = sample_submission[sample_submission[\"example_id\"].str.contains(\"_short\")].apply(lambda q: short_answers[q[\"example_id\"].replace(\"_short\", \"\")], axis=1)\n\n    sample_submission.loc[sample_submission[\"example_id\"].str.contains(\"_long\"), \"PredictionString\"] = long_prediction_strings\n    sample_submission.loc[sample_submission[\"example_id\"].str.contains(\"_short\"), \"PredictionString\"] = short_prediction_strings\n\n    # And finally, we write out our submission!\n    sample_submission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('submission.csv')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df[df['PredictionString'].isnull()].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_answers_df[\"long_answer_score\"].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}