{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"參考資料：  \nHappyWhale ArcFace Baseline (TPU)  \nhttps://www.kaggle.com/ks2019/happywhale-arcface-baseline-tpu  \nHappyWhale  Baseline ゆっくり実況　日本語＆English  \nhttps://www.kaggle.com/pixyz0130/eng-happywhale-baseline","metadata":{"id":"gS3X3mdeW8cj"}},{"cell_type":"markdown","source":"os.listdir('/kaggle/input') # 三個資料集  \nhappy-whale-and-dolphin #原始資料集  \nhappywhale-splits  \nindividual_ids.json所有ID進行編號、species.json所有物種進行編號、skf_species_10folds.csv [ 圖片 物種編號 ID編號 分到第幾Kfold ]  \nhappywhale-tfrecords-v1 # tfrecord版本資料集  ","metadata":{"execution":{"iopub.execute_input":"2022-03-09T13:28:15.033476Z","iopub.status.busy":"2022-03-09T13:28:15.033039Z","iopub.status.idle":"2022-03-09T13:28:15.041869Z","shell.execute_reply":"2022-03-09T13:28:15.04118Z","shell.execute_reply.started":"2022-03-09T13:28:15.033435Z"},"executionInfo":{"elapsed":2,"status":"ok","timestamp":1646574382985,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"22J0smqBW8cn"}},{"cell_type":"markdown","source":"Win10 64bit python 3.7.11  \n![image.png](attachment:2a538876-806e-4f1a-82ce-917fb0721545.png)","metadata":{},"attachments":{"2a538876-806e-4f1a-82ce-917fb0721545.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAloAAAFXCAYAAAB6LeCpAAAgAElEQVR4nOy9vW/jVvr3/dU6U3iR/QFpsl02x4aRwa6ATZlahAEBC1W3gvt5cN/PIoXAwoVhIFGhfyAq1AxcuBDUBtjNapthZmHAoIBsl16FgYF98hekSLKzCTBjPsU5lPhyKJGHpCxpvh/AYw9fr8NzePHi9XLY+PDDDwMQQgghhJDK+c1jC0AIIYQQsq/Q0CKEEEIIqQkaWoQQQgghNUFDixBCCCGkJmhoEUIIIYTUBA0tQgghhJCaoKFFCCGEEFITNLQIIYQQQmqChhYhhBBCSE3Q0CKEEEIIqYl3bHb66KOPcHh4CAAIAvUFn0D9J7YsufzhIcD330u8evXKWmBCCCGEkF3BytBqt9twHAcHBwcAlGG1MK2CILSrECBYGFl3d/d493fv4l8vXsD3ffz0009lZSeEEEII2WqsDC0AODk5wZMnT7B0XkWNLQCNhY2F3zSAH374AUdHR3jvvfdweHiIFy9e4McffywhOiGEEELIdmNlaAWREKEKDap/1NJGJISot280AAAHBwf4wwcfoPvppwiCBr755jl+/vnnMvITQgghhGwtVsnwsbyscNlyZfRXjNevXyMIAvz+/ffR6fwFn3zyic3pCSGEEEJ2AitD6+EhYkgZDKol8ZXPnz/HV199ha+//hrfffcd3nnyxOb0pehOJKRUP5Pu6nXNwQxSSswGzcLnKbMvIYTUTxcTKSGTinDlLhPqNUIKYufRwgOAwGBkNWK/wvUPbx7w548/xv/+f/5f/J//+//h//zf/4v/9b+6serE+mliMJMYHY3REQJCCPSmedaRbaE5mJVW8NtyjPpRYzp8cZBSQs4G2HapCSFk37BLho9N3xD+nc7NQqOB0No6PPwt9IwQQBDgnXeerPGGVUyzg1MB+P0h5jnXzYctiKHd6crsS5I0MZh5cAUgx7t+jE3QxUSe4a4jIBYDWsnuyVOMOy0MUzcBIYSQOrCesDQA4lWGsbytVXHFYDG3VrBJS+vpMYTNul1Eu/eLRAS2VQYVgvVwfDOG3PFjbI4peiJpTM0xPB9DQsC9eMyBQQghbxeWyfCRPyK2kprVQc/tEFkYmP+Dhw2GDpsnR1bryGOhQl/e8ZUK5Xq7fIwtYe7hRgI4OmEIkRBCNoTd9A5Ra6kRN7zCqRzMOVybjRaSXWaOYUugXPR1W46xZdy/TIfPCSGE1IJd1aH2ZEUihuqPRU7W0sjK8mapA9mcPYqumon+pEsJIaWE56rgoDOKJAavWhfZ1xz+Sicbx7ZbuW8OuaPbTbrG86USsvU55ciJtycmR8FjFpU7lwxvOc0BZotrve6ahn00Qfbly7MNlrmI1zaVHhWPGzmDeZeax2dBopXIq2UoInf5Pg2rmk3NUzKbru+qKkND8QRvWEIqwS50+KCrDiOf2IkmvmcZWWlvVglLqzuBlCM4fh9CVwoK0YfvjOLVVdMehBDojFVmjd/X27aGmK9at/bcHtz76LkFrttZDw8LuaMcnWEmL4Hz5bk6YwnhenFlqNsj+n68PaYqyrzHtLzeuWR42zm+wCxMWg+vaWcMGbumc3g3EoCDdtZzTxtQcvwMmZe4O4H0XAi/X64fKhk3Ap0x4HorHuZ1jc/cKMOjfS0Scq+QIbfc5ft07t1ARYGTLeui7QCAwGkn+SLWhgODoZ0hM5wRjS1CKsAuGT4AVB5WI+7FQurX4j9pIysoYWZ1MRk5gN+HiD01puj1fUC4uKyr/L45wMx4bmDaW1fNZSm3AG4SlWLzYQt9H4DTXu3FyKLQMR/xeu8xwjlK9QHmQ7QS13Q+vIIPwDkzGwzdCxcCPq5iB0p4d0ZAX4jUmC0udIFxs+JemQ9bi4e58Vn+6ONThYyTl2s+PMdYmmQoJrddn0YFUfl24rQT37/bhgMfvp9ep3JRfaQcmitlPlv/8kgIWYll1WEQ/zH8udzUXFsYANYOrebgDA4kxs8MD43pM4xNCqgilALMOPcarOX2r4wG3O3dmrfiVRQ45mNe770mow/S13SKax+AOEXSSRF6MNKejyl6IuqNuUZ7ZciunMymcbPuXlkYG6YBvLXjM/RGHSHlTCokt02fxuV4eZ/ev9t2AHmH6zuZWNdE51QA8g63yUPJG3gGmadKQBw/zRSCEJIDywlLF583XBQepgwsvTJlZAVB5BM+NpaWVhi4x8uNZ/Q2cXKETMW0bt+q5Z6/vK/mQCuPWcP1DvO4cuW+7BgF2ibvUo88TfgQPUb4jJs+U1MzJMNBoZFxs3ZATtETHYylgOsl8n4q6A/TuFl/r9zirmAVZCXj8xHGn+leLdunoSG03F8baDcept4NZNRICkORN146LYLFEYTUSrmPSpvXmsOHascKqw4djKTEKHO9ekhVq0Ce4lgA8MsopseQuwoqlHvag9jXfK262jb3cCNduO4FusOe9nRoI8Pv55yAdI7hlQ93pDwr01DOx+4PUcWYLzA+C7a3O5HQtR0RKphNrWyfTq/hjxw4x7pl3bby7HlzYA517LCjnx4r72LxN0RCSEmsDK2HtAVl+pXYJG5kBWkXWDHkGJ11SeuVo97AHf0GbnXuR5G7AnZV7p0k9AZFwzwGI6l7AVdIjM933Go1hbMKH6Pi8dkcYOa5aiJjvw8Rscyagxk8t4qTlO3TKa79ERynjS6muD05ingQtVdUr9OxSAtPPCGkLHY5Wg+RuR3CqR5M+VmawOjJsp0ZPh1W2Rxlzv2YcpdhV+XefkRm8ov2nCZDOtPrWAK1ysexCWNLZEYtKyPPuMloZ+XnKUoTg0uVjF5JAcEqSvapCh86aHeVJywaGlyuW4YUaWcRsnlKfFQ6KzcruXH2WluHlsptcHD2CDk9ofKyOfdjyl2GXZV768moXMtO7p7i2VgnOXcHOHMA/6qIF6eJwZmtcVacdeNmZRJ7hecpjjYAjZ62MCesKkr26cJQu8SpSOR16XVH7TaOANxvPqmVEALr6R1ymEg66X3VloHtJ3jmQ5zruWnSCaxNDGYlK6tWMe0t5sVJT0K45rybkvv2DqoCvaI5cGzkrlqGNYQTOJZJaK7iGMVwMEpMSKnCUgJyfG7M0ZnrJGd35ELIMUw2SnMwM8x/FH4Q20d/UyHgyLhJyhO20+9X8IHryu+rsCLQRfyzkF1MpPqoeJXk7lPj2AxlFRApA1qtE44DYZrWgRCyEUpUHWZM2pDDwIodyJL5sAWhZj1MVA9dAucVKO/C58533o3IHc7F5Iwqm5W9sNw1yLBvyHEHog+MItfTc4FxR6CVOX/SEFe+3j8jFDQfttC5O0v0k4fTmw6E6K2YMqB65sPWcuLQWDvv0a9wEtuq76tpT03aGf2ygdSTy4YTHFdGjj5dKeu13tkQgl2s86832u+EkCWNDz/8sLC589fPPkP/iy9wcHCQy7ll4tWr/+Dzzz/Hv7/91u4AhOwqOtEa4062QbVy95k2VDZrNJH6YJ8Ssr9YVR2GXquDg4NSJ7c10gh5e+niwhWAf8UH8t7APiVkn7Ge3mE6/ScaDV1RGIS/dSVhEOZfqWUPukIxuv71mzfq49SEkNxUlUBOtgf2KSH7jZWh9Y+//w2NRiPXtkHsj7hh9ebNG5vTE/J20hzg0hWQ406tOYhkg7BPCdl7rAytX3/9tWo5CCEZRGcm9/vVJZCTx4N9Ssjbg1UyPCGEEEIIWY/dPFqEEEIIIWQtNLQIIYQQQmqChhYhhBBCSE3Q0CKEEEIIqQkaWoQQQgghNUFDixBCCCGkJmhoEUIIIYTUBA0tQgghhJCaoKFFCCGEEFITVp/g+eijj3B4eAgg/Hh0+CnDILYsufzhIcD330u8evXKWmBCCCGEkF3BytBqt9twHAcHBwcAlGG1/Hh0ENpVCBAsjKy7u3u8+7t38a8XL+D7Pn766aeyshNCCCGEbDVWhhYAnJyc4MmTJ1g6r6LGFoDGwsbCbxrADz/8gKOjI7z33ns4PDzEixcv8OOPP5YQnRBCCCFku7EytIJIiFCFBtU/amkjEkLU2zcaAICDgwP84YMP0P30UwRBA9988xw///xzGfkJIYQQQrYWq2T4WF5WuGy5MvorxuvXrxEEAX7//vvodP6CTz75xOb0hBBCCCE7gZWh9fAQMaQMBtWS+Mrnz5/jq6++wtdff43vvvsO7zx5YnP6/aM7gZQSs0Fz4+dpDmbpczcHmG1CHkJICuM9+RairsMMu38ZuphICTnpFthlQ88EshHsQod4ABAYjKyGWq5/hblaD28e8OePP8Yf/9RcHOGX//6CL4df2spNdpYmBjMProgskmN0WkPMyxwDgN8X6E03eYxNkve6mduVovA1r5t9bx+pnvRYyHf/Lvfb3vud7BN2yfCx6RvCv9O5WWiEFhdwePhb6BkhgCDAO+88WeMNI5tgPmxBDDd1ti4m8gx3HQGxeAIqpefJU4w7LQzXPhm7mMgRHL8P0VpqyOZgBm8kIdt9iLWas4pjbJIi122OYUsgs0ubA8w8F/dX22SE7Hv7irHZe3JH6U4gRw4Qu4ebGMxmGNyu0SPdi/WGOiEVYj1haeiwWv4nmre1Kq4YJtAH0TrF3UK7dYt4gjfGNsuGKXoiqQTnGJ6PISHgXuQT+rovUobQfNhC3wfgnOUKNVRxjM1RzXUDmhhcuhByjGfbZEfufftIpTQHmI0cyHEncQ/PMWyte1lrYnDm1CwgIXEsk+Ejf0RsJRUp1PHCyMLA/B88mDLmydvH3MONBHB0gvX2zRTTjIfo9NoHIHDaWXeUKo6xBRS6bli8yfu74u3Z9/YRK7oXypg+X+/+TtEcXMIVPvp9vwbJCDFjZ2hFraVGxLsVBAgWRpYph4vRQrKC+5flHpC3d5BlZajiGJsm53Xrth1gF709+94+kp/mAGcOIG88C13RxYUrIMfPwCFCNold1aH2ZEUihuqPRiQLfjE7fIjB8HqwOTviVXH6bxn+zAbLt18dRjOuS9CdyPi2poqP8Hgj5Xp2RsttF6G6WMWerjaJ/hSO6RmOYarEySObueFrtqmiDWtodnAqAP+6pPp7egwB4P5lCXOt9DGaGMzi1yt2uaocH0Wum35Alff27GD7atAX2WLMMu8npWNMVXSJqjTjPRndJt0HZavTcuk/1cDqr2XyOKva8/QYAhI3noWZNRnBKegJq6Q/40dM9V3l+pRsHXYerQdddRj5xE408T3LyEp7s2wtLc3pJeQlcC4EhBAQnTGkcOHNBhhMJOTZHTqGdfH7Qg389rXeTv90xhLC9eI3wbSn1mu3s99fbp/KnT6+wCxM8I3K4IzyK/DuBDJM2o7JBrietJctL1W0YR3dCaTnQvj9ktU/Ye6FD3t7reQxuhNI6cG9j/fXddugjMte24LXrXvhQpS6Ntj99lWiL1Yz926gop2pNyG0HcAYlu624SCnQXl0hplqxGpdlZsC+i9KRdeyOZhBei4w7kTO38e968EzZKx32w6Ae7ycG4zDVX3VnWDkSIzPi71oVNqfGX0HZ0Rja8+xS4YPAJWH1Yh7sZD6tfhP2sgKyppZEOIe/WgJ93yI87EEhAvX8TPWnSJ+X6gqpqQynw/PMZYAnDZsbgHhHOEmWUU3H6LV9wHh4nLdG6hO+ISfroCbD1uLG7TO+7N0G4wkvBwjoC/SiemFjzpR5dplwgKljrGiv6a9dIJu8Wtb5rqph0KpkMketK8afbEGnVcmTjvxh363DQc+fD+9rnlyhNzGvUDqui6LOGx0lZ3+q+RaNge4dAXkuINWbKBM0RN9ZGdRHeFsJjFCP2ac+cKFJycGebuYjBzAv8pR1Zygyv5c2XfbVoBDqsSy6jCI/xj+XG5qri0MgNIOLfjXKcU6f3m/Zp3A8dM8B5/DU5m4SL3M5JIt46aePsPYdOMmUG/oEuOMhJP58ApKt9ZoaZVsg5kpeiL69nyNdlY4NBfqjXzkALBMkK3iGOv6K0Xha2t/3ZqDMziW4ZaQvWhfrfpisRfUbnGjQuWP3eH6LmlwNNE5FYC8w22ew2dc19s7CcBBdepgjf6r4FqGXsgr80DBs3EyW7KJkyMAEBD3SYN/il5nDAkHo3gsG4PZCA589K1e5irsT3kD0xANC3CKjTOyS1gmwy8+b7goPEwZWHplysgKgsgnfMpZWvLOoJp0MrNxnSbtBq6e7POHN+4xsu8rrVAybkzFLe6KVGRZUKgNyZyM3LkjU/REB2Mp4Hqmt9EVNAeYST1hod+HsJmgMs8x1rYtT3/FKTc+gPzXTSv+VW/z+96+VTIV1Rc5xnm6clV73G48TL0byOhDVeeh2SV3L1kYORui/LUMx1ROAzN+drPBH1apRrxwqspQYtzpWXtzK+vPssU+ZGcp91Fp81pz+FDtuNVVh92J9mrE2PIaNP3AevQbeNqDsI5LzTG88uGO1Bt51tQLMcIJC1Fidue8x1jbtqc4FgD8TSvSHNctTCi/WtGAfW9fleQZ59Nr+CMHzrG+M7ttOJAYe3NgDtxIF27YIJ3cPS7hbayKzeo/PaYKHT400nOu707g6dCklaM7ZEf7k2wPllWHySWRxHhT+BBIGVmBcaNHIFLxEo/560TF6k9Y4k3OQFXHKUTFbSjKwkCSGHfKGlkljrGgSu9itde22TmFgMQKJ0MO9r19VTPFdSRnqnlyFPEGamNAr1MhqPyewsrZuP4L0WOq6F65d1pOTCpcL+2FTFZnr0x23aH+JFuJXY7WQ2Ruh9C4yjKwoDxgpmT4x/dv6Zmk4VeSkB1FZAbc9ZvcSjdynvBKnuOUo1wbbMjzwNSJrfDRT80mnpcqjhElbzhsSbXXNuu6hfkiZRX/vrevelS4yUG7q2SMhpKW65YhqMcRvz79t571Y+rpcdp1tTpvLmrEqyR/ITJ+ktXZa9q+G/1JthXLHK0H/TvbuFpunL32sc2spfva9HatlbgtGdU6YeLuuqTi6TOV2HmWkeOU9zilKNmG/Oi3zxwPTHVuwO/b51xUcYwkobLN6q8UlVzbNdetovwfYP/bVznTa1WscnaJU5FI1NfrjtptHKHkvG+lqFH/5WDlmNLzoqV3UsUUzplhKoc6x8NO9CfZViynd8hhIumk91VbBo/+CR7tEhYu4p9T62IivewPj+qkz9UVfw5GiVLj5mCmcwbO13tRdEm0aS6b8Dh+3+CNySVbXkq2IUFzMDO46NXHg12RKAkPt08kGqu33PzzQVVxjFxMe4u5h+IFAPpDt6mnQv5rW/S6LahiAteQfW9f5YS6RUCkDEW1TjhO+bnNSmGp/yo7fQ9qtg8vPemtd4p73xQm1N/AFC686E76Y+LC7yemiiiGSV9oYXegP8m2YpcMD2SE/bImJl1xoEdm2hO4HczgjdTcPQqVt/OsM4PnGnaaD9HqH0OORpB6p2QytRx30Hp5oXMf4sfNqwfmwxbEsIuJXJ5HoUJexvs5h2x5qaINcdFa6Axmibao84hCB3QwismUxEdfrPNWVXGMOPNhC8IbYOZ5kItxIzFOzieFYtfW9rqp+Xyqy1/a9/ZVzfTax8hxjGHSxTrDVAibxEr/1X5+pd9uBzM4Jq/WfIiWeJnSi35foFXjxdyF/iTbSePDDz8sbO789bPP0P/iCxwcHORybpl49eo/+Pzzz/Hvb7+1O8C2ot+skJqEjxDs//jY9/YRQkhBrDxaYVjw4OCg1MkfPXJICCGEEFIjVobWQwBMp/9Eo6ErCoPwt64kDML8K7XsQVcoRte/fvNGfZyaEEJIhah0A1PULUr60zeEkDqwMrT+8fe/odFo5No2iP0RN6zevHljc3pCCCGZTNGznz2YEFIxVobWr7/+WrUchBBCCCF7h1UyPCGEEEIIWY/dPFqEEEIIIWQtNLQIIYQQQmqChhYhhBBCSE3Q0CKEEEIIqQkaWoQQQgghNUFDixBCCCGkJmhoEUIIIYTUBA0tQgghhJCaoKFFCCGEEFITVp/g+eijj3B4eAgg/Hh0+CnDILYsufzhIcD330u8evXKWmBCCCGEkF3BytBqt9twHAcHBwcAlGG1/Hh0ENpVCBAsjKy7u3u8+7t38a8XL+D7Pn766aeyshNCCCGEbDVWhhYAnJyc4MmTJ1g6r6LGFoDGwsbCbxrADz/8gKOjI7z33ns4PDzEixcv8OOPP5YQnRBCCCFku7EytIJIiFCFBtU/amkjEkLU2zcaAICDgwP84YMP0P30UwRBA9988xw///xzGfkJIYQQQrYWq2T4WF5WuGy5MvorxuvXrxEEAX7//vvodP6CTz75xOb0hBBCCCE7gZWh9fAQMaQMBtWS+Mrnz5/jq6++wtdff43vvvsO7zx5YnN6QgjZGpqDGaSUmA2ajy3Ko6Kuwwy7fxm6mEgJOekW2GXCMUAysQsd4gFAYDCyGmq5/hXmaj28ecCfP/4Yf/xTc3GEX/77C74cfmkrN6mI5mCGS5yjNZwX2QuDmQdXxJf6fYHetJ5zNgczeMkTLk8MkefEG8VwjeQYndYQ83XbmTDu+5jse/tI9aTHQj6dsdwvr44hZJuwS4aPTd8Q/p3OzUIjtLiAw8PfQs8IAQQB3nnnyRpvGKmXpfKS4yL7dTGRIzh+H6K11HjNwQzeSEK2Vxk9tucM8dEXPWy/nu1iIs9w1xEQC8tBtd2Tpxh3WljamHMMWwLDrEM1B5h5Lu6vtskI2ff2FWM+bEFkNpAAUB6fkaNeihZ6o4nBbIbBbXS8mPa9WG+oE7LFWE9YGjqslv+J5m2tiiuGCfRBtE6RbBDl4vdwfDOGtNj/ui9SxtR82ELfB+CcGUMHZc/59HiXNO0UPZF8eMwxPB9DQsC9yBuSaGJw6ULIMZ5tlXW57+0jldIcYDZyIMedhN6YY9haY2ShicGZU7OAhNSLZTJ85I+IraQihTpeGFkYmP+DB1PGPKmRJgYzCe/4CkII9DybY0wxzXgoTq99AAKnnailVcU5NfIOtyV2f3TmHm4kgKMT5Mrk0G/y/q54e/a9fcSK7oUyps8LpScomoNLuMJHv+/XIBkhm8HO0IpaS42IdysIECyMLFMOF6OFj4sK49SWz3R7Z/BWVXHOJk6OSuy+bdy/zGVYdNsOsIvenn1vH8lPc4AzB5A3noUx3cWFKyDHz3YgXYCQbOyqDrUnKxIxVH80Ilnwi9nhQwyG14PN2YF4VYjymEi5/DFWfuiqkKxCku5EpitmmgPMwuPpvxfnmQ2Wb+362MZ1pmOF8kd/YoKFbZogOwiTZ5sN8/QYAsD9y5r8Ezkf4MVIj594VxTpt3Wn6uBUAP51rooBnDlVeHt2sH1V3ndrxZhl6gWjTlBr4lVpRt1ioaMKoGTLccw6rmXyOKva8/QYAhI3noWZNRnBKegJq6Q/40dM9V2he4IQ2Hq0HnTVYeQTO9HE9ywjK+3Nsra0FEdnmMlL4FxACPXTGUsI16v2Zji9hDqNPk9nDClceLMBBhMJeXaHjmGdUfUcX2AWJhJH93FGEeU2h3cjAThoZzVDP9S2520vzKXwkec5W4ynOBYAnFHpB2uM7gRSenDv+8u+EALXbYMyztVva87luRB+P1fVVPfChSh7LXe9fVXedxnMvRuoaGdyry7aDpAOhQPotuEgp0FZuY5SD/72tYj16dpjVnQtm4MZpOcC407k/H3cu56xKrjbdgDc4+XcYByu6qvuBCNHYnxe7EWj0v7M6Ds4IxpbpBB2yfABoPKwGnEvFlK/Fv9JG1lBWTMLEMBNJ55MuUzKblfm6RHiHv1o6fl8iPOxBIQL1/Ez1p0ieT8DgHCOUjJjPkSr7wPCxaV+As6HV1C55WZlFD6orizyHuqgOwmrCesw/KboifiDZfEwWOGlXIlO0DVNDTHtpRN08/bbkoR3aAT0Rd4QqnoolLqWe9C+Ku+7THRemTjtxO+zbhsOfPh+el3z5Ai5Xygq11EqFJ+8zPPhOcYy+5iVXMvmAJeugBx3ElOzTNETfWRnUR3hbCYxQj9mnPnChWf0yHcxGTmAf7UmUd5Alf25su/MRT+EmLCsOgziP4Y/l5uaawsDoLRDK+tGvL1b4w0qfJ7r1ANh/vJ+zTqB46emY2Uoj+kzjGMKYgqVW256cFTwIK4M9YY9cgBYJrxaMR+ipZW7MyoePlWGqsQ4b4JQ7n5brEgYh9doy6zQRZzm4AyOZbglZC/aV+V9l8kcarf4fabyx+5wfZc0OJronIr8hRmb0lELL/gRUs4coJJrufrlbopn42SGZphbKSDukwb/FL3OGBIORqm0iREc+Ohb5XVW2J/yBqYhGhb9FBtn5G3GMhl+8XnDReFhysDSK1NGVhBEPuFT2qdlZKFAKkLeGVSqTvw2rtOk3dertg8VxDHC+3f6TJXLJ13dxgdVMseiolyQlTQHmEk9AaHfh9j4hJOhck88sNZeC/0AyFCkJor0W5asPdHBWAq43urcu86pWP02v+/tWyVT0fsux32RrpbVLzI3HqbeDWT0oRqG7K2Su5dUraPWUf5ahmPKpvI3w+APq1QjXjhVZSgx7tjPl1dZf9aSE0reRsp9VNq81hw+VDuy6rAIcw830oXrXqA7DBVP+KDqxx9U0x7EJt1b4QSEeNzZmsM3b/VA0Bdk7bXQOV/+phXpHMMrH+5IGYbGaTLChPKrFQ3Y9/ZVSZ77YnoNf+TAOX4KYK7DTBJjbw7Moe7BsEE6uXtcwttYFd2J9iTHsJmlLg96TBU6fGik51zfncDToclSjvEd7U+yv1hWHSaXRBLjTeFDIGVkBcaN3mZMb4zqwRXz2HQv1BvfY9bEL4wsiXFnOz6JUazS8RZ3ReZ7WkmZN33D0TqnEJBY4WTIwb63r2p0mF57VponRxFvoDYG9DoVgsrvKaycSMVfPOdJJ2rXhh5TRffKvdNyYlLhemkvpLYonVGeyr8d6k/yVmCXo/UQmdshNK6yDCwoD5gpGf4x/FumcN4mEZmBff3GmHRXT7vobRgAACAASURBVK9jSfGPrxh0oip89FOzg28elcha9MGdNxy2pHC/rSRL3jBfpGz/7nv7qkeFmxy0u0rGaChpuW4Zgnoc8fVM+vALFB5UxfoxZfp6w+q8uagRr+fbSxa9hD96wlK/r/+/pu270Z/kbcEyR+tB/842rpYbZ6/dqJmlDRbjA0XP6bMRMqqCwryrtKdK5yGJU3S6Vc2tZI+SE/D72/DNQTWhoc2DO1S2Z3lz2Ar3mwn91p4lb0X5P8D+t69yFi80lzgVifxHve6o3cYRapwnbi1h+M7kXdRGbI2sHFNZOlQXUxirp+scDzvRn+RtwXJ6hxwmkk56X7VlsNFP8GiDxRklEsS7mHhudhpB5TgYJUqam4OZzk04N3qI5jqB0x1t9rtw4eR/0eul3lrrmCsr+5xoDjBLhQr0x62RKE3Py7S3mHsoPh70h25TT4X8/dYczAyhjfCD2ivkrXLC131vX+WEVb4CImUoqnXCccrPbVaKUEYX8c9JdjEJi1JqPX0ParYPLz3prXeKe98UJtTfwBQuvOhO+mPiwu8npooohlFfKGF3oD/J24JdMjyQEfbLmph0xYE2yHzYQgczeK4H6YZLVZ7R9YUpsbR65LiD1ssLnWMRlyFT38yHuPJdjLbGze1gFJM/iY++qNjj5YwgZfyMctyBKKGk58MWhDfAzEuOh3RItEi/zYctdAazwvLahUGz2ff2Vc302sfIcYxh0sU6w1QIm2TaE7gdzOCN1NxlCtVPzzozeO6qves6v0ojuB3M4Jh06HyIlniJiYzfw35foFXjxdyF/iRvB40PP/ywsLnz188+Q/+LL3BwcJDLuWXi1av/4PPPP8e/v/3W7gC7hn6DQ2qyv7y7z+C599UbMGQ1Jftt69n39hFCyCNj5dEKw4IHBwelTr7RyOFOo3OR/CsaWYQQQsgOYWVoPQTAdPpPNBq6ojAIf+tKwiDMv1LLHnSFYnT96zdv1MepyVqKJSQTQnaLMN9wNelP3xBCdgErQ+sff/8bGo1Grm2D2B9xw+rNmzc2p3+7iHxfjDqWkH1kit5GZxsmhGwSK0Pr119/rVoOkiA66/NjzrxOCCGEEHuskuEJIYQQQsh67ObRIoQQQggha6GhRQghhBBSEzS0CCGEEEJqgoYWIYQQQkhN0NAihBBCCKkJGlqEEEIIITVBQ4sQQgghpCZoaBFCCCGE1AQNLUIIIYSQmrD6BM9HH32Ew8NDAOHHo8NPGQaxZcnlDw8Bvv9e4tWrV9YCE0IIIYTsClaGVrvdhuM4ODg4AKAMq+XHo4PQrkKAYGFk3d3d493fvYt/vXgB3/fx008/lZWdEEIIIWSrsTK0AODk5ARPnjzB0nkVNbYANBY2Fn7TAH744QccHR3hvffew+HhIV68eIEff/yxhOiEEEIIIduNlaEVREKEKjSo/lFLG5EQot6+0QAAHBwc4A8ffIDup58iCBr45pvn+Pnnn8vITwghhBCytVglw8fyssJly5XRXzFev36NIAjw+/ffR6fzF3zyySc2pyeEEEII2QmsDK2Hh4ghZTColsRXPn/+HF999RW+/vprfPfdd3jnyROb0+8szcEMUs4waD62JISQvKj7VmL2lt+4+6O/uphICTnpFthlwjFArLELHeIBQGAwshpquf4V5mo9vHnAnz/+GH/8U3NxhF/++wu+HH5pKzepieZghkucozWcF9kLg5kHV0QWyTE6rSHyHqU5mMGLHSCC34foTQvIswnyttmwnYmC16t+9r19pHrSY8HvC6y/dZf75duekN3CLhk+Nn1D+Hc6NwuN0OICDg9/Cz0jBBAEeOedJ2u8YWSzLJWdHBfZr4uJPMNdR0AsnqLqWJ48xbjTQn6bzUdf9LD9erZIm+cYtgSGWYdqDjDzXNxfbZMRsu/tK8Z82ILIbCABoDw+I0e9FLXCO7iJwWyGwe0aHdC9WG+oE7LDWE9YGjqslv+J5m2tiiuGCfRBtE6RPCIqJODh+GYMWXjvKXoiqUjnGJ6PISHgXuRzzz893iVNW02bgSYGly6EHOPZVlmX+94+UinNAWYjB3LcSXie5xi21r1oNTE4c2oWkJDHxTIZPvJHxFZSkUIdL4wsDMz/wYMpY55skCYGMwnv+ApCCPS8Cg8993AjARydIHdWg7zDbYUibJyibdZv8v6ueHv2vX3Eiu6FMqbPC6UbKJqDS7jCR7/v1yAZIduBnaEVtZYaEe9WECBYGFmmHC5GC7cLFfapNf/p/mWOh2wTJ0f1ibBxcrUZ6LYdYBe9PfvePpKf5gBnDiBvPAtjuosLV0COn+1AugAh9thVHWpPViRiqP5oRLLgF7PDhxgMrwebs0O5qhcVILqCJPqzopqkO0lsm1lJEq1MUZ6f9fvE5cu9fSG5YJSnSAHNRmh2cCoA/7qACs35AC8oyOprVWIspU9VoM36AVXe27OD7YvKlLxXZoOlt0xXehnX5RVZVwyamqnuOVMVXaIqTcsRP0YJ/ZCD3PqgjmtZRH89PYaAxI1nYWZNRnAKesIq6c/4EVN9V+ieICQHdh6tB111GPnETjTxPcvISnuzbC0tzfEFZmHSrtA/nTGkMzIoEnVDta8j2wqBzlhCuF72zXV0hpm8BM7z7dMczCA9Fxh3Iufp4971MqrqCsrVHGAmPZzedGLbX7e3yNjqTiA9F8Lv56wgeopjAcAZlX6wpuSQHtz7fuJaGZRxobGUca4Cbe5euBDwUcQONZ5zl9t3egl1a0XkES682QCDiYQ8u0PHsK7ImJh7N1DRzuReXbQdABA47STWddtwkNOgLKgf1mOppyq6lkX1V7ftALjHy7nBOFzVV90JRo7E+LzYi0al/ZnRd3BGNLZIpdglwweAysNqxL1YSP1a/CdtZAVlzSwI5wg3yaq2+RCtvg8IF5exp40KkyUfEvPhOcYSgNOG8dYSSJ1jPmyh7xv2aQ5w6QrIcScxPcIUPdGHOQuhmFzhA+wq8RY47T1mWXTCUzIC+qJISHKKnog/WBYPA1tvnU7QNU0NMe2lE3SLjSWgXJvVQ6FUyGQP2ifEPfrRKR/mQ5yPJSBcuI6fse4UyefoSnRemTjtxB/63TYc+PD99LrmyRGQ1wguoh/yCWylpyq5llb6CwCOcDaTGKEfM8584cKTE4O8XUxGDuBfFahIDmWvsD9X9t3ZHswXRrYFy6rDIP5j+HO5qbm2MABKO7Qyb9TpM4xNN6OROTyV4YvUS9KKc9zeSQAO2hEtkmUEaaHwbFykpm+VXPHzPj5JQ+kabZnlxs/JfIiWVu7OyKSsV6P6QmKcN0Go8Fiyb3NzcAbHMtwSshft869Thtj85f2adQLHT1cfNrEX1G5xo0Llj93h+i5pcDTRORX5CzMK6IdyrNNT5a9lcf0V5lYKiPukwT9FrzOGhINRPJaNwWwEBz76Vm+GFfanvIFpiE6vfRQfZ4RkY5kMv/i84aLwMGVg6ZUpIysIIp/wKWdpybssVRjejMeo615ZKLEFWunUXDk3faamYHBGa3KzkjkZFeWO5JQSPdHBWAq4XnEjKXocpdwTD6y1bQv7wqxITZQfS3nbrBX/qrf5fW/fKplu7yBXypsIG+UY5+GDcxlS0h63Gw9T7wYy+lDVeWh2yd1L0vqhXspfyzL6K8PgD6tUI144VWUoMe7Yz5dXWX/WkhNKSJpyH5U2rzWHD9WOW1F12J1IjFJTtxSfQSqOzjMqcZhccs2HaInhYiZ1Z6TCOil3/7QH8ailPHMMr3y4I2UkTS1lCd+81QNBt29t23Rf+JtWpDnaHCaUX61owL63r0ryjPPpNfyRA+f4KYC5DjNJjL05MAdupAs3bJBO7h6X8DZWRT16Kgsb/RUa6TnXdyfwdGjSYiaIJTvan+TtxbLqMLkkkhhvCh8CKSMrMG5UFYa3s0glTTyXQCdAluYWdzaHsZRrPmwtc5kACNfb6+9w3b8soih1XxSZwyuTaj2Vzc4pBCRWOBlysO/tq5opriM5U82To4g3UBsDep0KQeX3FFZO7XoqCzv9dZt7p+XEpML10l5IbVGGnvrVyeg71J+EwDZH6yEyt0NoXGUZWFAeMFMyfFn/lsgMouu3s4VrWM9QDb9gknYR1odg0rOfVyBXJJcpX07apin30FWJrEWPUTx0nH8s5SFL3jBfpKzi3/f2VY8KNzlod5WM0VDSct0yBPU44m9CT2Vho7/W5c1FjXg9Z1+y6CX80ROW+n39/zVt343+JERhmaP1oH9nG1fLjbPXlnZoZVTghMm4y7yB0C1uemvXD4cKCG/ws4z5btJfmqhKLv02ulU5B/oNttRDV01oaHOMlX1hIvdYWsWaNleU/wPsf/sqZ3qtCivOLnEqEon6et1Ru40jFPWeVslm9FQWxfUXFsUUzplhKoc6x8NO9CchCsvpHXKYSDrpfdWWQelP8DgYJcqHw9wlOT6P5AFoV7NwEf9MWxcT6VX3QdNpD6pa3ktPGumd4t5PutmLy9WdpKu+VIJpwclBLQgnC4yGKJuDmcHNH36gOlFWnnEMNAeYpY7RxUTq6qSWxaSe095i7qF4SFV/6NZQyZlvLBVv84Knx1DOowoU/763r3LCe01ApAxFtU44Tvm5zUqxIT2Vefqi+gtYfANTuPCiO+mPiQu/n5gqohhGfaGE3YH+JERhlwwPZIT9siYmXXGgEshxB62XFzqfYbEU445IJVtOewK3gxk8nTwe3fZZZwbPLSfL6vP46IsWbgczOE6e7bPlmvau1PxGsaXq+I+hT+bDFjqDGeRSeACqb0QRBeuMyh/DIJvwBph5HuTiOkqMk/NJodhYsm2zXRg0m31vX9VMr32MHMfo+V2sM0yFsEk2paeKnT9bfwHQ6QsvMZHxe9jvC7RqvJi70J+EAEDjww8/LGzu/PWzz9D/4gscHBzkcm6ZePXqP/j888/x72+/Lb6zfltCamI9Qgqy72Np39tHCCFbjpVHKwwLHhwclDp56cghIYQQQsgWY2VoPQTAdPpPNBq6ojAIf+tKwiDMv1LLHnSFYnT96zdv1MepCSFkrwnzDVeT/vQNIWQfsDK0/vH3v6HRaOTaNoj9ETes3rx5Y3N6QgjZIaboPe7swYSQR8TK0Pr111+rloMQQgghZO+wSoYnhBBCCCHrsZtHixBCCCGErIWGFiGEEEJITdDQIoQQQgipCRpahBBCCCE1QUOLEEIIIaQmaGgRQgghhNQEDS1CCCGEkJqgoUUIIYQQUhM0tAghhBBCasLqEzwfffQRDg8PAYQfjw4/ZRjEliWXPzwE+P57iVevXlkLTAghhBCyK1gZWu12G47j4ODgAIAyrJYfjw5CuwoBgoWRdXd3j3d/9y7+9eIFfN/HTz/9VFZ2QgghhJCtxsrQAoCTkxM8efIES+dV1NgC0FjYWPhNA/jhhx9wdHSE9957D4eHh3jx4gV+/PHHEqITQgghhGw3VoZWEAkRqtCg+kctbURCiHr7RgMAcHBwgD988AG6n36KIGjgm2+e4+effy4jPyGEEELI1mKVDB/LywqXLVdGf8V4/fo1giDA799/H53OX/DJJ5/YnJ4QQgghZCewMrQeHiKGlMGgWhJf+fz5c3z11Vf4+uuv8d133+GdJ09sTl85zcEMUkrMBs3oQsySy4rsXwXdST3HJYTsJLXpmh1DXYcZdv8ydDGREnLSLbALnwu7hl3oEA8AAoOR1VDL9a8wV+vhzQP+/PHH+OOfmosj/PLfX/Dl8EtbuclbQHMwwyXO0RrOC+7ZxUSO4ESW+H2B3jT3mTGYeXBFfGmxY2wSg7xyjE5riPm67UwY931M9r19pHrSYyHf/bvcb3vvd7Jr2CXDx6ZvCP9O52ahEVpcwOHhb6FnhACCAO+882SNN2xzzIctiOHj7U+SLJWdHBfddYCZ5wLjDkRooHUnkCOJ2Uknh9GmjTS/D9FaatnmYAZvJCHbfYit0r5dTOQZ7joCYtE0df08eYpxp4Vlk+cYtgQyh6q+dvdX22SE7Hv7ikFdk4PuBHLkALF7uInBbIbBbXS8mPa9WG+oE1IQ6wlLQ4fV8j/RvK1VccUwgT6I1inuJ9rFW8Qr/LajQgIejm/GkMX3xuDShfD7cYNq2kPfB4R7mSvUcN0XKWNqPmyh7wNwzrYsXDFFTyQfHnMMz8eQEHAv8g4+fe3kGM+2yY7c+/aRSmkOMBs5kONO4h6eY9haY2ShicGZs2oDQqywTIaP/BGxlVSkUMcLIwsD83/wYMqYJ28pTQxmEt7xFYQQ6HkWh+hewBUSY8OTdPpMPZhPO+uspCmmGQ/i6bUP5DrGFjD3cCMBHJ0gl7T6Td7fFW/PvrePWNG9UMb0eeF0A6A5uIQrfPT7fg2SkbcZO0Mrai01It6tIECwMLJMOVxbEy0kW4cK+5QJy3XbDoB7vDTp2PlL3AMQp518D2YTt3cWXrZH5v5lLsOi23aAXfT27Hv7SH6aA5w5gLzxLIzpLi5cATl+Bg4RUjV2VYfakxWJGKo/GpEs+MXs8CEGw+vB5uwhygMi5fInO0S3ZttCIT5dJRKtFEnur/8vR8oN7YzyyFiUiBxJeUxbTxLbZlatRKtgIueYDbSBEl2fvq6PVwnTxMkRAHmHW+P6W9xJAOIYT21P8fQYAsC90ZLLw5pxGKt0Lda/6VN1cCoA/zrHY0M/oMp7e3awfVGZ9N8yNeaxvKdN6/KKrCsGTc1U96epii5RlWbUVfXek7l1Rx3XMnmcVe15egwBiRvPwsyajOAU9IRV0p/xI6b6rtA9QbYWO4/Wg646jHxiJ5r4nmVkpb1ZlpZWc4CZ9HB604EQYvFz3TYM+u4EUnpw7/uJbW1Kg5sYzFQ1m2/I41kw7anzaBe031+et5I86u4EMkzYXrSpD98ZGZSWunnb1yLW/s5YQrhe9o18dIaZbOM63CdZpXV0hpm8BM4LHLNWnuJYWUEZxsIcL+/LHD/M3/CR59meosg4PL7ALEwAD7ftjCGN/ZtxLk/lquUZb90LF8K2XdFz7nL7Ti+hhnNEHuHCmw0wmEjIszt0DOuKqJC5dwMV7Uzu1UXbAYxh6W5b6Zs8nVP5PWmpOyq6ls3BDDIsbInouXvXg2fIWI96tFPG4aq+6k4wciTG58VeNCrtz4y+gzOisbUH2CXDB4DKw2rEvVhI/Vr8J21kBdYOrVBxXiXePqa9hCGjEyPhpyvFpr11iZFJtqXst4uJsU1T9FTGNy5jTzYVkkvKOx+eYywBOG0Yb2MB3HR62W50Adx04tdwmTCeccwdpjsJqyAtQgsFx6FwjlLXFvMhWsb+BVLeoRHQF3nDsOqhUCpksgftE+Ie/ejLxHyI87EEhAvX8TPWnaJQup7OK0uFr7ttOPDh++l1zZMj5DbuK78n7XRHJdeyOcClKyDHyUrhKXqij+wsqiOczSRGSLyECheenBjkDfXpVcHnAartz5V9t20FOKQollWHQfzH8OdyU3NtYQCUDB06aK/RHMogMydHFyM0siTGncedW6U5OIOT1abpM4xNN76ROTyVTYzUCxkAyBus9MBnKKbbO4k8fbM7qLf6kQPAMsm28DjMUvqZ/TtFT0S9Dtdoy6zQRZxwPNmEW0L2on3+dcoQm4cu0Mx1AseF4tDaq5owKlT+2B2u75IGRxOdU7EiHJ5sw6buyTW6o4JrmfUyrZji2TiZLalTByAg7g0voZ0xJByM4rFsHaHw0bdS6hX2Z4a+DQtwio0zsm1YJsMvPm+4KDxMGVh6ZcrICoLIJ3zsLC1VQbbMfTJ7VsOcnTUGw1qeRoysol6wqtE3albCd5XkTDJOMjfF55I5GY+ez5UTHaJ2BfScPIbQwtq2FR+H8i7rsRoq9nV5ZlP0RAdjKeB6prf4ED2eVr3N73v7Vsmkix+y5U2EjXKM83Tlqva43XiYejeQ0YeqzkOzS+5eYrwna6T8tVyXb7ny7GaDP6xSjXjhVJWhxHiV534NlfWnpb4lu0G5j0qb15rDh2rHaqoO50O0xFBNIukKZXCNkHAz65wdv9wAFu4ILgA5Pn9kIyuKg5GUGGWuVw+qpLjdifbMxNhAHd20B1G7F1Aluzu63D+zq/Iq73DSQ6wJFa9tWzXjsDhzDK98uCPlyTBOWREmlF+taMC+t69K8ozz6TX8kQPnWN+h3bbyUHtzYA7cSBdu2CCd3D0u96ZYCZvVHXpMFTp8aKTnXN+dwNOhyVJ6fUf7k2wWy6rD5JJIYrwpfAikjKzAuFEx5sPWMqESgHC9yNujrjLLO89OBnLc0ZNdPlaStwE5XiaTGn8ib2iRqp143oJOttwb1nlD9FtynjfHhZFVRai4mnGoKPOmbzha5xQCEiucDDnY9/ZVzRTXkZyp5slRxBuox7Bep0JQZT3yJXg03aHHVNG9cu+0nJhUuF7aC5msFl+p93eoP8mjYZej9RCZ2yE0rrIMLCgPmCkZvrKZ4edDtHSC5DK/I28YYj3T3rZUgBRtk54NG36B5OHdRSnajLyR3NMB6ORY+OinZiS3ofg4FJkJGesqK01kGRphvkhZxb/v7aseFW5y0O4qGaOhpOW6ZQjqccR/TN2xfkw9PU67rlbnzUWNeD1nX9aLarJafE3bd6M/yWNimaP1oH9nG1fLjbPXVjt5qX4LiijpcJCfVZALNB+2lsaWxRw6VaHy0/K2KXTBmzwEYb7X/jAfXsHP+CyL8m6sr95SydOA37fP20hSeBxmVHOtLIRIb63e2rMMjYryf4D9b1/lTK+hiskucSoSifp63VG7DeWAfSzpH1d3rBxTel609E6qmMI5M+jnOsfDTvQneUwsp3fIYSLppPdVWwaWn+DpTtLVRiqxMeGxmPYWc77EE6/1B0YLWksLYyvvHDo6AdSpsgRPl0On2wSk26Xd2sJF3PboYiK9nf54ajhZYPwa6GokZxRfrvMxksaT6RjqTbnknFJJCo9DB6NEKXqYj5jMFWwOZgYva1glmyilj1J68tUI+96+ygnvSwGRMhTVOuE45ec2K8Uj647F90m99KS33inufVOYUH8DU7jwojvpj4mnvoFaELPOAXajP8ljYpcMD2SE/bImJl1xIAumvSs1r05sqQr1pEqHhy0Ib4CZ50G64VL7CsL5sIUOZvBcF550VaJ09sZo9Y8hRyNIqVLXq5iDy9wmwNSuaU/gdjCDpwsGltsJPOvM4MX2333M10aFP/Jf9nXFBj76opjHq8g4lOMOWi8vdG5MdFuR2nY+bKEzmC3GV/QYYsUAV/P5VJe/tO/tq5rptY+R4xjDpIt1hqkQNslj6w7z+ZWevx3M4Ji8WvMhWuIlJnIUGzN+X6BV48Xchf4kj0fjww8/LGzu/PWzz9D/4gscHBzkcm6ZePXqP/j888/x72+/tTsAIfuGfvNGapLGPWHf20cIIQasPFphWPDg4KDUyW2NNEIIIYSQXcDK0HoIgOn0n2g0dEVhEP7WlYRBmH+llj3oCsXo+tdv3qiPU791dDGR6nuJq0h/eoIQQspC/UPIprEytP7x97+h0Wjk2jaI/RE3rN68eWNz+h1nil79s3cSQogB6h9CNo2VofXrr79WLQchhBBCyN5hlQxPCCGEEELWYzePFiGEEEIIWQsNLUIIIYSQmqChRQghhBBSEzS0CCGEEEJqgoYWIYQQQkhN0NAihBBCCKkJGlqEEEIIITVBQ4sQQgghpCZoaBFCCCGE1ITVJ3g++ugjHB4eAgg/Hh1+yjCILUsuf3gI8P33Eq9evbIWmBBCCCFkV7AytNrtNhzHwcHBAQBlWC0/Hh2EdhUCBAsj6+7uHu/+7l3868UL+L6Pn376qazshBBCCCFbjZWhBQAnJyd48uQJls6rqLEFoLGwsfCbBvDDDz/g6OgI7733Hg4PD/HixQv8+OOPJUQnhBBCCNlurAytIBIiVKFB9Y9a2oiEEPX2jQYA4ODgAH/44AN0P/0UQdDAN988x88//1xGfkIIIYSQrcUqGT6WlxUuW66M/orx+vVrBEGA37//Pjqdv+CTTz6xOT0hhBBCyE5gZWg9PEQMKYNBtSS+8vnz5/jqq6/w9ddf47vvvsM7T57YnJ5E6E4kpFQ/ky6A7gRSSswGzccWbeM0BzNIOUORptvsQwh5O1D64e3Up1H2R092MZESctItsEv5Z6pd6BAPAAKDkdVQy/WvMFfr4c0D/vzxx/jjn5qLI/zy31/w5fBLW7kJmhjMPLgYoyOGmIeLC4wfso90MZEjOJElfl+gN7U7WnMwwyXO0RrO122pxqMof97853ws0m2FHKPTityHWduZMO77mOx7+0j12N7/y/3K6Kltxy4ZPjZ9Q/h3OjcLjdDiAg4Pfws9IwQQBHjnnSdrvGFkJc0OTgXg96nAiKY5wMxzgXEHIjRSuhPIkcTspFPQcFkqQDles2l3AjlyAL8P0Qo1ZROD2QyD2xbyn7bAOR+NLibyDHcdAbFol5Lbk6cYd6LtnWPYEhhmHUr31/3VNt3D+96+YsyHLYjMBhIA5e7/7sV6Q30PsJ6wNHRYLf8TzdtaFVcME+iDaJ1icbQ7r4gHcK94eoy3YHyS3DQxuHQh/H7coJr20PcB4V7mdvurMIGH45sx5PqNMRs5kOMOROx1dI5hK7+RVeicj8oUPZFs1xzD8zEkBNyLvApJ95cc49lWvcXve/tIpZS6/5sYnDmrNtgbLJPhI39EbCUVKdTxwsjCwPwfPJgy5kkumidHjy0C2Sa6F3CFxNjwVJs+Uw/J0846S6uJwUzCO76CEAI9L89p1cP03DrMV/ycW8ncw40EcHSCXPasfpP3d8Xbs+/tI1aUuf+bg0u4wke/79cg2XZhZ2hFraVGxLsVBAgWRpYph4vRQkLqoNt2ANzjpUnfzV/iHoA47ax5SKpQkMibKNEc4MwB5I1X4mFa8Jzbzv3LXNei23aAXfT27Hv7SH5K3f9dBc++FgAAIABJREFUXLgCcvwMb8MQsas61J6sSMRQ/dGIZMEvZocPMRheDxYn1yFDOVIuR2eUqLqLb6wqDKI/xlhjtBJBvWFH98muNkhvmx3KNMiSWcURlSey32yApm6/pwPbi/bPBjneNPPJEFbamNqiqhxNcltUc8SOmfeaQ7mri2xvu491n9U1hjIbh5MjAPIOt8b1t7iTAMQxnhY99CqeHkNA4sbbVp/Fmmurx4Tqm7y6IutUOmfyOsdjQz+gynt7drB9UZmS92RUh4V63rQur8hV6DFjiortvZ6P3PqwjmtZRE+WuP+7kxGcgp6w6p9L6b6zeX7lwc6j9aCrDiOf2IkmvmcZWWlvloWlNe1BCAGh3Y1+X6j/i0TFQncCKUdw/P5ivRB9+M4oe6AdnWEmL4Hz5TE7YwnheukOaA4wkx5ObzqR4wtctw0DwSiLQGcMuN6Kzj06w0y2cR3u0xpirtvfGct4+9dV9RSQYe7dQEUJklepi7YDwBSG6rbhIKciXqAGevtaJGTKuObQN1uY8B3p13vXWxifVexTrs9qGEMreYpjgRXehjle3hc5Xj6iXrTUw8HiwVitcBNI6cG97yeurUEZH19gFiaAh9t2xpCrdEXyXJ7Kj8vjmOteuBDwUehWMZ1zl9t3egl1m0TkES682QCDiYQ8u0PHsK7ImKpdjxW513NRXB8CqOxaFtWT1vd/d4KRIzE+L/aiUWl/ZvQdnFEtxpZdMnwAqDysRtyLhdSvxX/SRlZg5dDKRxeTsAoiphmm6KnMYFyarHQB3HTiCXzzYQt9H4DTjs2cECqTq4RFPu0lDD6dLJiWRR077Fxj3wrgptMr71otKoPOx0iFmrptOPDh++l1Kmes6MNDhY2Syns+PMdYInXN0Rzg0hWQ42QF3RQ90Ycx0m+5T7k+q3gMbTVHOJtJjJB4oREuPDl5nNlGVvTftJdO0BXOUarPMB+ilakrEt6hEdAXecOf6qFQKmSyB+0T4h796MvhfIjzsQSEC9fxM9adYm2aYZS69ViBez2nwMX0YShGFdfSRk8CKH7/h8/mqwLVyKHsFfbnyr47q3y+MMuqwyD+Y/hzuam5tjAA7EKHOWgOzuDAnBiM6TOMTZ0FZHb+7Z0E4KCdGjWmZXHUwzRDFgDz4RXUfWk4kLxBFVGZ4jJoD0jiZlR5F3e4vkveqE10TsWK0FVR5vBU5i2iLy9ZholiimfjdL2a/T6WfVbDGNpOdLgSAuLe8ELTGUPCwegRyoLX9V+KLKWfqSum6Imo1+Ea7ZVh5SWhbioTbt2L9vnXKUNsHrpdM9cJHBeKfdesxwrf67aY9eFSjvLXsrietLn/mxjMRnDgo2/1Jllhf2Y8W6fXPoqPs/VYJsMvPm+4KDxMGVh6ZcrICoLIJ3zqsLT0xc1KDLZgboi7qEquZY6U+XkS5s6sMph0/oypmidn4ulq7GQIB9zSFavfVG88TL0byOhg1Pkb5ZKi87ajiDFXZh/LPsvAfgwhnWNRUS5INWQ88MMqNas3+xWsvRZ5+i+OvMsaHaFiX5fbNkVPdDCWAq63younddOqt/l9b98qmW7vIFfKmwgb5bgvHkOPme71Oil/LW305OLsue9/VWUoMS4RpamsPyt5tuansm8dRtYurC+jkWVzQiscjAw3oZR69tqyicHzIVqRXKnwYWn98Ks6UbmsDNNr+ABEOGq77eWbqr6JFh6dCpKikzH+dE6AzkMqhM0+BdjUGArzEhM/Sxd/TsOvMo8jsD7vq568sPXXYl2+Wl3MMbzysdKTkSehfN/bVyVrrxU2rseqYr0+rBIbPVnw/u9O4LkCcnxePGQYZUf707LqMLkkkhhvCh8CKSMrMG5UIXK8TAA0/lSQ+wQ9c3CYZAhAuJ6dsVXpQ9CSmAxTXEdyDZonR5G3aH0T6XXKdWsR5oxUuMRj/EvjY4k2Jgphs08BKuqz8mNonWdCv7FW/HC+rfXi2lLc25hNmTd9w9E6pxCQWOFkyMG+t69qNqDHqqKQPqwSOz2Z//5fTkwqXC/t/EjOILAy3WCH+jOCXY7WQ2Ruh9C4yjKwoDxgpmT4evxbed3hVZ92iJZOGlzmPOSRpe43VHsZlJvWQburQgJRF+xy3dJ1W0x+PXM0/JyJtuvb8TT1WlbPPrX1mXEM5UMpvYwcjpo8DavzZqp9iBeQqvD9LzITMmz6OcvQCPNFyir+fW9f9dSrx6qiqD6sEhs9WeT+13PlZTk9kjMIrGn7bvRnHMscrQf9O9u4Wm6cvbYuh5bKfXFwtvEcFv1mEFFc62RZmbhfEdYyaDetc3aJU5Fwwep1R+02lLOk6HDWSt74IA7z7JLiqJvI2A49d08l+zxqn6XHUB5Ugr75EynK01ByOgETOpnaOTOUcm8kby9DrFV9biIjj6xYP+u39ixDo8Lrse/tq5xa9VhVFNeHVWKjJx/t/t+J/oxjOb1DDhNJJ72v2jIo8wkenexnrPzSJazmEIz+2GVJG6w7MU30eak+OxF9okVkSbpEm4MZPFfA7xf58K4F1jJoN60QECkFq9YJx7F8iIfHdhG3DbqYSM/8odHFd/u89MSM3inufYMr22afDfVZ7jGUC10Z5IziY17nRvj9eKg8nPyvXEK9/gaecOFFr5P+mHDyu4vVnDMH095i7qH4ubLufQejRCl62M/JnJLmYGYIbYQfw06U0kfR3yatRPHve/sqp049VhUW+rDS01voyYL3f1Gy9cUu9Gecd2x2Ul6sjEkbzF/eyT6QLfMhWv1jyNEIUo4AKNdj6HWcD1sQ3gAzz4N0ozvKxBfo7Zj2rtRcM7GlPvqilS6nHbYghl1M5FLWVdvXga0M02sfI8cxelgW6wwlxHmY9gRuBzN4IzVXj0Ji3BF41pnBc/Puo9pwO5jBMXmoLPbZRJ8VGUN5MI95FYqobYzNh2iJl6nr5PcFWo+o5MzXwnzvy3EHrZcXOjcmuq1IbTsfttAZzBJjQh1DrFAqaj6f6vKX9r19VVOnHqsKG31Y//lX68nHuv93oT+jND788MPC5s5fP/sM/S++wMHBQS7nlolXr/6Dzz//HP/+9lu7AxBCSBn0mzdSkzTuCfvePkJ2BCuPVhgWPDg4KHXyMpFDQgghhJBtx8rQegiA6fSfaDR0RWEQ/taVhEGYf6WWPegKxej612/eqI9Tkz1DhdtMXuYo6U89EELIvkF9SCwNrX/8/W9oNBq5tg1if8QNqzdv3ticnmw1U/TEtkTGCSHkMaE+JJaG1q+//lq1HIQQQgghe4dVMjwhhBBCCFmP3TxahBBCCCFkLTS0CCGEEEJqgoYWIYQQQkhN0NAihBBCCKkJGlqEEEIIITVBQ4sQQgghpCZoaBFCCCGE1AQNLUIIIYSQmqChRQghhBBSE1af4Pnoo49weHgIIPx4dPgpwyC2LLn84SHA999LvHr1ylpgQgghhJBdwcrQarfbcBwHBwcHAJRhtfx4dBDaVQgQLIysu7t7vPu7d/GvFy/g+z5++umnsrITQgghhGw1VoYWAJycnODJkydYOq+ixhaAxsLGwm8awA8//ICjoyO89957ODw8xIsXL/Djjz+WEJ0QQgghZLuxMrSCSIhQhQbVP2ppIxJC1Ns3GgCAg4MD/OGDD9D99FMEQQPffPMcP//8cxn5CSGEEEK2Fqtk+FheVrhsuTL6K8br168RBAF+//776HT+gk8++cTm9IQQQgghO4GVofXwEDGkDAbVkvjK58+f46uvvsLXX3+N7777Du88eWJzehKhO5GQUv1MugC6E0gpMRs0H1u0HHQxkRJy0q33NM0BZjtzTQgh+42F3tspvU6S2IUO8QAgMBhZDbVc/wpztR7ePODPH3+MP/6puTjCL//9BV8Ov7SVm6CJwcyDizE6Yoh5uLhmm4WY6GIiR3AiS/y+QG9qd7TmYIZLnKM1nGdtofpexJcWP2e1ctdPVe2uE4OMcoxOK3KPEkLeKuzm0YpN3xDo7Kx0bhYaocUFHB7+Fv/zP/+jfn73P3j3d79b4w0jK2l2cCoA/4oK/FFpDjCTIxyNOxBCqJ++D2dk8/bZxGAm4SUtiRhdTKQH976/PJ8Q6IwlnFGBt+RK5d4EFbW7VrqYyEvgXERk7GAMF56cYSsvKyGkdqwnLA0dVsv/RPO2VsUVwwT6IFqnWBztSt0K/foYPD3Gqscx2QRNDC5dCL8f9z5Ne+j7gHAvcz9cm4MZpPRwfDOGXLPtdV9AJFw482ELfR+Ac5bjnNXJvUnKt7tupuiJFuKOyDmG52NICLgXb6uyIuTtxjIZPvJHxFZSfi3t3YosDMz/wYMpY57konly9NgikO4FXCExfpaOW02fqYfraWfd0197sY6vIIRAz1t30immGWGy6bUP5DlnJXJvmgra/VjMPdxIAEcn2FIJCSE1YmdoRa2lRsS7FQQIFkaWKYeL0UKyP3TbDoB7vDTFbucvcQ9AnHbWPFznGLbSnhorbu/WesOAquTeInK2+9G5f8kwPyFvIXZVh9qTFYkYqj8akSz4xezwIQbD68Hi5DpkKEcqhdcZJaru4hur6o7ojzHWGK0CUR6G6D7ZOSvpbbNDmQZZMvM2ovJE9psN0NTtD/N4Fu2fDXI8GPPJoMJY5raoKkeT3GUrCNPX0nisNSHjbPkScq4dE+vlPTkCIO9wa1x/izsJQBzjqcXRrdDh5HujBRVSk9yxys511zjs68mK2o0822hytfsRZQzzKa9tjGlb3RTZt7DOqeA8JXM6otXUq2UoInf5Pq1eN+bUe2SnsfNoPeiqw8gndqKJ71lGVtqbZWFpTXuLxF1AVRyFiacxp0B3AilHcPxo8mwfvjPKNkqOzjBLJLN2xhLC9dKDvznATHo4venEknOv24ab0CiLQGcMuN6KG+voDDPZxnW4T2uIuW5/Zyzj7V9X1VRAhrl3AxXpSF6lLtoOYAzTdNtwYPkwybjucEbVKp3jC8zkGe46kWTlzhhy1ZjI5CmO1dM947rP8fK+Aplz08TgzAHgY3UX1Cx3rms8h3cjAThoZ3WvNk7k+BlWj6i87X4kGbsTSE/lw5VyWhbRTeF5rXVO2fOs0bMrUYZH+1ok5F4hQ265y/dppbpxU3qPPDqWVYeAysNqxL1YSP1a/CdtZAVWDq18dDEZOYDfT4RkpuipbF9cmt6OBHDTiSezLpNt27E3nO6FCwEfV4kS/GkvYfA1B5gZZVHHDm8s430lgJtOb82DJgdFZdA5JanwUbcNBz58P71O5YwVeNhFWXndq0tyFs5R6jyYD9FaNSZ2hO5ETSmw3jCpl7zXeD68gupe88M46/5KbWfR7nplTHh3RkBfVBAaLqCbyuucnOex1bMrUaH05OWaD88xliYZisldetxVqRs3pPfI42NZdRjEfwx/Ljc11xYGgF3oMAfNwRkcmJN9MX2GselGAQD/Cqb76/Yu6y1oxZuRRt24GbIgcuObDiRv4FWQ1FFcBu3VEKeIvpx12w4g73B9JxPrmuicihXhqDVktDNMcj6uKvaW0b8rx8TWozwAIweAHON8jWFSO7mv8RSqe+NjTKG8A6uNpxLtrlXGKXoi6o25RntlyK6czCbdVErnFDiPtZ61IvRGHSHlTCokd9lxV6Fu3JTeI4+OZTL84vOGi8LDlIGlV6aMrCCIfMKnDktLD+ysZF8L5oZYiqrOWuZImT29YT7MKoNJ58SYKpIqSZ61kyFdyaWV0I2HqXcDGVUEobv9xrOTd0NJwvIuywwMlWckLynMBcyVJ/JI6PC1K6C8CnVNilngWhS5xlkVjuED/CZrwJZs90ZkXDBFT3QwlgKul8j7qWCMpXVTSZ1T4DyF9ewj3FPZutu+TyvTjSyOeGso91Fp81pz+FDtuMGqQwcjKTHKXK+UqfVAnw/REkM0BzN4rlAG1wiQ486KGb1XIErKUwVRGabX8EcOnGO9pNtWb6/eHJgDN9KF2+4C06lORtbr9oVpD2JltEc9rBz9sMpsua2Xbx3dyaIgpNjM6BZyr70Wlsw9NY7cC3SHYYhcP8D9vtnrZN3uDcqYPgiGVz7ckfKsLKapqOu65qUSnVNAzxZsb3eiPZYxKqgvLdunb7tuJIWxrDpMLokkxpvCh0DKyAqMG1WIHKMj4gmV8Z8Kcp+gYuqLZFoAwvXs3tDqeiBby6Bd7Dq/oXlyFHlL1m/+ep1ym1cT5nwc1lXimTB4wUzHrOOtdWFsSIw7RY2Nx5LbdI2VARIL7ayY46tcuzck4y5Rhc6pWs/qylApJUZIfwWgGsr26dukG0kV2OVoPUTmdgiNqywDC8oDZkqGr8e/te5BUhPzIVqiDxX+D/MS8siyrgqstGDWMigXuYN2V73tRd3fy3VLt/kmdUm66mc9IjPpwa4PVP5HRs5IqZL+VegEZPjop2Yhz0edche+xtPrWHJy9oOpfLvrl3EdEplRy8rYlM6pQ8/qLxbAr6aAYBUl+3SbdSPZPixztB7072zjarlx9tq6HFoqBu/gbOP5NDr3IaLA1smyMqG0IqxlWCijS5yKRO6CXnfUbkM5QDakSvR5jQ/L5gBnqVBDhIyKJds+UEnF5k+rNDunELZVmCtQsgJ+394jW6vcha/xFM/GOoG4q/rP9P3OKtpdt4zZ6CkoNuTZ2JTOqV7PagPQ6GkLc8KqomSfbqNuJFtL6Y9Kr9rG7MmKblLC1NKzQRsrZ+ZDnOs5VNJhvCYGs/IfeO1OTBN9XsJNegMisiTnRgnzu/x+uTf0tVjLEFboCIjUQ0KtE45Ti0GRjVaQzijRt11MPHfN9x8djBITEYbtl+Nziz7IkKU70dc0bhSEkx2WSf59eixQZBoN8zmLyV2M4td4rhOI3ZELIccwPf+raXe9MjYHM8P8R00MZh5c4aNfV7FCStgN6ZzK9Wyob1zE3wH0B8Ur/rhr7j41jqNt1I1kW7FLhgcywn5ZE5OuOJAt8yFa/WPI0QhSqlTMaHLsfNiC8AaYeR6kG91RYpycQ8eCae9KzZcTW6rCGsn7aj5sQQy7mMilrKu2rwNbGabXPkaOYwwzLNb51xudu2k+bKGDGTw32rcqb+f6wpRAq7cYd9B6eaHzP+L72Y4H8zhToY/6rsm6BGQf/TW5MXXJbXWN50Nc+S5Ga8Ms5dtdp4zzYQudwSxxf6nziQ1Pu7EpnVO1np32BG4HM3i6uGh5LIFnnRk8d9XeBck97jJk3ULdSLaTxocffljY3PnrZ5+h/8UXODg4yOXcMvHq1X/w+eef49/ffmt3AELI9tAcYOa5gGXVrfK03OcylKzZBRnJRmGfkk1g5dEKw4IHBwelTl4mckgI2Re6uHAF4F9t8cNuF2QkxWCfks1gZWg9BMB0+k80GrqiMAh/60rCIMy/UssedIVidP3rN2/Ux6nJnqHCFaty0oES842RvWMTBSFl2QUZSTHYp2RTWBla//j739BoNHJtG8T+iBtWb968sTk92Wqm6D3qLIxkp2gOcOkKyHGn3oKQMuyCjKQY7FOyQawMrV9//bVqOQghbxHRWb83MsO7BbsgIykG+5Q8BlbJ8IQQQgghZD1282gRQgghhJC10NAihBBCCKkJGlqEEEIIITVBQ4sQQgghpCZoaBFCCCGE1AQNLUIIIYSQmqChRQghhBBSEzS0CCGEEEJqgoYWIYT8/+2dPW/kRraG3155Ai1sA068mdclQZjBvQTWoeMmBDSwYLQ07r1Y34WDBgMFggC7A/6B7YDJQIGCRqcGdmd7k6FnIUAgAW/mvAMBA6n8CyawvXPXwEzzBlXs5kfxq0hK3fJ5AI00/CyyyKrDc95TRRAE0RNaU/A8fvwY+/v7AOLJo+OpDKPUsuzy1SrC999zvH79WrvABEEQBEEQu4KWoTUajWCaJvb29gAIw2ozeXQU21WIEK2NrJubW7z73rv4x4sXCIIAP/74Y9uyEwRBEARBbDVahhYAHB0d4dGjR9g4r5LGFoDB2sbCrwbAq1evcHBwgA8++AD7+/t48eIFfvjhhxZFJwiCIAiC2G60DK0oESIUoUHxj1g6SIQQ5faDAQBgb28Pv/3oI9iffYYoGuCbb57jp59+alN+giAIgiCIrUVLDJ/SZcXLNiuTv1K8efMGURThNx9+CMv6PT799FOd0xMEQRAEQewEWobWapUwpBQG1Yb0yufPn+Prr7/Gs2fP8N133+GdR490Tk8ksOccnIufuQ3AnoNzjtA17rtoBEF0gOGG9E4jvg8hdv822JhzDj63G+xC7fouoxc6xApApDCyBmK5/BVrtVZvV/jdJ5/gP/7TWB/h3//3b/x5+mfdchMw4IY+HMxgsSmW8eIG7+52IK+DpZcGE4bxoslhXIS+g81hOGbWENNlyT45bMy5B7NNOe4MxX3jM1jDxLNQtJ0K5b73yUO/PqJ78s9Cvfd3s9/2vu/ELqMnhk8N3xD/nddmYRBbXMD+/q8hR4QAogjvvPOowhtGlGJYOGZAMNnlzkMaNsEEbLhp3Qw3hO9x8NEErEarZ7ghfIeBzywwaVkZbgjf5zieWRjWsbakoYbEMWDPwT2O8KjmMe4MG3N+ghuLga2LJToLnx9nDMwlpkOGadGh5HXfXmzTc/TQr68Zy+kQrPACCQDyXTWBVFtiwA1DuNcVH1z2WbWhThAt0B6wNHZYbf6T1G2VxRVjAX2UzFNsjnSlNvG+PiieHOIhtA2XE5YzppbTISYBAPOkRpjAxpn4FE0ZQ/ExmHNWw8lnwD13wDLHwGIsj3G+ZeGKBcYs23ksMT2dgYPBOav7Usjr5jM83aqv+Id+fUSnGC5CzxQfWqm2ZInpsMqrbcA9Mcs2IIjWaIrhE38kbCURKZTxwsTCSP0frFSKeaIWxtHBfRehAxZYFHSAi8sAAMOxVWHh2COYAPjNdcExTIyq+mX7DA7jmCl648VT0blXlmMbWPq44gAOjlCrtPJLPtgVb89Dvz5CC/tMGNOnGl5nwz2HwwJMJkEPJSMIgZ6hlbSWBgnvVhQhWhtZKg0XRQuJmlzfgHe5XQn2yARwi5eqdnr5ErcA2LFVr3PfBm5f1jIs7JEJ7KK356FfH1Efw8WJCfArX8OYFt5wPnsKekSIPtHLOpSerETEUPwxSKjg16PDxygMr5XGyWXIkHvC3Wt6may79MYiuyP5o4w1JrNADLhhep/iTI/8tsWhTEVZCjNokuVJ7Be6MOT1+1JUsL7+0K1hCNQrQ5zlpLoWkeWoKrdGJk0ZMjR6q7R+Ekgvh8oQEgZUgMvSVtTA0QEAfoO8TwwArnHDAbBDPKlb9szxS58Rw0W4fsbqPq9Fp5K6vfILXp/3xOzC27OD15csk/ybc8V7FLc1qnV1i9zFu6SUSei2WfVIZjOXHrOPe5k9Ttn1PDkEA8eVr2FmzT2YDT1h3beN+brrrA0ltgY9j9ZKZh0mpthJCt+LjKy8N0vD0lqMwRgDk67eYMLE/1kmW8Seg8dCaxZvM0FgesUv+cEJQn4OnG6Oac04mOPnH37DRch9HF9ZieMzXI4UL6GyLAzWDHD8khfr4AQhH+Ey3mc4xVJevzXj6euvyqhqUIalfwURocmZLhiZgDKkJ0N4tTrBSmLdRJWRBKy1O8zBeaKFM9wQnskxs8YVX6tPcCgsuoL7t8TL20aF32DPwbkP53aSeUYUjfHhGcJYAB5va83Ay57X7Ll8oTOrkzVlnzlgte5vxTl3+fqOzyFe90R5mAM/dOHOOfjJDSzFuiYmTO/vUpM2qxai4x9dskw7UXHMju6l4YbgcVJKot2+dfz1x2WSpDc6ZxyW1ZU9F+3DabMPjU7rs6DuYHpkbD0w9MTwESB0WIO0Fwu5X+v/5I2sSMuhVQ8b8zgDJdUqLzAW6uZUp7yGAVeZIQE2wuxRSlQdN+QXma+hxThj8EmhZr4s4tjxi6V8rxhwVWko1KBpGYq8RPYIJgIEQX6d0Iy17Ljj08xFqnVtl/5yiqGwGNeNrGirmw7v0CEl93wxzpeLmQe5Zw/LKYaFz2vGO+QBE5ZPLFAjOoVWIZMHcH2M3WKS/EBZTnE64wBz4JhBwbpjNJLr9f0uNWizahYY02F+iIPl9BQzXnzMTu6l4eJcZg+ns3wXGLMJilVUBzgJOTxkPqqZA5/PFeWN+4eL5u1Dl/VZWnd1EoGIXUEz6zBK/yj+3Gyqzi2MAL3QYQ0M9wQm1OJmLJ5iVhBqKnrxrm841KLqaqG1MMgKygJgOb2AaBMVB+JX0PCId1AG6cXJNIRC83KDy5tsI2nAOmYl4be6iK9pzwTQwKVvzzm4f4yrlLfkCsdl3sKeqbrnOYoa/cLndYExS3odLjEqDUdviN8PnXBLzIO4vuAyZ4gtY/dl4TqGw0Yx5J7fpcZtli5L+CITATlnDtDJvSz6eBUs8HSWVWPKsD8Y2K3io9qagcOEl45lww09mAgw0Rowq8P6LGjf40SgZs8Zsc1oiuHX0xuuEw9zBpZcmTOyoigxhU8flpZ8sIvEzRosFbEjkY220Uip+/NY/1NmMEkNkCqTqqbotxy9MuSz/qSX4MrHwr8CTzYEUjuTEqRmNRlVWgsZinUY5Fg4NV36cQhA5S2RoanOR1OuvLY69zyNKmtSEDfsVRqxBcbMwowzOL7qKz5Gvh9lX/MP/frKyiSTK4rLmwkb1XjOW79LGqjarD5pfy+rtJKlZ1cb/HGWasILJ7IM60gKiumsPjtp34ldoLO5DhNr19aX0sjSOaEWJjxFA8jjzlxb3CxZTjFMaKVig0u7U29bni5IlmFxiQAAi1sMe7TxEsgGbO0BUwlSYy1d5kc58Gesv4HUnNX+0pRarqIOv0Qov6HE0E2S7AAqr61K99UXS0wvKoa0qCMof+jX1yV1nvO279I9kdU8qTRS3SGfqUZU6Scz6+25HNj4tJ2kYEfrk7g/NLMOs0sSwnhV+BDIGVmRcqMO4bON+FLXWJD9AAAQs0lEQVT504H2CXLU5ljgCYA5vp6x1Trs1gGpMixwmdB5GEcHCYNGNmBynXCba4Y54xGdwTGzmk5/UVPIXmrEVm0jv7QbGRU1jbdatPnSVxzNOgYDR4mToQYP/fq65o7epS5IZPylNU+bj8p+kM9U071q77QZmJQltJwbDWAmi71UcrBD9UlsBXoarVVibIfYuCoysCA8YCoxfD/+rbqhiK5PO8VQCjY3HpQ6ZenbO6Bfhs2AnyIck3R/b9Zt3OY649jMPZFdOMmNBF6Hqg6/XicuGusC7YmWh6T5M8gKBRk6z0eRoRHrRdo2/A/9+rqn/3epC+RI+ggaJB50RfUz9UTh8irXzSXffyHyL/zwzmaxV1z7btQnsS1oarRW8nexcbXZuHhtXw4toZ8ycXLnaRuy4090GlVlKRXud4R2GaSL3Dw5xzHLhgbFuoPRCMLhozMq84lIe57oehelQLcoE6ym3kUkA6indhEekubZlHFjW/sZLMjmavZ8VIRSO9L/AA//+jqn53epG6TRq/wwibWv/VH6TMlx0fI7iWQK80QxlEOfz8NO1CexLWgO71DDRJKi97ItozZT8EihpTJbT6YPq8N4cqLRljaYPVcN9HkupvxI9sqJsmTd0fFkyMGk52EItMsgXeSMgeU6N7GOmab2eEziC7X+vvFggck6FWnnDI6fqQ85mXB2ag7VMdYZTVnhvNR0aBmCi/F67KH0uYqePxNeJhV9M1l2WlNiuKEitCEmXXZYJpU+Sd1BYOvw0K+vc/p9l7ohLqOD9DeHjXmsbe319PHcon5+0Fv/GLeBKky4GUfPT+4Uv//Z+Usbom4vgN2oT2JbeEdnJ+HFKhi0QT3zTvGBdFlOMZwcgnseOPcACLdv7PFdTodgvovQ98Gd5I6KDDUNFuMLMc5PaqkIgeVSmadDsKmNOd+UtWz7PtAtw+IygGeaytDOep0ifbs+ImnBK1wfYFKqpxMhAd8N4fscyarmMwusZkWrnxcRQtG9NvUx1c8fn1kYvjyT2pjktiy37XI6hOWGmXqsvl4xnk93+qWHfn1d0/+71J7FmOHaDeF7Yuwygainp1YI3ynbu6/zizbq2g1hqrxayymG7GWubQsmDMMeb+Yu1CexHQw+/vjjxubOn774ApOvvsLe3l4t55aK16//hS+//BL//PZbvQMQxENAfnkjN0jjA+GhXx9BEEQFWh6tOCy4t7fX6uRtIocEQRAEQRDbjpahtYqAxeLvGAxkRmEU/5aZhFGsvxLLVjJDMbn+zdu3YnJqgiAIokeEZEAVdUuSn/qGIIgu0DK0/vbXv2AwGNTaNkr9kTas3r59q3N6giAIojYLjBkphQjivtAytH7++eeuy0EQBEEQBPHg0BLDEwRBEARBENXojaNFEARBEARBVEKGFkEQBEEQRE+QoUUQBEEQBNETZGgRBEEQBEH0BBlaBEEQBEEQPUGGFkEQBEEQRE+QoUUQBEEQBNETZGgRBEEQBEH0BBlaBEEQBEEQPaE1Bc/jx4+xv78PIJ48Op7KMEotyy5frSJ8/z3H69evtQtMEARBEASxK2gZWqPRCKZpYm9vD4AwrDaTR0exXYUI0drIurm5xbvvvYt/vHiBIAjw448/ti07QRAEQRDEVqNlaAHA0dERHj16hI3zKmlsARisbSz8agC8evUKBwcH+OCDD7C/v48XL17ghx9+aFF0giAIgiCI7UbL0IoSIUIRGhT/iKWDRAhRbj8YAAD29vbw248+gv3ZZ4iiAb755jl++umnNuUnCIIgCILYWrTE8CldVrxsszL5K8WbN28QRRF+8+GHsKzf49NPP9U5PUEQBEEQxE6gZWitVglDSmFQbUivfP78Ob7++ms8e/YM3333Hd559Ejn9PeC4YbgPIRr3HdJCIIgmmBjzjn43G6wyxycc4TU4BFEa/Q8WlgBiBRG1iD1K16/ervC7z75BP/13/+DP37+v/jj55/jD3+wU9mJhC4G3JCDc44m7Wh23/hHfYz8djx00bwJlg1+5fm2BcV1b12Zu6obgiAIog/0xtFKDd8QSXVWXpuFwQCxtbW//2u8//774ue99/Hue+9VeMOIWthncJjOfnNw7sO5nYAxJn8s3JxkvXY25vwcOGWp7WZw4Dfx8BkuQu7hYGZtjjMJYHrb+tVsY567PwzWjMP0GnoHeqOjuiEIgiB6Q3vA0ghIZxmmdFtlccVoPbZW1MbSkq7trejv7g0D7ompsZuL0DPBZxbYeJFYscR0OMR0mdx4gTHLLltiejoDB4NzVqcCDLjnDlgwwTB5oMUYkwBgzvlWGgWXE5a5P8ByOsQkAGCebEGZu6gbgiAIok80xfCJPxK2kvBrSe9WYmGk/g9WFDpsheGew2EBJpOg0X72mQPGZzhN99DNWPq44gAOjqrDVPYZHMYxe7rIrVo8FUbBsXXvVkuGBRb54oo1lwGwlWWWNKkbgiAIolc0NVoJA2mQ8G5FEaKkQEthR5Fp1RU2zhwGPnuKAntAjeHixAT4lY8WZtaG25eVx7FHJoBbvFRtuHyJWwDs2Nodo+D6Bvy+y1CHGnVDEARB9Ite1qH0ZCUihuKPtSZrY2QVebPEgTROLkOG3BMhM9MrESkbLsKMkJnP7fIMQsU+hRoiuW3oGvn9koLkuMwdipXtuQdTxyv15BAMHFd+yy7YsHDMgOCyyswzcHQAgN/gWrn+GjccADvEk0bnT9x7hcg+raGKBeNzFAfT6mwjeXIIBuBWaTluQRlr142KZIZaXmhfrqdTXGOhVqzj87TUENjzfNKDugxNyt2+TkVbpZZIiDKr7m9ZlqEieeKXrb8giN7R82itZNZhYoqdpPC9yMjKe7M0LK3FeC2kBoBgshECJ+U0hhuC+w6QFF+zCQLTg1+gHi/a59bxC/cBAByfQ2iS5T7WDJw58EMX7pyDn9zAUqzTNrbsOTyTY3Y6beyxSHqXcp1L3TLZc3BfaK7GlX35ExwKq6SgrEu8vG10CWkOzxDyE9xYCUG4NQM3vcT1LOFfcQAmRkV9ijROqj2EsS4uQG075i7L2KhuSjg4QZgR2lszDub46o7ZnoNzD2aQTR4AHL+kM+/kPOK91vuAEYbH6JLlkh4Ky1C73O3rdOlfQUSBs1dmY2QCyhC2PYIJhaFdUGaYHhlbBNEjmlmHgNBhDdJeLOR+rf+TN7IiLYdWLQwX5w4Dn1lp8TUWGLMJlIomnX0kjN1iMkwYPcspTmccYA4cMyhYdww9iY+NuWcCwQX0JVYHOAk5PGQ6K+bAV35ZZzwIHjBheaH4fcDMA1xZGUH4coqhUNnjXH7uL6cXEBp2dWdsnzlgCHBRcVPtuQ+nlkF2V2XsqW4YcmXeJAKM0s+ITK5AMFEmD8SdubIvb3Ke9bOfPc8C48y9rM8S0yHLGaXL6SlmXFWGZuVu/dxJvV0utG6PYCJAEOTXGUcHUH4IlJZ5G5I7COJhopl1GKV/FH9uNlXnFkaAXuiwBuWN1wJPZ3mFjc4+a4LLXKe7jN00hesYDhvFygDx9e3BRICJVkcqw3hgYLeKzsqagcOEl+sRFxiz5Bf/JUalYaE7pMjgXDzFLNVBLSA07CoDV3gHyo0n4fnwTABNQ7a9lrGnuiko8/VN3kMj3h11sgOQMDZUbp0G5zHcE5hF58ndy7bE3qgD5JxJjcrd9rmTHt/M/vbIBPgNLm+yH20GrGOmDtXzK6gUA3FyR/P2iCCIOmiK4dfTG64TD3MGllyZM7KiKDGFTx+WVpUmqKt9NvAbxV5SMK1cJ0mFA7I6LoXmQ2QZcsyscTMBfL7E6s4qzlYr+opfs8CYWZhxBsevoWlqSo17EVN8f+MOaqP9KspwjDvwQt2a4SLkwpOFYAI2bBayvZMyrimpmwb3tYhlLs4bvzvqTlwgdXgNsiBV57GOGQqTKlR0cL1NyZe7fZ3ms1ylgXblY+FfgSeNpDgUqUp2oeQIgrgX2k0qrV6rDh+KHe8g61Bqghqlhens0zGLMViZ9WTP4cvQpn7IMO7Yddent51eBHA88fVeNBRC3MmaspMtLHrSyK26F7osfVxxB45zBnsaG6uyAw8m6vtqz9eJF8EkH2LaijLmD6Kum77ua12kQdmuszfhcQ6vcH3iHA2v155Lj2WKDhqFtnW6uETgmTAP5ZXZI+HZ85fAEuLYcUXLZJdZ22QXgiA6QzPrMLskIYxXhQ+BnJEVKTfqAvn13Ps+d8lmYFLm+Pmv9GwGZomw9frOLzTvtUkjPSKdf22rvJTCAEmFdkrG+NoYWRwzqw8jq4My7hKaHuP0MWabxBLlT0NvbyJbOK1ZlELxTmhbpzL8KD3NxtFBwoMo3y+5ToQUy7yLBEHcNXoarVVibIfYuCoysCA8YCoxfD/+raqOHXhymHXZ6OxzlwjBbmHnks3ALLEIyvVhOiFUjpLoKIDYuCvQurQaigBghcKSgmzHxWVKnFzcMUnhNQJMcqOvb0sZq6ium/ZUvzvVmaddnacpcsYCBP0nd7SsUxE+NDGyhScsGRrcrNuEFMnOIojtocWk0kXarOzGxWv7CiPGDc+JSoshB+zsYp+dRIqGlVlQZfqOHNLLVqOzEGJo9ZQwhnUM1mSohCwFerJi4bRMbGDHsGxRr8FFXnMl9geCSVs9XH9lLKZ+3XSB0CAVvDuoELF3eJ7mxJIB1YdFrAnripZ1ujbUznHMMrouue5gNIJwDpOZRRDbROtJpcu2UXuykpu0MLWk2FyZyRRnIDl+OqXccBH6h7hShQTW8+6p9jnGbbDVsUUl8WCHafGvnAuPOfDnqZQuhH5+PkLDDRWhSANu6IvpfzLCcPU5ZSdjeunlUnfWzpgx4WWGpDDcUOrZTpWeqKUUEDuemIpI1f8LD2Z9A1B93f2WsWnd9IYcskQ17lR8ncGknVcwe578fTbghk0zLeOMQAfpbwA5oXjHTuzadap8juKyMrCcAS3WMdNs99FCEEQv6InhgYKwX9HApCUH0mU5xXByCO554FxIYzdiZRFq890QvifGFgIg9R1jwA3hKA65GDNcZ/eRoaNrN4T5ULxayymG7CXmfHPvAHH/hrnxhIaw3DC1HQAxIXWDnnM5HYL5LkLfB1/ffBGyadMv8JmF4cszqbFZL8XMYsUd+3KKi8CBVxlmqRJeB5jU0AT1Vcau6qYLltMh2NTOPVPx+9NV369+jgBxP5sbc+p3XtTNUyuEr2oodKn93BWU9TKAZ5rKEOx6nWI4GYIg7pfBxx9/3Njc+dMXX2Dy1VfY29ur5dxS8fr1v/Dll1/in99+q3eAFoivbGg1zMSWID1wyA0wW3f3EL5zW8tQ0mYXykjcKVSnBPHLQ8ujFYcF9/b2Wp28TeSQIPQRE3IjuNjizm4Xykg0g+qUIH6JaBlaqwhYLP6OwUBmFEbxb5lJGMX6K7FsJTMUk+vfvH0rJqcmiDumK3F2n+xCGYlmUJ0SxC8TLUPrb3/9CwaDQa1to9QfacPq7du3OqcnCH0Sc1pubdh4F8pINIPqlCB+sWgZWj///HPX5SCIXkmO+n0nI7xrsAtlJJpBdUoQhJYYniAIgiAIgqhGbxwtgiAIgiAIohIytAiCIAiCIHqCDC2CIAiCIIieIEOLIAiCIAiiJ9ZZh8Ph8D7LQRAEQRAE8eBIDe8QhuF9lYMgCIIgCOLBQaFDgiAIgiCIniBDiyAIgiAIoif+H20ctc4OkBMAAAAAAElFTkSuQmCC"}}},{"cell_type":"code","source":"import os\nIS_COLAB = not (os.path.exists('/kaggle/input'))and (not os.path.exists(r'D:\\DATA\\dolphin'))\nIS_PC = os.path.exists(r'D:\\DATA\\dolphin') # My PC\nprint(IS_COLAB) # 判斷是否有'/kaggle/input'路徑以判斷是否為colab\nprint(IS_PC)","metadata":{"executionInfo":{"elapsed":6,"status":"ok","timestamp":1646574382984,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"5a2w5y92W8cm","outputId":"de996799-36f9-4f29-9882-f45d61c9c572","scrolled":true,"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\ntry:\n    # 嘗試tpu 實例化\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\nelse:\n    # 預設策略 CPU 或 單GPU.\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE # 自動分配訓練與預處理時間\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync) # 多少設備","metadata":{"executionInfo":{"elapsed":16417,"status":"ok","timestamp":1646574399400,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"AZobaTEDW8cn","outputId":"3801ae7c-1032-4cf7-9443-4137718096e5","tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"from : https://tf.wiki/zh_hant/appendix/tpu.html  \n在 TPU 上進行 TensorFlow 分散式訓練的核心 API 是 tf.distribute.TPUStrategy ，可以用簡單幾行程式碼就實作出 TPU 上的分散式訓練，同時也可以很容易的遷移到 GPU 單機多卡、多機多卡的環境。以下是如何實例化 TPUStrategy ：\n```\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nstrategy = tf.distribute.experimental.TPUStrategy(tpu)\n```","metadata":{"id":"2eOWYboJW8co"}},{"cell_type":"markdown","source":"時間消耗是這樣的：先是打開文件，然後從文件中獲取數據項，然後使用數據進行訓練。這種執行方式，當數據進行預處理，模型就空閒；當模型開始訓練，管道又空閒下來了。預處理和訓練這兩部分明顯可以重疊。  \ntf.dataAPI提供了tf.data.Dataset.prefetch轉換。它可以用於將數據生成時間與數據消耗時間分開。轉換使用後台線程和內部緩衝區預取元素。要預取的元素數量應等於（或可能大於）單個訓練步驟消耗的批次數量。  \n將預取的元素數量設置為tf.data.experimental.AUTOTUNE，這將提示tf.data運行時在運行時動態調整值。","metadata":{"id":"Z5mgWZuNW8co"}},{"cell_type":"code","source":"if IS_COLAB:\n    from google.colab import drive\n    drive.mount('/content/drive') # 若為colab 連接Google雲端\nelif IS_PC:\n    None\nelse:\n    from kaggle_datasets import KaggleDatasets","metadata":{"executionInfo":{"elapsed":1854,"status":"ok","timestamp":1646574401247,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"gjVvKrXcKs1o","outputId":"56da4a77-5e2b-4ef6-f315-ea44dbe48e40","tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not IS_PC:\n    !pip install -q efficientnet==1.1.1\n    !pip install tensorflow_addons==0.12.1\nimport re\nimport os\nimport numpy as np\nimport pandas as pd\nimport random\nimport math\nimport tensorflow as tf\nimport efficientnet.tfkeras as efn\nfrom sklearn import metrics\nfrom sklearn.model_selection import KFold, train_test_split\nfrom tensorflow.keras import backend as K\nimport tensorflow_addons as tfa\nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\nimport pickle\nimport json\nimport tensorflow_hub as tfhub\nfrom datetime import datetime","metadata":{"executionInfo":{"elapsed":15423,"status":"ok","timestamp":1646574416668,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"74V8FJWGK3cF","outputId":"0230f85b-f5c5-4f65-9a89-6a79907f9a19","scrolled":true,"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CONFIG","metadata":{"id":"Sx-kD3nOW8cp"}},{"cell_type":"code","source":"os.path.abspath(\".\")","metadata":{"executionInfo":{"elapsed":6,"status":"ok","timestamp":1646574416669,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"ttqv8EtAW8cq","outputId":"2b0452d9-db9b-4faa-93df-3597f3681947","tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_dir = '.' # kaggle工作資料夾\nEXPERIMENT = 0\nrun_ts = datetime.now().strftime('%Y%m%d-%H%M%S') # 現在時間\nprint(run_ts)\nif IS_COLAB:\n    save_dir = f'/content/drive/MyDrive/Kaggle/HappyWhale-2022/experiments-{EXPERIMENT}/{run_ts}'\n    !mkdir -p {save_dir}\nelif IS_PC:\n    save_dir = f'D:\\\\DATA\\\\dolphin\\\\output\\\\experiments-{EXPERIMENT}\\\\{run_ts}'\n    !mkdir -p {save_dir}\n    print(save_dir)","metadata":{"executionInfo":{"elapsed":432,"status":"ok","timestamp":1646574417097,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"lK8RntkELDWW","outputId":"ce6f0c60-4954-403f-ae98-1dfc72c445d3","tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class config:    \n    \n    SEED = 77\n    FOLD_TO_RUN = 0\n    FOLDS = 5\n    DEBUG = False\n    EVALUATE = True\n    RESUME = False\n    RESUME_EPOCH = None\n    \n    BATCH_SIZE = 16 * strategy.num_replicas_in_sync # GPU 12GB USE 2 BATCH_SIZE # TPU USE 16 or 8\n    IMAGE_SIZE = 512\n    N_CLASSES = 15587\n    \n    model_type = 'effnetv1'  \n    EFF_NET = 7\n    EFF_NETV2 = 'xl-21k-ft1k'\n    FREEZE_BATCH_NORM = False\n    head = 'arcface' \n    EPOCHS = 30\n    LR = 0.001\n    message='baseline'\n    \n    CUTOUT = False\n    \n    save_dir = save_dir\n    \n    KNN = 50\n    \ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)\n\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    tf.random.set_seed(seed)\n    \ndef is_interactive():\n    return 'runtime'    in get_ipython().config.IPKernelApp.connection_file\nIS_INTERACTIVE = is_interactive()\nprint(IS_INTERACTIVE)","metadata":{"executionInfo":{"elapsed":11,"status":"ok","timestamp":1646574417097,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"OG4ej4ohLJQT","outputId":"764acd2f-33a7-4261-e1e2-e9b6d0f01bf0","tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_NAME = None\nif config.model_type == 'effnetv1':\n    MODEL_NAME = f'effnetv1_b{config.EFF_NET}'\nelif config.model_type == 'effnetv2':\n    MODEL_NAME = f'effnetv2_{config.EFF_NETV2}'\n\nconfig.MODEL_NAME = MODEL_NAME\nprint(MODEL_NAME)","metadata":{"executionInfo":{"elapsed":10,"status":"ok","timestamp":1646574417097,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"4PyoOrFSMY76","outputId":"5488b731-90a1-411e-b9d0-b859b060c6a9","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(config.save_dir+'/config.json', 'w') as fp:\n    json.dump({x:dict(config.__dict__)[x] for x in dict(config.__dict__) if not x.startswith('_')}, fp)","metadata":{"executionInfo":{"elapsed":10,"status":"ok","timestamp":1646574417098,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"5XHg_etYMPGn","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config.__dict__ # python class __dict__屬性","metadata":{"executionInfo":{"elapsed":10,"status":"ok","timestamp":1646574417098,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"d9OBsIaQW8cr","outputId":"f00b175f-f2d6-4852-ec6a-dda90b9f49bb","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_PATH = \"gs://kds-5998360efc5fefbd84f6f7ce20d9040a838bf5e72a85d1e34c10b2a4\"  # Get GCS Path from kaggle notebook if GCS Path is expired\nif IS_PC:\n    GCS_PATH = \"E:/happywhale-tfrecords-v1\"\nelif IS_COLAB:\n    None\nelse:\n    GCS_PATH = KaggleDatasets().get_gcs_path('happywhale-tfrecords-v1')\n    \ntrain_files = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/happywhale-2022-train*.tfrec')))\ntest_files = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/happywhale-2022-test*.tfrec')))\nprint(GCS_PATH)\nprint(len(train_files),len(test_files),count_data_items(train_files),count_data_items(test_files))","metadata":{"executionInfo":{"elapsed":8,"status":"ok","timestamp":1646574417098,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"BkwheRd7W8cr","outputId":"6ad138bd-26cc-41ca-84fb-c30cbdf65ae0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files","metadata":{"executionInfo":{"elapsed":7,"status":"ok","timestamp":1646574417098,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"nWuqTpomW8cs","outputId":"25802d7c-1182-4e2d-bd0f-4c0c4f0ad650","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_files","metadata":{"executionInfo":{"elapsed":6,"status":"ok","timestamp":1646574417099,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"D1DF-8UfW8cs","outputId":"95465194-850d-4e9d-ad83-284c46d8a62a","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DATA","metadata":{"id":"vv7GSIMqW8cs"}},{"cell_type":"code","source":"def arcface_format(posting_id, image, label_group, matches):\n    return posting_id, {'inp1': image, 'inp2': label_group}, label_group, matches\n\ndef arcface_inference_format(posting_id, image, label_group, matches):\n    return image,posting_id\n\ndef arcface_eval_format(posting_id, image, label_group, matches):\n    return image,label_group\n\ndef data_augment(posting_id, image, label_group, matches):\n\n    if tf.random.uniform([])>0.5 and config.CUTOUT:\n        N_CUTOUT = 6\n        for cutouts in range(N_CUTOUT):\n            if tf.random.uniform([])>0.5:\n                DIM = config.IMAGE_SIZE\n                CUTOUT_LENGTH = DIM//8\n                x1 = tf.cast( tf.random.uniform([],0,DIM-CUTOUT_LENGTH),tf.int32)\n                x2 = tf.cast( tf.random.uniform([],0,DIM-CUTOUT_LENGTH),tf.int32)\n                filter_ = tf.concat([tf.zeros((x1,CUTOUT_LENGTH)),tf.ones((CUTOUT_LENGTH,CUTOUT_LENGTH)),tf.zeros((DIM-x1-CUTOUT_LENGTH,CUTOUT_LENGTH))],axis=0)\n                filter_ = tf.concat([tf.zeros((DIM,x2)),filter_,tf.zeros((DIM,DIM-x2-CUTOUT_LENGTH))],axis=1)\n                cutout = tf.reshape(1-filter_,(DIM,DIM,1))\n                image = cutout*image\n\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_hue(image, 0.01)\n    image = tf.image.random_saturation(image, 0.70, 1.30)\n    image = tf.image.random_contrast(image, 0.80, 1.20)\n    image = tf.image.random_brightness(image, 0.10)\n    return posting_id, image, label_group, matches\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels = 3)\n    image = tf.image.resize(image, [config.IMAGE_SIZE,config.IMAGE_SIZE])\n    image = tf.cast(image, tf.float32) / 255.0\n    return image\n\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64),\n#         \"matches\": tf.io.FixedLenFeature([], tf.string)\n    }\n\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    posting_id = example['image_name']\n    image = decode_image(example['image'])\n#     label_group = tf.one_hot(tf.cast(example['label_group'], tf.int32), depth = N_CLASSES)\n    label_group = tf.cast(example['target'], tf.int32)\n#     matches = example['matches']\n    matches = 1\n    return posting_id, image, label_group, matches\n\ndef load_dataset(filenames, ordered = False):\n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False \n        \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)\n#     dataset = dataset.cache()\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls = AUTO) \n    return dataset\n\ndef get_training_dataset(filenames):\n    dataset = load_dataset(filenames, ordered = False)\n    dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    dataset = dataset.map(arcface_format, num_parallel_calls = AUTO)\n    dataset = dataset.map(lambda posting_id, image, label_group, matches: (image, label_group))\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(config.BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_val_dataset(filenames):\n    dataset = load_dataset(filenames, ordered = True)\n    dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    dataset = dataset.map(arcface_format, num_parallel_calls = AUTO)\n    dataset = dataset.map(lambda posting_id, image, label_group, matches: (image, label_group))\n    dataset = dataset.batch(config.BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_eval_dataset(filenames, get_targets = True):\n    dataset = load_dataset(filenames, ordered = True)\n    dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    dataset = dataset.map(arcface_eval_format, num_parallel_calls = AUTO)\n    if not get_targets:\n        dataset = dataset.map(lambda image, target: image)\n    dataset = dataset.batch(config.BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_test_dataset(filenames, get_names = True):\n    dataset = load_dataset(filenames, ordered = True)\n    dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    dataset = dataset.map(arcface_inference_format, num_parallel_calls = AUTO)\n    if not get_names:\n        dataset = dataset.map(lambda image, posting_id: image)\n    dataset = dataset.batch(config.BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","metadata":{"executionInfo":{"elapsed":232,"status":"ok","timestamp":1646574417326,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"-qGFiNhlW8cs","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nprint(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.config.list_physical_devices('GPU')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = 10; col = 4; # if BATCH_SIZE < col 讓col比 BATCH_SIZE小 否則會報錯\nrow = min(row,config.BATCH_SIZE//col)\nN_TRAIN = count_data_items(train_files)\nprint(N_TRAIN)\nds = get_training_dataset(train_files)\n\nfor (sample,label) in ds:\n    img = sample['inp1']\n    plt.figure(figsize=(25,int(25*row/col)))\n    for j in range(row*col):\n        plt.subplot(row,col,j+1)\n        plt.title(label[j].numpy())\n        plt.axis('off')\n        plt.imshow(img[j,])\n    plt.show()\n    break\nprint(img.shape)","metadata":{"executionInfo":{"elapsed":30078,"status":"ok","timestamp":1646574447402,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"1fggnMDxW8ct","outputId":"f2ed3432-49f8-4448-bdf6-2b99590042db","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = 10; col = 4;# if BATCH_SIZE < col 讓col比 BATCH_SIZE小 否則會報錯\nrow = min(row,config.BATCH_SIZE//col)\nN_TEST = count_data_items(test_files)\nprint(N_TEST)\nds = get_test_dataset(test_files)\n\nfor (img,label) in ds:\n    plt.figure(figsize=(25,int(25*row/col)))\n    for j in range(row*col):\n        plt.subplot(row,col,j+1)\n        plt.title(label[j].numpy())\n        plt.axis('off')\n        plt.imshow(img[j,])\n    plt.show()\n    break\nprint(img.shape)","metadata":{"executionInfo":{"elapsed":19714,"status":"ok","timestamp":1646574467113,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"Yrx_Qb-qW8ct","outputId":"206fdc1f-cd56-4ed1-c0cd-b13d696c50fe","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MODEL","metadata":{"id":"1snuExNVW8ct"}},{"cell_type":"code","source":"# Arcmarginproduct class keras layer\nclass ArcMarginProduct(tf.keras.layers.Layer):\n    \n    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n                 ls_eps=0.0, **kwargs):\n\n        super(ArcMarginProduct, self).__init__(**kwargs)\n\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps\n        self.easy_margin = easy_margin\n        self.cos_m = tf.math.cos(m)\n        self.sin_m = tf.math.sin(m)\n        self.th = tf.math.cos(math.pi - m)\n        self.mm = tf.math.sin(math.pi - m) * m\n\n    def get_config(self):\n\n        config = super().get_config().copy()\n        config.update({\n            'n_classes': self.n_classes,\n            's': self.s,\n            'm': self.m,\n            'ls_eps': self.ls_eps,\n            'easy_margin': self.easy_margin,\n        })\n        return config\n\n    def build(self, input_shape):\n        super(ArcMarginProduct, self).build(input_shape[0])\n\n        self.W = self.add_weight(\n            name='W',\n            shape=(int(input_shape[0][-1]), self.n_classes),\n            initializer='glorot_uniform',\n            dtype='float32',\n            trainable=True,\n            regularizer=None)\n\n    def call(self, inputs):\n        X, y = inputs\n        y = tf.cast(y, dtype=tf.int32)\n        cosine = tf.matmul(\n            tf.math.l2_normalize(X, axis=1),\n            tf.math.l2_normalize(self.W, axis=0)\n        )\n        sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = tf.where(cosine > 0, phi, cosine)\n        else:\n            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n        one_hot = tf.cast(\n            tf.one_hot(y, depth=self.n_classes),\n            dtype=cosine.dtype\n        )\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n        return output","metadata":{"executionInfo":{"elapsed":5,"status":"ok","timestamp":1646574467113,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"DMfX7OqrW8ct","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n        efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6, efn.EfficientNetB7]\n\nDS_GCS_PATH = KaggleDatasets().get_gcs_path(\"efficientnetv2-tfhub-weight-files\")\n\ndef freeze_BN(model):\n    # Unfreeze layers while leaving BatchNorm layers frozen\n    for layer in model.layers:\n        if not isinstance(layer, tf.keras.layers.BatchNormalization):\n            layer.trainable = True\n        else:\n            layer.trainable = False\n\n# Function to create our EfficientNetB3 model\ndef get_model():\n\n    if config.head=='arcface':\n        head = ArcMarginProduct\n    else:\n        assert 1==2, \"INVALID HEAD\"\n    \n    with strategy.scope():\n        \n        margin = head(\n            n_classes = config.N_CLASSES, \n            s = 30, \n            m = 0.3, \n            name=f'head/{config.head}', \n            dtype='float32'\n            )\n\n        inp = tf.keras.layers.Input(shape = [config.IMAGE_SIZE, config.IMAGE_SIZE, 3], name = 'inp1')\n        label = tf.keras.layers.Input(shape = (), name = 'inp2')\n        \n        if config.model_type == 'effnetv1':\n            x = EFNS[config.EFF_NET](weights = 'noisy-student', include_top = False)(inp)\n            embed = tf.keras.layers.GlobalAveragePooling2D()(x)\n        elif config.model_type == 'effnetv2':\n            FEATURE_VECTOR = f'{DS_GCS_PATH}/tfhub_models/efficientnetv2-{config.EFF_NETV2}/feature_vector'\n            embed = tfhub.KerasLayer(FEATURE_VECTOR, trainable=True)(inp)\n            \n        embed = tf.keras.layers.Dropout(0.2)(embed)\n        embed = tf.keras.layers.Dense(512)(embed)\n        x = margin([embed, label])\n        \n        output = tf.keras.layers.Softmax(dtype='float32')(x)\n        \n        model = tf.keras.models.Model(inputs = [inp, label], outputs = [output])\n        embed_model = tf.keras.models.Model(inputs = inp, outputs = embed)  \n        \n        opt = tf.keras.optimizers.Adam(learning_rate = config.LR)\n        if config.FREEZE_BATCH_NORM:\n            freeze_BN(model)\n\n        model.compile(\n            optimizer = opt,\n            loss = [tf.keras.losses.SparseCategoricalCrossentropy()],\n            metrics = [tf.keras.metrics.SparseCategoricalAccuracy(),tf.keras.metrics.SparseTopKCategoricalAccuracy(k=5)]\n            ) \n        \n        return model,embed_model","metadata":{"executionInfo":{"elapsed":6,"status":"ok","timestamp":1646574467114,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"ZwSwED7dW8ct","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback(plot=False):\n    lr_start   = 0.000001\n    lr_max     = 0.000005 * config.BATCH_SIZE  \n    lr_min     = 0.000001\n    lr_ramp_ep = 4\n    lr_sus_ep  = 0\n    lr_decay   = 0.9\n   \n    def lrfn(epoch):\n        if config.RESUME:\n            epoch = epoch + config.RESUME_EPOCH\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n        \n    if plot:\n        epochs = list(range(config.EPOCHS))\n        learning_rates = [lrfn(x) for x in epochs]\n        plt.scatter(epochs,learning_rates)\n        plt.show()\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback\n\nget_lr_callback(plot=True)","metadata":{"executionInfo":{"elapsed":612,"status":"ok","timestamp":1646574467721,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"AIEl0PgmW8cu","outputId":"789681b7-c12f-4dc7-ef02-1f127b2f90a3","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Snapshot(tf.keras.callbacks.Callback):\n    \n    def __init__(self,fold,snapshot_epochs=[]):\n        super(Snapshot, self).__init__()\n        self.snapshot_epochs = snapshot_epochs\n        self.fold = fold\n        \n        \n    def on_epoch_end(self, epoch, logs=None):\n        # logs is a dictionary\n#         print(f\"epoch: {epoch}, train_acc: {logs['acc']}, valid_acc: {logs['val_acc']}\")\n        if epoch in self.snapshot_epochs: # your custom condition         \n            self.model.save_weights(config.save_dir+f\"/EF{config.MODEL_NAME}_epoch{epoch}.h5\")\n        self.model.save_weights(config.save_dir+f\"/{config.MODEL_NAME}_last.h5\")","metadata":{"executionInfo":{"elapsed":11,"status":"ok","timestamp":1646574467722,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"OKGT_tG9W8cu","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TRAIN","metadata":{"id":"ItnIGuTJW8cu"}},{"cell_type":"code","source":"TRAINING_FILENAMES = [x for i,x in enumerate(train_files) if i%config.FOLDS!=config.FOLD_TO_RUN]\nVALIDATION_FILENAMES = [x for i,x in enumerate(train_files) if i%config.FOLDS==config.FOLD_TO_RUN]\nprint(len(TRAINING_FILENAMES),len(VALIDATION_FILENAMES),count_data_items(TRAINING_FILENAMES),count_data_items(VALIDATION_FILENAMES))","metadata":{"executionInfo":{"elapsed":4,"status":"ok","timestamp":1646574467722,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"iSzreZr_W8cu","outputId":"c38f55b1-1bf3-4daa-acad-602ad41ac6c7","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if config.DEBUG:\n    TRAINING_FILENAMES = [TRAINING_FILENAMES[0]]\n    VALIDATION_FILENAMES = [VALIDATION_FILENAMES[0]]\n    print(len(TRAINING_FILENAMES),len(VALIDATION_FILENAMES),count_data_items(TRAINING_FILENAMES),count_data_items(VALIDATION_FILENAMES))\n    test_files = [test_files[0]]","metadata":{"executionInfo":{"elapsed":3,"status":"ok","timestamp":1646574467722,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"JGuShz0GW8cu","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything(config.SEED)\nVERBOSE = 1\ntrain_dataset = get_training_dataset(TRAINING_FILENAMES)\nval_dataset = get_val_dataset(VALIDATION_FILENAMES)\nSTEPS_PER_EPOCH = count_data_items(TRAINING_FILENAMES) // config.BATCH_SIZE\ntrain_logger = tf.keras.callbacks.CSVLogger(config.save_dir+'/training-log-fold-%i.h5.csv'%config.FOLD_TO_RUN)\n# SAVE BEST MODEL EACH FOLD        \nsv_loss = tf.keras.callbacks.ModelCheckpoint(\n    config.save_dir+f\"/{config.MODEL_NAME}_loss.h5\", monitor='val_loss', verbose=0, save_best_only=True,\n    save_weights_only=True, mode='min', save_freq='epoch')\n# BUILD MODEL\nK.clear_session()\nmodel,embed_model = get_model()\nsnap = Snapshot(fold=config.FOLD_TO_RUN,snapshot_epochs=[5,8])\nmodel.summary()\n\nif config.RESUME:   \n    model.load_weights(config.resume_model_wts)","metadata":{"executionInfo":{"elapsed":61824,"status":"ok","timestamp":1646574529543,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"Xtt8_BpmW8cu","outputId":"bb02c594-4797-46bc-cdf8-532a863a5c55","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('#### Image Size %i with EfficientNet B%i and batch_size %i'%\n      (config.IMAGE_SIZE,config.EFF_NET,config.BATCH_SIZE))\n\nhistory = model.fit(train_dataset,\n                validation_data = val_dataset,\n                steps_per_epoch = STEPS_PER_EPOCH,\n                epochs = config.EPOCHS,\n                callbacks = [snap,get_lr_callback(),train_logger,sv_loss], \n                verbose = VERBOSE\n                   )","metadata":{"executionInfo":{"elapsed":7149649,"status":"ok","timestamp":1646581679190,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"q02A1zqUW8cv","outputId":"7abc0562-ff1d-48bf-8af5-fc2e52007a1e","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_weights(config.save_dir+f\"/{config.MODEL_NAME}_loss.h5\")","metadata":{"executionInfo":{"elapsed":11794,"status":"ok","timestamp":1646581690974,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"8k2ZxCOLW8cv","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EVALUATION","metadata":{"id":"Otn6tjaFW8cv"}},{"cell_type":"code","source":"def get_ids(filename):\n    ds = get_test_dataset([filename],get_names=True).map(lambda image, image_name: image_name).unbatch()\n    NUM_IMAGES = count_data_items([filename])\n    ids = next(iter(ds.batch(NUM_IMAGES))).numpy().astype('U')\n    return ids\n\ndef get_targets(filename):\n    ds = get_eval_dataset([filename],get_targets=True).map(lambda image, target: target).unbatch()\n    NUM_IMAGES = count_data_items([filename])\n    ids = next(iter(ds.batch(NUM_IMAGES))).numpy()\n    return ids\n\ndef get_embeddings(filename):\n    ds = get_test_dataset([filename],get_names=False)\n    embeddings = embed_model.predict(ds,verbose=0)\n    return embeddings\n\ndef get_predictions(test_df,threshold=0.2):\n    predictions = {}\n    for i,row in tqdm(test_df.iterrows()):\n        if row.image in predictions:\n            if len(predictions[row.image])==5:\n                continue\n            predictions[row.image].append(row.target)\n        elif row.confidence>threshold:\n            predictions[row.image] = [row.target,'new_individual']\n        else:\n            predictions[row.image] = ['new_individual',row.target]\n\n    for x in tqdm(predictions):\n        if len(predictions[x])<5:\n            remaining = [y for y in sample_list if y not in predictions]\n            predictions[x] = predictions[x]+remaining\n            predictions[x] = predictions[x][:5]\n        \n    return predictions\n\ndef map_per_image(label, predictions):\n    \"\"\"Computes the precision score of one image.\n\n    Parameters\n    ----------\n    label : string\n            The true label of the image\n    predictions : list\n            A list of predicted elements (order does matter, 5 predictions allowed per image)\n\n    Returns\n    -------\n    score : double\n    \"\"\"    \n    try:\n        return 1 / (predictions[:5].index(label) + 1)\n    except ValueError:\n        return 0.0\n\nif IS_COLAB:\n    f = open ('/content/drive/MyDrive/Kaggle/HappyWhale-2022/archive/individual_ids.json', \"r\")\nelif IS_PC:\n    f = open ('E:/HappyWhale Splits/individual_ids.json', \"r\")\nelse:\n    f = open ('../input/happywhale-splits/individual_ids.json', \"r\")\n\ntarget_encodings = json.loads(f.read())\ntarget_encodings = {target_encodings[x]:x for x in target_encodings}\nsample_list = ['938b7e931166', '5bf17305f073', '7593d2aee842', '7362d7a01d00','956562ff2888']","metadata":{"executionInfo":{"elapsed":645,"status":"ok","timestamp":1646583188168,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"OcDhBJf0W8cv","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_targets = []\ntrain_embeddings = []\nfor filename in tqdm(TRAINING_FILENAMES):\n    embeddings = get_embeddings(filename)\n    targets = get_targets(filename)\n    train_embeddings.append(embeddings)\n    train_targets.append(targets)\ntrain_embeddings = np.concatenate(train_embeddings)\ntrain_targets = np.concatenate(train_targets)","metadata":{"executionInfo":{"elapsed":325516,"status":"ok","timestamp":1646583535137,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"v_YhqB6PW8cv","outputId":"1dcea430-2d54-41f6-dd02-67bca51b147d","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.neighbors import NearestNeighbors\nneigh = NearestNeighbors(n_neighbors=config.KNN,metric='cosine')\nneigh.fit(train_embeddings)","metadata":{"executionInfo":{"elapsed":339,"status":"ok","timestamp":1646583535469,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"sYPF0CU1W8cv","outputId":"3ada67ef-a99a-45e9-862f-33e0ef73fb68","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ids = []\ntest_nn_distances = []\ntest_nn_idxs = []\nval_targets = []\nval_embeddings = []\nfor filename in tqdm(VALIDATION_FILENAMES):\n    embeddings = get_embeddings(filename)\n    targets = get_targets(filename)\n    ids = get_ids(filename)\n    distances,idxs = neigh.kneighbors(embeddings, config.KNN, return_distance=True)\n    test_ids.append(ids)\n    test_nn_idxs.append(idxs)\n    test_nn_distances.append(distances)\n    val_embeddings.append(embeddings)\n    val_targets.append(targets)\ntest_nn_distances = np.concatenate(test_nn_distances)\ntest_nn_idxs = np.concatenate(test_nn_idxs)\ntest_ids = np.concatenate(test_ids)\nval_embeddings = np.concatenate(val_embeddings)\nval_targets = np.concatenate(val_targets)","metadata":{"executionInfo":{"elapsed":116096,"status":"ok","timestamp":1646583651562,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"35z1vTIoW8cv","outputId":"08fa4d93-43b2-4861-8615-2626a95fe32c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"allowed_targets = set([target_encodings[x] for x in np.unique(train_targets)])\nval_targets_df = pd.DataFrame(np.stack([test_ids,val_targets],axis=1),columns=['image','target'])\nval_targets_df['target'] = val_targets_df['target'].astype(int).map(target_encodings)\nval_targets_df.loc[~val_targets_df.target.isin(allowed_targets),'target'] = 'new_individual'\nval_targets_df.target.value_counts()","metadata":{"executionInfo":{"elapsed":425,"status":"ok","timestamp":1646583651984,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"1UC25n_aW8cv","outputId":"56ddaf22-d770-4805-b0c1-26e741565ffa","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = []\nfor i in tqdm(range(len(test_ids))):\n    id_ = test_ids[i]\n    targets = train_targets[test_nn_idxs[i]]\n    distances = test_nn_distances[i]\n    subset_preds = pd.DataFrame(np.stack([targets,distances],axis=1),columns=['target','distances'])\n    subset_preds['image'] = id_\n    test_df.append(subset_preds)\ntest_df = pd.concat(test_df).reset_index(drop=True)\ntest_df['confidence'] = 1-test_df['distances']\ntest_df = test_df.groupby(['image','target']).confidence.max().reset_index()\ntest_df = test_df.sort_values('confidence',ascending=False).reset_index(drop=True)\ntest_df['target'] = test_df['target'].map(target_encodings)\ntest_df.to_csv('val_neighbors.csv')\ntest_df.image.value_counts().value_counts()","metadata":{"executionInfo":{"elapsed":12644,"status":"ok","timestamp":1646583664625,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"a90EtrUTW8cw","outputId":"da430290-4695-4246-a6ea-5c7cc6e6c007","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_th = 0\nbest_cv = 0\nfor th in [0.1*x for x in range(11)]:\n    all_preds = get_predictions(test_df,threshold=th)\n    cv = 0\n    for i,row in val_targets_df.iterrows():\n        target = row.target\n        preds = all_preds[row.image]\n        val_targets_df.loc[i,th] = map_per_image(target,preds)\n    cv = val_targets_df[th].mean()\n    print(f\"CV at threshold {th}: {cv}\")\n    if cv>best_cv:\n        best_th = th\n        best_cv = cv","metadata":{"executionInfo":{"elapsed":195359,"status":"ok","timestamp":1646583859976,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"4gpRz8AKW8cw","outputId":"48d33212-4425-4568-94fc-9dc2e99e37a8","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best threshold\",best_th)\nprint(\"Best cv\",best_cv)\nval_targets_df.describe()","metadata":{"executionInfo":{"elapsed":17,"status":"ok","timestamp":1646583859977,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"5MYN1ZRpW8cw","outputId":"c8eb6257-8983-44b6-c20c-11685c1fec21","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_targets_df['is_new_individual'] = val_targets_df.target=='new_individual'\nprint(val_targets_df.is_new_individual.value_counts().to_dict())\nval_scores = val_targets_df.groupby('is_new_individual').mean().T\nval_scores['adjusted_cv'] = val_scores[True]*0.1+val_scores[False]*0.9\nbest_threshold_adjusted = val_scores['adjusted_cv'].idxmax()\nprint(\"best_threshold\",best_threshold_adjusted)\nval_scores","metadata":{"executionInfo":{"elapsed":9,"status":"ok","timestamp":1646583859977,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"jLS1Om_0W8cw","outputId":"70750d88-de71-401d-c87d-88b9de8581e8","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# INFERENCE","metadata":{"id":"hBhy_3ugW8cw"}},{"cell_type":"code","source":"train_embeddings = np.concatenate([train_embeddings,val_embeddings])\ntrain_targets = np.concatenate([train_targets,val_targets])\nprint(train_embeddings.shape,train_targets.shape)","metadata":{"executionInfo":{"elapsed":7,"status":"ok","timestamp":1646583859977,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"FoP0J3bKW8cw","outputId":"14275b1b-06a4-467c-f6c1-1e2c2e935c4c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.neighbors import NearestNeighbors\nneigh = NearestNeighbors(n_neighbors=config.KNN,metric='cosine')\nneigh.fit(train_embeddings)","metadata":{"executionInfo":{"elapsed":402,"status":"ok","timestamp":1646583860374,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"JfAvRfpmW8cw","outputId":"7a32c21c-57dc-486b-e20e-7ff2253df1e7","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ids = []\ntest_nn_distances = []\ntest_nn_idxs = []\nfor filename in tqdm(test_files):\n    embeddings = get_embeddings(filename)\n    ids = get_ids(filename)\n    distances,idxs = neigh.kneighbors(embeddings, config.KNN, return_distance=True)\n    test_ids.append(ids)\n    test_nn_idxs.append(idxs)\n    test_nn_distances.append(distances)\ntest_nn_distances = np.concatenate(test_nn_distances)\ntest_nn_idxs = np.concatenate(test_nn_idxs)\ntest_ids = np.concatenate(test_ids)","metadata":{"executionInfo":{"elapsed":407037,"status":"ok","timestamp":1646584267408,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"8EwOFJiyW8cw","outputId":"3a9f2979-1459-479d-a629-6dffdb687284","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if IS_COLAB:\n    sample_submission = pd.read_csv('/content/drive/MyDrive/Kaggle/HappyWhale-2022/sample_submission.csv',index_col='image')\nelif IS_PC:\n    sample_submission = pd.read_csv('E:/happy-whale-and-dolphin/sample_submission.csv',index_col='image')\nelse:\n    sample_submission = pd.read_csv('../input/happy-whale-and-dolphin/sample_submission.csv',index_col='image')\nprint(len(test_ids),len(sample_submission))\ntest_df = []\nfor i in tqdm(range(len(test_ids))):\n    id_ = test_ids[i]\n    targets = train_targets[test_nn_idxs[i]]\n    distances = test_nn_distances[i]\n    subset_preds = pd.DataFrame(np.stack([targets,distances],axis=1),columns=['target','distances'])\n    subset_preds['image'] = id_\n    test_df.append(subset_preds)\ntest_df = pd.concat(test_df).reset_index(drop=True)\ntest_df['confidence'] = 1-test_df['distances']\ntest_df = test_df.groupby(['image','target']).confidence.max().reset_index()\ntest_df = test_df.sort_values('confidence',ascending=False).reset_index(drop=True)\ntest_df['target'] = test_df['target'].map(target_encodings)\ntest_df.to_csv('test_neighbors.csv')\ntest_df.image.value_counts().value_counts()","metadata":{"executionInfo":{"elapsed":45684,"status":"ok","timestamp":1646584456786,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"i4uhrPhdW8cw","outputId":"d46668ee-8d14-4ae4-893a-d67f31a34ea5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_list = ['938b7e931166', '5bf17305f073', '7593d2aee842', '7362d7a01d00','956562ff2888']","metadata":{"executionInfo":{"elapsed":10,"status":"ok","timestamp":1646584456786,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"sjcceXS9W8cx","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = {}\nfor i,row in tqdm(test_df.iterrows()):\n    if row.image in predictions:\n        if len(predictions[row.image])==5:\n            continue\n        predictions[row.image].append(row.target)\n    elif row.confidence>best_threshold_adjusted:\n        predictions[row.image] = [row.target,'new_individual']\n    else:\n        predictions[row.image] = ['new_individual',row.target]\n        \nfor x in tqdm(predictions):\n    if len(predictions[x])<5:\n        remaining = [y for y in sample_list if y not in predictions]\n        predictions[x] = predictions[x]+remaining\n        predictions[x] = predictions[x][:5]\n    predictions[x] = ' '.join(predictions[x])\n    \npredictions = pd.Series(predictions).reset_index()\npredictions.columns = ['image','predictions']\npredictions.to_csv('submission.csv',index=False)\npredictions.head()","metadata":{"executionInfo":{"elapsed":51060,"status":"ok","timestamp":1646584507836,"user":{"displayName":"張育禔","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi6XYj7uLdZWcjhb2_xCnpJVK8VzCb1CdL5OfjLrF0=s64","userId":"09450846008256654552"},"user_tz":-480},"id":"5LhZKFQnW8cx","outputId":"fd3f8513-b465-41ce-e496-8c15243e144f","trusted":true},"execution_count":null,"outputs":[]}]}