{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\npd.set_option(\"max_colwidth\", None)\n\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nplt.rcParams.update({'font.size': 14})\nplt.rc('legend',fontsize=10)\nimport seaborn as sns\n\nimport re\n\nimport spacy\nfrom spacy import displacy\n\nimport nltk\nfrom nltk import word_tokenize\nfrom nltk import tokenize\nfrom collections import Counter\n\n\nimport os\nos.environ[\"WANDB_SILENT\"] = \"true\"\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # Disable tensorflow debugging logs\n\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input, Dropout\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom transformers import TFBertModel\nimport transformers\n\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import log_loss\n\nimport warnings # Supress warnings\nwarnings.filterwarnings(\"ignore\")\n\n\nSEED = 42\n# 1. Set the `PYTHONHASHSEED` environment variable at a fixed value\nos.environ['PYTHONHASHSEED'] = str(SEED)\n\n# 2. Set the `python` built-in pseudo-random generator at a fixed value\nimport random\nrandom.seed(SEED)\n\n# 3. Set the `numpy` pseudo-random generator at a fixed value\nnp.random.seed(SEED)\n\n# 4. Set the `tensorflow` pseudo-random generator at a fixed value\ntf.random.set_seed(SEED)\n\n# 5. Configure a new global `tensorflow` session\nfrom keras import backend as K\nsession_conf = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)\nsess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph(), config=session_conf)\ntf.compat.v1.keras.backend.set_session(sess)\n\nhighlight_color = 'cornflowerblue'\ncmap = 'binary'\neffectiveness_colors = ['salmon', 'yellow', 'lightgreen']\neffectiveness_order = ['Ineffective', 'Adequate', 'Effective']","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-19T15:45:23.050006Z","iopub.execute_input":"2022-07-19T15:45:23.050558Z","iopub.status.idle":"2022-07-19T15:45:23.070544Z","shell.execute_reply.started":"2022-07-19T15:45:23.050523Z","shell.execute_reply":"2022-07-19T15:45:23.069497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feedback Prize - Predicting Effective Arguments\n\n![thumb76_76(3).png](attachment:cdc1967e-f707-464e-b759-e4dc04278a72.png)\n\nThis notebook contains:\n\n* Analysis of [Duplicates](#Duplicates)\n* [Exploratory Data Analysis](#Exploratory-Data-Analysis):\n* [Preprocessing](#Preprocessing): Label Encode Target and concatenate `discourse_type` [SEP] `discourse_text`\n* [Model](#Model): Experiment Tracking with W&B, BERT variant using TensorFlow based on [【Tensorflow】FeedBack BERT-Baseline by IMvision12](https://www.kaggle.com/code/imvision12/tensorflow-feedback-bert-baseline) (don't forget to upvote the original baseline if this notebook was helpful to you)","metadata":{},"attachments":{"cdc1967e-f707-464e-b759-e4dc04278a72.png":{"image/png":"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCADwAPADASIAAhEBAxEB/8QAHgABAAICAgMBAAAAAAAAAAAAAAUIBgcCBAEDCQr/xABXEAACAgICAgAEAwMFCAoOCwAEBQIDAQYABwgREhMUIRUWMUFRYSIlVXGBCRcjMjaRlaEkU3WTlrHR0+HwGDU4QkNUYne1trfB1NYnNEVSZXJ2srPS1f/EABsBAQADAQEBAQAAAAAAAAAAAAABAgUDBAYH/8QAQBEAAQMCBAQCBwQJAgcAAAAAAQACEQMhBBIxQRMiUWEFcQYUMlKBkdEjQlOSFRYkQ2KTocHwM9JFVWRzwtPx/9oADAMBAAIRAxEAPwD9/HHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwicc8ZzjGPf9mMfvz+zH9v8OdIhkCHmMSzBRZTxmUYEEV0yliOcYzmMbJRzL1nOPfr9M5xjPILg27iAOpsLmNdFBIAJJAA1JMAea73HIXOxosf/bCv+1gJj+H+2554/MaL+mFX+kBf6/8AbP3ff+r7858alMCpTOuj29u/f4KvEp++z8w+qm+OQv5jRf0wq/T37/ERM49fp794t9evf2/r+3H5iReveXKv1jHvOcMBc4xj+OfmYxjHv7e/fr36x+3jjUpjiMnpmH17pxKfvs/M36qa45CfmNF/TKrOPf2z+IC/87zz+Ykf9MK/9IC/85wa9EWNWnew52/XunEp++z8zfqprjkJ+Y0X9Mqv9Iif85x+Y0X6fjCr3+78RF/53jj0fxaf529u/cJxKfvs/M3691N8chPzGi/phV/pET/X/hftzz+YkX7HKrP9TAX/AI82Yx7/AIfrn9mOTxqUxxGT0zDz6pxKf4jPzN+qmuOQn5jR5/Rwq/r/ABAX1/8Ay4/Z/H+PH5jRf0yq+2Pf/bET9P8AfeQK1I6VGTaQHtMTGt+/+QU4lP32fmb9e6m+OQn5kQ/f+elP2+2f5xF+2fXv1n/Cese8ff8AqzjP6cY2NFn36cqft/8AiAuf0/X9LP7Pv69Z441L8RvTVOJT99n5m/VTfHITGxoc+/55U/b9f5xF+2P359Wfpj/3Z4/MaL+mFX+kRP8AneONR3qMHm4DXzKCpTNxUZ19pv1U3xyF/MaL+mFXv7/b8RF9/b9f/Cf9POVT9LdZXVU1W22WyjCuus4ac7Jyz6jGEI2ZlOWc/bEY4znOftjkitSMRUZcwOYXNu/cJxGe+zb7zd/j3Uxxxxzor6pxxxwiccccInHHHCJxxxwiwjsOVkdWKxVcRRKxlrdObBSLxbvl37MmouhC8ayq6vFtNllNma7IZlXZOGc/DLOM0v8AMVIXtG16xr4ZDyR8tAetFCxLtT/V7Sig+0uqRmpGLETdRaREbWzm1Jdltl/0a4sqdEI3W1/Hc/sT/Ja//dbV/wD1rR8rB5R99E9NbHq9ZITC7XTdV2FszNT2rZMwWAj7XE6amCw22JbMZqU3+itkroIuW2ZqYG4ivpvznLx4luIBBP2GHMCCZ4tbYlovABuDHkvHjBNGsL3ZSFoM/aOMQXNnvcWKqrgzvorUUlQMD0zvGynjHlm6brVzKCClZrNVdMlJpxIMAQ3Zu1Do3eLSGmwJNeUtWGbrGJN7HXUd58jZbL+HHa80XlWgEsFwlOqojGf0QlSkJj9Cs+tGWHoyNq/AAz3JdrQtIl2N0Yua1kgFCJtvBeasGFsxxdf7ExYNmn6/6kZIvqAzQHg/ZPjsZRCgVZrlUqpZHD+Mt5C3FoQwuMesz2w+Wheu7c0025JuJjEJswRCHD2azSqZMQ3N+ugYmyKlWKhXtXwTQGs58QDECC+LAsfAx1Eq/nwxgcZY7M6SDw2NLRY6iuCIi2Yib2hYrRRaTLHEkzGRkiS08s1SeoggzIladXt+/k+0oJ7DbbCW4m6nrdoRSZJSiFYvLaTHxqymgy2yR2krKDMYqrliLlAmYMToXH1FGW5Owo7le7Xtt887MoSpNb7ep1LK9WnXj42NoFcr0qSkmJNtu11ho6VjVI2c0YhXtzbYV19NcF9FQM8H5pWtSLAkiTbGZ0hgLxfbfRxBMxbREDCiQzJNiANOOxG066fGgoqsK2F5pd9NA5A9fHVPMZzt92tqlSnYIOG7nUFrIs2SwBFSG5eotZeNVUypVmuIhbA1uUrli6LA62usVwRmYFltODm08zTleCRlnhMNwWmQeNbpDiZBt0VHMpzIDxAEzTpnpeOKIkWmTaLmVq0TZfLEggYa3XKghrzHVGCbtRCuJHhioWkT6nF0RhJUAhT/ABdUxsvpHbPJMExpT2oaANEbsPZXkNqxtlLldevBvLToteNM1TVc/mja4kXChLxKsEiw/CdurlWw2G9eUc21zFU5qB08Y3rK7FofLCt/uedNHxsVE4OdhTZezuQkJbLUaqxxRMSwWNlhpDgQZhkEAXNl2crS/kXH2UFUhYyX5qyConfdru8TzD8JhDApOnsc2OthD/GNVU0YWXnX2EOEM/xaw+ge0ZFbLKdh6Y4nCNg2jmg0CZEg8Fgk26VwCTGgJJ/or/Zk3YeYWHDaDJIuZrGd5iL7iSsK1p35LzL1crZFd8Qrdv15Q9Ti6cmCtmhLIdZ2FwazIhglZSonAEaN0F4AZ6kYUgI2LRlSW35E7D5LzfbVav18oVIqfmYQDHarrl83Cq5hSujCsudokrx1C2JOxqbKJkWty8ir2Lc8aZAFGwNk8tn2rbDKo5Q7t0yrT0W13PBSU+HgcNjVomK+JWvEZrJ/wJLqlQQIHEkzJN2DY+ggi8zkmPlixXJ9Ucy1/drh9lSsnZtPztWBK1sdSzcLDqmA7W4EhlbCevuSaMrR7JFCDDSxXVewoohQNoEtcKJEjlmk2Nj+NcwNzMX2lVy0okEieVv2bOxMfbRpNug1stZFMPIxQh65E1NJc9tPRvLdsO3JQpDcrW2diffgtrsXDSqpdecrYJWBSoKd4q0dWWuomLica6oIo3yVMLFY4WOz0nytkTQWn62j1x7cqLUrLtf2FmpVsqYDPz3/AMC90NWUN+VkAbUtCPBicTjm3R/L5gxdodfXKNige9bawGPa0baqOvyDsjfW40XClBWmVlM4apsYe1Wpa5wYgYxlYaPmyg4oXgV5enrimQRCfaz/AMNbPl8mAF+qCiE06zO8vYiaAjmEjq4oEeBDJwIjG58VZYGoHrtj8uvplbJOQkkzJpMDrxFxWAPQEeWqmKQaBDptJ4bLmRFuL0kWInftr7W967fC7C1rT9xs+pKapXLqtUPq+rg2Mxq9KcuiZm3is6yksUG6CrdRRTphFc6omdB8w+uhSdjG1Gz+S+SkTl7qm22/hUnubk4elI/ktVB06qBTGmKi1F1u1CkypEBRRWhg1JBidihTi+y8u/chPmYOGGXeQBtdZw4GWggGS9UnaxTXDLTqDRDhriqcSJHd6ti9TCFzMRnsC1XeFYXQROuJH81bqVxx7VfssbbLbp6+uVYUHXMwKzcjVzmRXMj4SsUG68azqgHjCn8fGGjWymLbdM0UoltNzXaEmkJMa5Yqt6tgCSY1tKNFEQQxxkNbdjCDpJH2wINxvoAL2Kwzrlt3lr+xL6d5Xbcdqx1jYhxEnX1ptagliuTwUQAcjzKMuW0u7SB5KqiBzAplEF/E3UjE1persdnkiic9hH6nF66CN2s0JUG119K0oXa/S1ZmgutViPWQWfRWvKBTDg5AIrODyQSZRlkspMG3cz8r7VzfTE1Im0HMt415I7TCRL1hTfSXsE4fhyBxY3sFF18ywaJhOWLUodd8IVsa43wlK6rBwfNVnVFRQ6SbTlq3x7EVoakreR4xLAqS50EUGYZTEIhNRG6SwuMC/r/ixljUHZTLMBrPayvAcLzTblJMXA4xIsLCRuoyUiSRnuBA4bC1oJaTbinSNjI0lRG9w76y02fOqSbL1x6LWJywnVpmU8trtYTL38NZvblEWqLEzKTY8akWE4smlFMTbjapZwRDLG/lMyHvKOCgntHIMsqXV6Ugng3MIPT8ATvZ3U3QBrMWp0AJ4VGMkq3sz/r2DCkdtTsAjzXHySQOrUb46jUJXfQSNDXhAzLfZEyhqj2Ng4FWRBgWVkrLi8SIIWFLx6c3Sqslyo80pXlkr69d36RgFk6WVUZ6pWOGQAMUx2MbLO2+tQXaiUjxPj9EaRN1K/AQFVd1MrLIaymWjkc7fM6nTzeZmuDN7WvppZMtICADqBJpsJF27CsDOh/sRMYMhz5C7Gq2v81Td644LF1IbX8KkKqhenID7REI2M8aoquqLMoPXbyR6MnVzW7FqS+i+/8AFrIklsMi6u2ryPM7a0enYdSYh6vZvOriOI51hGPCtfaAbQ4KqbVfNrETiGDhOoz+cQZdYf8Ahwmw1m0flK3MdZ8rXDzX27AlU7EepYaZVNFU31fEGxm7WTpAsVtTZDA0pqjKiBrHbK6sb6gYgSzFDKiwKNxfGneGfaPWAe9OAxaPxXZNuih+GuiZmdbV7CcrQEsbYXEww1KAEgUdCn6esW6/IkKIfT5st9mBosq1gDPKc4Y+i0CzmaEVDsRBAIiTBK9GEoNqVgJ9kB8Gm0DlLTAIqEzHUR/Qqwdf+JH7es+vv/X+39cY/t/jznxxz6dfQJxxxwiccccIuga0Xrs1/Xmhh4txn5X1RYwvzMx/xvg+otq+P4cZx8Xw+/h949/rzo/mbX/6bTf6WX//ABHMLeI0rzsRXS7UK29NGmN7aKmi8RhXRZPYUkJ2VQLpuhXOUP5MpwjGWY/bOfXr1obs3uboLrQBzdPqlvuzhN22o6RxqfXvUq/Ztoab+46/C7PHXqF0oLqDAKNLOrbGNpHUAj5qIDzZIyr5M/LnrvdUFMUg1j8nO5wceVrpMNI+9prbdc5qEugMgGBJdMQ07W0MfHZWt/MyD+nEv+ll/wD8T/yf289tD9KRdXRQ3VX3Wy+GuqhkFbbbL75+Guqu+U5y+HGc/DHGc5xjOft65RbdPKnxa0u/Z1t3Wex7Hsmmbzq2gbTqOm9H2bPtKl1s3ULXvG0j8JAA+Joo1rr9BsBO1sEd7XCh+hbahGkvZxMq52PLWdctNc0TdNJTapYueP8Arx7r+wJUqwb61Q6ZKzV7AEugOi/A564uq6uUcwlMe/GJxxiUo8o9+Lpsc9zaJDRmID3TAiQPsx33GmokEHOqNBJFOBrBdO0xNp1+fZZ12J99Wv8AX3/nbVvX/CtH/ZzRXfbzqEPcdLUdhqtbYbEag2wvV7dgJXAWxW12rV+yhqi2ARHzLShmA+GQQ1sJkLfnSvpsGpszjefYPrOqXYx9/wCdtX+2P/1Wkzj7f1c0D5IdRax2YyRWPtl/L1wiZgALPEo/MxiTpS5gXTialnCUxzlYmZV25jVbVidVlNkJ5lHzY10trEOEmjhbnIQDxaxMh8tsJsfguGLJFOpBAMUokNIBzuvz8vz01WmYbN4xKzSwK1PWC5kGrgKeFXtWrCnip0gBMKhix4gwIqXqlv1ddY1leKRRM21QrjXL4Mxm27H427PUwVNFqBiaetfCksEe2rL3wEGQwDxmJW3HBuuWsNmqmERQERZjOzzlKm2g6qRMeQdninoM1twee1SIHXkWsL3sc0UsbTibrSDCrYDauMNWOZdddEgMfAw+RrMiU2UQxCcepR4h9VxH1+qjseeb9VGZhqTqSI/UDya0MwWlpGKdZ+mtvJUO2yi2FtGK6wGFua66i4UE14rTUFy6mb+1kw3a0ydvPcRqsUioObNTEx93Dfw2O/wA87wsqUsvFymtRqiXVdBZZ2vBWnirFztG7IfQ19N85oubxoXFEMaVq1RXFvcxyTComIeDJZJJqlLj+c/HAB+5HK68UJCdP2GubDaGlC9Nr6/ZVSaEMkC7SSrpGpco1VYq+d2bRWFA9wcFkS12cl0+lJ416ZrrlQ9RdofhxCilVVmmFAhQxlqJAdrie4jJep2202CL3De2yAdg0DyWWLC42VAhVQ5uvHDRth3MzdWXZA9xtrNayGCqHDppDysJiZQuLPp1SLl8uGIx6S0bEexv1YIg1WhIEXsLqcxmfJLjSggXLcNJNjBGYCNJI0g2SKpOtEDLeW4aZME9ug1m215yhQ08efzCLlDrugR2m5tdUJNPsmvRezdvV1TO+NNoQGD8sW6idR1uYT+caFKFss2wzHGcWR7x4rbEvbKlev8AX01YZ9apopJZpV4thTq8iFUJqy1VECKWpxx4tREhs1mMLGdOMyswRnMen8Xuv0jzT3kOz4k40FrNqhWn1DSCDqtOE2Eoa++nWRjbYkbGBS+ywMJtODqh+Frrg18YThHtvErrR3QLA3sfF8h3jR3RdmMLK7K3WARy1tlX5b+mLCkpBgtGuIiRfXCyRspEExjweJmlz6MEcstwwjcHW0H4mDBtYQ+DzMkRHLhgItIsSflOmsa7Nu2Toix8WqY69qdOzj0RUkrmexJaXli1NZHA9H0hAUjrUwl1vwgwnXAam2FmKKKvlZxGJN3/AMaUKemN4mgCKUoJogIA21JLYCAKcx+vCVqgwLbMjr/xL4yhABpQpiZZdOHw32zlGbD429d7Ky2ZqfvdNJm0PM7AXYFeTTMI+Q/0sZLr569aRRCNUrIYxbbfZnF10c2erZ45jYfiP1iuHcDgdgVgxZm5KHsohH5quFBZpi0IWd2tXZJpUzOtqokfIid41Q498cV1fDOAHQJfSOmjcMZMAEm9pOpG3bUWvsJpC3u4U6ZbG46X1sLLNwHXjVMwKkTWOuhXBRtK0MDD7VxnMXLm8bbK1URaQJFRcHGxGf8A0dUpk3HQqPj8ZUY2Y4NexPHdU0NXHI9duOXr92fPrBGoDHCITWKlEdxJ2CwRbdctL+F4qoKHvpsNYTvlXZVmse6VcEi8YdAQnr2A/ZOC8psDxT1GfKhWsGq2pfu99GLBtXouM+s2kK1hcQbdYQOMfcqEsoDHEjVLCePmlAEbWf8A3y6vi2NDsuuZxZLFA6JBsbGbq0UKUNdqyTYraEHmBsXEzSbajcisLCBhRcV2h0zNMiG8oGGBJmbGbamxkfMqAHzANIkZbZcN/Ba19J16DYLs6/sPjbtyoMVFqmjMwH038h0cGiP5ht4x4rbaKbkf4XK36ypiOvauhrqPnZKqAPMjmdI9lfThvni9mElc1PX9OGobi1ktufoYQrquBSSei7BTBfYKESUExSjsAT7IWH4yOPiJURMxrjUnjH14i21Bug2/i3tNdW0r4RLjOQhcRFBSgMu/ENeq+iKpiYwMssCzRkm0mdEpVh0xpnAneIfWhaxath2VYNWporootxiu/F2aGx7iiZlVmtRqJlUawvzKHuELaY1UXwshGWZwCQTLqUH2eXDSNB1F/K+86q5FSImmACDGXDS2S066nXXSwGyz0vYvGRreWMwUdXsy1KuCI4cra9VMuWpRmdSmtMRXaFZIRUO4tpWxAlisOg6UBsUwsj8MfDDbvHWnW1e5HalrU9ZtciArXd1w0wh26d0erAlVK1LiwaxY/ALGBvor9hFx+fR8sfMyYYW08SOrmaoBVHfaRKlVolwFtUfmZpmHXdRVG+mWtV0G1ZEJKFkPfCNEvn5uzXmVcIYysjoHWi9TC0sntEf8GSvIOtYuoFhUwSRrpafAvnZPWrAmow5Tcy2mw4HNkh4DjE5urhbiwS52Qh9OARmBZhdwLAAnzvY36KGh7SL0435MMdY7mdpvseynZO/HuVihjlDoGbnFLRQgPlsuvYm1HVlFnOVqcqS/EjBgDbjTGQwlk6qCSCCCYRtIlOzqj7P4323nMhlfXFhcFS8pkfTtmtyKglY/WTU3HERB+fFcfI0/IP1E8VkyLJxVicr54ngZ3izqLNHrunl9pUW6grXbuG2CjTOlg1r2t7rj4FYIVXr0q9fSJDUNV4gaPANuMUDAwnFXdeHH3D+KXXYyYNWP2RkYoIl6wqcBZyEVazfoQEhRlo42t1jzrHsW0NghsS9DNcWXfVSzPMoyA4RLqVz0w1hax5viJBj5qoa+16cGCRlw9pLZF46d9J0Ky8dz43XRPSA6t1+QHstVdjoMJ3rdq1pChfF6Fh9TUBigvNq6djNdEykjM6aSSIZqzRZPFuelH+qvtWLxpuFMUqp4xX5gnaCthamE5xNZVzvDrrpqKwWXORFPw4niU82SzL5uJSo1q3hFpB31AKnfym8RRVE4hRKHAyNEIxdOTHNYuuC5vMbYU5UMWea/qiEhFycgi+NYt9Fx/Hnoxd0NrD3W1pmTaHGzEbBiUvlfENVYACtDXfHUIJO+teEvGFqIJxaTdXXiVtkfUaq9Lw6lWFZtUtik5hBc1lMBxgAczJdEtiDy+Rle7A06raoeWjhua4FwZRaPuwJYAScw2t5TCsBxxxzeWwnHHHCJxxxwiwO/79kg49Z/yIa+/WM5+35iS+/7cYxnP9XKPd8+BrXu3YN8vM7aSY0fbe2NJ7uXdbbx0H1z2vqarsLVeulvVx5DgbbjYV7ehdauuquoSNw6ZIH/AMpusYyyNWPy+GwaUi2YsU5nFjAoIYgMchW7dI78DFW0X3023JmIEiKZXDUW4rI+bGuyv4q/hzKWcwmOqdU/bft3/D/ef/mHnk/aGPqcOlTc19QvDnVS0mQxvsim4CADqb6rl9qC6GNcC6QS8tMQBpkPTqvnsX/czRl2x09i9d9+9jaD2u4a7O67F3OKVPtqvbm+11djKrmaLr/a2TDSutSNb1ftbctc08fSloihaMUvMcKthNCskVeVpoWtar1f1/1kvCnfqWsFdXaYuXsrZmWTQ66cjTLhjCJYhMq2IK8eN9+cRldbGVssY+PMeZJ/ep1P/b9u/wCH+8//ADDz2j9X6oKWEbH8wkXLzRmAsGO4bazEiWJZi0a6wFg7IDvlRbjFtWL6bIwtjCzEfijHOK1Bi6lN1N1GjDxlP2xNjE24QBnmsdbbKrxVe0sdSpw6xmoSIOtuHe22+iw/butNFSpq2qzV1ALAJ3qlwpg9Fkb6LI7Wkxidc5WZxifw5l6z6z69/pnGeaJ8uNP7u2rYOvMdUEsF60QTYq9lNWurltscHTDHohYLGi2om6AsyiFhX07G4FhTC2qkCXwH5th2Hj1q1+P3NdWx/m2tH/1/XlSPLzupv1W+00JSnrd2stZ21zJXQ5YgPmH4OxRiYijAEEtFPpWYa5ebKQaWJhbrq44gfM7pYnV5cZTp02V2tpNDTTw7iGtYJPEqtkyWtOgEmdTFlwxVNraNUNptiKRyta1sEVDLosDbrsFW6PSHfRdG/DuR3zYo7Tqdb1Vm32EloNbSLviJ+oosWzVQtXMQkye/GxvDfqpbS0JostxbmN8oejSulvJjT7diGBMIgrJRvjFq4+JbVeTsjpkU0JhG6u4Jgm2ApjcUTXt/0JYSwA4ECwVxUsFBUwGfLjbZUAZzr6oYlnRnNQlu6PLjZX1KLGNFAkA1hgRxW9ZjKzq0YY8mhyICysaEZyLjE/cn8mOxLleXbFVMcRBseq6Q6DLZFVnOCdmj/L2SAQMiWC9iH+BMTF+qiD5m2WbaoMGDOkprqaYcMaDNN0EgRNEieUCB2ibAjU6SVhkBrTmpvIc5upomTygQOlrwIB36RIPQ/lkGXQ5oOfjOj8rx3JsNiuLJvCWXRgrgzHNneCwvGrrWSckZKc3tVyq5WRJ7Iius2Uf9X+R+nJm73LHspqKNlXI1WjeOSXzkUtRTDZLxbV2smHU7MTs1YMF+yREJKVa/RYPVEdZ8+Me/u/k3uGpbzs2vREHYBAOhdYCWjt3n4+LdU7frT259k11iypkyEgmfIVMCGot+sZgwqFNnefaJ7l3ffY53Xu0MbhmoewakBqDIlpDJcvxQbbtozWDAJVaD9LUTnWRGsDLIUEVgniU35GnEjMYyS2Wl1P2sonLR7DTLI/yNlNnNa51N2SASfsri2sCT/bZd3bOlu/dxAtr+dsoa95oekKnKmDIpPho2t1Rcs3rGw0iKvZl8yPrqw7J3ZhTfbfaJAeNmIQg49QeWYAqJIjuaJFGuQiAHINrRbWYPmLG5XbIS1ePkdNr41wyExUXWyvZ2wFPEWXjr6CafU58sN8TzvoloRNMgk7y+69vthise5rrRN6t39LgqAEi1C5svNotJHqhAsc9CRE1TUfHN2a9pdybyiTaTtGlknsFzpY1ctwoBtGN1VOmBq972dd6uroOAON1Bdtmsi0Fjjm0Pbg5QFg2GgNyMzGuawsdzOkSKLgDAmTBgRAiYjbRQMshmRzgSCL04FgBBjQCNbi6gGHUflVVHIahtu1sR2S36RwTuGCLpa5VMmWVhi89QNM59U7uHdstkKPxg7Uhvyf8ARu8ywpnkkOuvJenUFQkqdlY7IFtO8lTs/N9yggwA2ijOlGv3MEzYcxOuKw3gZqSoWqu2g5MTUElkHcvR6vL8iu5tVHDp2AG1yerLflH3rZmiUOFIrNyopob03A0ja+SqMVzPLLUssB/l6mRBV2S7rLh9gvPIbsFYDqjUDTMtAGulJNtcjVbRsFjaq9nl7dao12C1WxVs7bA0EsrCCza4FFNlsZxkPC22+SKbTIpSDYECiJgDQgAjsCQeyl4aS0GmZvBHBEwATzQDeNDHx26iTpvyOu1Dbl+3k7U4Nd6Y/QBKSNlLshQXgrXyNauHYREPrVPJj17EKx2STBsRG20CZpjKFFF8eY/S3kS+R9lIt2JfsAtlV67RrAhrKxgOukHsahtgY8oUb6opguThFLNtZTCGA3kyzJHy2VZWbhcKc+UnYheJLEg6EMzMAbKX4brZNhVMq7jDlpFimq4AaVA8zZqlqwlyPXUxdDOgQbzpDXChZu/8it0WJ9P2kdETkZu27ISmadY2c52N2ZphKBL9Fq1sV9mbnsNiztf0ABcBwmuvp72lt2LIxqorlYHTkcMzmkXowC0h1omJIM6SO+lYa2Bkcc0CZokgi/YDS8EAgR2XQ1LpDyGqYtnm4Vu27ll1/tusVllt7CcVND4VzU2HgUVFLmAOTYmzVkX2GFIwWNYltQ9FU14eUf3oPIOzUAE12w7zW5r2Opsc3F2E+llYBdqq2k1XFrFP8z8JhtuW5QCiFMaBhPpMUyiNCuiOuYeW2/wULnluh1XhMCQgKrg91ZWCfV/ErIOOvYWVxT0oThGY61CTW4LMnsNtYhkLa84on11Hkv2QsX3VbV8uxkI0keVKZj0NicrLY234D15Ov1QmsjX9XXVUrtsPOs/HBL2K60NkQSRVGw5jSWuLHElwAvRcC4CIIII3g+RiN7OGY5jTdaBrSi2k3IHfsYsLHIQup/KrJ7Ql+/29YnCpX3C2p37BiQTJfMYRifJfUqLJMhdrmCzMoYBAVHbRVC2AB5sqWzCRe6D5TbBqnWpKXGyqXF+puWW8yFZ3oi6H2xUGl0JxlzIUemeU9Ji4JPeVV9QqJAtqmXrZMZN7ekm8huxnOnKtqFU5tYPt6t1delIath0YdGNUVW/VlNQQGrMpV+N2MGgxgmc3WU2ULsnlD1ZlZ39k8lNuV7Mk1tJqxDgltpOu7fYPc82Og0O1omF2MtXmAq+6Jdn4ZaQApkOPioh9TAViWHL6kAVlpkiKJntwA2WkWIiCbzPy2mA5uYRTcSJmRRDXaAk2AN9CY1sBAWMhdbeYjGdd4zDaNesEfMA2EGDe1wGSaH84de7WinAVXw0YYXH4cYlJw6M2ljOrZ6RTZUxawy/TeqPIZTvIjckPchddZRYWuwdk3ojeMUXy0hekUXHVUrwZt2a10sHtxXUMcpuCIvuyVQxFwzaYBX5j7TKsGX5YqxkhWrYGzhtLkmobBewBLbrQpCwKg2pNWn1EpBFV5xRLCk0Gu9mzEsQwz/a/Jlzqm3brqt6QtjnXmtCtHEJ84mfsJJA6m4ZdXGoMgWp8wqLbkjJcTgdUEnmdMUpZfBrbL2tNuE4SD7Joi3KJMeYI11MzYqHtBMGk8E2hppb5W3MkxcESdZ6qxfiP1t2Nquzb9tPZtLf8Vc61pSBVY7fW7KcOIjP2g1oP+I/QBD5HNctr3QcB4fEKsYLkxf8AhEkK6b0/FjOfX3++Pf3xnH/HjH3/AIfrz5CIPIDeCA9/dtPwFkkS66XsOtNNdebaYlJvUrtFHYBTLKSAkmoyGe1SKFbUxtLp+icj3TgrEGaVWR8OO6mvbZu9Vt02E1qZTo5w9EGjBx7g/CPuJiVffVkUIik0W6NAGbYn4XyFMtgYESE1N2MBjL0sK2kYBINTM0iCM5ECDIJgzPXutbBYoHLhxSdDSQXlzYFibtA1tFuoO6vXxxxzbWqnHHHCJxxxwif2/wDF/n/T/o/hxxxwiccccIsH7E/yXv8A91tW/wDWtHzR3fnYej6BsWtEbECuIbmoXsQirNcfvmA6qtkowSL81AORJeK0cTThA0m4qy62CxYsX5vOnVTzd/Y84V6oROycIQi21bMp2SjCEcfmxH95ylnEYx9+veZZxjH65zjmmO/et9G3pprpG027NQUCtZCDT1otsDK0e5iuPpydNS1W5ItVuVy5yi+qxfhc7CGZi4iRTiMsvHZvt8rww8LDXOUfva1peC29tRGq8eMnhVb5QW0hMgCOI6QZBA7nfRVtI8melbyqbLx0RJKWcygb8aDvxMhzChbltoysitfOFza2kk9PctCnadEilursojeAwqolY+SnVOVtTbH4DYJcZEGuuvTt0sZTKzTSVCmKaIsmvzMg5qZQ9iRj+HfKN+KI2I2R7Y3RXSI9ItNI+y4gCXSwqzMdtZbg2m5iXksiyb6Vlt9pTVgUXZdmWSCC7rr/AIp5x66UujOiB3gpGZ7RQ1spKsDXzucQoukMMIrOY1J57Fgcg4cEpesKY/TyIpEvBGnPHzq5WYXML8YEAi5dhYHsxHLP9z8b4sk2D2WiHZ6JGxsMpPwIj5yvRLyZ6vpXSYkCI8DVEwBzcHq+2MV0z7vnwGDFeDU4TklWV1iwJqqM+Jbc0WBH/JsNrzKIV+SfTQ9RjqSVGlN2C4di1q/J23lsSTqVw48aWEwKCYfjSXXRgjHqiqywnVUnyDmcBwrfq7ZO3xq6FIiIPIfdbA1xlRQizBWw5VjHSxTi+2tdHYpCVGMcihZa3RrrLYyAExfd8qu6q/o29P8AjZqplI7Ns7BKtpxXQucM2kvn2bLCeqWH1hE7HnNjTZab6dauawryW1rFXAQslYBT6DMRaq0mzpBw+WBlFjlJNyYg20hSSSAOIwHs6jGws0t23BF57wsz2XvDTdRfh629QA0lMkc3IN4Wp7I6BJFheyqvGhcmmfL6ikVKexIjbCsaC6iFs78ZtpqliUPKrqGbIEXGNbrEZiKrwHd2v7EKrNNam5GqV0XkfLjcas9YvfWwlYNr05xFb2iGyxVnKH3T/VDoWm7aWO6MPwnXG+t2tGETBbIJmEjbDosb6XIw1pAcDSqR2Bcc2Lh5SpzP5WZ4ljK7x08ffhVHrIbeQKvmOWmtrJeFADfQlWTNsCqs2GY1NTYvMpbPKiFUnN1UPrrI/Jr+GWgkQ6qxxgzzYaCZmdJggi3/AJKrSWgB1RhdBBINH2jtoDYanS/ZTrfyI6vQu2Wttsa8vaqDor2w9+sbZ8lfP5WJTLMNrrmFQpqn8Idjq6+Kr8QlECJc77IRnG2+THWoSRbsDpYrQhtiNlrXVs9S2uo+yjV9ihq5ZJgVcfqU8r2VokQgG8AmFuCY1wHnZQTirwz6V8ffx9ma3sdyddlmM5EDHxPxDYr4342RsMtW4fV0zqqtFsaF1CV+h6M2VZnUHkcauRu6X6VbJMa4QXuLUZRMkfJBMmrJ6Kc0dBbpeQQ5vd3nzbFM4jMYFEW2E4EvkLR8sIiVWYGYgHijRpgOw/TaWkQSNTrHmFYEtaDnp7E5jR1tJGoi3XqBsuhHyd6fssqrpK1S+y7D6QWBtc2giZ+NbnbI+SzFEZTPhbcOXFTMWNkWxAR0F+bbQycVQgvkr02jRhUaygVmpgLE4IoutaFvBAYwzVkANVcNbWLbSVmFza26A9F5B7Q4J2ABgxuCbRXIEeNHQJRJploW2fVtcHTNvhJxEgqsr5Ga4zty+lO+hT9LX+DwukRUB7ujb9VGyUMSY/SHRiWtDSPfti8ZcYvXqhcEOqgzWMmeWCQc4Wex/JesRmWS7EdR+CbhrC2EBavZMsQhpJNqwN7h3qoBMC9hJMa3iBe5QOmIey15PBgG2lr77b3XY3Dv/TdKzp9jHWBDV+61fVrGiTWHzYKhXD8HnhkThfcSXGqVLwEygWkWwqQcTCfl11hXZj6c+R3V80pOxAR1VmmD2T8qHmLFL86+De0Ag8agAATJLB3lrAeFSn8LGurafOjaLbbCqeMy9vUXVZgmujGnbkwG1WDajXpW0HZwEvZ00i2rM2DuYYMXrQRRBFlJv1Fi2oSmyFmLsZsnFp+mOiwlGUyab6adbt8NigKOOWVQr2VHjAtVVVkHksiUKcXfKynlPI1Fl8vqqp2WYlyRMAcRhM658PBbobRIIkXktHS5VeaAOJTkb5qUObPs6TF9Y6XUGP5WdPX2W0TijHJjeRQKJfqe1WGsh6D6Vopa0MWF5RYzGy6V66jFMS5gCHnWDUDAlWU9wLyc6pYskadVSkZONiZo1oSobVdppYwm8d1phCThD/pCQR6LSYtiY3wgRBBZF7VTNfdC+frC6J8fAXmbhZ7BBxQWOFML42ds6mMBIvQ6bl+X8pQMrW4uYiUW4jn8ONOKrqlWX82MpR0l0Mp2NK8+qdom7J9qixUwImeBQTsUHayrVcYza7xUSzmwFDUixsxb9WJbcpzRbEy2M5GbNDajXSYa0uwxFyNw0EkyPjoACAbXJAblcTGUZ6MEmAIBF5toDr2tFR8ourKQpkuA0yDNLJ8pJoZ6vs8I1E63azlKUS6ISBIFYUrCiUJY19ozO2NogVthYpcaZfZfIbr7VakVrNciqlsSxc8W1z19+LGxIWYMNexvtIlXSPYrCvvZlqr5wb4WgG5pClPFMLejLpHxh3LdfwVfvIbXeUlTOf4Lr+zFfjglI7NmOaTlYBsOCM/hzUplGVtdXoS+UqzYTFqpqhlR/ir0xGCNkfY8Fo0vXAECkiNjsEdUkTzKmDOdgrqn3eHYYTPB054n8dmJSx8Y4uR/AfE8HTflfjqTXaOaTQBEai4B1EQ6Ra19dFvhHib8rm4WpB9og0Y2iLWNrXMm+xCzfpjsbrjtrYC0oAKgyQ+t0bBXGOvv1GZCkTXRxiu9rCoe+8Shuuy1XjzmWqw1WZOqprYi5ttWrQJk0rpqwKA5EYrjdKr485sxV8Xy8TzOc85+D4pYj+7GfWPt9uVZ6i6n6m6Pvv2vWjT6wCkEllZRrAu9LAU05eWeyEma2IVUmPC16qTlgFXTlteEtjPP+x6apWZVbIubyGkBm22gweBQxWK/Yl411ECaLaroylC2u8eyF9Eo5zi2mULY5zCWM82PDfGPCXClRGKw78VUqOYAA0Pc5zuVrS1mUnSBIgLRw/h+Jo0c1bDOa5gcXPcxgcGAxMsnlGkzrYrJeORywzJtF1mavlZpOODzH4sSxn6Mq0fFmM4jH1izFeJ/BnH8j4vh+KXw+89/4o4+36ev4c+ipVqdemytTdmp1G5mHSWnQwYI+MbroQQS0iCDBFjfpaR8ly45x+OP7/8AVnj44/xx/Xj/AJPfOkjqPmEg9D/n/wBHzXLjnH44/v8A9Wf+TjE45zjHv75znGMfv9f9f2+uJHUfMKFy4445KJxxxwi6xgYjAe0M8Uc0S/GI3Cl0VEj2xxLE8RtovhOqyOJxjLGJwljEoxljHuOM4pt5T9K7p2m01kvVCkMcLdd2VPbjYMtswWkOjFN02CipcxFFsOZrQzdeZWOQ2oIqphaTBY0shNYXdLlQ/JbZO5NfYJCOrteG2EcbX3RbBdekvY2sGVhq4BcNQxpOokqwuGJMfWU1gsLm/wBBBf8ALjXbKFmN49XOF8NxGIZSZVe00gGvqCkDNRou8tcBlkkS0zpurMwrMY4Yeo91Nj5lzG5nAt5hAsLkCSdB2uKa/wDYcd5TCRr7N7Apzr/01a1wMa6uaDZtCPHPYZEMayWlMAsmYF13GML7AaKamUGK8mFqwyb1HxH7P1ihtmbZAYeb1xsemLzyzWjEpZc/uSSpTxtvMCEt0xRlMQYDUKrCZxNfMIzB+hFqHM6Me+PIQphdrwOs0sdrDC10hkgV9dMrpKaTCxKhznJd5tbBfZtq0i84lWfr6i/UDqKgyblYhGDYy9mx+UFuqjqR1Oy1GwiIxzuedQKzsdlmbyVBg9yCxh8jJNV6wvYLVEV+VVqxyooWA4uyLZH89d49XYWtfg8OA7KZOMYZbykODhhjYRNrGwG67/q1hYkYnEbEn1emYggbbeVjeLa9MrxL7VY06Frp1wM1OqaMChavcN2H1rPNe6vGZ+trRRWoNUVTnWjgFbw9gLVbMZWuGXVUjxmKHDX+E/Zhs5fUbIvAxXMewTK2xjZMM5dicgdlqsYPLbY7CWxldtLL1PAQuysmFivMq5YJI22f2p3YfpWhbLrGp1MmOz7htAR9KnVmGxJhta15kPrQ7K+9WcSUOK6KGebemvonXA5FWIBWzqJpiG6xRf2v5N3QoxZ14feRNZrhQ0L+tGwo7B1MUyqeunkSKXySjbg0pxdZteacgaQj+lufr9Wc3EJ5VZ6Q14EYPDiHFoHrzGmQQCQfVvI6ecaqw9GcILjFV7Cf9CmdQDYSRm21+JGmB3+EPZhCdUsk7RynVaRh1Mw1mwxarnr4aQLWg77iaPjQjVWPxbAawUMLV7nOZRsrquUkTwXh32fEwYhg+W2zCnk0W0Zg+ySQ2r9XI7zfnP40Zq0OUblmn4przLKk0m5jXA2NcJSDzsPyUshkFLru2S/GaJxv2SvqZstmvrp12WTbVaRizbQWnIncbAhcX0LRtrhGtlXWxvKrFnlui7n5FbXt2pibdrpOnLZN9usIgJpDylURXHWNtpDp2d2b9AOuWavsktXGWj4oshveCqzwGhliskqI+kVRrS92Fw+9vXmFziBNh6sZmCBBifK/MejOFN/WsRcgGKNO5zCdgR2Gtr6oP4wbfZ2XsW5t/oLwm1e/iBijGE1ucD75USPgwom9vkERskVkC6qDJNTSLPXUCWzOKj680i62Z+G/cTFeWMVti1oyZrFsT27Ax0IZFlVheTs9CuoduevBRb+wqJjsdRSxxYArCSiL1xN8brgpMjb/ACw26KeRGsvutrLmtauzKnUSGnsYy2g0UxwNa2uHqFCoZCDnFxmHnJutOALjqFbOVec92Z95F0v9WeIV7EvNHWad050uWoSjqzfbbCma52km0/EaSETI+y5cXRTaYbZrwYlZA9JQ95JHDfSGoHNacNh3EtMD1xroyiQyRhYJInex1jRX/VfCmXetYgiWgDhUiSDpAnS5Jk62lYh/2J3a0NRda1jYw7jGHY+t7svbGnsDiKoKAYCn2tbSWEy3ZxhOMH0LGF5QVExqaanwdkFZiCJl4ddoOZYxtZ2rvKJ2K85nIrZ7C14qIaIgQKjNux0CRqdXjrth2Ys2i0iD5SLUrhkeWDadj/nPyJM1LrpwHrMB9mPq3X801X6g5sCrDUG0Foai9bMJ16xWy2KgGSgcskXExoF3mq6pQJjm/Edf3ryvK+almg+n/DahYZfv+uGJB7D4ATMZuvHgSkTytaMbZZt+iYm2r6EtFf0YdTuBEzfSCsQ5wwmHkOMg41gIyEMJj1bSwgzcadpPozhRH7ViCIm1CmYgNttvAgHWZg2XU3PxO7h2fbcbANsikOqpTogNdRN5nzbTtUigsvZtB6C5iOJYPXumAIpt2a75O81QzrzIaewEct48ROyn+xuXusPwtXy/3QjcGV1V5OWVJ2G5ZapoDdSzsEi3VqCPwSMq6131VGY+2IwtVojDLtS7C77u3hZDsTXz9f05U0tXMrk2jtcwdZYauyXoSzifpyqwl7PY8AO5xWnsfy2wsGSNbQxLa4WTXYj/ALuSq7QtRBfkEut+7nAmaJq1myNlGu0yJs0Fonnb6UArAq5TJXxY2VWs6KBh1EGZEJpWUD0grCo2mMDRJyghwxYLW5uYguGFsQWgkD3gB3gejOFJbGKxAguEcClNgDpMCcwn43BWoBfDHsbJd8zWSOhI0I1eTbUlJj+KOlfrYuAi1ai03Ycnzo2yi6+7Z/xC34CTBlP0NQQA1wc53q7oXuvUH5lThBA9ZR2T0lfa9vdjTabEv0vfq9ic7ldQVs7AesJejkOp+HNIDclalVA1pyThLp4n4bx5PWSEFoQ0zLICk1Iss6qPGGCCioteqgYE27HGgl2/axJ0Rgr+ZhlrE1g+xXhRvdiD2w3XXdPbGzbqmsI+gORPtxRdeX14QyWK07KIRm8bbNb/AD6dB/ZpygVrrjDYRLMVNwidVcXCjRquyV1Z6QVxmqeoU3GmGvBGKH3SHCIwwIDspBJiIAMEyrN9G8K1zJxNexaQHUWC7XAicpBmQIG/UDXIHPjP2juCvZF9rrXtTXHbS82ZbVKwm3Y72kbnur1rTmYNzFOJo206mbIk60IU99dSzpWME9UASRyYmrxL7VH2GFVW6rL1N9TBjnZ722ylbApbRq08Kcx5zbj3MGGzhLGKo2TLLAJLrsmQ686ubMJGtimW8eUFdhutLNZfGq72m7C17MRp7xdsdHp0beAoBMhg2gkdCksBPQbrdNUr2mu3KUeRF62YuZVd2T5VRBdQx1jRaQHaQApFMTOfn0fRa8yfCmGszbaoOaG4qGtZC8Il0RTtm4qlhZMbRJL4Y9SsXl1TgtIec2X1ygSwvIdInC5pBbBDpLYAAvfeDXcrTWPKBpSdsAII4o1t0tfZT2t+I22JNW7G1gvZkDAfbFGmLtfXW2Nc66o/J79Y5BruVSzVaMMJBVWItsCJPLowcTj2KPRAYjY/SPjTsnWPaIG0R2/BOpqtUY6+v1oUtpChZAz5U8hDiW3FDWLCWH1D+/55+cjuZx+lXZxKRtOr2uyeRZkernq0doMXnSm7raNcF0Y29K4cKz9kiOA0pPmjZoWuyJ867ANWZ9DaAad9eEB8xdminc3jHtfbez7NvBXY6p0iVwUrqtYXH6hbrot8adh2ORTCy+3GbPxfIJSlYaBmywaQ6sZgCQwhbe0N0vActbxXCEtyzVFRxNelUc11MOIDWeqU9TTE5HNPMSSTC82LJbhqxD5kQeRzS4Zmt14rhEXAgjrqZtuh+wp+cfrh26x98/vZX/t9fb/3ft98ihL9kYVTLpLUU0yKNqqqsWnX3QrHMvFjiycGdeJyzinEpSjVCHvP2jjGMZz1l2Ni9MMAXJoiZduPggWMfYRjP4jd8XxzoJhXn3P38OMRx/J9YznOffumnmR1V2p3P1L1PofW+R4sWffCmzcjSGPYSzWFuojouzonttkz1tuuhbqQnXvbdfKDDA2kGNmwwQzNnIWm2cPrMDUdjz4Vgi3xLCUqeCxlR72POGp4h9J+DYwipSe5zw0VHloIFnSQCBGW8Bja1QGk4mpSABAe5ocHl3K4QJIF51sN1d35ez+veGSLP2/k5ioPlief/I9Ns/FnHrPuMfizj9vr1nj5W05/Rikl9s/fCdhmPv8A/NhtmP7/AHj37/TnwL3/AKV7/Wgbvqp6zyr2fZkkNmVd9drQVb92LqPdmlAdh9d2+Opeu9c6z2xpjbsNSJpSZlT2ppvSGxaZ2jriI7fRDmx2ztQKNqzTHj/5EbIsC7C2DRO6k/ZCrrTvetKDjtff3ZSB3dqHjON1mpwVjcxBtnZDlX9yjI2rwAttYbLaCTS4uLJHkbP6FpX/AGzxO4P/ABHFdr+0Y8p3jZefin3aU/8AZp627f5A+H2/+XtHr/tmix79es/hB/39/tx7b/eOc+sRlj3jOftj3nPrnkIhzS4GCZXLiaiRDiITEDIEsrsFmHHEZZuMKjZCUSpY/wAWEoyr9/F/3ufgA46x80Ri57D07q3kcr3TSFO8bP3At3PbndGrd9eRa3Qe/BWomqXN9hLTE6D2DrGwo9b0/atZpWaOr287o6QcFrfrF0Ih+5XWQzcLVepQ9gwyw+G60UDO/wAanK1x+LjINVpZ4a2ztInNlg6BGD5yvtlIvF0pW2ZzmeeFfw8YI4avTxePLhjcGwtqY2vVpvZVxFOm9jqb3Frw5jiL6TIuBF2Pz52ubSg06htTY10tYXAhwAIgjbXey2/xxxz6FedOOOOETlae8O6aOrdj1VaajaNgnqZ+wlYkpWlMxrUpqb6iz6VsWuCkuGTktmptv18S/iWjhgCGFG114stzDNm0HVdvJEM2FMuaEgDGBh3GAik2jDMYxrPposvqsnVWZXHFZEI5xG2v+RPEo+scyfG8LiMb4dWw+FE16hp5DxeDEVGkuzwSIaDYXOxGo9GFfTp12Pqew3NPLm1aQOXe6ouq8wdVtySTnQNxhsd1NGHI6bXtZJxdIAA02ymWwksVgjL8EWZX+qriMkYse0ULh50Ds7Rcp1zytQ7Xs4euKde2SrEJufzE1bKFgQGvR13UXGzOqsTHIOi4OTMFWdfYirLbszJKizWWHKoZKlYwboDqYK4gkLSNbEIKhGsi6hKvhO2uH1PquWcUesQz9YVicYYj8yN8o2fHGNWK+0F0d1ouaZdgaojDcZIpL/ExVIFJuCqAj1tZGLo0+8WYXtWQU84xjFwxxNZOLsWYzH4U+jPjhDgHVQ4shpd4i5wBAaGRyTDDmcJ0IF7GdT13CfwxM2w4kXMg3Ezt/XZVEo8xNcVLDLG+k7YkpDdfl5MvBS64ZYSQVQOx1hHgZe2sqGdvNayVs1dI0bka9YNEMhpU6tgr5lN/lIsmhq2hbrz4tOLvp+obFUWsULnylUBpjndCNt/BSzqSYrq1Sys+S9j9C6KVTsKWrz7ZhVk74r8aekqh5i1dcajUPYCStnVUgXVxkAWzg5JF+KFGLI13tK4H2ZjONn1EcZhZCv3VmUl0J1PMOa+ek67MGzY6dvmJJOuyNLaRoZrGf5o+m+VFmPXmVVN0IQhVRZaPXXAe66qyD6M+NnLBrWcCZ8SeQWmA4AZScxuZkibgbIMdhRaGG1nerCQZGozXGvyvran8/NpJaSMWFo23latcDm/LyS5MIwuOKtZUoQ16Eoisg+rYpUpZr2FdsRRoO5wYzEvVHVQy135Qkjo9T2TW+v8AZn6zatrda8JWVTraRraOgKRBGMgVrBpVL0cW2MXrh3F6ciotXdYzpFAuoJssFb439M31Sou6/wBXsqn+IYlDKUDH2aXFksMYlirFkIlXnmWSxXOPypE2YG+RHFeK8hn0317YOoFnriqQ2vn3tEY+VwOKFDIqM4ksFtWB8QDLIxbZ866nEZWZnmUsZliOcQ70Z8dzUyC85SA4u8QqOD2je8ODoPlbRBjMHMmNBH7O2/WbkxuI1OqrRrPlZqW2bQv1dOp2ciZztKom2ip1ySsIfYLFQihuZZBzMiApbdwImkGOMW2BNhfYyADFHturxmjzHRjjLqnmqbQG9OPbAVKVwOrnfVWCxItQkL5WvR75BbXTTitUSdSDAM6Fw7n8OjmiVlrk/Q/Vmvk2GI9N19QZYUMZMpcoXi35KDKoODIjOoeOa7BDhhzRfl4hAcumBFVcbcZnL1X9AdSkmhsidH1q9iv+HAJ1qRbIkWMM4lXiizI/8nNMveaJZxLNGZ2/JzX8675j9WfHQ6A6qWZbT4i7MHAD72QWN502JOsvXsKBbKTI1wzRbe8/1v8ANV3A8nsbAl3M1FqT1ex1JFLYaqdzASBrHQgjNIKdUITr7d0XXbAR4vIqmQONCUzqI5xZkU6iq8OARv2j0Z++f/Bwxn7Z9e/8TPr3+31+vvmoXXQPXDlUapwnrUQOp+nmciHAWtKKJzE+orFNiFbmjBg4dQJMo1ylIPMoQzXbGq6vt9u9+9N9DJg3fbnY2raMM1umIgEcscfjm1MY/Bn8J0/WAazdn3FzP5kPlJtVTOWtvxRxUHPOcc+m9HfB8TgW4sY9janFfSNDiVRiSxrWkPgvby5iQTAuQdoA8WMxFOqaXBJGUHMWs4UuIbsCSYIJE6THUnamAh8Z94HHxn7/AH+VD9v6/wDeft9/f9/7ePoRv0+mH/3qv1/+z9Pf39f1ftx75SrHk/3l2DZCPQnh92U2UW2WV0b75FbAr8ZNMJr+GOaShNZdLN476tpnGWZxwx6VQxsxHGI3xxPEo+KtT/uhW1Qsk37h8Ven6p2ylFfpnSvZ/crOqnOJxjXHat37Z6wV2Wwx8Eo3Y6+qrzLEvjHnDOI4+l9Vw/4NLr/pU/8AZdeLO/33fmP1V1voRv8Axcf3j9M4qr94z+zOM5h+uPX6+/3fpzoD6+lExXgVKrG+TeYVTkdeFR8kljKc2BFWKqIfLvPlOcjboZjYXKcpESszLPumUPH3zBu9zN8+NmHtl7zmCHxr6AAEjnOffqqp0r2crFeP/u2n22Zx692+/eec6+j/ADYWTlan84ULifw+oUdi+KOgOQsz/fZ/e933q43OM4+2fgLj6+2cYz6ziQYagP3NL+VT+fs6+aZ3e8T5uNv6/wCQrs/Qjfb0MP8A71X/AB/8j9P0z/DP+fnj6Eb/AMWG/wB5h/8A0/r/AM+f38pVJ1/dCtHmVey0nxY7+V0QxmMdM23szx424iMfi+KwJHuiru/TyjJRjHEBC9818Sy6f8o4WmGc49tHnPoWplCq/I/r/tPxPYX3ihfjfduvL/70txpMfUahe/8Ar5xvHSg2Lbf8GHTtO66o0KxmPpXXZn5WIGFw4/c0v5VL/Zvumd/vO/Mfqro/QjZ+/wBMP7/jVXnP3z9/f8j+P3/fn7/w5yiPVTnOa6a4fF/jShGEft+zGc4xjOfv/D1j9nrnpWNljoAJonYBNFjISg9cxXFUHAMASa43DmAmi2WjGCX1ThZSSNbbRbCUZQslHOM57+ce8es/t5YYei05m0qYcNCGMaR8WtBHTyJ6oXONi5xHQk/VQSH7in+vv/Pbr7Yz+vplf7x7z/mz/q5EiD7SBXMWhelvogUbbTda0LpsnWSaQTDM6YK7o12RhdiMsRtnH4sZ9SzjPvmSrwfoKr6/m5t+ecabmWY4j8OTCbCM14xjOf5NfzPgxnOc5liPxZ++c+u/zKo+FufhfD+JXxGExGEoPpE4d9MEioWF7H56dVrhNNhERBGsErqaoa+pDWvY8gxUDolswbOaZGZwuTr1WKfHtmfftQgz7+2f54M++MZ949/zN+z9fX6e8c8/M2z9PwhB6/3YM/8A8f8AdjH+b+GOTrJkvTrzmzY4NYrWBksGTJgTQEAvACosJMOOMJsqGEDEGqtIKKItroHorsuushXCUsdmq2q+qu+iyF1N1cLara5xsrsrsjiddlc4ZzGcJwliUJxzmMo5xKOc4zjPOv6Lqf8ANfE/5mH7f9N2/wAgQ44/AofKp/7FjHx7X9/5oQ/fOc5/ngz75z79+/5n+/v3nGffv7ZzjnIIZ7c2GNZirRRxQzaI4DOvLsnYVYJLGfVoQmK64RGl7ziU5ZzOOMRjjGc8ynjkfomX0nVcfj67aNalWFKrUoGm59JzXszhuHa4gOaDAcJIAmENeQQKNFpIIzND8wBBBjNUIuDFwY2hOOOOa64JxxxwiccccInHHHCJxxxwiccccInPXbbXTXO22cK664ynOc5RhCEIRzOcpSlnEYxjCMpSlLOIxjGUpZxHGc455ziOMyzn1jGM5znP6YxjHvOc/wBWOUN7GFO8vezNp6LFMuE8a+qGACvyFMXE3UXd3b6SGI8p8dRGYkoSo641lQaqd+QEgiYXbaS617puRGFOe1VkiL1W9zdu+UjAxL4pMgdB6XGLJVu/Ll6mB2evabw7ZjMgPF7RGsfwffYBk02h2d5bvAjqAQ6GfyRrXdFdTKSndXTvit0/0u0N29KlO2vtZ2P9PtPd3ZLczf8AubaYShDFw7TsLYM3twUsrIfMF0/Vo63oibEvpkGqqQ4wGjYFatATgBq1YYq9cvFHCABCHpEDCDEqgOKIIKPCocUQUeuscUUeqscYeuuiiuumuEI652Hu3qXU+xtT6i2TsHVUvZe8gEs9T0pi2HHfvQhosJZvBBnn4pYIioc5X12yqua/grrCqs2ShlgUi2liMY/pjGPf6+sfrn9+f35/jn788805r/kJ0ftn4NjWO19A2DOwp+vNgRYT7QqYZbou2rtjH6xcrsDE2ZKVb6Rp+10amfV7Fd3627pCtssWlRrxbY/Lnxm1JOpf7H3j1soTPK9evUsC9oAwOcNtBuzL0pdUq52SiGQRpW45vMtjWItG1TZDml4ICNkUMRWM45qFr390qkUPn7btDSQU2sa5te2vmN78HAqvWtG2QvTtvdF2Qsn8C7XdrBL11oRHE4jOaJr84kTjFecML8v/ABkA00PsAzu3r0bUTtb3HbxXFr2qMLNa672JJqW/uZB/L/EqgdG2XZEKTcbLwa5ayxbAjOYByvjnBFZHnWKDENoIFLHpJGLqsoKGvrhcOVRbXKq2gmiyMqiKbK5yhZTdCdc4SzGcJYz65XLUPMPxn34l6Jpncen7KRrOj29kP6lBBxclOlUq1zux8ZmAGI1DYTtlbPFHxZPmCxBKgHKoqmU/Gu+Y/i5tejFdl6/3r1uy0MLFMitpr2GipRTAkTYjhbZEEwolmkwPUtmJDvhXKkypE0mNO3AdvwkWtW/h/LrM5hufhxt1Xjxsl95DRp1dFaQ/8YewD7J13k17N0wOYvF0Rq0nTmN+/wDSR2g7Xgu7DHYht6GHyhL2F0r5Hw3vY2fUnaGoF9N+QmsrJunvWLhtQ8W7JrFZNQWex+nt3oDWL+0+sbTrxw73YC9TtGoHlhJezdN0Z6cAvN3J1v2Zovb2ort7642QDbNSbWG1Lni2JUQyrF5doJka8GDiEe6C6LaJ5lRHGZwz8GZR9SzgvfXRGvd460uFIYs9Q3vTW0ds6p7T1jAtW7dW70MNcKHtOsFFVWD3xuEIIUbRq7SsnWN81Q1tpu2rmKFsUPgi3rxyuvjt29sHYqXZdR7KWJtb7x6ieVaX25rqG++zXyGloFLTW+wNIwfOTK3rjtHXLhdu02ZsiDFGL3GkuS7tn03YMV2K4RU68/dNf7x4h95qELzXk/4fpLDbnq/bdUL3TU951LRPh3XbOrdtQL9o044rUu1USAzr/bJhPxCqta2BrimBErM0T2V4vdf7V1X0B1Z19ue80djP9W1URbbto2uWamMYsxcQRrqwZFe/2goUbWNcvU6qNedsLZgyHSVNGBcjTL4Q4+Vn/cv+R3/mG7h/9nWyc29qn+TGu/7hJ/8A0aLwin+OOOETjjjhE4444ROOOOETjjjhE4444ROOOOEWou++ypdPdM9mdmUjVHsNN0x66Sq7vm/A52OkKynWEn+Bxmz262S9SpxmPrOMm4z7xjHPR0B1XV0z1HpfX8zPxd0qXWH7jsVkrLCtt7D2IwnYuxdzPtuzmy5jt+8Nn+wmWz9Z+YxxVGMaqq4R1z5ofBjoRlK7/wCqV9leP9zP7+sYTUeQfVtriU85ziMaYL43zJlP+RgeNvx5+H37tRHMcYzn3jGPin/DHv4pe/8AX74RMzjj379/b9fUZZ++f0x9sZ++fePt+v3x+/nz6778b+9tv7E7Md9Pbr1pq6HujQNQ1nb3266/sbzetHc6Eq7JRpmfWsF1WE9dzIPsOB1DZiYIbpTpEWwXgbJjaLa0dVvMrx37m7K8kO5mmrdbtNhW754uouuOu91/vfaZtSzXNqmN2wLslEN/c9t6jsfXVM6NgTxd66q0pou7CDMrWG7KqhaRev152/4F9oolfcTjWdP0LcKK9i8XdX1RDpXj9pZJrnR9In1W47OYaxqTrunXlNvWRTRftGGPQbF4kVlNgzdkH2BuQVSCSRbl62/uZ+29Y9p9e7qn7eEZqOsOy9GqRVutWohsOw+O2pM+3NyG6o2fOrKtY1YRtrHZHYaHZdFaa2kXJE4euNltimuO1MPgz3cPBLcU7bsXa+lmnVy7YuxOyPIJkwW74h2g7WQtJ8ieu9O09zcJWBg8yOyaS+1Ura1OuAjgafsI+0bmguL1crbDtiorqx8Wdh37UPILTtm8UNnP3XcOpNjs0vs11sGu6fqtmuZ0vq+ro/phRoKbsNxFRsPXOxa81DA1Uy6OsdQsULzada3Jvb2Fca/6XcXiJ5HXIu9dQ681R4HoO8DeQGxqEKTYVHyhGHVeNrWeNGoa5rpG2KBPwfsZV2lCVaKo1AnHA6aBTbIwRQYrbpEW9U39z87Jq0ft3q9r2DqsdTI6f3LpHptyvq3CrcrtY3/uFh2862LscqqtfFRtKS8kPWU9+jGG4JyFbtuT1BpgqVbjY/8Ac1ux1+wsnYneVt9aL++/pmjDvF7PYGWz9dd5g91T3hv3PuD4ZpvmzduXNe19YjjY0e1rtadidQaOZsCIw+I9SeqW/wDin5dsNLc39S9d7Wm1XbdETaBsXV9arTOnKw5ap1r5QMk+96l1An7X2rVtcYlb1u6vRtw10Df64b2t3fVdjZMao6dbBZON/Gvug0vU86r47dmBAFdAdh6A9/BtU6j6Mt1xQXX27g8fqxOq7r2oDrHfdmb7Mkg46f22HaHTPkIIfr22bA+6y2PVLmCci+oXRnj/ANmdVd/doby1apm2gdgr9VvGwN2T3PIxe213qnp3r2+iXTjKd3SoFtzHrdqz/Pqqsbdi1rQNadOyGTvjqP2H/cxdq3XTUoFXZ1Y23a51T1V14pvrf9rqNMLI0DUfIrX2zN1pqLYh0DcpmV3Usv1/YmaJltOp1Jjbk7IEua3EcaedDdzNvH3xzQ7B0Nsr6zW9J7/1zVevdIYrdAs6g7m2rc1RXjb3u41hp3BsQXWIWm60E3NvTajvu/V+Pt76WtaIEalxQEryzXvBvc8+DPd/XHYOn6fvHbHau0u95mpD1dTrGwkbDT2G1ZJjdo26ncGUdz2T8OOJZh7cUZr5dStjUmtWjfT/AE+CL6UePWp7pofVqHVOwJDT2VVc0gTYJ2V2t2/GwYhiQUFO3f8AufNvYjy7NF+MTw7utqXYxBarn+HCjRju2M4yznGM/ePr3jOM4zj3j7e8ZxjOPePvjGcfp6z+mcZ58JN88TvKytqYImzuLjq2ndfJiGp9cdfutQ6xe6DoWxdokF6YIg38vaTR2F+x6V9PdpVT1DBZo1VIqk0OgO0qin6HeBGpbtonjbq2rb5o1fXLJY93TCfWMqhdUKp1YzamhuuHMevVW69i6z1kzagkfXm9eaZuTXTdbuuzDWhNeWlVaynIu93ALX1v5FdBdzASiGH2AeV4zdl4riRmDNTtArnc+nmhkISxRYRqPZqVhrKa234ciidv7FVH44k4jXcDGcZxjOM+8ZxjOM/vxnHvGeU/84tkD0/pRHsxIb9jYj798YGQa7U9cc7htR5IvkN1tdMXXtU1wJjsOxMphQMzhUlAMYXixKlWPOuFvI3HnN1vj7Y6g8x/XvPr14UeUOcevefXrOOr/WcevXr19vXCLa3lZ/3L/kd/5hu4f/Z1snNvap/kxrv+4Sf/ANGi8+enfXl1qe+9Hdy6PrPS3mKbse5dUdjapr4Vnhh5MBwMebHpzpMoEmYb1qOEJAlgaPTIo0gcMeM83FEDj12XQ+iWvDXBoUghNeaiBVCwe+qWcZzXdQCPVbDOY5zHOYWQlHOY5zjOcZ9ZzjhFMccccInHHHCJxxxwiccccInHHHCJxxxwiccccItWd39bC9wdQ9k9YFFRX43rS9h1sVrKMpZSs2S6+lO+qxDMZ/PQt/oHI+Y594vArz6l6+HNU9q8kN62Hwhb9k6OobDeQDJbX07HVES1Iz2DTPJhjtg/TbtQOm2EsPXjL9G7LJPdjU7Gas11miUiM2ZwevsPreX/AM4xnGcZxjOM49Zxn74zjP64zj9uM8ox2qA/8ZezH3kdpqV5svTm+4Bs8oOvdZXlu3aFmnXDJlPkro+tgVEsdgaINaXgav3PpiQa95tuhJ9d3HWBGG19eka9u5F8sB/LfzJbP2e6uXPZfWfXRyanr7Yj2iDSthXaBtOodZdH194STdbJ+smbxL2b1ywad2d1LNuedi7Z1vsuuaPsurrdXczVLR7e+580++S+tds1LrTuHayewGwxtHRmx7Vr3We5bHu2naiL5Up9q7cSN9J1eGi9m6QldaZ1gWfvqNYIqNHH1qbFeqP3Wuh5+gXVtn1jeNdR7bqD5LtOr7IpAe67sWvMw3SJ4kaDwLVuEzZfdeEyWHi2QJBODutHvqniyqefv6n/AJdf2/k4/k4ziP6/ycZzjOcY/dj7R+2Pt/Jjj16jH0RfnGC80fI7be5FZOy9qNOsNDbY2dppemHvtE6fIbMdR6/6M10jVg126dLb477ecN+0IdovwOvk77QW5ge0CYH2X8JoWDh5vrvkL5amajr+8dk9gdvrtv6R2ILUuxtE0PSNNRad3szR+Dku+1jgEFx167c4S9ydj40qNENe2VDhAduO09bWlr2CT5y79Afw4x9vv6/rz/y88/DHHv1j9ffv+39f8+c5zn+Oc5/XOc8Ivg1o+/8AmRsC+BT3uXu3We0kumdWdUbVo7rTuoA9VD7iYeVlPj3svYJadHpWzh0k7HoGdf7PwHr3Yr7R1MtoIYgSsV14oEkOr/KTu3sPyf8AH+O69mk9dde9kqLy1HVTPc9I6tk5eahtuOoNpU1attPS21bf2uU+33Utse1a3r+89dlKETlSyHZHqzV+cfdTEI4/SOP+j939X7PX6evt+n24xGOP0x6/f+v3/r/f+77/ALPt+mPXCL4VdLeY+2qO3jqu3fIJTBKz8m+9Osmus7F2B149H1cJNPu4DqLWiNKT9SaG/wCoTXTDWNaE1xLs28dht99uBqICfU5uuDYVs1vzR8tROo9Oc6z3Cw7B2nfemu4EDsihvoHds9A8g9jT9SmdXO3FumdI9YjdeT0WQPZc2fVGxqth/CZbCJDYdjffKXfR/pp+HH/XOfXr93r369fw/T9n6cfDj7fr9v0z7z7/AM/v3wi/NkL5keV6nQdx7K1ntdl2I20vZPJ/Y+0etwGGjdrbVrvXek6r2Kt1Opd15qvS+kbB1Fs+lup612Ar0XYHnZDfsBNob5RF2RYxgOTeTSw/Jincv9j+R/bOyZq8yVGkV65vK7qGnVJ9bi9daH2RtINxYXWWu7A312nXzOzkSNchMzsDbameitCdoBW6S2wf9aMxxn9fef685z6/b7x7z9s4z98Zx+mfXr16xyu/fvfq7p8JHr2uIrex+7OxbWCjp/p5MfUG93Z0BCn8QatWEqSatN6w07BIzLszstqPJNp6mdI41Tnb3Gras+IsD7MMx2l5NdMdRLq4GqOmY2eSHad3wzsHXGzX7FoPRmrlWVxlXFlsGzsd07AEpnKudIfVMCp/yDwcXXFxjGMYxj7YxjGMY/hj9OV+8d+mjupdXcG7g+q3Xt/sh9dvfc2/1h3g07VvJwQa6I6JeVeSQk0LTEK5To/W+tWEX2ItMQrKzyDnxbtowsFwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJzjOEbI/DPHvHvGf1zjOMxziUc4zjOM4zGWMZxnGcZxnGM4zjOOcuOEVHdh8eOx+mdiedjeHrfWkY2wtS9h3/AMZ96IPX9I7+7YFSMc7No7lSA3bePvZLwi4kxy91ZDsfXO4tLyGm7dZF7OwJ3gbJNH8z+r2uxq+t+2gtg8be5GVmAw+se86VuqX7MdGVVM/72G/jMGHVncQdt85ZHs623N81ro+GTlEkLzYBTb7mJbtoWk9k64w0/sLUdZ3nU20YwaaxuCFTs2vMoQlicInpXYhy0r4J4xOGbhpzrnjE65RljGcEWV4lHP29+s59+sZ9xzn1n19sS9Z9fx9es/s5y5SCvwc1fSswn489zd/eNtFEb8jat1/2DXunVtFts82YqE6g7xT9q9foF/8AiVyX6Up1CmNMI1jyGz7lz1x0f+6CamLUOj798au1KKs5z/8ASX497xoOxXxxmXqonYusO5ztcxKWM4/2SL10NGHr19FPOfi4RXj45R8Rx/dHaY5iZ114TsZYzPEbh+6e/UsZY9/yJZHv6FfSjn1j+VX9VP75+1nrHuXXyR/dJWN0qvwnwi02iXw/LP8AzF5BdlW0fr8Upp8ax1PWX69Y9Qw7D9+8+549e8kV585xjHvOcYx+/OfWP9fMS3ff9H611thuXYe4avouoqYQsabRuD9TrOuroWS+GuRrp2WCtHxOX8mGLCYznL1GEZSljGan2dFeXm6X5l2L5oZ1FWQNmspL419CaN10TjMs+8107h3I68iH1PxYzmFha4RMXmOIzGkHZiWZ5PpfhD49arsaze32sN+4ezFGaLV/Z3f227J3hu6ooefzKy9ZP7HYvVmjW4ziMcV6Am1QWEY4jUPD174RYNb5L9o99Ziq8PuvrSdaOjiu7ye7sQbBqnSq8acoYsY9a6KZnXuz/II6Ec2yX2JKNC6pYfDC+Pb9+I/h5W6OkPHLWOnin+3sXb3s/ubeago9j9277MEredwgBO65ekGqWBr9f0jQEV5JMtV6z0ZSi0vXpXkn1rTdiYOH7Sw0Y4jj1j+3Oc5znP8AXLPvOf7c85cInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccIv//Z"}}},{"cell_type":"markdown","source":"# Data Overview","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"../input/feedback-prize-effectiveness/train.csv\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T15:45:23.085476Z","iopub.execute_input":"2022-07-19T15:45:23.086082Z","iopub.status.idle":"2022-07-19T15:45:23.250850Z","shell.execute_reply.started":"2022-07-19T15:45:23.086038Z","shell.execute_reply":"2022-07-19T15:45:23.249523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are **no missing values** and all columns are of data type **object**.","metadata":{}},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T15:45:23.253489Z","iopub.execute_input":"2022-07-19T15:45:23.254031Z","iopub.status.idle":"2022-07-19T15:45:23.294701Z","shell.execute_reply.started":"2022-07-19T15:45:23.253993Z","shell.execute_reply":"2022-07-19T15:45:23.293649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T15:45:23.296065Z","iopub.execute_input":"2022-07-19T15:45:23.297059Z","iopub.status.idle":"2022-07-19T15:45:23.391677Z","shell.execute_reply.started":"2022-07-19T15:45:23.297015Z","shell.execute_reply":"2022-07-19T15:45:23.390499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Duplicates\nThe first thing, you can notice is that there are not 36765 unique `discourse_texts` as one would expect. Instead, we have a few duplicates, as the number of unique `discourse_text`s indicates with 36691.","metadata":{}},{"cell_type":"code","source":"def highlight_duplicate(val):\n if val.discourse_text == \"Big States \":\n return ['background-color: yellow']*len(val)\n else:\n return ['background-color: white']*len(val) \n\nduplicates = train[train.discourse_text.duplicated(keep=False)].sort_values(by=\"discourse_text\")\nduplicates.head(10).style.apply(highlight_duplicate, axis=1)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-19T15:45:23.394505Z","iopub.execute_input":"2022-07-19T15:45:23.394952Z","iopub.status.idle":"2022-07-19T15:45:23.423560Z","shell.execute_reply.started":"2022-07-19T15:45:23.394915Z","shell.execute_reply":"2022-07-19T15:45:23.422546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You can also see, that for the same `discourse_text` the `discourse_effectiveness` can differ for different essays.\n\nThere are 10 discourse texts that are duplicates in the dataset which have difference `discourse_effectiveness`.","metadata":{}},{"cell_type":"code","source":"true_duplicates = duplicates.groupby([\"discourse_type\", \"discourse_text\"]).discourse_effectiveness.nunique().to_frame()\ntrue_duplicates.columns = [\"nunique_discourse_effectiveness\"]\ntrue_duplicates = true_duplicates[true_duplicates.nunique_discourse_effectiveness>1].reset_index(drop=False)\ntrue_duplicates","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-19T15:45:23.425090Z","iopub.execute_input":"2022-07-19T15:45:23.425514Z","iopub.status.idle":"2022-07-19T15:45:23.444881Z","shell.execute_reply.started":"2022-07-19T15:45:23.425477Z","shell.execute_reply":"2022-07-19T15:45:23.444039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"duplicates = duplicates[duplicates.discourse_text.isin(true_duplicates.discourse_text.unique())]\nduplicates","metadata":{"execution":{"iopub.status.busy":"2022-07-19T15:45:23.445959Z","iopub.execute_input":"2022-07-19T15:45:23.446242Z","iopub.status.idle":"2022-07-19T15:45:23.466495Z","shell.execute_reply.started":"2022-07-19T15:45:23.446219Z","shell.execute_reply":"2022-07-19T15:45:23.465532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here is an **example**:","metadata":{}},{"cell_type":"code","source":"def get_text(ids):\n with open(f'../input/feedback-prize-effectiveness/train/{ids}.txt', 'r') as file: data = file.read()\n return data\n\ndef display_sample(essay_id):\n text = get_text(essay_id)\n discourse_text = duplicates[duplicates.essay_id == essay_id].discourse_text.values[0]\n begin = text.find(discourse_text)\n end = begin + len(discourse_text)\n label = duplicates[duplicates.essay_id == essay_id].discourse_effectiveness.values[0]\n\n ex = [{\"text\": text,\n \"ents\": [{\"start\": begin, \"end\": end, \"label\": label}],\n \"title\": f\"Essay ID: {essay_id}\"}]\n\n displacy.render(ex, style=\"ent\", manual=True,\n jupyter=True)\n\ndisplay_sample(\"331CA007D0AD\")\ndisplay_sample(\"F52B9A0882BB\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-19T15:45:23.468160Z","iopub.execute_input":"2022-07-19T15:45:23.468524Z","iopub.status.idle":"2022-07-19T15:45:23.488120Z","shell.execute_reply.started":"2022-07-19T15:45:23.468492Z","shell.execute_reply":"2022-07-19T15:45:23.486722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**This means, that we should probably take more information into consideration than only the `discourse_text`**. A few notebooks in this competition additionally use the whole essay text.","metadata":{}},{"cell_type":"markdown","source":"# Exploratory Data Analysis","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(24, 6))\nsns.countplot(data = train, x = \"discourse_type\", ax = ax[0], color=highlight_color)\nsns.countplot(data = train, x = \"discourse_effectiveness\", order = effectiveness_order, ax = ax[1], palette=effectiveness_colors)\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-19T15:45:23.489692Z","iopub.execute_input":"2022-07-19T15:45:23.490424Z","iopub.status.idle":"2022-07-19T15:45:23.854380Z","shell.execute_reply.started":"2022-07-19T15:45:23.490398Z","shell.execute_reply":"2022-07-19T15:45:23.853496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = train.groupby([\"essay_id\"]).discourse_type.value_counts().to_frame()\ntemp.columns = ['amount']\ntemp.reset_index(drop = False, inplace=True)\ntemp = temp.pivot(index=\"essay_id\", columns = \"discourse_type\").amount\ntemp","metadata":{"execution":{"iopub.status.busy":"2022-07-19T15:45:23.855702Z","iopub.execute_input":"2022-07-19T15:45:23.857123Z","iopub.status.idle":"2022-07-19T15:45:23.909289Z","shell.execute_reply.started":"2022-07-19T15:45:23.857084Z","shell.execute_reply":"2022-07-19T15:45:23.908260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"On average, one essay has 8 `discourse_text`s.","metadata":{}},{"cell_type":"code","source":"sns.boxplot(temp.fillna(0).sum(axis=1), color=highlight_color)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-19T15:45:23.914429Z","iopub.execute_input":"2022-07-19T15:45:23.914692Z","iopub.status.idle":"2022-07-19T15:45:24.033781Z","shell.execute_reply.started":"2022-07-19T15:45:23.914668Z","shell.execute_reply":"2022-07-19T15:45:24.032779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp.mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T15:45:24.035194Z","iopub.execute_input":"2022-07-19T15:45:24.035713Z","iopub.status.idle":"2022-07-19T15:45:24.045694Z","shell.execute_reply.started":"2022-07-19T15:45:24.035677Z","shell.execute_reply":"2022-07-19T15:45:24.044526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(16, 6))\nsns.countplot(data = train, x = 'discourse_type', hue='discourse_effectiveness', hue_order = effectiveness_order, palette = effectiveness_colors)\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-19T15:45:24.047241Z","iopub.execute_input":"2022-07-19T15:45:24.047915Z","iopub.status.idle":"2022-07-19T15:45:24.370687Z","shell.execute_reply.started":"2022-07-19T15:45:24.047861Z","shell.execute_reply":"2022-07-19T15:45:24.369758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Effective `discourse_text`s tend to have both more characters and words than ineffective `discourse_text`s.","metadata":{}},{"cell_type":"code","source":"train[\"discourse_num_chars\"] = train.discourse_text.apply(lambda x: len(x))\ntrain[\"discourse_num_words\"] = train.discourse_text.apply(lambda x: len(x.split()))\ntrain[\"discourse_num_sentences\"] = train.discourse_text.apply(lambda x: len(tokenize.sent_tokenize(x)))","metadata":{"execution":{"iopub.status.busy":"2022-07-19T15:45:24.372216Z","iopub.execute_input":"2022-07-19T15:45:24.372940Z","iopub.status.idle":"2022-07-19T15:45:28.339189Z","shell.execute_reply.started":"2022-07-19T15:45:24.372902Z","shell.execute_reply":"2022-07-19T15:45:28.338206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=3, ncols=1, figsize=(16, 12))\nsns.boxplot(data = train, \n y = 'discourse_num_words', \n x='discourse_type', \n hue='discourse_effectiveness', \n hue_order = effectiveness_order, \n palette = effectiveness_colors,\n ax=ax[0])\n\nax[0].set_ylim([0,300])\n\nsns.boxplot(data = train, y = 'discourse_num_chars', \n x='discourse_type', \n hue='discourse_effectiveness', \n hue_order = effectiveness_order, \n palette = effectiveness_colors,\n ax=ax[1])\nax[1].set_ylim([0,1800])\n\nsns.boxplot(data = train, y = 'discourse_num_sentences', \n x='discourse_type', \n hue='discourse_effectiveness', \n hue_order = effectiveness_order, \n palette = effectiveness_colors,\n ax=ax[2])\nax[2].set_ylim([0,15])\n\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-19T15:45:28.340736Z","iopub.execute_input":"2022-07-19T15:45:28.341413Z","iopub.status.idle":"2022-07-19T15:45:30.600286Z","shell.execute_reply.started":"2022-07-19T15:45:28.341368Z","shell.execute_reply":"2022-07-19T15:45:30.599115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Copied and edited from https://www.kaggle.com/code/cdeotte/tensorflow-longformer-ner-cv-0-633/notebook\nplt.hist(train[\"discourse_num_words\"], bins=100, color=highlight_color)\nplt.title('Histogram of Train Word Counts',size=16)\nplt.xlabel('Train Word Count',size=14)\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-19T15:45:30.601937Z","iopub.execute_input":"2022-07-19T15:45:30.602347Z","iopub.status.idle":"2022-07-19T15:45:30.901942Z","shell.execute_reply.started":"2022-07-19T15:45:30.602309Z","shell.execute_reply":"2022-07-19T15:45:30.901017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Edited from https://www.kaggle.com/code/cdeotte/tensorflow-longformer-ner-cv-0-633/notebook\n\n> From the histogram of train word counts above, we see that **using a transformer width `MAX_LEN` of 256 is a good comprise of capturing most of the data's signal** but not having too large a model. (Note that analyzing a histogram of train token counts would be better but we don't do that here).","metadata":{}},{"cell_type":"markdown","source":"As discussed in the [Duplicates](#Duplicates) section, it might be a good idea to have some additional information to the `discourse_text` as input. If you use the full essay text, we would need a `MAX_LEN` of probably around 6000 according to the following histogram. That however it not feasible. A lot of notebooks in this competition are using `MAX_LEN = 512` when using the essay text as well. This is something we could consider later during the experiments.","metadata":{}},{"cell_type":"code","source":"# Copied from https://www.kaggle.com/code/tanlikesmath/feedback-prize-effectiveness-eda-deberta-baseline\nessay_len = []\nfor file in os.listdir(\"../input/feedback-prize-effectiveness/train/\"):\n with open(f\"../input/feedback-prize-effectiveness/train/{file}\") as f:\n essay_len.append(len(f.read()))\n \n\nplt.hist(essay_len, bins=100, color=highlight_color)\nplt.title('Histogram of Essay Word Counts',size=16)\nplt.xlabel('Essay Word Count',size=14)\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-19T15:45:30.903315Z","iopub.execute_input":"2022-07-19T15:45:30.904060Z","iopub.status.idle":"2022-07-19T15:45:33.050795Z","shell.execute_reply.started":"2022-07-19T15:45:30.904022Z","shell.execute_reply":"2022-07-19T15:45:33.049808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## POS Tags\nSome writing and editing tips can give us some idea what good writing. Here are some examples:\n- A good introductory paragraph catches the readers attention. Some easy methods to do so it to use quotes, questions, or statistics/numbers. For the latter we could check if a \"Lead\" is more effective if it contains a number.\n- Some editing guidelines recommend to keep the amount of adjectives small. We could check if the amount of adjectives in a discourse_text has an effect on its effectiveness.\n\n\nPart-of-speech tagging (POS tags):\n- ADJ - adjective, e.g. new, good, high, special, big, local\n- ADP - adposition, e.g. on, of, at, with, by, into, under\n- ADV - adverb, e.g. really, already, still, early, now\n- CONJ - conjunction, e.g. and, or, but, if, while, although\n- DET - determiner article, e.g. the, a, some, most, every, no, which\n- NOUN - noun, e.g. year, home, costs, time, Africa\n- NUM - numeral, e.g. twenty-four, fourth, 1991, 14:24\n- PRT - particle, e.g. at, on, out, over per, that, up, with\n- PRON - pronoun, e.g. he, their, her, its, my, I, us\n- VERB - verb, e.g. is, say, told, given, playing, would\n- . - punctuation marks, e.g. . , ; !\n- X - other, e.g. ersatz, esprit, dunno, gr8, univeristy\n","metadata":{}},{"cell_type":"code","source":"%%time\ndef get_pos_tags(x):\n tokens=word_tokenize(x)\n tags = nltk.pos_tag(tokens, tagset='universal')\n return Counter( tag for word, tag in tags)\n\ntrain[\"pos_tags\"] = train.discourse_text.apply(lambda x: get_pos_tags(x))\n\ntrain = pd.concat([train, train[\"pos_tags\"].apply(pd.Series).fillna(0)], axis=1)\npos_tags_cols = train[\"pos_tags\"].apply(pd.Series).columns\ntrain.drop(\"pos_tags\", axis=1, inplace=True)\n\nfig, ax = plt.subplots(nrows=len(pos_tags_cols), ncols=1, figsize=(16, 30))\nfor i, tag in enumerate(pos_tags_cols):\n train[f\"{tag}_ratio\"] = train[tag]/train[\"discourse_num_words\"]\n sns.boxplot(data = train, \n y = f\"{tag}_ratio\", \n x='discourse_type', \n hue='discourse_effectiveness', \n hue_order = effectiveness_order, \n palette = effectiveness_colors,\n showfliers = False,\n ax=ax[i])\n\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-19T15:45:33.052100Z","iopub.execute_input":"2022-07-19T15:45:33.052560Z","iopub.status.idle":"2022-07-19T15:47:38.114766Z","shell.execute_reply.started":"2022-07-19T15:45:33.052492Z","shell.execute_reply":"2022-07-19T15:47:38.113817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- As expected a lead is more effective if it contains a NUM tag\n- Contraty to our expectation a discourse_text seems to be more effective when it has more ADJ tags\n- The higher the ratio of PRON tags, the less effective a discourse_text","metadata":{}},{"cell_type":"code","source":"# Delete train that was modified for EDA purposes and start with a fresh train dataframe\ndel train \ntrain = pd.read_csv(\"../input/feedback-prize-effectiveness/train.csv\")\ntest = pd.read_csv(\"../input/feedback-prize-effectiveness/test.csv\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-19T15:47:38.116248Z","iopub.execute_input":"2022-07-19T15:47:38.116808Z","iopub.status.idle":"2022-07-19T15:47:38.260086Z","shell.execute_reply.started":"2022-07-19T15:47:38.116773Z","shell.execute_reply":"2022-07-19T15:47:38.259071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing\n\nUsually, when approaching NLP problems, you would apply some preprocessing to the text such as lowercasing, removing punctuation, removing stop words and so on. \nA lot of the Notebooks in this competition however don't apply any preprocessing to the text. \nThis is because those Notebooks mostly use a BERT variant.\nAccording to the answers to [this discussion post](https://www.kaggle.com/competitions/feedback-prize-effectiveness/discussion/335030), \n\n> **BERT handles a lot of the pre-processing for you**!\n> No need to do anything apart for maybe converting to lower case (depends on what BERT version you use). \n\nIn the pretrained Huggingface BERT variants, you can see variants with the ending \"cased\" and \"uncased\". We will be using an \"uncased\" variant, so we don't have to take care of lowercasing as explained in [this StackOverflow post](https://stackoverflow.com/questions/62466514/shall-we-lower-case-input-data-for-pre-training-a-bert-uncased-model-using-hug)\n\n---\n\nThe only preprocessing, we will do, is to label encode the `discourse_effectiveness`.","metadata":{}},{"cell_type":"code","source":"train[\"label\"] = train[\"discourse_effectiveness\"].replace({\"Ineffective\": 0, \"Adequate\": 1, \"Effective\": 2})","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-19T15:47:38.261547Z","iopub.execute_input":"2022-07-19T15:47:38.261921Z","iopub.status.idle":"2022-07-19T15:47:38.290623Z","shell.execute_reply.started":"2022-07-19T15:47:38.261859Z","shell.execute_reply":"2022-07-19T15:47:38.289769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model\n\nFor the baseline, we will be using [【Tensorflow】FeedBack BERT-Baseline by IMvision12](https://www.kaggle.com/code/imvision12/tensorflow-feedback-bert-baseline). Please make sure to upvote the original baseline if this notebook is helpful.⬆️\n\n## A Note on Efficiency\nAs the notebooks title mentions, the baseline uses BERT (Bidirectional Encoder Representations from Transformers). One disadvantage of BERT is that **it is very compute-intensive at inference time** (see https://blog.marketmuse.com/google-bert-update/). \n\nIn this competition, there will be an \"Efficiency Prize\":\n> We are hosting a second track that focuses on model efficiency, **because highly accurate models are often computationally heavy**. Such models have a stronger carbon footprint and frequently prove difficult to utilize in real-world educational contexts. We hope to use these models to help educational organizations, which have limited computational capabilities.\n>\n> For the Efficiency Prize, we will evaluate submissions on both runtime and predictive performance.\n\nThis is just a thought for you if you want to enter the efficiency track as well: **BERT might not be the best choice for this competition**. \nHowever, in this notebook we will continue using BERT as this is intended as a starter notebook for NLP beginners and BERT currently seems to be all the hype.","metadata":{}},{"cell_type":"code","source":"# Configuration\nBATCH_SIZE = 16\nMAX_LEN = 256 \nDROPOUT = 0.1 # 0.2\nLEARNING_RATE = 1e-5\nEPOCHS = 1#8\nAUTO = tf.data.experimental.AUTOTUNE\nMODEL = \"distilbert\" #\"bert\"\nMODEL_PATH = f\"../input/huggingface-bert-variants/{MODEL}-base-uncased/{MODEL}-base-uncased\"","metadata":{"execution":{"iopub.status.busy":"2022-07-19T15:47:38.292163Z","iopub.execute_input":"2022-07-19T15:47:38.292526Z","iopub.status.idle":"2022-07-19T15:47:38.297893Z","shell.execute_reply.started":"2022-07-19T15:47:38.292489Z","shell.execute_reply":"2022-07-19T15:47:38.296909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Experiment Tracking with W&B\nBefore we begin with the modeling, let's setup Weights & Biases (W&B) for experiment tracking.\n\n**Disclaimer:** I am a W&B Ambassador\n\n![wandb_logo.png](attachment:b2e6faf9-375c-4bc8-857f-c2bbcfc59678.png)","metadata":{},"attachments":{"b2e6faf9-375c-4bc8-857f-c2bbcfc59678.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAioAAACCCAYAAABo4OMZAAAACXBIWXMAABYlAAAWJQFJUiTwAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAB5pSURBVHgB7Z3f0x1Fmcc7K6WlBWWCW1Ckapek1sJyaxUCf0ASrzdouAt7AXgRvIug1yZ4jQlcolX+usA7QdlrAveaILhqUmt+rEVYXTWEYDAu5Oz5vGc79Pu83TPdM3Nm+rzv91M1lTfnzJmZ7ukf33766ae3zeY4IYQQQoj6mP2dE0IIIYSoFAkVIYQQQlSLhIoQQgghqkVCRQghhBDVIqEihBBCiGqRUBFCCCFEtUioCCGEEKJaJFSEEEIIUS0SKkIIIYSoFgkVIYQQQlSLhIoQQgghqkVCRQghhBDVIqEihBBCiGqRUBFCCCFEtUioCCGEEKJaJFSEEEIIUS0SKkIIIYSoFgkVIYQQQlSLhIoQQgghqkVCRQghhBDVIqEihBBCiGqRUBFCCCFEtUioCCGEEKJaJFSEEEIIUS0SKkIIIYSoFgkVIYQQQlSLhIoQQgghqkVCRQghhBDVIqEihBBCiGqRUBFCCCFEtUioCCGEEKJabnFT84cfLY7rl5z76E7ntu91budh5z5ym1sp3r/q3P/M0/H7+fHB/O9P3OPcHQ8796l/dUIIIYToxrbZHDcVb33buUvf3vg5nfxnn3crxa/mouS9sxs/R3TdddgJIYQQopjZdFM/WFBiIgWunV1YJlaFP70UFylAGq/+3AkhhBCinOmEyp/+vfn7d1eoc29Ly7WzTgghhBDl1OtMi8/HqtDmT/PBCqVFCCGEqIjphMptDzR/j5/KqvDxlmdtS6sQQgghoky36ofOmyPmv8HqnzsfdoOBPwwri668Ord+3LoQFnc97tzH7nKDcOehxfTP3y5t/I57SagIIYQQnZh21Q9TIr87vnBG9dw679R3H5uLlYFEBCLl7Fc2igjE0D3PDSdWuP4Zcx/S8umnV2+ptRBCCFEHs2mFiofO/fpbC/EwlHDwnH08veoGSwdiZUhwnPVxVCRQhBBCiD5UIlSWBYLhtf3N59x3UoJCCCGEqJPZ5g6hn7Pa5v13nRBCCCHqZHMLFaaSmqwly5hqEmIC3n77bffKK6+4CxcuOCGE2Exs/k0JdzaEr9c+PCIBHf9TTz3l9u/f7w4ePOh+8IMfuNrgGZ999tm1Z9y9e/fav0888YQTYkoQzI899thaeeRfiWfRl+k3JVw2dxxaBI97y4TrZ/+dndqDR2yEhpVGNmxgX3zxxbX/Hz161E0JncCrr7669jyvvfbahu+3b9/uhJiKF154wT300EPrPkPkv/zyy27fvn1OiC5M60zLChmWJhPfBDHBNAzTMYiLoWOPsLLIr8jZvm94B1pWFhGrhT1/fFqGjtciRgGrxDPPPBP97vLly5OKgW3btjV+/+ijj7rvfe97TgwDwvAXv/jFmvWKd79jxw537733uvvuu8/t2rXLiQ+hK7n//vujAhrhj1gRogOz6SwqsZ2Tr11diIm3X1lMy/zD14YTFAggjqFBlFx8avHMIT4tBIKbcAdlGo0u0wF0xoyOQr761a+uNdqlHDlyxH3pS1/q/Ex0vo888ogbi1hDG3435ciQe/MMdJxiOfgpNcRqUz7zLhCFUwuWIaZXqO+IL0QY6eoqxlN1B8EnRFemESoxkWJZi/T61vBxTobm3Nfbd0f2aZ1ArHgny1JijS8ipcu1EBp9nmlsYUCDnXq+qTulkydP3vybToFOKuwcNPXTDzp8fJJ8npKfiGTfeVMHEDCcRxnBN+j73//+qELaMpQTNdOJHuos05yl5Z3zY89CnRKiK+NP/RAp9pcP5p+PVYWpoBph2urCU/nnI7pGDqePKGCO+Pz58+7ixYtJkUADEx6MrEIrCHj/CBoiDq5lR1A05v4ad99991pD/sUvfnFdgxc+Ew2/v569hjev8/sxGzqeZc+ePRtG01iGUlNCU4FJPXyfx44dm9yPZpWhvPqySNlDGMY6azry0MGa86aytCEwwnpk65PHDhiAMs65KUtIaXk6ceKEe/LJJ9d9xnQllqcpxZxYaSYI+EbHHobMb4NQ+p8rOH9M3ngwvr9PCqazdh1zU2M7N+ja0Nprdb0OougnP/nJ2t9T+4EAjTfTU74Bp5Gl0a6NqYUK1gcv6OikVnnkzCqv8B0jpFMWBdKMmPWCgDIfWrumhGdCcFmamnrSg+AhD6zIKS1TWJgQcVyH/LNTv0IUMoGPSts0iYXpH8Lr1+aQiv9JiUiB0rQvCRoNK1QQCaUCw5u/Q2iguggV7/vSZ358SGhgrY+OWA/vP5wuWHW/GTpYD+WwadqDMoqFwgsb7zdUS9ktxaeHdIdTX0AauWauRYTrxKw3QnRl/DgqpZ17198sm5yotxZEVwUwlWIJO5xcYlNIXa5DoxiOTMVqYB2rV9k/JiyDkFMOQ+sRImUzODinBDqWFiGmYnyh0mUVzy0V7sXTxcJTyZ5C3ockxPuclBALgtbFeRe/F8/evXudWA1i/kmrSheR8clPftJtRmgbrFCLWU+FGIvxhQq7CpdA5/7xwt/EYBkxS4g5rg9g2WgLzx9jiHQMRGzO2PuI5NDUcJVcB7wVho5OFpXV4fTp026zkiNcNpNFyRJrH7qEJhBiCMYXKqVh67cPMMImEBsrjX779cXxywMbI9V2oTQtf3/A1UJs+qdkxBSeax0ow7n+NkILjETKarGZOmprYcypC+E0Zy2+VUNhrUU44ip2j5iK8Z1pP3VgESMlx7EUq8XOx10vECm/+9bGz4ltgkWkz9JnArm9/WqeDw3WlIr2FkJc0LCGjQ8Nb25UUz/tw3VwnCMYnMeLjxzhEU779FkZwDREGPtiSOuMX8LZZ0WL94Hw+R0uBW+7N0cXB8nYtcKlqH4JeJcO1l8rpG9HbZ/Pv8euz1gCeRHWB56hqQwjxkMxI+fROOSjDzfQFV8m/LsJwxcMRarsDTl4it1jyHRMWX+WzmwKrr85m71+YDb72QPpg++vnZn1puk+p/fNevOXM+1p+Y9D8zRfmtXGvBKyXnHdcf78+dbfcY4//9ixY7PLly9vuM5cuMxyeOSRR4ruHXLy5MnZkSNHZvNKuOH+/ph3IMXXDa8/F083r196nfnUSOvzzRuq2TPPPLPu2uQnn4XvJ4V9h7yPWDo4L/YcfBb7TewacxG7di55Om/8NlyLe6SOtmvHyqK99lwczJYJZdbeMwbvivfmz+Md10YsD0ug/IW/3bZt2+yFF17I+i3ll3P9O929e/eslJy6zXd96jdQppdd9trKN2WpbztVQ/1ZIjemESqeN5/b2MkjHs4fHaZjRxA1CQiOvw50n/PH4kKINL7/zqxGbGPEwWdtULn9+XTGYCsKlS8H3+Hlng80hLZTaTtyOmN/bc6NdcQlDQnXiDWsvuOONVb8JtXgpGgSKrybtgYsFHRNhB1zlyOV17nPF+ZTbodZCg1+W32wIgWhXSMlZSjGfGp4g1BpK/8pcbFnz55ZLn/+85+L6zZCqLSTP3fuXHHZ4z4lZa/0Hn7QUlMaKmFioeLBKvHOzxb/Dtmpc602oTL0/UiHT0vlhJYRf7SNfsFXjlBcxEQPDVfu/ds6yvA3ttPkebgXHR/QQSOm7HlN98jp1P3127ANLc9h84JrHT16NLuBSZESKrH30XY0jbiGFiqk316T/Ajfox+Zh1a33DLaFXuvsBzzb9gJN1lSODfn6GMNaKKkDFli7UKThdRa/+yxf//+rPvS8YZlAnFk8yhVt+mAc+unvQ/vlLLHtW0bYtP12GOPZd2D31rBRvsTlm/+jpW3nEFVKg2x+oNVuEsaKqISobJMzhxOi5TfHJ5tdWyFp8A3kRIXsemftgoXWmZyVH5MpFA5m863lpHUM5GW0s62LU1epDR1RidOnNhwHxp+rhMeKWJCxVpzOIfP/LXoYGPCg7zKxd6jxCIGtoFue//h8+aK2i7EBBR1IhSfMeEZEuvoXQdx2Ieu5Rdsx9YkAm7cuNGaxhyhwnVsWW6aMqKjDoUAoibHGsF9wvTxO28VTp0f3ueJJ55ovcePf/zjdenYsWNHY3k5derUhrQ0nc8zhfWnNA0SKjWS8odhWqZCv5GxiZlZmypJ2BHb80qnf0Lzcs5oqESkeOg07MgmJhz86Ck25VPS0NtnzOmIcvxMcn9rfSdS+RoTfam8idFHqNiOPMdCEpaVZQoViE0BecFCmWsrqzGBnLresszwXcpvzNeBaZumMuE7fn6XsrrlCJXvfve7G37XJAq4r227cu7z8ssvr/tNW6fNfcJ32VY3YwKqSUSknqtNpIXnHjx4cNaWhvC9SqjUivchQbBwDOUDswmINcpNlTE27eOJTTc0NXK+Qud0VH06RvvbHEffmI9JGzYv26xTHptvJWlL+bu0TbvFnjdXWEGf92HTmyM8wt8sW6ikfGfosHKnF/x1vBNy+OyIk2VN+XhigiE29eQtbDa9uU7WFtux5wiImDWlzaoAVtxguWi7jxU3bWm0v2k631prSkSBzYOm9Fsr7JBpqJQb48dRmQKWOe866tznfro42Bjwo5XtHTQRsaVrqRgSbTFPYnuBxKLX+nv45YZty/M4z8ZmKdkg0D5XTpyXLsv5wqXWkLvs0Ma06RIl2ON3/C0NAR/ee9l0uUe4vHWIpdopnn322bUN/Siftgyw7JN9cHLxy1vnHejNz/wGfctMQwo2sLTHvCNdS7NPL8usCVHAhoxdN7fsUndsmYhFx7XY8htbMm8hXSFXrlxpPJ+dnz//+c+7HMjDMLYOv/3GN76R9VvODduBeefsfvjDH244j89t/KKLFy+6tmvnpqFWtoZQEUn8OvuQUESEhJUwFjAuFncgJXrCTj12LXvfsAHiPiVbxttOIadB6xIePdbYdqWrUGnbTC8kFt9iDKFise83BmWERppjGbtDk9903MQD4m/qBJF3bTmjPLOrdgl+75yhY38Mjd9BmcEF0aW7loXScm87dITd8ePHW38XE0RtzxyKRmDQ0lbXEHNtZY/v7KCspC5Cl6CDkFN/ctJQM9MLFULan318ETn21w8vgsGtKkS7JR0cpGlF0hILtNa04SANRCo4m/2c68Qqkb9+TlAl2wCUNvZ2TxqYokOukSlG9jHhiEiIvacxoCzs2bPnZpmkDGOV4jnnU04byhuflWzS50X51JGXSZM9EFFYJ0mzD3hHPmBZwbJEBzdGXeE+86myNXF47ty5XsEfUyCIbNnjnvfff3/vNJJvYTtlLSQ52GeIPRPXvffee9d9Rhq+8IUvbO42bTYll56Lr8b5z6/NVo4Lx+JpIY2VkxOwLTyHediSa1lv/NxredqerY1Sh2Gwq3dyqoq9T64vBc529l65vhB2br/Uf6Pr7/v4qJC2VCAvnmfMOA92lQ9/27xPOR7nBsrz5+c4VQ5FLG9ziC39zU0r4A9hV9DlLk8ugXdkfVRy/Fqs06ot+zm+XTnXzXmWEOLH2HznGqlzl5GGirkxfgh9z/VLizD2MbCyYI2oKOR8I396ybk/vhT/jjTefqDbbssj4ad/whEt1pMTJ06s+7+nabTjLSShRYbfMjfvKQmbHxtlc21GYG340eFYe5TYkU6u6dam0Ydz36yQNszPsSkU8ozD+yhQbpY5XcJUTzgSxT/D5r33+8HqEpYlLBF81zQN6f2hap/28WBJwRLACD0sl94nbMppA8oFbYcvI12gTPHOsYpZeFccvCvygPNyLY7Wb2Teua5Z3VI+eiGl7RTlMycNpJX3NYXVdHBmU/HfzzcHYhvaqkIEWh+M7a8Dr/jhWZvSQlorJ2Z1CEeWJUuJY6t/wt+EMQDaRpmppaJ9j7b7drGoxKxJOaPoPlaRVbSoeHIjkC4r/LctW21xZLD0xJ4v9Wzh0vixw5fHnrOE1Ai/bbQ+tEXFR4qOxbaxK2xyrRg2Dklb2ctJc2k03ZyjKaJvyT1z0lA5E1pUPrja/P37Ld/nwuaH+I7YTRA/cY9zdzw8jtXmg4HSskQYQViFjkMdo0WUPn9Dzi6x/CbcpBAYWXirireo5IwyY/OuWGH6jk6XMbr1Tr7hKIpRlXemjGE3t4NVdHbrAhY7rFDkUdP8uh9Bcww5QrSrv9qse3zP/a1/CmWddKSc0tusLjWC0yn1NbR6zTsM981vfnMUXxvyjnwO64avX7wHnoEVPKGlNxf8PLCc8V7afI18ucPS1FT2YtYQykUfy2hTOScN1J+77757bdVWTv1pS0PVzKYCy0aTFeK/np71JuUDM7QPyZst97l8crYKxEI+Q2hdyB0Z2pG6j5VSGjY/ZtkYY3TaxaICscimqfl9G5KdgxgJJayyRcVDmSDdOWH62yL9lmDzPnfUGRvJci1rPfPpmWIzuFjelRLzhcBq0WRR7WtRie1fk7II2MBnpX4h/hpYV3LKXmpPoViac/ZGGgruQyDCPmmonAnjqNz2wMKqEYO4J3c+7HqB30jKByaEc/7wI9eLOw8595Hb4t+Rlu373Cqwd+/edf/3o5Vw1GLPSRFb/RPGYYEcr3i7nBBq9m5nBOVXjHiY32cFBaM3RvGMgHwci3AkxgjJWqK2AuQV6WaETN4xak6NRHn3Nt+6gP9F12vwnqxVgWsRY8WXTd4zf6+iNcXDO4i9h2WtziLPWIETrggkr3PjAnWBesl9KXtYPpusapyD706s3MTyaSzfOMoYbUxYf1I0paFmpl2e/E9Pb+zE6dg//XS/gGxMtVz6Tv75iJU+0zOIlHueWzx7yK1zMfaZ59yqQAcQ4oVFOO2TazZMBX8LRU/O9It1UIXal+F558swr3hmGhOcgOmUQ8FGvrIscyuKFAt54TsOb563kJeIvT70bajp1Gz59SLKT1tASWDCGhmrAyY/qRv+2gxQqENj1glECs/RVPb4zpY9pmGYgrFYB9sxyKk/sTTUzrRChY4dsfLZ5xf/0tkTOfbj97hesGrob5fyz0ekXH7V9QLrEM9OGoiCS5oQKSsUATcmLsI56pLYBrH4KIgU759CI58jemIrYLrMS4+N97tgXpgGlw6Lv/3B/D8NCTEQ+H4VVoSMiY+SmoqQGlvx0JeSjoXno1OLBcxDrKy6NcWTaz3oA+/4ySefvPl/738xVZ3gvVH2Tp06FRVKfuAWYgdU8+mKDZGqx8SngZg0sfqTsxqpJuqITEsnj2WF6aAhuHbWFfPeGTcIpOFTB9LTWhUTExehmTd32scTi1LrG74SU24sVHbX5Ylj4Kd4/DJcv0wQYeIPOloaks28DHkoEHm2w8iJLtxELN9LBbC3nKXeYZMT9SqQ2sphaAERCwpZ2tYsAx8ht6lN9MS2IqE8TT3Fguij7bGC+UKPbTqmYHOG0P/gXVfMCqzMGYOUgOgSB6JpNFkStTFmySmJDBrSt4Nrg0bXm/tLQ2jXQG7e9Gnk6ABoQDlyrxPG4RmCmKWOd1fqf9EkVsKpjKkhr0uJ7TWTs+qvBCwPr7/++obPl1FvuJcvd7fffnvWbzg3p+zFtiLBWtp1iiVVbrqmIdYWS6hMTZfpFutfskVJjWS6hLT2QYdilFhUYs6VoR9ALj5U+xDOmCnCqbIprCVj+e/0yb8wX/o4ZvbpzGIdC3QRF1wnNkVQuolhTTAdY6fXUh1eX2L5nVOO+5T1kgGL9T+J1evUBoS0UaVlnLwnuOCXv/zlxvNIQ+61bV2J7fNVM5tTqHSZQhpq2mnFSY2YSvetCK+X81kTPhKjBctFrljxvgNU7JiZdghsw1G70y90fUabf11Nybl+IfY5h1gFEpu75/2V7m9DGUz5zORGUa6J1MoQLAs5QqUk72L778wiG/xZyHOe0VJSBnN9SKwYSA3aKJO2XJKWhx56KFtQYJ3z+/bEHHQhzK/cNCyj/ozKbLNy5nB7DBV/vH5gJj7ExgToGyPDmbX8du+fXIgc6gojL/rIlj5mRm4cDhsnJKeqxNJKXhLRlOdrO7rsBZOKV5OLjb2Q+/tY9OHYOyD/bX6H98yN22LTOdR+QKnonjwj8U9SMUNIk42Yyt/zzjx6vdL9qfoQi5Cc2jcmhPcXK/cl74mYIvZdtcVRie2/w/PG2gnO9e1Azp5EYdnj2cLYOXPr56wNG6G3LT4KcVlS+1iRh6k6Tt6H0XJTUWlJQ/g8cxE8GzoNFXJjWqHyv+8sAq4hKt6Yi4VfHZqXrKPDhLi//uZsdnpfu0jhnOsDh9RfcWyws9IgYhbbcHXdnC21OZw/aCC4lz9iYbeb7k0Dz/c0kPaZfSNIB5mq5E2b7ZUcbSHjuT/PwXPa+/F/PqfhS3WyfM73XX/vrxHr3L2A4Bm5Dp9xzTDPwrzlu6YgXXxn3wXBrYakLZw69ydcO/UgVq582n0aU+LHX4dyRB1ry+NcwvKAULKh5cPyaw9/fqpe8X5yghCSDv8MsXrXVKZi4iZWp8OySuCyWHC4HTt23BwYkC7O8/e0nTzfNbUHoSgK87CNU6dONbYDtp2y5zYFZbNpIL1N9adrGipjQqHy++ebhcQQkWn/cmZhLWmypFw7MxPrsZ1Q330iwlFaH+sMUIFTDXHTwX2bGqUXX3yx6HqpXZ9p1EufLXXQMdqGPdWgpw5reSjdk6TJcpF7LWvFCveNso13KAhijf3QIsXDdbu+Jzp7+55y95LpW7cgJyJp6UHekydtI+/SfW4YzcfKFKIjNx1YaPxzNe2GzHHw4MF1zxqzytqyxzkxAV8SNbokPTZtbZF/Y21ATv0h70sjX1fCREIlJ7Q9B9aVvrw/t9r88acLqw3CCHHym8OLsPd8NyRYiNgagPvx7wrjKwOFvS+lYfNzYESamgqKNbhtI9dYyPymo2mKJDccfM5h86tUqFjLjJ3WK/29pa2TouGM5T2dVYngHGNjNcpprsCgXCFQmjryHPHT1boY0reskRauwfsgTSWWnlj4+KaDzjJVpmIWEvucMWvA8ePHo+enOn3qem49ynnPTeS2BW1W1L5p4N2uYOh8z421iUs3JtcvOffLB/PP33VsnI0D+0Lslt9+fX2gOVYSEQDuY6sT9M3jYxvgNNllxY/FxxTAkXXIOAw8o19WeuXKlZufcw+CME3lNEYcFZwoybscB7+mkO7LDCE+BLwDnB9Dhz3yvykUvsfHxMGp1jr84TTo3+GYK6iansmXq1yH7KbyibPkEHVrs0F+4SRaUp58Pl+8eHHtnJy6z294L0O85zZ8ebJOtVyflZZd28Qa688SmI0vVC48tdiHJxei1P7z865qEF+//rd4LBbEChFrxZaAxgKBQuNBA4HIyIVGjOXNNpgdK56I1CmEEFuQ2fjLk68VRoB9b26puP6Wqxo2NUwFjMPCcvXnTmx+EBrEP/BCgwi0JTCqillPVmGZsxBCLIvxhcp7HcLbl+zbMwVtaZJQ2fQgJgju5advMOl3Dahk43us2k6nQggxJOMLFXYaLuWWDr8Zk7Y0faTy5xe9IVS2nVPvyipFjBRCiGUzvlDpsllf392U4f2rC4fXZVg3tu9t/n7HPic2N31CwVusBUUOl0KIrcwtbmxYwVMiFoZY8YMPyaVvf+hHgoPrnYecu+OQGwR2S756Ku4kfNfhbnsPiZXCios+0zU2fHjX7QuEEGIzML5FhU4916qCoNj5uOsF4uF331rv7IrPC58hYIZi19HFwZ5BPPetDyyWJu887MTmx071sDy5i1jBMhPuG4O/iqaChBBbmfGXJwNC4cxXmp1k8ev4zHP9p33eeDB9H+5xX/7yUSFSsNKHTQ9DHn300bVlxbkxDPBzYaNFL3AQP6dPn3ZCCLGFmU0jVADxcOk78ekSrBG7j/WfMuEeb7QEl/uXl1YyIJuoD2Kg2F10sYYQByUWPMoHnELkMN0TWmAIbsW1VjxQkxBC9GVCoeL54P+dXBEVWDiYOhlqlQzXfm1/8zlYVLQqRwwEFhG2oO8KwoZrIFSEEELUIFSWzdnH0867qxD1VqwcWEqwhhAGPGc1EFYTgrwdOXKk6lD5QggxAVtAqGCtQazYyLFYURApWpEjlgjTOX4fH+tci0BhOkjOskIIkWQLCBVgWok9hrCsIFCwpPzj14aJzyKEEEKIZbFFhIoQQgghVpEJNiUUQgghhMhEQkUIIYQQ1SKhIoQQQohqkVARQgghRLVIqAghhBCiWiRUhBBCCFEtEipCCCGEqBYJFSGEEEJUi4SKEEIIIapFQkUIIYQQ1SKhIoQQQohqkVARQgghRLVIqAghhBCiWiRUhBBCCFEtEipCCCGEqBYJFSGEEEJUi4SKEEIIIapFQkUIIYQQ1SKhIoQQQohqkVARQgghRLVIqAghhBCiWiRUhBBCCFEtEipCCCGEqBYJFSGEEEJUi4SKEEIIIapFQkUIIYQQ1SKhIoQQQohqkVARQgghRLVIqAghhBCiWiRUhBBCCFEtEipCCCGEqBYJFSGEEEJUyy1OCCGEEGI4Zm44Zv8H/wAs69kXwxIAAAAASUVORK5CYII="}}},{"cell_type":"code","source":"!pip install --upgrade -q wandb","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-19T15:47:38.299310Z","iopub.execute_input":"2022-07-19T15:47:38.299898Z","iopub.status.idle":"2022-07-19T15:47:47.881491Z","shell.execute_reply.started":"2022-07-19T15:47:38.299847Z","shell.execute_reply":"2022-07-19T15:47:47.880355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\n\nimport wandb\nfrom wandb.keras import WandbCallback\n\ntry:\n # Setup user secrets for login\n user_secrets = UserSecretsClient()\n wandb_api = user_secrets.get_secret(\"WANDB_API_KEY\") \n\n # Login\n wandb.login(key = wandb_api)\n anon = None\nexcept:\n anon = \"must\"\n print(\"If you want to use your W&B account, go to Add-ons -> Secrets and provide your W&B access token. Use the Label name as wandb_api. \\nGet your W&B access token from here: https://wandb.ai/authorize\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-19T15:47:47.885392Z","iopub.execute_input":"2022-07-19T15:47:47.885694Z","iopub.status.idle":"2022-07-19T15:47:48.692202Z","shell.execute_reply.started":"2022-07-19T15:47:47.885665Z","shell.execute_reply":"2022-07-19T15:47:48.691172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = dict(competition = \"Feedback Prize Effectiveness\", \n _wandb_kernel = \"iamleonie\",\n dropout = DROPOUT,\n learning_rate = LEARNING_RATE,\n epochs = EPOCHS,\n batch_size = BATCH_SIZE,\n model = MODEL\n )\n","metadata":{"execution":{"iopub.status.busy":"2022-07-19T15:47:48.693853Z","iopub.execute_input":"2022-07-19T15:47:48.694225Z","iopub.status.idle":"2022-07-19T15:47:48.699322Z","shell.execute_reply.started":"2022-07-19T15:47:48.694187Z","shell.execute_reply":"2022-07-19T15:47:48.698335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"TODO Experiments:\n* Experiment 1:`discourse_type` [SEP] `discourse_text` with `MAX_LEN`= 256 vs. `discourse_type` [SEP] `discourse_text` [SEP] `essay` with `MAX_LEN`= 512\n* Experiment 2: BERT vs. Distilbert","metadata":{}},{"cell_type":"markdown","source":"For the baseline, we will be using [【Tensorflow】FeedBack BERT-Baseline by IMvision12](https://www.kaggle.com/code/imvision12/tensorflow-feedback-bert-baseline). Please make sure to upvote the original baseline if this notebook is helpful.⬆️\n","metadata":{}},{"cell_type":"markdown","source":"## Tokenizer","metadata":{}},{"cell_type":"code","source":"tokenizer = transformers.BertTokenizer.from_pretrained(MODEL_PATH)\n#tokenizer.save_pretrained('.')","metadata":{"execution":{"iopub.status.busy":"2022-07-19T15:47:48.701258Z","iopub.execute_input":"2022-07-19T15:47:48.702110Z","iopub.status.idle":"2022-07-19T15:47:48.754617Z","shell.execute_reply.started":"2022-07-19T15:47:48.702071Z","shell.execute_reply":"2022-07-19T15:47:48.753669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Get the `sep_token` from the tokenizer and create the input sequences from `discourse_type` and `discourse_text`.\n\n> `sep_token` - A special token separating two different sentences in the same input (see https://huggingface.co/docs/transformers/main_classes/tokenizer)","metadata":{}},{"cell_type":"code","source":"sep = tokenizer.sep_token\nprint(sep)\n\ntrain['inputs'] = train.discourse_type + sep + train.discourse_text\ntrain.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T15:47:48.756255Z","iopub.execute_input":"2022-07-19T15:47:48.756666Z","iopub.status.idle":"2022-07-19T15:47:48.791692Z","shell.execute_reply.started":"2022-07-19T15:47:48.756588Z","shell.execute_reply":"2022-07-19T15:47:48.790483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's have a look at what the tokenizer does with one sample:\n\nThe function `bert_encode()` returns two arrays: `input_ids` and`attention_mask`\n\nhttps://huggingface.co/docs/transformers/main_classes/tokenizer\n\n> `input_ids` — List of token ids to be fed to a model.\n\n> `attention_mask` — List of indices specifying which tokens should be attended to by the model (when return_attention_mask=True or if “attention_mask” is in self.model_input_names).\n","metadata":{}},{"cell_type":"code","source":"print('Sample input sequence:')\nsample_sequence = train['inputs'].iloc[0]\nprint(sample_sequence)\n\nprint('\\nTokenized sequence:')\nprint(tokenizer.tokenize(sample_sequence))\n\ntoken = tokenizer(sample_sequence, \n max_length = MAX_LEN, \n truncation = True, \n padding = 'max_length',\n add_special_tokens = True,\n return_tensors = \"np\"\n )\n \nprint('\\ninput_ids:')\nprint(token['input_ids'])\nprint('\\nattention_mask:')\nprint(token['attention_mask'])","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-19T15:47:48.793394Z","iopub.execute_input":"2022-07-19T15:47:48.794101Z","iopub.status.idle":"2022-07-19T15:47:48.809560Z","shell.execute_reply.started":"2022-07-19T15:47:48.794058Z","shell.execute_reply":"2022-07-19T15:47:48.808412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def bert_encode(texts, tokenizer, max_len = MAX_LEN):\n input_ids = np.zeros((len(texts), max_len), dtype=\"int32\")\n #token_type_ids = np.zeros((len(texts), max_len), dtype=\"int32\")\n attention_mask = np.zeros((len(texts), max_len), dtype=\"int32\")\n \n for i, text in enumerate(texts):\n token = tokenizer(text, \n max_length = max_len, \n truncation = True, \n padding = \"max_length\",\n add_special_tokens = True,\n return_tensors = \"np\")\n \n input_ids[i] = token['input_ids']\n #token_type_ids[i] = token['token_type_ids']\n attention_mask[i] = token['attention_mask']\n return input_ids, attention_mask #token_type_ids, \n","metadata":{"execution":{"iopub.status.busy":"2022-07-19T15:47:48.813197Z","iopub.execute_input":"2022-07-19T15:47:48.813519Z","iopub.status.idle":"2022-07-19T15:47:48.821313Z","shell.execute_reply.started":"2022-07-19T15:47:48.813467Z","shell.execute_reply":"2022-07-19T15:47:48.820322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**A comment on token_type_ids**\nThe copied baseline [【Tensorflow】FeedBack BERT-Baseline by IMvision12](https://www.kaggle.com/code/imvision12/tensorflow-feedback-bert-baseline) uses `token_type_ids` in the model.\n\n> `token_type_ids` — List of token type ids to be fed to a model (when return_token_type_ids=True or if “token_type_ids” is in self.model_input_names).\n\nHowever, according to this post https://www.analyticsvidhya.com/blog/2021/09/an-explanatory-guide-to-bert-tokenizer/\n> Token_type_ids are 0s for the first sentence and 1 for the second sentence. Remember if we are doing a classification task then the token_type_ids will not be useful there because the input sequence is not paired(only zeros essentially not required there)\n\nWe ran below experiment and saw that not incorporating the `token_type_ids` makes no difference in the model's performance. \n**Therefore, we `token_type_ids` from this model.**","metadata":{}},{"cell_type":"code","source":"%%html\n","metadata":{"execution":{"iopub.status.busy":"2022-07-19T15:47:48.828108Z","iopub.execute_input":"2022-07-19T15:47:48.829003Z","iopub.status.idle":"2022-07-19T15:47:48.837844Z","shell.execute_reply.started":"2022-07-19T15:47:48.828965Z","shell.execute_reply":"2022-07-19T15:47:48.836925Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The fundamental idea of this baseline is to get the trained BERT model and fine-tune it to our classification problem.\n\nhttps://wandb.ai/jack-morris/david-vs-goliath/reports/Does-Model-Size-Matter-A-Comparison-of-BERT-and-DistilBERT--VmlldzoxMDUxNzU","metadata":{}},{"cell_type":"code","source":"input_ids = Input(shape = (MAX_LEN, ), dtype = tf.int32, name = \"input_ids\")\nattention_mask = Input(shape = (MAX_LEN, ), dtype = tf.int32, name = \"attention_mask\")\n\ntransformer_layer = (TFBertModel.from_pretrained(MODEL_PATH))\n\nsequence_output = transformer_layer(input_ids, \n attention_mask = attention_mask)[0]\n\n# intermediate_layer = Dense(512, activation='relu', name='intermediate_layer')(sequence_output)\nclf_output = sequence_output[:, 0, :]\nclf_output = Dropout(DROPOUT)(clf_output)\nout = Dense(3, activation='softmax')(clf_output)\n\nmodel = Model(inputs = [input_ids, attention_mask], \n outputs = out)\n\nmodel.compile(Adam(learning_rate = LEARNING_RATE, \n #decay=1e-6\n ), \n loss = 'sparse_categorical_crossentropy', \n metrics = ['accuracy'])","metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-19T15:47:48.839569Z","iopub.execute_input":"2022-07-19T15:47:48.840513Z","iopub.status.idle":"2022-07-19T15:47:57.920458Z","shell.execute_reply.started":"2022-07-19T15:47:48.840454Z","shell.execute_reply":"2022-07-19T15:47:57.919500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T15:47:57.921934Z","iopub.execute_input":"2022-07-19T15:47:57.922503Z","iopub.status.idle":"2022-07-19T15:47:57.943935Z","shell.execute_reply.started":"2022-07-19T15:47:57.922465Z","shell.execute_reply":"2022-07-19T15:47:57.942937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"TODO: Verify if this is a good idea.\n\nFor the validation, we use GroupKFold with 5 splits based on the `essay_id`.\n\nThe BERT authors fine-tune with 4 epochs (https://github.com/google-research/bert). \nTo be in a similar range of epochs, we will train each fold for 1 epoch.","metadata":{}},{"cell_type":"code","source":"X = train['inputs']\ny = train['label']\n\nkf = GroupKFold(n_splits = 5)\n\n \n# Initialize a W&B run for logging\nwith wandb.init(name = f\"Run_{MODEL}_{DROPOUT}_{LEARNING_RATE}_{EPOCHS}_{BATCH_SIZE}_no_token_type_id\", \n project = \"Feedback Prize Effectiveness\", \n config = CONFIG, \n anonymous = anon):\n config = wandb.config\n\n for i, (train_index, val_index) in enumerate(kf.split(X, y, train[\"essay_id\"])): \n print(f\"Fold {i+1}: Train Set: {train.loc[train_index, 'essay_id'].nunique()}, Validation Set: {train.loc[val_index, 'essay_id'].nunique()}\")\n\n X_train = X.loc[train_index].values\n X_train = bert_encode(X_train.astype(str), tokenizer)\n\n X_valid = X.loc[val_index].values\n X_valid = bert_encode(X_valid.astype(str), tokenizer)\n\n y_train = y[train_index].values\n y_valid = y[val_index].values\n\n train_dataset = (\n tf.data.Dataset\n .from_tensor_slices((X_train, y_train))\n .repeat()\n .shuffle(SEED)\n .batch(config.batch_size)\n .prefetch(AUTO)\n )\n valid_dataset = (\n tf.data.Dataset\n .from_tensor_slices((X_valid, y_valid))\n .batch(config.batch_size)\n .cache()\n .prefetch(AUTO)\n )\n\n print(f\"Steps per Epoch: {len(train_index) // BATCH_SIZE}\")\n\n train_history = model.fit(\n train_dataset,\n steps_per_epoch = len(train_index) // BATCH_SIZE,\n validation_data=valid_dataset,\n epochs=config.epochs, \n callbacks=[WandbCallback()], # Add WandbCallback() to the fit function\n verbose = 2,\n )\n\n model.save_weights(f\"model_{i}.h5\")\n\n # Validation\n y_valid_pred = model.predict(X_valid, verbose=1)\n print(f\"Validation Log Loss {log_loss(y_valid, y_valid_pred):.2f}\")\n","metadata":{"execution":{"iopub.status.busy":"2022-07-19T15:47:57.945133Z","iopub.execute_input":"2022-07-19T15:47:57.945917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = bert_encode(train['inputs'].astype(str), tokenizer)\npreds = model.predict(X_train, verbose=1)\n\ntrain[\"label_pred\"] = preds.argmax(axis=1)\ntrain['Ineffective'] = preds[:,0]\ntrain['Adequate'] = preds[:,1]\ntrain['Effective'] = preds[:,2]\n\ncols = 4\nrows=2\nfig, ax = plt.subplots(nrows=rows, ncols=cols, figsize=(22, 10))\ni=0\nfor n, t in enumerate(train.discourse_type.unique()):\n \n j = n % (cols)\n try:\n y_train = train[train.discourse_type == t].label\n y_pred = train[train.discourse_type == t].label_pred\n y_pred_classes = train[train.discourse_type == t][['Ineffective', 'Adequate','Effective']].values\n cf_matrix = confusion_matrix(y_train, y_pred)\n sns.heatmap(cf_matrix, annot = True, fmt=\".0f\", ax=ax[i,j])\n ax[i,j].set_title(f\"{t}: Log Loss {log_loss(y_train.values, y_pred_classes):.2f}\")\n \n except:\n pass\n if j == (cols-1):\n i = i+1\nplt.tight_layout()\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction and Submission","metadata":{}},{"cell_type":"code","source":"# Load test data\ntest['text'] = test.discourse_type + sep +test.discourse_text\n\n# Encode\nX_test = bert_encode(test.text.astype(str), tokenizer)\n\n# Predict\ny_pred = model.predict(X_test, verbose=1)\ny_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load submission template\nsubmission = pd.read_csv(\"../input/feedback-prize-effectiveness/sample_submission.csv\")\n\n# Replace template with predictions\nsubmission['Ineffective'] = y_pred[:,0]\nsubmission['Adequate'] = y_pred[:,1]\nsubmission['Effective'] = y_pred[:,2]\n\n# Save submission file\nsubmission.to_csv(\"submission.csv\", index=False)\n\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}