{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\npd.set_option(\"max_colwidth\", None)\n\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport re\n\nimport spacy\nfrom spacy import displacy\n\nimport nltk\nnltk.download('stopwords')\nfrom nltk.corpus import stopwords\n\nimport os\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow as tf\n\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import log_loss\n\nimport warnings # Supress warnings\nwarnings.filterwarnings(\"ignore\")\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-21T15:26:40.497546Z","iopub.execute_input":"2022-07-21T15:26:40.498059Z","iopub.status.idle":"2022-07-21T15:26:48.199898Z","shell.execute_reply.started":"2022-07-21T15:26:40.497938Z","shell.execute_reply":"2022-07-21T15:26:48.198943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feedback Prize - Predicting Effective Arguments\n\n![thumb76_76(3).png](attachment:cdc1967e-f707-464e-b759-e4dc04278a72.png)\n\nThis notebook contains:\n\n* Analysis of [Duplicates](#Duplicates)\n* [Exploratory Data Analysis](#Exploratory-Data-Analysis):\n* [Preprocessing](#Preprocessing): Label Encode Target and concatenate `discourse_type` [SEP] `discourse_text`\n* [Model](#Model): Experiment Tracking with W&B, BERT variant using TensorFlow based on  [【Tensorflow】FeedBack BERT-Baseline by IMvision12](https://www.kaggle.com/code/imvision12/tensorflow-feedback-bert-baseline) (don't forget to upvote the original baseline if this notebook was helpful to you)","metadata":{},"attachments":{"cdc1967e-f707-464e-b759-e4dc04278a72.png":{"image/png":"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCADwAPADASIAAhEBAxEB/8QAHgABAAICAgMBAAAAAAAAAAAAAAUIBgcCBAEDCQr/xABXEAACAgICAgAEAwMFCAoOCwAEBQIDAQYABwgREhMUIRUWMUFRYSIlVXGBCRcjMjaRlaEkU3WTlrHR0+HwGDU4QkNUYne1trfB1NYnNEVSZXJ2srPS1f/EABsBAQADAQEBAQAAAAAAAAAAAAABAgUDBAYH/8QAQBEAAQMCBAQCBwQJAgcAAAAAAQACEQMhBBIxQRMiUWEFcQYUMlKBkdEjQlOSFRYkQ2KTocHwM9JFVWRzwtPx/9oADAMBAAIRAxEAPwD9/HHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwicc8ZzjGPf9mMfvz+zH9v8OdIhkCHmMSzBRZTxmUYEEV0yliOcYzmMbJRzL1nOPfr9M5xjPILg27iAOpsLmNdFBIAJJAA1JMAea73HIXOxosf/bCv+1gJj+H+2554/MaL+mFX+kBf6/8AbP3ff+r7858alMCpTOuj29u/f4KvEp++z8w+qm+OQv5jRf0wq/T37/ERM49fp794t9evf2/r+3H5iReveXKv1jHvOcMBc4xj+OfmYxjHv7e/fr36x+3jjUpjiMnpmH17pxKfvs/M36qa45CfmNF/TKrOPf2z+IC/87zz+Ykf9MK/9IC/85wa9EWNWnew52/XunEp++z8zfqprjkJ+Y0X9Mqv9Iif85x+Y0X6fjCr3+78RF/53jj0fxaf529u/cJxKfvs/M3691N8chPzGi/phV/pET/X/hftzz+YkX7HKrP9TAX/AI82Yx7/AIfrn9mOTxqUxxGT0zDz6pxKf4jPzN+qmuOQn5jR5/Rwq/r/ABAX1/8Ay4/Z/H+PH5jRf0yq+2Pf/bET9P8AfeQK1I6VGTaQHtMTGt+/+QU4lP32fmb9e6m+OQn5kQ/f+elP2+2f5xF+2fXv1n/Cese8ff8AqzjP6cY2NFn36cqft/8AiAuf0/X9LP7Pv69Z441L8RvTVOJT99n5m/VTfHITGxoc+/55U/b9f5xF+2P359Wfpj/3Z4/MaL+mFX+kRP8AneONR3qMHm4DXzKCpTNxUZ19pv1U3xyF/MaL+mFXv7/b8RF9/b9f/Cf9POVT9LdZXVU1W22WyjCuus4ac7Jyz6jGEI2ZlOWc/bEY4znOftjkitSMRUZcwOYXNu/cJxGe+zb7zd/j3Uxxxxzor6pxxxwiccccInHHHCJxxxwiwjsOVkdWKxVcRRKxlrdObBSLxbvl37MmouhC8ayq6vFtNllNma7IZlXZOGc/DLOM0v8AMVIXtG16xr4ZDyR8tAetFCxLtT/V7Sig+0uqRmpGLETdRaREbWzm1Jdltl/0a4sqdEI3W1/Hc/sT/Ja//dbV/wD1rR8rB5R99E9NbHq9ZITC7XTdV2FszNT2rZMwWAj7XE6amCw22JbMZqU3+itkroIuW2ZqYG4ivpvznLx4luIBBP2GHMCCZ4tbYlovABuDHkvHjBNGsL3ZSFoM/aOMQXNnvcWKqrgzvorUUlQMD0zvGynjHlm6brVzKCClZrNVdMlJpxIMAQ3Zu1Do3eLSGmwJNeUtWGbrGJN7HXUd58jZbL+HHa80XlWgEsFwlOqojGf0QlSkJj9Cs+tGWHoyNq/AAz3JdrQtIl2N0Yua1kgFCJtvBeasGFsxxdf7ExYNmn6/6kZIvqAzQHg/ZPjsZRCgVZrlUqpZHD+Mt5C3FoQwuMesz2w+Wheu7c0025JuJjEJswRCHD2azSqZMQ3N+ugYmyKlWKhXtXwTQGs58QDECC+LAsfAx1Eq/nwxgcZY7M6SDw2NLRY6iuCIi2Yib2hYrRRaTLHEkzGRkiS08s1SeoggzIladXt+/k+0oJ7DbbCW4m6nrdoRSZJSiFYvLaTHxqymgy2yR2krKDMYqrliLlAmYMToXH1FGW5Owo7le7Xtt887MoSpNb7ep1LK9WnXj42NoFcr0qSkmJNtu11ho6VjVI2c0YhXtzbYV19NcF9FQM8H5pWtSLAkiTbGZ0hgLxfbfRxBMxbREDCiQzJNiANOOxG066fGgoqsK2F5pd9NA5A9fHVPMZzt92tqlSnYIOG7nUFrIs2SwBFSG5eotZeNVUypVmuIhbA1uUrli6LA62usVwRmYFltODm08zTleCRlnhMNwWmQeNbpDiZBt0VHMpzIDxAEzTpnpeOKIkWmTaLmVq0TZfLEggYa3XKghrzHVGCbtRCuJHhioWkT6nF0RhJUAhT/ABdUxsvpHbPJMExpT2oaANEbsPZXkNqxtlLldevBvLToteNM1TVc/mja4kXChLxKsEiw/CdurlWw2G9eUc21zFU5qB08Y3rK7FofLCt/uedNHxsVE4OdhTZezuQkJbLUaqxxRMSwWNlhpDgQZhkEAXNl2crS/kXH2UFUhYyX5qyConfdru8TzD8JhDApOnsc2OthD/GNVU0YWXnX2EOEM/xaw+ge0ZFbLKdh6Y4nCNg2jmg0CZEg8Fgk26VwCTGgJJ/or/Zk3YeYWHDaDJIuZrGd5iL7iSsK1p35LzL1crZFd8Qrdv15Q9Ti6cmCtmhLIdZ2FwazIhglZSonAEaN0F4AZ6kYUgI2LRlSW35E7D5LzfbVav18oVIqfmYQDHarrl83Cq5hSujCsudokrx1C2JOxqbKJkWty8ir2Lc8aZAFGwNk8tn2rbDKo5Q7t0yrT0W13PBSU+HgcNjVomK+JWvEZrJ/wJLqlQQIHEkzJN2DY+ggi8zkmPlixXJ9Ucy1/drh9lSsnZtPztWBK1sdSzcLDqmA7W4EhlbCevuSaMrR7JFCDDSxXVewoohQNoEtcKJEjlmk2Nj+NcwNzMX2lVy0okEieVv2bOxMfbRpNug1stZFMPIxQh65E1NJc9tPRvLdsO3JQpDcrW2diffgtrsXDSqpdecrYJWBSoKd4q0dWWuomLica6oIo3yVMLFY4WOz0nytkTQWn62j1x7cqLUrLtf2FmpVsqYDPz3/AMC90NWUN+VkAbUtCPBicTjm3R/L5gxdodfXKNige9bawGPa0baqOvyDsjfW40XClBWmVlM4apsYe1Wpa5wYgYxlYaPmyg4oXgV5enrimQRCfaz/AMNbPl8mAF+qCiE06zO8vYiaAjmEjq4oEeBDJwIjG58VZYGoHrtj8uvplbJOQkkzJpMDrxFxWAPQEeWqmKQaBDptJ4bLmRFuL0kWInftr7W967fC7C1rT9xs+pKapXLqtUPq+rg2Mxq9KcuiZm3is6yksUG6CrdRRTphFc6omdB8w+uhSdjG1Gz+S+SkTl7qm22/hUnubk4elI/ktVB06qBTGmKi1F1u1CkypEBRRWhg1JBidihTi+y8u/chPmYOGGXeQBtdZw4GWggGS9UnaxTXDLTqDRDhriqcSJHd6ti9TCFzMRnsC1XeFYXQROuJH81bqVxx7VfssbbLbp6+uVYUHXMwKzcjVzmRXMj4SsUG68azqgHjCn8fGGjWymLbdM0UoltNzXaEmkJMa5Yqt6tgCSY1tKNFEQQxxkNbdjCDpJH2wINxvoAL2Kwzrlt3lr+xL6d5Xbcdqx1jYhxEnX1ptagliuTwUQAcjzKMuW0u7SB5KqiBzAplEF/E3UjE1persdnkiic9hH6nF66CN2s0JUG119K0oXa/S1ZmgutViPWQWfRWvKBTDg5AIrODyQSZRlkspMG3cz8r7VzfTE1Im0HMt415I7TCRL1hTfSXsE4fhyBxY3sFF18ywaJhOWLUodd8IVsa43wlK6rBwfNVnVFRQ6SbTlq3x7EVoakreR4xLAqS50EUGYZTEIhNRG6SwuMC/r/ixljUHZTLMBrPayvAcLzTblJMXA4xIsLCRuoyUiSRnuBA4bC1oJaTbinSNjI0lRG9w76y02fOqSbL1x6LWJywnVpmU8trtYTL38NZvblEWqLEzKTY8akWE4smlFMTbjapZwRDLG/lMyHvKOCgntHIMsqXV6Ugng3MIPT8ATvZ3U3QBrMWp0AJ4VGMkq3sz/r2DCkdtTsAjzXHySQOrUb46jUJXfQSNDXhAzLfZEyhqj2Ng4FWRBgWVkrLi8SIIWFLx6c3Sqslyo80pXlkr69d36RgFk6WVUZ6pWOGQAMUx2MbLO2+tQXaiUjxPj9EaRN1K/AQFVd1MrLIaymWjkc7fM6nTzeZmuDN7WvppZMtICADqBJpsJF27CsDOh/sRMYMhz5C7Gq2v81Td644LF1IbX8KkKqhenID7REI2M8aoquqLMoPXbyR6MnVzW7FqS+i+/8AFrIklsMi6u2ryPM7a0enYdSYh6vZvOriOI51hGPCtfaAbQ4KqbVfNrETiGDhOoz+cQZdYf8Ahwmw1m0flK3MdZ8rXDzX27AlU7EepYaZVNFU31fEGxm7WTpAsVtTZDA0pqjKiBrHbK6sb6gYgSzFDKiwKNxfGneGfaPWAe9OAxaPxXZNuih+GuiZmdbV7CcrQEsbYXEww1KAEgUdCn6esW6/IkKIfT5st9mBosq1gDPKc4Y+i0CzmaEVDsRBAIiTBK9GEoNqVgJ9kB8Gm0DlLTAIqEzHUR/Qqwdf+JH7es+vv/X+39cY/t/jznxxz6dfQJxxxwiccccIuga0Xrs1/Xmhh4txn5X1RYwvzMx/xvg+otq+P4cZx8Xw+/h949/rzo/mbX/6bTf6WX//ABHMLeI0rzsRXS7UK29NGmN7aKmi8RhXRZPYUkJ2VQLpuhXOUP5MpwjGWY/bOfXr1obs3uboLrQBzdPqlvuzhN22o6RxqfXvUq/Ztoab+46/C7PHXqF0oLqDAKNLOrbGNpHUAj5qIDzZIyr5M/LnrvdUFMUg1j8nO5wceVrpMNI+9prbdc5qEugMgGBJdMQ07W0MfHZWt/MyD+nEv+ll/wD8T/yf289tD9KRdXRQ3VX3Wy+GuqhkFbbbL75+Guqu+U5y+HGc/DHGc5xjOft65RbdPKnxa0u/Z1t3Wex7Hsmmbzq2gbTqOm9H2bPtKl1s3ULXvG0j8JAA+Joo1rr9BsBO1sEd7XCh+hbahGkvZxMq52PLWdctNc0TdNJTapYueP8Arx7r+wJUqwb61Q6ZKzV7AEugOi/A564uq6uUcwlMe/GJxxiUo8o9+Lpsc9zaJDRmID3TAiQPsx33GmokEHOqNBJFOBrBdO0xNp1+fZZ12J99Wv8AX3/nbVvX/CtH/ZzRXfbzqEPcdLUdhqtbYbEag2wvV7dgJXAWxW12rV+yhqi2ARHzLShmA+GQQ1sJkLfnSvpsGpszjefYPrOqXYx9/wCdtX+2P/1Wkzj7f1c0D5IdRax2YyRWPtl/L1wiZgALPEo/MxiTpS5gXTialnCUxzlYmZV25jVbVidVlNkJ5lHzY10trEOEmjhbnIQDxaxMh8tsJsfguGLJFOpBAMUokNIBzuvz8vz01WmYbN4xKzSwK1PWC5kGrgKeFXtWrCnip0gBMKhix4gwIqXqlv1ddY1leKRRM21QrjXL4Mxm27H427PUwVNFqBiaetfCksEe2rL3wEGQwDxmJW3HBuuWsNmqmERQERZjOzzlKm2g6qRMeQdninoM1twee1SIHXkWsL3sc0UsbTibrSDCrYDauMNWOZdddEgMfAw+RrMiU2UQxCcepR4h9VxH1+qjseeb9VGZhqTqSI/UDya0MwWlpGKdZ+mtvJUO2yi2FtGK6wGFua66i4UE14rTUFy6mb+1kw3a0ydvPcRqsUioObNTEx93Dfw2O/wA87wsqUsvFymtRqiXVdBZZ2vBWnirFztG7IfQ19N85oubxoXFEMaVq1RXFvcxyTComIeDJZJJqlLj+c/HAB+5HK68UJCdP2GubDaGlC9Nr6/ZVSaEMkC7SSrpGpco1VYq+d2bRWFA9wcFkS12cl0+lJ416ZrrlQ9RdofhxCilVVmmFAhQxlqJAdrie4jJep2202CL3De2yAdg0DyWWLC42VAhVQ5uvHDRth3MzdWXZA9xtrNayGCqHDppDysJiZQuLPp1SLl8uGIx6S0bEexv1YIg1WhIEXsLqcxmfJLjSggXLcNJNjBGYCNJI0g2SKpOtEDLeW4aZME9ug1m215yhQ08efzCLlDrugR2m5tdUJNPsmvRezdvV1TO+NNoQGD8sW6idR1uYT+caFKFss2wzHGcWR7x4rbEvbKlev8AX01YZ9apopJZpV4thTq8iFUJqy1VECKWpxx4tREhs1mMLGdOMyswRnMen8Xuv0jzT3kOz4k40FrNqhWn1DSCDqtOE2Eoa++nWRjbYkbGBS+ywMJtODqh+Frrg18YThHtvErrR3QLA3sfF8h3jR3RdmMLK7K3WARy1tlX5b+mLCkpBgtGuIiRfXCyRspEExjweJmlz6MEcstwwjcHW0H4mDBtYQ+DzMkRHLhgItIsSflOmsa7Nu2Toix8WqY69qdOzj0RUkrmexJaXli1NZHA9H0hAUjrUwl1vwgwnXAam2FmKKKvlZxGJN3/AMaUKemN4mgCKUoJogIA21JLYCAKcx+vCVqgwLbMjr/xL4yhABpQpiZZdOHw32zlGbD429d7Ky2ZqfvdNJm0PM7AXYFeTTMI+Q/0sZLr569aRRCNUrIYxbbfZnF10c2erZ45jYfiP1iuHcDgdgVgxZm5KHsohH5quFBZpi0IWd2tXZJpUzOtqokfIid41Q498cV1fDOAHQJfSOmjcMZMAEm9pOpG3bUWvsJpC3u4U6ZbG46X1sLLNwHXjVMwKkTWOuhXBRtK0MDD7VxnMXLm8bbK1URaQJFRcHGxGf8A0dUpk3HQqPj8ZUY2Y4NexPHdU0NXHI9duOXr92fPrBGoDHCITWKlEdxJ2CwRbdctL+F4qoKHvpsNYTvlXZVmse6VcEi8YdAQnr2A/ZOC8psDxT1GfKhWsGq2pfu99GLBtXouM+s2kK1hcQbdYQOMfcqEsoDHEjVLCePmlAEbWf8A3y6vi2NDsuuZxZLFA6JBsbGbq0UKUNdqyTYraEHmBsXEzSbajcisLCBhRcV2h0zNMiG8oGGBJmbGbamxkfMqAHzANIkZbZcN/Ba19J16DYLs6/sPjbtyoMVFqmjMwH038h0cGiP5ht4x4rbaKbkf4XK36ypiOvauhrqPnZKqAPMjmdI9lfThvni9mElc1PX9OGobi1ktufoYQrquBSSei7BTBfYKESUExSjsAT7IWH4yOPiJURMxrjUnjH14i21Bug2/i3tNdW0r4RLjOQhcRFBSgMu/ENeq+iKpiYwMssCzRkm0mdEpVh0xpnAneIfWhaxath2VYNWporootxiu/F2aGx7iiZlVmtRqJlUawvzKHuELaY1UXwshGWZwCQTLqUH2eXDSNB1F/K+86q5FSImmACDGXDS2S066nXXSwGyz0vYvGRreWMwUdXsy1KuCI4cra9VMuWpRmdSmtMRXaFZIRUO4tpWxAlisOg6UBsUwsj8MfDDbvHWnW1e5HalrU9ZtciArXd1w0wh26d0erAlVK1LiwaxY/ALGBvor9hFx+fR8sfMyYYW08SOrmaoBVHfaRKlVolwFtUfmZpmHXdRVG+mWtV0G1ZEJKFkPfCNEvn5uzXmVcIYysjoHWi9TC0sntEf8GSvIOtYuoFhUwSRrpafAvnZPWrAmow5Tcy2mw4HNkh4DjE5urhbiwS52Qh9OARmBZhdwLAAnzvY36KGh7SL0435MMdY7mdpvseynZO/HuVihjlDoGbnFLRQgPlsuvYm1HVlFnOVqcqS/EjBgDbjTGQwlk6qCSCCCYRtIlOzqj7P4323nMhlfXFhcFS8pkfTtmtyKglY/WTU3HERB+fFcfI0/IP1E8VkyLJxVicr54ngZ3izqLNHrunl9pUW6grXbuG2CjTOlg1r2t7rj4FYIVXr0q9fSJDUNV4gaPANuMUDAwnFXdeHH3D+KXXYyYNWP2RkYoIl6wqcBZyEVazfoQEhRlo42t1jzrHsW0NghsS9DNcWXfVSzPMoyA4RLqVz0w1hax5viJBj5qoa+16cGCRlw9pLZF46d9J0Ky8dz43XRPSA6t1+QHstVdjoMJ3rdq1pChfF6Fh9TUBigvNq6djNdEykjM6aSSIZqzRZPFuelH+qvtWLxpuFMUqp4xX5gnaCthamE5xNZVzvDrrpqKwWXORFPw4niU82SzL5uJSo1q3hFpB31AKnfym8RRVE4hRKHAyNEIxdOTHNYuuC5vMbYU5UMWea/qiEhFycgi+NYt9Fx/Hnoxd0NrD3W1pmTaHGzEbBiUvlfENVYACtDXfHUIJO+teEvGFqIJxaTdXXiVtkfUaq9Lw6lWFZtUtik5hBc1lMBxgAczJdEtiDy+Rle7A06raoeWjhua4FwZRaPuwJYAScw2t5TCsBxxxzeWwnHHHCJxxxwiwO/79kg49Z/yIa+/WM5+35iS+/7cYxnP9XKPd8+BrXu3YN8vM7aSY0fbe2NJ7uXdbbx0H1z2vqarsLVeulvVx5DgbbjYV7ehdauuquoSNw6ZIH/AMpusYyyNWPy+GwaUi2YsU5nFjAoIYgMchW7dI78DFW0X3023JmIEiKZXDUW4rI+bGuyv4q/hzKWcwmOqdU/bft3/D/ef/mHnk/aGPqcOlTc19QvDnVS0mQxvsim4CADqb6rl9qC6GNcC6QS8tMQBpkPTqvnsX/czRl2x09i9d9+9jaD2u4a7O67F3OKVPtqvbm+11djKrmaLr/a2TDSutSNb1ftbctc08fSloihaMUvMcKthNCskVeVpoWtar1f1/1kvCnfqWsFdXaYuXsrZmWTQ66cjTLhjCJYhMq2IK8eN9+cRldbGVssY+PMeZJ/ep1P/b9u/wCH+8//ADDz2j9X6oKWEbH8wkXLzRmAsGO4bazEiWJZi0a6wFg7IDvlRbjFtWL6bIwtjCzEfijHOK1Bi6lN1N1GjDxlP2xNjE24QBnmsdbbKrxVe0sdSpw6xmoSIOtuHe22+iw/butNFSpq2qzV1ALAJ3qlwpg9Fkb6LI7Wkxidc5WZxifw5l6z6z69/pnGeaJ8uNP7u2rYOvMdUEsF60QTYq9lNWurltscHTDHohYLGi2om6AsyiFhX07G4FhTC2qkCXwH5th2Hj1q1+P3NdWx/m2tH/1/XlSPLzupv1W+00JSnrd2stZ21zJXQ5YgPmH4OxRiYijAEEtFPpWYa5ebKQaWJhbrq44gfM7pYnV5cZTp02V2tpNDTTw7iGtYJPEqtkyWtOgEmdTFlwxVNraNUNptiKRyta1sEVDLosDbrsFW6PSHfRdG/DuR3zYo7Tqdb1Vm32EloNbSLviJ+oosWzVQtXMQkye/GxvDfqpbS0JostxbmN8oejSulvJjT7diGBMIgrJRvjFq4+JbVeTsjpkU0JhG6u4Jgm2ApjcUTXt/0JYSwA4ECwVxUsFBUwGfLjbZUAZzr6oYlnRnNQlu6PLjZX1KLGNFAkA1hgRxW9ZjKzq0YY8mhyICysaEZyLjE/cn8mOxLleXbFVMcRBseq6Q6DLZFVnOCdmj/L2SAQMiWC9iH+BMTF+qiD5m2WbaoMGDOkprqaYcMaDNN0EgRNEieUCB2ibAjU6SVhkBrTmpvIc5upomTygQOlrwIB36RIPQ/lkGXQ5oOfjOj8rx3JsNiuLJvCWXRgrgzHNneCwvGrrWSckZKc3tVyq5WRJ7Iius2Uf9X+R+nJm73LHspqKNlXI1WjeOSXzkUtRTDZLxbV2smHU7MTs1YMF+yREJKVa/RYPVEdZ8+Me/u/k3uGpbzs2vREHYBAOhdYCWjt3n4+LdU7frT259k11iypkyEgmfIVMCGot+sZgwqFNnefaJ7l3ffY53Xu0MbhmoewakBqDIlpDJcvxQbbtozWDAJVaD9LUTnWRGsDLIUEVgniU35GnEjMYyS2Wl1P2sonLR7DTLI/yNlNnNa51N2SASfsri2sCT/bZd3bOlu/dxAtr+dsoa95oekKnKmDIpPho2t1Rcs3rGw0iKvZl8yPrqw7J3ZhTfbfaJAeNmIQg49QeWYAqJIjuaJFGuQiAHINrRbWYPmLG5XbIS1ePkdNr41wyExUXWyvZ2wFPEWXjr6CafU58sN8TzvoloRNMgk7y+69vthise5rrRN6t39LgqAEi1C5svNotJHqhAsc9CRE1TUfHN2a9pdybyiTaTtGlknsFzpY1ctwoBtGN1VOmBq972dd6uroOAON1Bdtmsi0Fjjm0Pbg5QFg2GgNyMzGuawsdzOkSKLgDAmTBgRAiYjbRQMshmRzgSCL04FgBBjQCNbi6gGHUflVVHIahtu1sR2S36RwTuGCLpa5VMmWVhi89QNM59U7uHdstkKPxg7Uhvyf8ARu8ywpnkkOuvJenUFQkqdlY7IFtO8lTs/N9yggwA2ijOlGv3MEzYcxOuKw3gZqSoWqu2g5MTUElkHcvR6vL8iu5tVHDp2AG1yerLflH3rZmiUOFIrNyopob03A0ja+SqMVzPLLUssB/l6mRBV2S7rLh9gvPIbsFYDqjUDTMtAGulJNtcjVbRsFjaq9nl7dao12C1WxVs7bA0EsrCCza4FFNlsZxkPC22+SKbTIpSDYECiJgDQgAjsCQeyl4aS0GmZvBHBEwATzQDeNDHx26iTpvyOu1Dbl+3k7U4Nd6Y/QBKSNlLshQXgrXyNauHYREPrVPJj17EKx2STBsRG20CZpjKFFF8eY/S3kS+R9lIt2JfsAtlV67RrAhrKxgOukHsahtgY8oUb6opguThFLNtZTCGA3kyzJHy2VZWbhcKc+UnYheJLEg6EMzMAbKX4brZNhVMq7jDlpFimq4AaVA8zZqlqwlyPXUxdDOgQbzpDXChZu/8it0WJ9P2kdETkZu27ISmadY2c52N2ZphKBL9Fq1sV9mbnsNiztf0ABcBwmuvp72lt2LIxqorlYHTkcMzmkXowC0h1omJIM6SO+lYa2Bkcc0CZokgi/YDS8EAgR2XQ1LpDyGqYtnm4Vu27ll1/tusVllt7CcVND4VzU2HgUVFLmAOTYmzVkX2GFIwWNYltQ9FU14eUf3oPIOzUAE12w7zW5r2Opsc3F2E+llYBdqq2k1XFrFP8z8JhtuW5QCiFMaBhPpMUyiNCuiOuYeW2/wULnluh1XhMCQgKrg91ZWCfV/ErIOOvYWVxT0oThGY61CTW4LMnsNtYhkLa84on11Hkv2QsX3VbV8uxkI0keVKZj0NicrLY234D15Ov1QmsjX9XXVUrtsPOs/HBL2K60NkQSRVGw5jSWuLHElwAvRcC4CIIII3g+RiN7OGY5jTdaBrSi2k3IHfsYsLHIQup/KrJ7Ql+/29YnCpX3C2p37BiQTJfMYRifJfUqLJMhdrmCzMoYBAVHbRVC2AB5sqWzCRe6D5TbBqnWpKXGyqXF+puWW8yFZ3oi6H2xUGl0JxlzIUemeU9Ji4JPeVV9QqJAtqmXrZMZN7ekm8huxnOnKtqFU5tYPt6t1delIath0YdGNUVW/VlNQQGrMpV+N2MGgxgmc3WU2ULsnlD1ZlZ39k8lNuV7Mk1tJqxDgltpOu7fYPc82Og0O1omF2MtXmAq+6Jdn4ZaQApkOPioh9TAViWHL6kAVlpkiKJntwA2WkWIiCbzPy2mA5uYRTcSJmRRDXaAk2AN9CY1sBAWMhdbeYjGdd4zDaNesEfMA2EGDe1wGSaH84de7WinAVXw0YYXH4cYlJw6M2ljOrZ6RTZUxawy/TeqPIZTvIjckPchddZRYWuwdk3ojeMUXy0hekUXHVUrwZt2a10sHtxXUMcpuCIvuyVQxFwzaYBX5j7TKsGX5YqxkhWrYGzhtLkmobBewBLbrQpCwKg2pNWn1EpBFV5xRLCk0Gu9mzEsQwz/a/Jlzqm3brqt6QtjnXmtCtHEJ84mfsJJA6m4ZdXGoMgWp8wqLbkjJcTgdUEnmdMUpZfBrbL2tNuE4SD7Joi3KJMeYI11MzYqHtBMGk8E2hppb5W3MkxcESdZ6qxfiP1t2Nquzb9tPZtLf8Vc61pSBVY7fW7KcOIjP2g1oP+I/QBD5HNctr3QcB4fEKsYLkxf8AhEkK6b0/FjOfX3++Pf3xnH/HjH3/AIfrz5CIPIDeCA9/dtPwFkkS66XsOtNNdebaYlJvUrtFHYBTLKSAkmoyGe1SKFbUxtLp+icj3TgrEGaVWR8OO6mvbZu9Vt02E1qZTo5w9EGjBx7g/CPuJiVffVkUIik0W6NAGbYn4XyFMtgYESE1N2MBjL0sK2kYBINTM0iCM5ECDIJgzPXutbBYoHLhxSdDSQXlzYFibtA1tFuoO6vXxxxzbWqnHHHCJxxxwif2/wDF/n/T/o/hxxxwiccccIsH7E/yXv8A91tW/wDWtHzR3fnYej6BsWtEbECuIbmoXsQirNcfvmA6qtkowSL81AORJeK0cTThA0m4qy62CxYsX5vOnVTzd/Y84V6oROycIQi21bMp2SjCEcfmxH95ylnEYx9+veZZxjH65zjmmO/et9G3pprpG027NQUCtZCDT1otsDK0e5iuPpydNS1W5ItVuVy5yi+qxfhc7CGZi4iRTiMsvHZvt8rww8LDXOUfva1peC29tRGq8eMnhVb5QW0hMgCOI6QZBA7nfRVtI8melbyqbLx0RJKWcygb8aDvxMhzChbltoysitfOFza2kk9PctCnadEilursojeAwqolY+SnVOVtTbH4DYJcZEGuuvTt0sZTKzTSVCmKaIsmvzMg5qZQ9iRj+HfKN+KI2I2R7Y3RXSI9ItNI+y4gCXSwqzMdtZbg2m5iXksiyb6Vlt9pTVgUXZdmWSCC7rr/AIp5x66UujOiB3gpGZ7RQ1spKsDXzucQoukMMIrOY1J57Fgcg4cEpesKY/TyIpEvBGnPHzq5WYXML8YEAi5dhYHsxHLP9z8b4sk2D2WiHZ6JGxsMpPwIj5yvRLyZ6vpXSYkCI8DVEwBzcHq+2MV0z7vnwGDFeDU4TklWV1iwJqqM+Jbc0WBH/JsNrzKIV+SfTQ9RjqSVGlN2C4di1q/J23lsSTqVw48aWEwKCYfjSXXRgjHqiqywnVUnyDmcBwrfq7ZO3xq6FIiIPIfdbA1xlRQizBWw5VjHSxTi+2tdHYpCVGMcihZa3RrrLYyAExfd8qu6q/o29P8AjZqplI7Ns7BKtpxXQucM2kvn2bLCeqWH1hE7HnNjTZab6dauawryW1rFXAQslYBT6DMRaq0mzpBw+WBlFjlJNyYg20hSSSAOIwHs6jGws0t23BF57wsz2XvDTdRfh629QA0lMkc3IN4Wp7I6BJFheyqvGhcmmfL6ikVKexIjbCsaC6iFs78ZtpqliUPKrqGbIEXGNbrEZiKrwHd2v7EKrNNam5GqV0XkfLjcas9YvfWwlYNr05xFb2iGyxVnKH3T/VDoWm7aWO6MPwnXG+t2tGETBbIJmEjbDosb6XIw1pAcDSqR2Bcc2Lh5SpzP5WZ4ljK7x08ffhVHrIbeQKvmOWmtrJeFADfQlWTNsCqs2GY1NTYvMpbPKiFUnN1UPrrI/Jr+GWgkQ6qxxgzzYaCZmdJggi3/AJKrSWgB1RhdBBINH2jtoDYanS/ZTrfyI6vQu2Wttsa8vaqDor2w9+sbZ8lfP5WJTLMNrrmFQpqn8Idjq6+Kr8QlECJc77IRnG2+THWoSRbsDpYrQhtiNlrXVs9S2uo+yjV9ihq5ZJgVcfqU8r2VokQgG8AmFuCY1wHnZQTirwz6V8ffx9ma3sdyddlmM5EDHxPxDYr4342RsMtW4fV0zqqtFsaF1CV+h6M2VZnUHkcauRu6X6VbJMa4QXuLUZRMkfJBMmrJ6Kc0dBbpeQQ5vd3nzbFM4jMYFEW2E4EvkLR8sIiVWYGYgHijRpgOw/TaWkQSNTrHmFYEtaDnp7E5jR1tJGoi3XqBsuhHyd6fssqrpK1S+y7D6QWBtc2giZ+NbnbI+SzFEZTPhbcOXFTMWNkWxAR0F+bbQycVQgvkr02jRhUaygVmpgLE4IoutaFvBAYwzVkANVcNbWLbSVmFza26A9F5B7Q4J2ABgxuCbRXIEeNHQJRJploW2fVtcHTNvhJxEgqsr5Ga4zty+lO+hT9LX+DwukRUB7ujb9VGyUMSY/SHRiWtDSPfti8ZcYvXqhcEOqgzWMmeWCQc4Wex/JesRmWS7EdR+CbhrC2EBavZMsQhpJNqwN7h3qoBMC9hJMa3iBe5QOmIey15PBgG2lr77b3XY3Dv/TdKzp9jHWBDV+61fVrGiTWHzYKhXD8HnhkThfcSXGqVLwEygWkWwqQcTCfl11hXZj6c+R3V80pOxAR1VmmD2T8qHmLFL86+De0Ag8agAATJLB3lrAeFSn8LGurafOjaLbbCqeMy9vUXVZgmujGnbkwG1WDajXpW0HZwEvZ00i2rM2DuYYMXrQRRBFlJv1Fi2oSmyFmLsZsnFp+mOiwlGUyab6adbt8NigKOOWVQr2VHjAtVVVkHksiUKcXfKynlPI1Fl8vqqp2WYlyRMAcRhM658PBbobRIIkXktHS5VeaAOJTkb5qUObPs6TF9Y6XUGP5WdPX2W0TijHJjeRQKJfqe1WGsh6D6Vopa0MWF5RYzGy6V66jFMS5gCHnWDUDAlWU9wLyc6pYskadVSkZONiZo1oSobVdppYwm8d1phCThD/pCQR6LSYtiY3wgRBBZF7VTNfdC+frC6J8fAXmbhZ7BBxQWOFML42ds6mMBIvQ6bl+X8pQMrW4uYiUW4jn8ONOKrqlWX82MpR0l0Mp2NK8+qdom7J9qixUwImeBQTsUHayrVcYza7xUSzmwFDUixsxb9WJbcpzRbEy2M5GbNDajXSYa0uwxFyNw0EkyPjoACAbXJAblcTGUZ6MEmAIBF5toDr2tFR8ourKQpkuA0yDNLJ8pJoZ6vs8I1E63azlKUS6ISBIFYUrCiUJY19ozO2NogVthYpcaZfZfIbr7VakVrNciqlsSxc8W1z19+LGxIWYMNexvtIlXSPYrCvvZlqr5wb4WgG5pClPFMLejLpHxh3LdfwVfvIbXeUlTOf4Lr+zFfjglI7NmOaTlYBsOCM/hzUplGVtdXoS+UqzYTFqpqhlR/ir0xGCNkfY8Fo0vXAECkiNjsEdUkTzKmDOdgrqn3eHYYTPB054n8dmJSx8Y4uR/AfE8HTflfjqTXaOaTQBEai4B1EQ6Ra19dFvhHib8rm4WpB9og0Y2iLWNrXMm+xCzfpjsbrjtrYC0oAKgyQ+t0bBXGOvv1GZCkTXRxiu9rCoe+8Shuuy1XjzmWqw1WZOqprYi5ttWrQJk0rpqwKA5EYrjdKr485sxV8Xy8TzOc85+D4pYj+7GfWPt9uVZ6i6n6m6Pvv2vWjT6wCkEllZRrAu9LAU05eWeyEma2IVUmPC16qTlgFXTlteEtjPP+x6apWZVbIubyGkBm22gweBQxWK/Yl411ECaLaroylC2u8eyF9Eo5zi2mULY5zCWM82PDfGPCXClRGKw78VUqOYAA0Pc5zuVrS1mUnSBIgLRw/h+Jo0c1bDOa5gcXPcxgcGAxMsnlGkzrYrJeORywzJtF1mavlZpOODzH4sSxn6Mq0fFmM4jH1izFeJ/BnH8j4vh+KXw+89/4o4+36ev4c+ipVqdemytTdmp1G5mHSWnQwYI+MbroQQS0iCDBFjfpaR8ly45x+OP7/8AVnj44/xx/Xj/AJPfOkjqPmEg9D/n/wBHzXLjnH44/v8A9Wf+TjE45zjHv75znGMfv9f9f2+uJHUfMKFy4445KJxxxwi6xgYjAe0M8Uc0S/GI3Cl0VEj2xxLE8RtovhOqyOJxjLGJwljEoxljHuOM4pt5T9K7p2m01kvVCkMcLdd2VPbjYMtswWkOjFN02CipcxFFsOZrQzdeZWOQ2oIqphaTBY0shNYXdLlQ/JbZO5NfYJCOrteG2EcbX3RbBdekvY2sGVhq4BcNQxpOokqwuGJMfWU1gsLm/wBBBf8ALjXbKFmN49XOF8NxGIZSZVe00gGvqCkDNRou8tcBlkkS0zpurMwrMY4Yeo91Nj5lzG5nAt5hAsLkCSdB2uKa/wDYcd5TCRr7N7Apzr/01a1wMa6uaDZtCPHPYZEMayWlMAsmYF13GML7AaKamUGK8mFqwyb1HxH7P1ihtmbZAYeb1xsemLzyzWjEpZc/uSSpTxtvMCEt0xRlMQYDUKrCZxNfMIzB+hFqHM6Me+PIQphdrwOs0sdrDC10hkgV9dMrpKaTCxKhznJd5tbBfZtq0i84lWfr6i/UDqKgyblYhGDYy9mx+UFuqjqR1Oy1GwiIxzuedQKzsdlmbyVBg9yCxh8jJNV6wvYLVEV+VVqxyooWA4uyLZH89d49XYWtfg8OA7KZOMYZbykODhhjYRNrGwG67/q1hYkYnEbEn1emYggbbeVjeLa9MrxL7VY06Frp1wM1OqaMChavcN2H1rPNe6vGZ+trRRWoNUVTnWjgFbw9gLVbMZWuGXVUjxmKHDX+E/Zhs5fUbIvAxXMewTK2xjZMM5dicgdlqsYPLbY7CWxldtLL1PAQuysmFivMq5YJI22f2p3YfpWhbLrGp1MmOz7htAR9KnVmGxJhta15kPrQ7K+9WcSUOK6KGebemvonXA5FWIBWzqJpiG6xRf2v5N3QoxZ14feRNZrhQ0L+tGwo7B1MUyqeunkSKXySjbg0pxdZteacgaQj+lufr9Wc3EJ5VZ6Q14EYPDiHFoHrzGmQQCQfVvI6ecaqw9GcILjFV7Cf9CmdQDYSRm21+JGmB3+EPZhCdUsk7RynVaRh1Mw1mwxarnr4aQLWg77iaPjQjVWPxbAawUMLV7nOZRsrquUkTwXh32fEwYhg+W2zCnk0W0Zg+ySQ2r9XI7zfnP40Zq0OUblmn4przLKk0m5jXA2NcJSDzsPyUshkFLru2S/GaJxv2SvqZstmvrp12WTbVaRizbQWnIncbAhcX0LRtrhGtlXWxvKrFnlui7n5FbXt2pibdrpOnLZN9usIgJpDylURXHWNtpDp2d2b9AOuWavsktXGWj4oshveCqzwGhliskqI+kVRrS92Fw+9vXmFziBNh6sZmCBBifK/MejOFN/WsRcgGKNO5zCdgR2Gtr6oP4wbfZ2XsW5t/oLwm1e/iBijGE1ucD75USPgwom9vkERskVkC6qDJNTSLPXUCWzOKj680i62Z+G/cTFeWMVti1oyZrFsT27Ax0IZFlVheTs9CuoduevBRb+wqJjsdRSxxYArCSiL1xN8brgpMjb/ACw26KeRGsvutrLmtauzKnUSGnsYy2g0UxwNa2uHqFCoZCDnFxmHnJutOALjqFbOVec92Z95F0v9WeIV7EvNHWad050uWoSjqzfbbCma52km0/EaSETI+y5cXRTaYbZrwYlZA9JQ95JHDfSGoHNacNh3EtMD1xroyiQyRhYJInex1jRX/VfCmXetYgiWgDhUiSDpAnS5Jk62lYh/2J3a0NRda1jYw7jGHY+t7svbGnsDiKoKAYCn2tbSWEy3ZxhOMH0LGF5QVExqaanwdkFZiCJl4ddoOZYxtZ2rvKJ2K85nIrZ7C14qIaIgQKjNux0CRqdXjrth2Ys2i0iD5SLUrhkeWDadj/nPyJM1LrpwHrMB9mPq3X801X6g5sCrDUG0Foai9bMJ16xWy2KgGSgcskXExoF3mq6pQJjm/Edf3ryvK+almg+n/DahYZfv+uGJB7D4ATMZuvHgSkTytaMbZZt+iYm2r6EtFf0YdTuBEzfSCsQ5wwmHkOMg41gIyEMJj1bSwgzcadpPozhRH7ViCIm1CmYgNttvAgHWZg2XU3PxO7h2fbcbANsikOqpTogNdRN5nzbTtUigsvZtB6C5iOJYPXumAIpt2a75O81QzrzIaewEct48ROyn+xuXusPwtXy/3QjcGV1V5OWVJ2G5ZapoDdSzsEi3VqCPwSMq6131VGY+2IwtVojDLtS7C77u3hZDsTXz9f05U0tXMrk2jtcwdZYauyXoSzifpyqwl7PY8AO5xWnsfy2wsGSNbQxLa4WTXYj/ALuSq7QtRBfkEut+7nAmaJq1myNlGu0yJs0Fonnb6UArAq5TJXxY2VWs6KBh1EGZEJpWUD0grCo2mMDRJyghwxYLW5uYguGFsQWgkD3gB3gejOFJbGKxAguEcClNgDpMCcwn43BWoBfDHsbJd8zWSOhI0I1eTbUlJj+KOlfrYuAi1ai03Ycnzo2yi6+7Z/xC34CTBlP0NQQA1wc53q7oXuvUH5lThBA9ZR2T0lfa9vdjTabEv0vfq9ic7ldQVs7AesJejkOp+HNIDclalVA1pyThLp4n4bx5PWSEFoQ0zLICk1Iss6qPGGCCioteqgYE27HGgl2/axJ0Rgr+ZhlrE1g+xXhRvdiD2w3XXdPbGzbqmsI+gORPtxRdeX14QyWK07KIRm8bbNb/AD6dB/ZpygVrrjDYRLMVNwidVcXCjRquyV1Z6QVxmqeoU3GmGvBGKH3SHCIwwIDspBJiIAMEyrN9G8K1zJxNexaQHUWC7XAicpBmQIG/UDXIHPjP2juCvZF9rrXtTXHbS82ZbVKwm3Y72kbnur1rTmYNzFOJo206mbIk60IU99dSzpWME9UASRyYmrxL7VH2GFVW6rL1N9TBjnZ722ylbApbRq08Kcx5zbj3MGGzhLGKo2TLLAJLrsmQ686ubMJGtimW8eUFdhutLNZfGq72m7C17MRp7xdsdHp0beAoBMhg2gkdCksBPQbrdNUr2mu3KUeRF62YuZVd2T5VRBdQx1jRaQHaQApFMTOfn0fRa8yfCmGszbaoOaG4qGtZC8Il0RTtm4qlhZMbRJL4Y9SsXl1TgtIec2X1ygSwvIdInC5pBbBDpLYAAvfeDXcrTWPKBpSdsAII4o1t0tfZT2t+I22JNW7G1gvZkDAfbFGmLtfXW2Nc66o/J79Y5BruVSzVaMMJBVWItsCJPLowcTj2KPRAYjY/SPjTsnWPaIG0R2/BOpqtUY6+v1oUtpChZAz5U8hDiW3FDWLCWH1D+/55+cjuZx+lXZxKRtOr2uyeRZkernq0doMXnSm7raNcF0Y29K4cKz9kiOA0pPmjZoWuyJ867ANWZ9DaAad9eEB8xdminc3jHtfbez7NvBXY6p0iVwUrqtYXH6hbrot8adh2ORTCy+3GbPxfIJSlYaBmywaQ6sZgCQwhbe0N0vActbxXCEtyzVFRxNelUc11MOIDWeqU9TTE5HNPMSSTC82LJbhqxD5kQeRzS4Zmt14rhEXAgjrqZtuh+wp+cfrh26x98/vZX/t9fb/3ft98ihL9kYVTLpLUU0yKNqqqsWnX3QrHMvFjiycGdeJyzinEpSjVCHvP2jjGMZz1l2Ni9MMAXJoiZduPggWMfYRjP4jd8XxzoJhXn3P38OMRx/J9YznOffumnmR1V2p3P1L1PofW+R4sWffCmzcjSGPYSzWFuojouzonttkz1tuuhbqQnXvbdfKDDA2kGNmwwQzNnIWm2cPrMDUdjz4Vgi3xLCUqeCxlR72POGp4h9J+DYwipSe5zw0VHloIFnSQCBGW8Bja1QGk4mpSABAe5ocHl3K4QJIF51sN1d35ez+veGSLP2/k5ioPlief/I9Ns/FnHrPuMfizj9vr1nj5W05/Rikl9s/fCdhmPv8A/NhtmP7/AHj37/TnwL3/AKV7/Wgbvqp6zyr2fZkkNmVd9drQVb92LqPdmlAdh9d2+Opeu9c6z2xpjbsNSJpSZlT2ppvSGxaZ2jriI7fRDmx2ztQKNqzTHj/5EbIsC7C2DRO6k/ZCrrTvetKDjtff3ZSB3dqHjON1mpwVjcxBtnZDlX9yjI2rwAttYbLaCTS4uLJHkbP6FpX/AGzxO4P/ABHFdr+0Y8p3jZefin3aU/8AZp627f5A+H2/+XtHr/tmix79es/hB/39/tx7b/eOc+sRlj3jOftj3nPrnkIhzS4GCZXLiaiRDiITEDIEsrsFmHHEZZuMKjZCUSpY/wAWEoyr9/F/3ufgA46x80Ri57D07q3kcr3TSFO8bP3At3PbndGrd9eRa3Qe/BWomqXN9hLTE6D2DrGwo9b0/atZpWaOr287o6QcFrfrF0Ih+5XWQzcLVepQ9gwyw+G60UDO/wAanK1x+LjINVpZ4a2ztInNlg6BGD5yvtlIvF0pW2ZzmeeFfw8YI4avTxePLhjcGwtqY2vVpvZVxFOm9jqb3Frw5jiL6TIuBF2Pz52ubSg06htTY10tYXAhwAIgjbXey2/xxxz6FedOOOOETlae8O6aOrdj1VaajaNgnqZ+wlYkpWlMxrUpqb6iz6VsWuCkuGTktmptv18S/iWjhgCGFG114stzDNm0HVdvJEM2FMuaEgDGBh3GAik2jDMYxrPposvqsnVWZXHFZEI5xG2v+RPEo+scyfG8LiMb4dWw+FE16hp5DxeDEVGkuzwSIaDYXOxGo9GFfTp12Pqew3NPLm1aQOXe6ouq8wdVtySTnQNxhsd1NGHI6bXtZJxdIAA02ymWwksVgjL8EWZX+qriMkYse0ULh50Ds7Rcp1zytQ7Xs4euKde2SrEJufzE1bKFgQGvR13UXGzOqsTHIOi4OTMFWdfYirLbszJKizWWHKoZKlYwboDqYK4gkLSNbEIKhGsi6hKvhO2uH1PquWcUesQz9YVicYYj8yN8o2fHGNWK+0F0d1ouaZdgaojDcZIpL/ExVIFJuCqAj1tZGLo0+8WYXtWQU84xjFwxxNZOLsWYzH4U+jPjhDgHVQ4shpd4i5wBAaGRyTDDmcJ0IF7GdT13CfwxM2w4kXMg3Ezt/XZVEo8xNcVLDLG+k7YkpDdfl5MvBS64ZYSQVQOx1hHgZe2sqGdvNayVs1dI0bka9YNEMhpU6tgr5lN/lIsmhq2hbrz4tOLvp+obFUWsULnylUBpjndCNt/BSzqSYrq1Sys+S9j9C6KVTsKWrz7ZhVk74r8aekqh5i1dcajUPYCStnVUgXVxkAWzg5JF+KFGLI13tK4H2ZjONn1EcZhZCv3VmUl0J1PMOa+ek67MGzY6dvmJJOuyNLaRoZrGf5o+m+VFmPXmVVN0IQhVRZaPXXAe66qyD6M+NnLBrWcCZ8SeQWmA4AZScxuZkibgbIMdhRaGG1nerCQZGozXGvyvran8/NpJaSMWFo23latcDm/LyS5MIwuOKtZUoQ16Eoisg+rYpUpZr2FdsRRoO5wYzEvVHVQy135Qkjo9T2TW+v8AZn6zatrda8JWVTraRraOgKRBGMgVrBpVL0cW2MXrh3F6ciotXdYzpFAuoJssFb439M31Sou6/wBXsqn+IYlDKUDH2aXFksMYlirFkIlXnmWSxXOPypE2YG+RHFeK8hn0317YOoFnriqQ2vn3tEY+VwOKFDIqM4ksFtWB8QDLIxbZ866nEZWZnmUsZliOcQ70Z8dzUyC85SA4u8QqOD2je8ODoPlbRBjMHMmNBH7O2/WbkxuI1OqrRrPlZqW2bQv1dOp2ciZztKom2ip1ySsIfYLFQihuZZBzMiApbdwImkGOMW2BNhfYyADFHturxmjzHRjjLqnmqbQG9OPbAVKVwOrnfVWCxItQkL5WvR75BbXTTitUSdSDAM6Fw7n8OjmiVlrk/Q/Vmvk2GI9N19QZYUMZMpcoXi35KDKoODIjOoeOa7BDhhzRfl4hAcumBFVcbcZnL1X9AdSkmhsidH1q9iv+HAJ1qRbIkWMM4lXiizI/8nNMveaJZxLNGZ2/JzX8675j9WfHQ6A6qWZbT4i7MHAD72QWN502JOsvXsKBbKTI1wzRbe8/1v8ANV3A8nsbAl3M1FqT1ex1JFLYaqdzASBrHQgjNIKdUITr7d0XXbAR4vIqmQONCUzqI5xZkU6iq8OARv2j0Z++f/Bwxn7Z9e/8TPr3+31+vvmoXXQPXDlUapwnrUQOp+nmciHAWtKKJzE+orFNiFbmjBg4dQJMo1ylIPMoQzXbGq6vt9u9+9N9DJg3fbnY2raMM1umIgEcscfjm1MY/Bn8J0/WAazdn3FzP5kPlJtVTOWtvxRxUHPOcc+m9HfB8TgW4sY9janFfSNDiVRiSxrWkPgvby5iQTAuQdoA8WMxFOqaXBJGUHMWs4UuIbsCSYIJE6THUnamAh8Z94HHxn7/AH+VD9v6/wDeft9/f9/7ePoRv0+mH/3qv1/+z9Pf39f1ftx75SrHk/3l2DZCPQnh92U2UW2WV0b75FbAr8ZNMJr+GOaShNZdLN476tpnGWZxwx6VQxsxHGI3xxPEo+KtT/uhW1Qsk37h8Ven6p2ylFfpnSvZ/crOqnOJxjXHat37Z6wV2Wwx8Eo3Y6+qrzLEvjHnDOI4+l9Vw/4NLr/pU/8AZdeLO/33fmP1V1voRv8Axcf3j9M4qr94z+zOM5h+uPX6+/3fpzoD6+lExXgVKrG+TeYVTkdeFR8kljKc2BFWKqIfLvPlOcjboZjYXKcpESszLPumUPH3zBu9zN8+NmHtl7zmCHxr6AAEjnOffqqp0r2crFeP/u2n22Zx692+/eec6+j/ADYWTlan84ULifw+oUdi+KOgOQsz/fZ/e933q43OM4+2fgLj6+2cYz6ziQYagP3NL+VT+fs6+aZ3e8T5uNv6/wCQrs/Qjfb0MP8A71X/AB/8j9P0z/DP+fnj6Eb/AMWG/wB5h/8A0/r/AM+f38pVJ1/dCtHmVey0nxY7+V0QxmMdM23szx424iMfi+KwJHuiru/TyjJRjHEBC9818Sy6f8o4WmGc49tHnPoWplCq/I/r/tPxPYX3ihfjfduvL/70txpMfUahe/8Ar5xvHSg2Lbf8GHTtO66o0KxmPpXXZn5WIGFw4/c0v5VL/Zvumd/vO/Mfqro/QjZ+/wBMP7/jVXnP3z9/f8j+P3/fn7/w5yiPVTnOa6a4fF/jShGEft+zGc4xjOfv/D1j9nrnpWNljoAJonYBNFjISg9cxXFUHAMASa43DmAmi2WjGCX1ThZSSNbbRbCUZQslHOM57+ce8es/t5YYei05m0qYcNCGMaR8WtBHTyJ6oXONi5xHQk/VQSH7in+vv/Pbr7Yz+vplf7x7z/mz/q5EiD7SBXMWhelvogUbbTda0LpsnWSaQTDM6YK7o12RhdiMsRtnH4sZ9SzjPvmSrwfoKr6/m5t+ecabmWY4j8OTCbCM14xjOf5NfzPgxnOc5liPxZ++c+u/zKo+FufhfD+JXxGExGEoPpE4d9MEioWF7H56dVrhNNhERBGsErqaoa+pDWvY8gxUDolswbOaZGZwuTr1WKfHtmfftQgz7+2f54M++MZ949/zN+z9fX6e8c8/M2z9PwhB6/3YM/8A8f8AdjH+b+GOTrJkvTrzmzY4NYrWBksGTJgTQEAvACosJMOOMJsqGEDEGqtIKKItroHorsuushXCUsdmq2q+qu+iyF1N1cLara5xsrsrsjiddlc4ZzGcJwliUJxzmMo5xKOc4zjPOv6Lqf8ANfE/5mH7f9N2/wAgQ44/AofKp/7FjHx7X9/5oQ/fOc5/ngz75z79+/5n+/v3nGffv7ZzjnIIZ7c2GNZirRRxQzaI4DOvLsnYVYJLGfVoQmK64RGl7ziU5ZzOOMRjjGc8ynjkfomX0nVcfj67aNalWFKrUoGm59JzXszhuHa4gOaDAcJIAmENeQQKNFpIIzND8wBBBjNUIuDFwY2hOOOOa64JxxxwiccccInHHHCJxxxwiccccInPXbbXTXO22cK664ynOc5RhCEIRzOcpSlnEYxjCMpSlLOIxjGUpZxHGc455ziOMyzn1jGM5znP6YxjHvOc/wBWOUN7GFO8vezNp6LFMuE8a+qGACvyFMXE3UXd3b6SGI8p8dRGYkoSo641lQaqd+QEgiYXbaS617puRGFOe1VkiL1W9zdu+UjAxL4pMgdB6XGLJVu/Ll6mB2evabw7ZjMgPF7RGsfwffYBk02h2d5bvAjqAQ6GfyRrXdFdTKSndXTvit0/0u0N29KlO2vtZ2P9PtPd3ZLczf8AubaYShDFw7TsLYM3twUsrIfMF0/Vo63oibEvpkGqqQ4wGjYFatATgBq1YYq9cvFHCABCHpEDCDEqgOKIIKPCocUQUeuscUUeqscYeuuiiuumuEI652Hu3qXU+xtT6i2TsHVUvZe8gEs9T0pi2HHfvQhosJZvBBnn4pYIioc5X12yqua/grrCqs2ShlgUi2liMY/pjGPf6+sfrn9+f35/jn788805r/kJ0ftn4NjWO19A2DOwp+vNgRYT7QqYZbou2rtjH6xcrsDE2ZKVb6Rp+10amfV7Fd3627pCtssWlRrxbY/Lnxm1JOpf7H3j1soTPK9evUsC9oAwOcNtBuzL0pdUq52SiGQRpW45vMtjWItG1TZDml4ICNkUMRWM45qFr390qkUPn7btDSQU2sa5te2vmN78HAqvWtG2QvTtvdF2Qsn8C7XdrBL11oRHE4jOaJr84kTjFecML8v/ABkA00PsAzu3r0bUTtb3HbxXFr2qMLNa672JJqW/uZB/L/EqgdG2XZEKTcbLwa5ayxbAjOYByvjnBFZHnWKDENoIFLHpJGLqsoKGvrhcOVRbXKq2gmiyMqiKbK5yhZTdCdc4SzGcJYz65XLUPMPxn34l6Jpncen7KRrOj29kP6lBBxclOlUq1zux8ZmAGI1DYTtlbPFHxZPmCxBKgHKoqmU/Gu+Y/i5tejFdl6/3r1uy0MLFMitpr2GipRTAkTYjhbZEEwolmkwPUtmJDvhXKkypE0mNO3AdvwkWtW/h/LrM5hufhxt1Xjxsl95DRp1dFaQ/8YewD7J13k17N0wOYvF0Rq0nTmN+/wDSR2g7Xgu7DHYht6GHyhL2F0r5Hw3vY2fUnaGoF9N+QmsrJunvWLhtQ8W7JrFZNQWex+nt3oDWL+0+sbTrxw73YC9TtGoHlhJezdN0Z6cAvN3J1v2Zovb2ort7642QDbNSbWG1Lni2JUQyrF5doJka8GDiEe6C6LaJ5lRHGZwz8GZR9SzgvfXRGvd460uFIYs9Q3vTW0ds6p7T1jAtW7dW70MNcKHtOsFFVWD3xuEIIUbRq7SsnWN81Q1tpu2rmKFsUPgi3rxyuvjt29sHYqXZdR7KWJtb7x6ieVaX25rqG++zXyGloFLTW+wNIwfOTK3rjtHXLhdu02ZsiDFGL3GkuS7tn03YMV2K4RU68/dNf7x4h95qELzXk/4fpLDbnq/bdUL3TU951LRPh3XbOrdtQL9o044rUu1USAzr/bJhPxCqta2BrimBErM0T2V4vdf7V1X0B1Z19ue80djP9W1URbbto2uWamMYsxcQRrqwZFe/2goUbWNcvU6qNedsLZgyHSVNGBcjTL4Q4+Vn/cv+R3/mG7h/9nWyc29qn+TGu/7hJ/8A0aLwin+OOOETjjjhE4444ROOOOETjjjhE4444ROOOOEWou++ypdPdM9mdmUjVHsNN0x66Sq7vm/A52OkKynWEn+Bxmz262S9SpxmPrOMm4z7xjHPR0B1XV0z1HpfX8zPxd0qXWH7jsVkrLCtt7D2IwnYuxdzPtuzmy5jt+8Nn+wmWz9Z+YxxVGMaqq4R1z5ofBjoRlK7/wCqV9leP9zP7+sYTUeQfVtriU85ziMaYL43zJlP+RgeNvx5+H37tRHMcYzn3jGPin/DHv4pe/8AX74RMzjj379/b9fUZZ++f0x9sZ++fePt+v3x+/nz6778b+9tv7E7Md9Pbr1pq6HujQNQ1nb3266/sbzetHc6Eq7JRpmfWsF1WE9dzIPsOB1DZiYIbpTpEWwXgbJjaLa0dVvMrx37m7K8kO5mmrdbtNhW754uouuOu91/vfaZtSzXNqmN2wLslEN/c9t6jsfXVM6NgTxd66q0pou7CDMrWG7KqhaRev152/4F9oolfcTjWdP0LcKK9i8XdX1RDpXj9pZJrnR9In1W47OYaxqTrunXlNvWRTRftGGPQbF4kVlNgzdkH2BuQVSCSRbl62/uZ+29Y9p9e7qn7eEZqOsOy9GqRVutWohsOw+O2pM+3NyG6o2fOrKtY1YRtrHZHYaHZdFaa2kXJE4euNltimuO1MPgz3cPBLcU7bsXa+lmnVy7YuxOyPIJkwW74h2g7WQtJ8ieu9O09zcJWBg8yOyaS+1Ura1OuAjgafsI+0bmguL1crbDtiorqx8Wdh37UPILTtm8UNnP3XcOpNjs0vs11sGu6fqtmuZ0vq+ro/phRoKbsNxFRsPXOxa81DA1Uy6OsdQsULzada3Jvb2Fca/6XcXiJ5HXIu9dQ681R4HoO8DeQGxqEKTYVHyhGHVeNrWeNGoa5rpG2KBPwfsZV2lCVaKo1AnHA6aBTbIwRQYrbpEW9U39z87Jq0ft3q9r2DqsdTI6f3LpHptyvq3CrcrtY3/uFh2862LscqqtfFRtKS8kPWU9+jGG4JyFbtuT1BpgqVbjY/8Ac1ux1+wsnYneVt9aL++/pmjDvF7PYGWz9dd5g91T3hv3PuD4ZpvmzduXNe19YjjY0e1rtadidQaOZsCIw+I9SeqW/wDin5dsNLc39S9d7Wm1XbdETaBsXV9arTOnKw5ap1r5QMk+96l1An7X2rVtcYlb1u6vRtw10Df64b2t3fVdjZMao6dbBZON/Gvug0vU86r47dmBAFdAdh6A9/BtU6j6Mt1xQXX27g8fqxOq7r2oDrHfdmb7Mkg46f22HaHTPkIIfr22bA+6y2PVLmCci+oXRnj/ANmdVd/doby1apm2gdgr9VvGwN2T3PIxe213qnp3r2+iXTjKd3SoFtzHrdqz/Pqqsbdi1rQNadOyGTvjqP2H/cxdq3XTUoFXZ1Y23a51T1V14pvrf9rqNMLI0DUfIrX2zN1pqLYh0DcpmV3Usv1/YmaJltOp1Jjbk7IEua3EcaedDdzNvH3xzQ7B0Nsr6zW9J7/1zVevdIYrdAs6g7m2rc1RXjb3u41hp3BsQXWIWm60E3NvTajvu/V+Pt76WtaIEalxQEryzXvBvc8+DPd/XHYOn6fvHbHau0u95mpD1dTrGwkbDT2G1ZJjdo26ncGUdz2T8OOJZh7cUZr5dStjUmtWjfT/AE+CL6UePWp7pofVqHVOwJDT2VVc0gTYJ2V2t2/GwYhiQUFO3f8AufNvYjy7NF+MTw7utqXYxBarn+HCjRju2M4yznGM/ePr3jOM4zj3j7e8ZxjOPePvjGcfp6z+mcZ58JN88TvKytqYImzuLjq2ndfJiGp9cdfutQ6xe6DoWxdokF6YIg38vaTR2F+x6V9PdpVT1DBZo1VIqk0OgO0qin6HeBGpbtonjbq2rb5o1fXLJY93TCfWMqhdUKp1YzamhuuHMevVW69i6z1kzagkfXm9eaZuTXTdbuuzDWhNeWlVaynIu93ALX1v5FdBdzASiGH2AeV4zdl4riRmDNTtArnc+nmhkISxRYRqPZqVhrKa234ciidv7FVH44k4jXcDGcZxjOM+8ZxjOM/vxnHvGeU/84tkD0/pRHsxIb9jYj798YGQa7U9cc7htR5IvkN1tdMXXtU1wJjsOxMphQMzhUlAMYXixKlWPOuFvI3HnN1vj7Y6g8x/XvPr14UeUOcevefXrOOr/WcevXr19vXCLa3lZ/3L/kd/5hu4f/Z1snNvap/kxrv+4Sf/ANGi8+enfXl1qe+9Hdy6PrPS3mKbse5dUdjapr4Vnhh5MBwMebHpzpMoEmYb1qOEJAlgaPTIo0gcMeM83FEDj12XQ+iWvDXBoUghNeaiBVCwe+qWcZzXdQCPVbDOY5zHOYWQlHOY5zjOcZ9ZzjhFMccccInHHHCJxxxwiccccInHHHCJxxxwiccccItWd39bC9wdQ9k9YFFRX43rS9h1sVrKMpZSs2S6+lO+qxDMZ/PQt/oHI+Y594vArz6l6+HNU9q8kN62Hwhb9k6OobDeQDJbX07HVES1Iz2DTPJhjtg/TbtQOm2EsPXjL9G7LJPdjU7Gas11miUiM2ZwevsPreX/AM4xnGcZxjOM49Zxn74zjP64zj9uM8ox2qA/8ZezH3kdpqV5svTm+4Bs8oOvdZXlu3aFmnXDJlPkro+tgVEsdgaINaXgav3PpiQa95tuhJ9d3HWBGG19eka9u5F8sB/LfzJbP2e6uXPZfWfXRyanr7Yj2iDSthXaBtOodZdH194STdbJ+smbxL2b1ywad2d1LNuedi7Z1vsuuaPsurrdXczVLR7e+580++S+tds1LrTuHayewGwxtHRmx7Vr3We5bHu2naiL5Up9q7cSN9J1eGi9m6QldaZ1gWfvqNYIqNHH1qbFeqP3Wuh5+gXVtn1jeNdR7bqD5LtOr7IpAe67sWvMw3SJ4kaDwLVuEzZfdeEyWHi2QJBODutHvqniyqefv6n/AJdf2/k4/k4ziP6/ycZzjOcY/dj7R+2Pt/Jjj16jH0RfnGC80fI7be5FZOy9qNOsNDbY2dppemHvtE6fIbMdR6/6M10jVg126dLb477ecN+0IdovwOvk77QW5ge0CYH2X8JoWDh5vrvkL5amajr+8dk9gdvrtv6R2ILUuxtE0PSNNRad3szR+Dku+1jgEFx167c4S9ydj40qNENe2VDhAduO09bWlr2CT5y79Afw4x9vv6/rz/y88/DHHv1j9ffv+39f8+c5zn+Oc5/XOc8Ivg1o+/8AmRsC+BT3uXu3We0kumdWdUbVo7rTuoA9VD7iYeVlPj3svYJadHpWzh0k7HoGdf7PwHr3Yr7R1MtoIYgSsV14oEkOr/KTu3sPyf8AH+O69mk9dde9kqLy1HVTPc9I6tk5eahtuOoNpU1attPS21bf2uU+33Utse1a3r+89dlKETlSyHZHqzV+cfdTEI4/SOP+j939X7PX6evt+n24xGOP0x6/f+v3/r/f+77/ALPt+mPXCL4VdLeY+2qO3jqu3fIJTBKz8m+9Osmus7F2B149H1cJNPu4DqLWiNKT9SaG/wCoTXTDWNaE1xLs28dht99uBqICfU5uuDYVs1vzR8tROo9Oc6z3Cw7B2nfemu4EDsihvoHds9A8g9jT9SmdXO3FumdI9YjdeT0WQPZc2fVGxqth/CZbCJDYdjffKXfR/pp+HH/XOfXr93r369fw/T9n6cfDj7fr9v0z7z7/AM/v3wi/NkL5keV6nQdx7K1ntdl2I20vZPJ/Y+0etwGGjdrbVrvXek6r2Kt1Opd15qvS+kbB1Fs+lup612Ar0XYHnZDfsBNob5RF2RYxgOTeTSw/Jincv9j+R/bOyZq8yVGkV65vK7qGnVJ9bi9daH2RtINxYXWWu7A312nXzOzkSNchMzsDbameitCdoBW6S2wf9aMxxn9fef685z6/b7x7z9s4z98Zx+mfXr16xyu/fvfq7p8JHr2uIrex+7OxbWCjp/p5MfUG93Z0BCn8QatWEqSatN6w07BIzLszstqPJNp6mdI41Tnb3Gras+IsD7MMx2l5NdMdRLq4GqOmY2eSHad3wzsHXGzX7FoPRmrlWVxlXFlsGzsd07AEpnKudIfVMCp/yDwcXXFxjGMYxj7YxjGMY/hj9OV+8d+mjupdXcG7g+q3Xt/sh9dvfc2/1h3g07VvJwQa6I6JeVeSQk0LTEK5To/W+tWEX2ItMQrKzyDnxbtowsFwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJzjOEbI/DPHvHvGf1zjOMxziUc4zjOM4zGWMZxnGcZxnGM4zjOOcuOEVHdh8eOx+mdiedjeHrfWkY2wtS9h3/AMZ96IPX9I7+7YFSMc7No7lSA3bePvZLwi4kxy91ZDsfXO4tLyGm7dZF7OwJ3gbJNH8z+r2uxq+t+2gtg8be5GVmAw+se86VuqX7MdGVVM/72G/jMGHVncQdt85ZHs623N81ro+GTlEkLzYBTb7mJbtoWk9k64w0/sLUdZ3nU20YwaaxuCFTs2vMoQlicInpXYhy0r4J4xOGbhpzrnjE65RljGcEWV4lHP29+s59+sZ9xzn1n19sS9Z9fx9es/s5y5SCvwc1fSswn489zd/eNtFEb8jat1/2DXunVtFts82YqE6g7xT9q9foF/8AiVyX6Up1CmNMI1jyGz7lz1x0f+6CamLUOj798au1KKs5z/8ASX497xoOxXxxmXqonYusO5ztcxKWM4/2SL10NGHr19FPOfi4RXj45R8Rx/dHaY5iZ114TsZYzPEbh+6e/UsZY9/yJZHv6FfSjn1j+VX9VP75+1nrHuXXyR/dJWN0qvwnwi02iXw/LP8AzF5BdlW0fr8Upp8ax1PWX69Y9Qw7D9+8+549e8kV585xjHvOcYx+/OfWP9fMS3ff9H611thuXYe4avouoqYQsabRuD9TrOuroWS+GuRrp2WCtHxOX8mGLCYznL1GEZSljGan2dFeXm6X5l2L5oZ1FWQNmspL419CaN10TjMs+8107h3I68iH1PxYzmFha4RMXmOIzGkHZiWZ5PpfhD49arsaze32sN+4ezFGaLV/Z3f227J3hu6ooefzKy9ZP7HYvVmjW4ziMcV6Am1QWEY4jUPD174RYNb5L9o99Ziq8PuvrSdaOjiu7ye7sQbBqnSq8acoYsY9a6KZnXuz/II6Ec2yX2JKNC6pYfDC+Pb9+I/h5W6OkPHLWOnin+3sXb3s/ubeago9j9277MEredwgBO65ekGqWBr9f0jQEV5JMtV6z0ZSi0vXpXkn1rTdiYOH7Sw0Y4jj1j+3Oc5znP8AXLPvOf7c85cInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccInHHHCJxxxwiccccIv//Z"}}},{"cell_type":"markdown","source":"# Data Overview","metadata":{}},{"cell_type":"code","source":"# Delete train that was modified for EDA purposes and start with a fresh train dataframe\ntrain = pd.read_csv(\"../input/feedback-prize-effectiveness/train.csv\")\ntest = pd.read_csv(\"../input/feedback-prize-effectiveness/test.csv\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-21T15:26:48.202136Z","iopub.execute_input":"2022-07-21T15:26:48.202804Z","iopub.status.idle":"2022-07-21T15:26:48.461940Z","shell.execute_reply.started":"2022-07-21T15:26:48.202764Z","shell.execute_reply":"2022-07-21T15:26:48.461004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"label\"] = train[\"discourse_effectiveness\"].replace({\"Ineffective\": 0, \"Adequate\": 1, \"Effective\": 2})","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-21T15:26:48.463276Z","iopub.execute_input":"2022-07-21T15:26:48.463642Z","iopub.status.idle":"2022-07-21T15:26:48.499596Z","shell.execute_reply.started":"2022-07-21T15:26:48.463607Z","shell.execute_reply":"2022-07-21T15:26:48.498759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train[['discourse_text','discourse_type','label']]\ntest_df = test.drop(['discourse_id','essay_id'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:26:48.502366Z","iopub.execute_input":"2022-07-21T15:26:48.503072Z","iopub.status.idle":"2022-07-21T15:26:48.512899Z","shell.execute_reply.started":"2022-07-21T15:26:48.503043Z","shell.execute_reply":"2022-07-21T15:26:48.511832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Text cleaning","metadata":{}},{"cell_type":"code","source":"stop_words = stopwords.words('english')\ndata_cleaning_re = 'https:\\S*|http:\\S*|[^a-zA-Z0-9]'","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:26:48.514681Z","iopub.execute_input":"2022-07-21T15:26:48.515100Z","iopub.status.idle":"2022-07-21T15:26:48.525843Z","shell.execute_reply.started":"2022-07-21T15:26:48.515060Z","shell.execute_reply":"2022-07-21T15:26:48.524874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stop_words[:10]","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:26:48.528774Z","iopub.execute_input":"2022-07-21T15:26:48.529101Z","iopub.status.idle":"2022-07-21T15:26:48.539469Z","shell.execute_reply.started":"2022-07-21T15:26:48.529077Z","shell.execute_reply":"2022-07-21T15:26:48.538557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Removing stopwords from texts\ndef rmv_stopwords(text):\n    word_list = text.split()\n    tokens =[]\n    for word in word_list:\n        if word not in stop_words:\n            tokens.append(word)\n    return \" \".join(tokens)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:26:48.541169Z","iopub.execute_input":"2022-07-21T15:26:48.541409Z","iopub.status.idle":"2022-07-21T15:26:48.547776Z","shell.execute_reply.started":"2022-07-21T15:26:48.541384Z","shell.execute_reply":"2022-07-21T15:26:48.546616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def standardizer(text):\n    text = text.lower()\n    text = re.sub(data_cleaning_re,\" \",text)\n    text = text.strip()\n    return text","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:26:48.549501Z","iopub.execute_input":"2022-07-21T15:26:48.550053Z","iopub.status.idle":"2022-07-21T15:26:48.556481Z","shell.execute_reply.started":"2022-07-21T15:26:48.550009Z","shell.execute_reply":"2022-07-21T15:26:48.555482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cleaning the data\ntrain_df['discourse_text'] = train_df['discourse_text'].apply(standardizer).apply(rmv_stopwords)\ntest_df['discourse_text'] = test_df['discourse_text'].apply(standardizer).apply(rmv_stopwords)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:26:48.557815Z","iopub.execute_input":"2022-07-21T15:26:48.558155Z","iopub.status.idle":"2022-07-21T15:26:52.073954Z","shell.execute_reply.started":"2022-07-21T15:26:48.558121Z","shell.execute_reply":"2022-07-21T15:26:52.072984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:26:52.079131Z","iopub.execute_input":"2022-07-21T15:26:52.079407Z","iopub.status.idle":"2022-07-21T15:26:52.093009Z","shell.execute_reply.started":"2022-07-21T15:26:52.079382Z","shell.execute_reply":"2022-07-21T15:26:52.091896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def concat(df):\n    discourse_text = []\n    for text,Type in zip(df['discourse_text'],df['discourse_type']):\n        text = f\"[{Type.upper()}]\"+\" \"+text\n        discourse_text.append(text)\n    return discourse_text","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:26:52.094480Z","iopub.execute_input":"2022-07-21T15:26:52.095138Z","iopub.status.idle":"2022-07-21T15:26:52.101581Z","shell.execute_reply.started":"2022-07-21T15:26:52.095100Z","shell.execute_reply":"2022-07-21T15:26:52.099584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_discourse_text = concat(train_df)\ntest_discourse_text = concat(test_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:26:52.102966Z","iopub.execute_input":"2022-07-21T15:26:52.103937Z","iopub.status.idle":"2022-07-21T15:26:52.141948Z","shell.execute_reply.started":"2022-07-21T15:26:52.103902Z","shell.execute_reply":"2022-07-21T15:26:52.140990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['discourse_text']=train_discourse_text\ntest_df['discourse_text']=test_discourse_text","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:26:52.143302Z","iopub.execute_input":"2022-07-21T15:26:52.143966Z","iopub.status.idle":"2022-07-21T15:26:52.155487Z","shell.execute_reply.started":"2022-07-21T15:26:52.143927Z","shell.execute_reply":"2022-07-21T15:26:52.154559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dropping discourse type\ntrain_df.drop(['discourse_type'],axis=1,inplace=True)\ntest_df.drop(['discourse_type'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:26:52.156821Z","iopub.execute_input":"2022-07-21T15:26:52.157647Z","iopub.status.idle":"2022-07-21T15:26:52.165149Z","shell.execute_reply.started":"2022-07-21T15:26:52.157612Z","shell.execute_reply":"2022-07-21T15:26:52.164083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Tokenizer","metadata":{}},{"cell_type":"code","source":"VOCAB_SIZE = 100000\nSEQUENCE_LENGTH = 300\n\ntokenizer = tf.keras.layers.TextVectorization(\n    standardize = None,\n    max_tokens = VOCAB_SIZE,\n    output_mode = 'int',\n    output_sequence_length = SEQUENCE_LENGTH\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:26:52.166471Z","iopub.execute_input":"2022-07-21T15:26:52.167301Z","iopub.status.idle":"2022-07-21T15:26:56.121900Z","shell.execute_reply.started":"2022-07-21T15:26:52.167205Z","shell.execute_reply":"2022-07-21T15:26:56.120800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"adapting_ds = tf.data.Dataset.from_tensor_slices(train_df['discourse_text'])\ntokenizer.adapt(adapting_ds)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:26:56.123485Z","iopub.execute_input":"2022-07-21T15:26:56.123826Z","iopub.status.idle":"2022-07-21T15:27:50.905600Z","shell.execute_reply.started":"2022-07-21T15:26:56.123791Z","shell.execute_reply":"2022-07-21T15:27:50.904499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"length of vocabulary of input data :\",len(tokenizer.get_vocabulary())+1)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:50.909112Z","iopub.execute_input":"2022-07-21T15:27:50.910011Z","iopub.status.idle":"2022-07-21T15:27:50.982790Z","shell.execute_reply.started":"2022-07-21T15:27:50.909971Z","shell.execute_reply":"2022-07-21T15:27:50.981491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transformer implementation","metadata":{}},{"cell_type":"markdown","source":"### positional encoding","metadata":{}},{"cell_type":"code","source":"def get_angles(pos,d_model,i):\n    return (pos) * 1 / (np.power(10000, (2 * (i//2)/np.float32(d_model))))\n\ndef positional_encoding(SEQUENCE_LENGTH,EMBEDDING_DIM):\n    angles = get_angles(pos=np.arange(SEQUENCE_LENGTH)[...,np.newaxis],\n                       d_model=EMBEDDING_DIM,\n                       i=np.arange(EMBEDDING_DIM)[np.newaxis,...])\n    \n    angles[:,0::2] = np.sin(angles[:,0::2])\n    angles[:,1::2] = np.cos(angles[:,1::2])\n\n    pos_encoding = angles[np.newaxis,...]\n    return tf.cast(pos_encoding,dtype=tf.float32) #(1,30,30)\n\npositional_encoding(30,100).shape","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:50.984407Z","iopub.execute_input":"2022-07-21T15:27:50.984788Z","iopub.status.idle":"2022-07-21T15:27:50.998733Z","shell.execute_reply.started":"2022-07-21T15:27:50.984752Z","shell.execute_reply":"2022-07-21T15:27:50.997665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n, d = 2048, 512\npos_encoding = positional_encoding(n, d)\nprint(pos_encoding.shape)\npos_encoding = pos_encoding[0]\n\n# Juggle the dimensions for the plot\npos_encoding = tf.reshape(pos_encoding, (n, d//2, 2))\npos_encoding = tf.transpose(pos_encoding, (2, 1, 0))\npos_encoding = tf.reshape(pos_encoding, (d, n))\n\nplt.pcolormesh(pos_encoding, cmap='RdBu')\nplt.ylabel('Depth')\nplt.xlabel('Position')\nplt.colorbar()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:51.000362Z","iopub.execute_input":"2022-07-21T15:27:51.000785Z","iopub.status.idle":"2022-07-21T15:27:51.724640Z","shell.execute_reply.started":"2022-07-21T15:27:51.000750Z","shell.execute_reply":"2022-07-21T15:27:51.723684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### scaled dot product attention","metadata":{}},{"cell_type":"code","source":"def scaled_dot_product_attention(query,key,value):\n    matmul_qk = tf.matmul(a=query,b=key,transpose_b=True)\n    \n    # scale the matmul_qk\n    dk = tf.cast(tf.shape(query)[-1],dtype=tf.float32)\n    scaled_attention_logits = matmul_qk/ tf.sqrt(dk)\n    \n    # we are not using either padding_mask or look_ahead_mask\n    # padding_mask will be applied in Embedding layer as mask_zero=True\n    # look_ahead_mask only applied in decoder and we don't need any decoder\n    \n    attention_weights = tf.nn.softmax(scaled_attention_logits,axis=-1)\n    \n    output = tf.matmul(attention_weights,value) # (...,seq_len,emb_dim)\n    \n    return output,attention_weights","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:51.725738Z","iopub.execute_input":"2022-07-21T15:27:51.727713Z","iopub.status.idle":"2022-07-21T15:27:51.734797Z","shell.execute_reply.started":"2022-07-21T15:27:51.727672Z","shell.execute_reply":"2022-07-21T15:27:51.733896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Multi-head-attention","metadata":{}},{"cell_type":"code","source":"class MultiHeadAttention(tf.keras.layers.Layer):\n    def __init__(self,num_heads,d_model):\n        super(MultiHeadAttention,self).__init__()\n        self.num_heads= num_heads\n        self.d_model = d_model\n        self.depth = self.d_model // self.num_heads\n        \n        # multiplying x with these three outputs from three dense layer\n        # we will get three thing \n        # query = matmul(x, trans(wq)) --> (seq_len,embedding-dim)\n        # key = matmul(x, trans(wk))\n        # value = matmul(x, trans(wv))\n        self.wq = tf.keras.layers.Dense(d_model) # --> (emb_dim X emb_dim)\n        self.wk = tf.keras.layers.Dense(d_model)\n        self.wv = tf.keras.layers.Dense(d_model)\n        \n        # this is the linear layer in the above picture\n        self.dense = tf.keras.layers.Dense(d_model)\n        \n    def split_heads(self,x,batch_size):\n        \"\"\" this is a logical split of q,k,v we are not actually splitting\"\"\"\n        \n        # we are reshaping the q,k,v (seq_len,seq_len) mean(30,30)\n        # to shape of ()\n        x = tf.reshape(x,(batch_size,-1,self.num_heads,self.depth)) \n        # shape --> TensorShape([16, 30, 4, 25]) # (batch_size,seq_len,heads,depth)\n        \n        # we are just bringing 4 inplace of 30 \n        # so we want to do (16,4,30,25)\n        # forget about batch_size 16\n        # we have (30,4,25)\n        # just assume we are feeding (seq_len,depth) feeding to heads(like batches)\n        # so we write the batch_size(here num_heads) in first place\n        # so now we get (4,30,25)\n        \n        x = tf.transpose(x,perm=[0,2,1,3]) # see we just swapped 1 and 2\n        # shape --> TensorShape([16, 4, 30, 25])\n        return x\n    \n    def call(self,q,k,v):\n        batch_size = tf.shape(q)[0]\n        q = self.wq(q) # shape (batch_size,seq_len,embedding_dim)\n        k = self.wk(k)\n        v = self.wv(v)\n        \n        q = self.split_heads(q,batch_size) # (batch_size,num_heads,seq_len,depth)\n        k = self.split_heads(k,batch_size)\n        v = self.split_heads(v,batch_size)\n        \n        scaled_output,attention_weights = scaled_dot_product_attention(query=q,key=k,value=v)\n        # again swap back to previous state (16,30,4,25)\n        scaled_output = tf.transpose(scaled_output,perm=[0,2,1,3])\n        #concatenating outputs of all heads\n        concat_output = tf.reshape(scaled_output,shape=(batch_size,-1,self.d_model))\n        \n        output = self.dense(concat_output)\n        \n        return output, attention_weights","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:51.736577Z","iopub.execute_input":"2022-07-21T15:27:51.737313Z","iopub.status.idle":"2022-07-21T15:27:51.751812Z","shell.execute_reply.started":"2022-07-21T15:27:51.737264Z","shell.execute_reply":"2022-07-21T15:27:51.750822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE =32","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:51.753093Z","iopub.execute_input":"2022-07-21T15:27:51.753720Z","iopub.status.idle":"2022-07-21T15:27:51.764993Z","shell.execute_reply.started":"2022-07-21T15:27:51.753679Z","shell.execute_reply":"2022-07-21T15:27:51.763920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding_dim = 100\nmha = MultiHeadAttention(4,100)\nx = tf.random.uniform((BATCH_SIZE,SEQUENCE_LENGTH,embedding_dim),minval=0,maxval=100)\nsplit_output = mha.split_heads(x,BATCH_SIZE)\nprint('Shape of output of split head fuction :',split_output .shape)\n\n# when output is passed through scaled_dot_product_attention\noutput,attention_weights = scaled_dot_product_attention(split_output ,split_output ,split_output )\nprint('Shape of output of scaled dot attention :',output.shape)\n\n# output of multi-head-attention\nmha_output,_ = mha.call(x,x,x)\nprint('Shape of attention output from mha :',mha_output.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:51.767011Z","iopub.execute_input":"2022-07-21T15:27:51.767370Z","iopub.status.idle":"2022-07-21T15:27:52.739431Z","shell.execute_reply.started":"2022-07-21T15:27:51.767334Z","shell.execute_reply":"2022-07-21T15:27:52.738432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Encoder layer","metadata":{}},{"cell_type":"code","source":"## point wise feed forward network\ndef point_wise_feed_forward_network(d_model,dff):\n    return tf.keras.models.Sequential([\n        tf.keras.layers.Dense(dff,activation='relu'),\n        tf.keras.layers.Dense(d_model)\n    ])","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:52.741176Z","iopub.execute_input":"2022-07-21T15:27:52.741555Z","iopub.status.idle":"2022-07-21T15:27:52.746926Z","shell.execute_reply.started":"2022-07-21T15:27:52.741511Z","shell.execute_reply":"2022-07-21T15:27:52.745889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EncoderLayers(tf.keras.layers.Layer):\n    def __init__(self,num_heads,d_model,dff,rate=0.1):\n        super(EncoderLayers,self).__init__()\n        self.num_heads = num_heads\n        self.d_model = d_model\n        self.dff = dff\n        self.mha = MultiHeadAttention(num_heads=self.num_heads,d_model=self.d_model)\n        self.fnn = point_wise_feed_forward_network(d_model=self.d_model,dff=self.dff)\n        \n        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n\n        self.dropout1 = tf.keras.layers.Dropout(rate)\n        self.dropout2 = tf.keras.layers.Dropout(rate)\n\n    def call(self,x,training):\n        attention_output,attention_weights = self.mha(x,x,x)\n        attention_output = self.dropout1(attention_output,training=training)\n        out1 = self.layernorm1(x + attention_output)\n        \n        fnn_output = self.fnn(out1)\n        fnn_output = self.dropout2(fnn_output,training=training)\n        out2 = self.layernorm2(out1 + fnn_output)\n        \n        return out1,attention_weights","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:52.748809Z","iopub.execute_input":"2022-07-21T15:27:52.749249Z","iopub.status.idle":"2022-07-21T15:27:52.760998Z","shell.execute_reply.started":"2022-07-21T15:27:52.749115Z","shell.execute_reply":"2022-07-21T15:27:52.759881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_encoder_layer = EncoderLayers(num_heads=4,d_model=embedding_dim,dff=2048)\nx = tf.random.uniform((BATCH_SIZE,SEQUENCE_LENGTH,embedding_dim),minval=0,maxval=100)\n\nencoder_output,_ = test_encoder_layer.call(x,training=False)\nprint('Shape of encoder output : ',encoder_output.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:52.762276Z","iopub.execute_input":"2022-07-21T15:27:52.762762Z","iopub.status.idle":"2022-07-21T15:27:52.834440Z","shell.execute_reply.started":"2022-07-21T15:27:52.762726Z","shell.execute_reply":"2022-07-21T15:27:52.833489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Encoder","metadata":{}},{"cell_type":"code","source":"class Encoder(tf.keras.layers.Layer):\n    def __init__(self,num_layers,d_model,num_heads,dff,input_vocab_size,rate=0.1):\n        super(Encoder,self).__init__()\n        self.d_model = d_model\n        self.num_layers = num_layers\n        \n        self.embedding = tf.keras.layers.Embedding(input_vocab_size,d_model,mask_zero=True)\n        self.pos_encoding = positional_encoding(SEQUENCE_LENGTH=SEQUENCE_LENGTH,EMBEDDING_DIM=d_model)\n        self.enc_layers = [EncoderLayers(num_heads=num_heads,d_model=self.d_model,dff=dff) for _ in range(self.num_layers)]\n        self.dropout = tf.keras.layers.Dropout(rate)\n    \n    def call(self,x,training):\n        \n        x = self.embedding(x)\n        x *= tf.sqrt(tf.cast(self.d_model,dtype=tf.float32))\n        x += self.pos_encoding[:,:tf.shape(x)[1],:] # tf.shape(x)[1] gives sequence length\n        \n        x = self.dropout(x,training=training)\n        for i in range(self.num_layers):\n            x,attention_weights = self.enc_layers[i](x,training)\n        return x # (batch_size,seq_len,d_model)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:52.836504Z","iopub.execute_input":"2022-07-21T15:27:52.837064Z","iopub.status.idle":"2022-07-21T15:27:52.846403Z","shell.execute_reply.started":"2022-07-21T15:27:52.837029Z","shell.execute_reply":"2022-07-21T15:27:52.845526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_encoder = Encoder(num_layers=8,d_model=100,num_heads=4,dff=512,input_vocab_size=10000,rate=0.2)\nx = tf.random.uniform((BATCH_SIZE,SEQUENCE_LENGTH),minval=0,maxval=100)\nsample_enc_output= sample_encoder.call(x,training=False)\nprint('Shape of sample encoder output : ',sample_enc_output.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:52.853122Z","iopub.execute_input":"2022-07-21T15:27:52.853466Z","iopub.status.idle":"2022-07-21T15:27:53.194375Z","shell.execute_reply.started":"2022-07-21T15:27:52.853434Z","shell.execute_reply":"2022-07-21T15:27:53.193379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data preparation","metadata":{}},{"cell_type":"code","source":"X = train_df['discourse_text']\ny = train_df['label']","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:53.195589Z","iopub.execute_input":"2022-07-21T15:27:53.195918Z","iopub.status.idle":"2022-07-21T15:27:53.201308Z","shell.execute_reply.started":"2022-07-21T15:27:53.195882Z","shell.execute_reply":"2022-07-21T15:27:53.200391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_vec = tokenizer(np.array(test_df)).numpy()\ntest_dataset = (\ntf.data.Dataset\n.from_tensor_slices(test_vec)\n.shuffle(10000)\n.batch(BATCH_SIZE)\n.cache()\n.prefetch(tf.data.AUTOTUNE))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:53.202855Z","iopub.execute_input":"2022-07-21T15:27:53.203501Z","iopub.status.idle":"2022-07-21T15:27:53.237013Z","shell.execute_reply.started":"2022-07-21T15:27:53.203467Z","shell.execute_reply":"2022-07-21T15:27:53.236195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:53.238239Z","iopub.execute_input":"2022-07-21T15:27:53.238564Z","iopub.status.idle":"2022-07-21T15:27:53.243607Z","shell.execute_reply.started":"2022-07-21T15:27:53.238532Z","shell.execute_reply":"2022-07-21T15:27:53.241963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train,X_val,y_train,y_val = train_test_split(X,y,test_size=0.3,random_state=43)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:53.245174Z","iopub.execute_input":"2022-07-21T15:27:53.245533Z","iopub.status.idle":"2022-07-21T15:27:53.258867Z","shell.execute_reply.started":"2022-07-21T15:27:53.245491Z","shell.execute_reply":"2022-07-21T15:27:53.258020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Cross validation","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import log_loss\ncv = StratifiedKFold(n_splits=5, shuffle=True, random_state=1121218)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:53.260323Z","iopub.execute_input":"2022-07-21T15:27:53.260683Z","iopub.status.idle":"2022-07-21T15:27:53.265904Z","shell.execute_reply.started":"2022-07-21T15:27:53.260649Z","shell.execute_reply":"2022-07-21T15:27:53.264409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE=128\nX_train_vec = tokenizer(np.array(X_train)).numpy()\nX_val_vec = tokenizer(np.array(X_val)).numpy()\ny_train_fold = np.array(y_train)\ny_val_fold = np.array(y_val)\n\ntrain_dataset = (\ntf.data.Dataset\n.from_tensor_slices((X_train_vec, y_train_fold))\n.shuffle(10000)\n.batch(BATCH_SIZE)\n.cache()\n.prefetch(tf.data.AUTOTUNE))\n\nval_dataset = (\ntf.data.Dataset\n.from_tensor_slices((X_val_vec, y_val_fold))\n.shuffle(10000)\n.batch(BATCH_SIZE)\n.cache()\n.prefetch(tf.data.AUTOTUNE))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:53.267819Z","iopub.execute_input":"2022-07-21T15:27:53.268153Z","iopub.status.idle":"2022-07-21T15:27:53.636604Z","shell.execute_reply.started":"2022-07-21T15:27:53.268121Z","shell.execute_reply":"2022-07-21T15:27:53.635628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def LogLoss(y_true,y_pred):\n    return log_loss(y_true,y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:27:53.638071Z","iopub.execute_input":"2022-07-21T15:27:53.638415Z","iopub.status.idle":"2022-07-21T15:27:53.644217Z","shell.execute_reply.started":"2022-07-21T15:27:53.638379Z","shell.execute_reply":"2022-07-21T15:27:53.642999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Creating model","metadata":{}},{"cell_type":"code","source":"\nEMBEDDING_DIM = 64\nNUM_HEADS = 8 ## this have to be a factor of 100 else it will through reshape error\nNUM_LAYERS =2\nDFF = 512\nMAX_SEQ_LEN = 300\nINP_VOCAB_SIZE = len(tokenizer.get_vocabulary())+1\n\n#Encoder layer\nencoder = Encoder(num_layers=NUM_LAYERS,d_model=EMBEDDING_DIM,\n           num_heads = NUM_HEADS,dff=DFF,input_vocab_size=INP_VOCAB_SIZE,rate=0.25)\n\n# Building model\nInputs = tf.keras.layers.Input(shape=(MAX_SEQ_LEN,))\nx = encoder.call(Inputs,training=True)\nx = tf.keras.layers.GlobalAveragePooling1D()(x)\nx = tf.keras.layers.Dropout(0.25)(x)\nx = tf.keras.layers.Dense(32,activation=tf.keras.activations.relu)(x)\nx = tf.keras.layers.Dropout(0.25)(x)\nOutputs = tf.keras.layers.Dense(3,activation='softmax')(x)\n\nmodel = tf.keras.models.Model(inputs=Inputs,outputs=Outputs)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T16:58:48.374275Z","iopub.execute_input":"2022-07-21T16:58:48.374657Z","iopub.status.idle":"2022-07-21T16:58:48.815567Z","shell.execute_reply.started":"2022-07-21T16:58:48.374627Z","shell.execute_reply":"2022-07-21T16:58:48.814604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# compiling model\nmodel.compile(loss=tf.keras.losses.sparse_categorical_crossentropy,\n              optimizer=tf.keras.optimizers.Adamax(beta_1=0.5,beta_2=0.555),\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-07-21T16:58:48.817344Z","iopub.execute_input":"2022-07-21T16:58:48.817718Z","iopub.status.idle":"2022-07-21T16:58:48.829002Z","shell.execute_reply.started":"2022-07-21T16:58:48.817682Z","shell.execute_reply":"2022-07-21T16:58:48.827938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(train_dataset,validation_data=val_dataset,epochs=4)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T16:58:49.308989Z","iopub.execute_input":"2022-07-21T16:58:49.309322Z","iopub.status.idle":"2022-07-21T17:00:01.770001Z","shell.execute_reply.started":"2022-07-21T16:58:49.309294Z","shell.execute_reply":"2022-07-21T17:00:01.769052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_val_pred = model.predict(X_val_vec)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T16:29:00.145578Z","iopub.execute_input":"2022-07-21T16:29:00.146480Z","iopub.status.idle":"2022-07-21T16:29:03.435177Z","shell.execute_reply.started":"2022-07-21T16:29:00.146441Z","shell.execute_reply":"2022-07-21T16:29:03.434215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('log loss - ',log_loss(y_val_fold,y_val_pred))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T16:05:34.843136Z","iopub.execute_input":"2022-07-21T16:05:34.843518Z","iopub.status.idle":"2022-07-21T16:05:34.853792Z","shell.execute_reply.started":"2022-07-21T16:05:34.843482Z","shell.execute_reply":"2022-07-21T16:05:34.852653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction and Submission","metadata":{}},{"cell_type":"code","source":"# Predict\ny_pred = model.predict(test_vec, verbose=1)\ny_pred","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:48:25.122681Z","iopub.execute_input":"2022-07-21T15:48:25.123097Z","iopub.status.idle":"2022-07-21T15:48:25.175358Z","shell.execute_reply.started":"2022-07-21T15:48:25.123062Z","shell.execute_reply":"2022-07-21T15:48:25.174526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load submission template\nsubmission = pd.read_csv(\"../input/feedback-prize-effectiveness/sample_submission.csv\")\n\n# Replace template with predictions\nsubmission['Ineffective'] = y_pred[:,0]\nsubmission['Adequate'] = y_pred[:,1]\nsubmission['Effective'] = y_pred[:,2]\n\n# Save submission file\nsubmission.to_csv(\"submission.csv\", index=False)\n\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T15:48:25.177452Z","iopub.execute_input":"2022-07-21T15:48:25.177792Z","iopub.status.idle":"2022-07-21T15:48:25.195905Z","shell.execute_reply.started":"2022-07-21T15:48:25.177758Z","shell.execute_reply":"2022-07-21T15:48:25.195083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}