{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport torch\nfrom torch import nn\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom torchvision import datasets\nfrom torchvision.transforms import ToTensor\nfrom torchvision.utils import make_grid\nfrom torch import nn\nfrom sklearn.model_selection import KFold\n\nimport matplotlib.pyplot as plt\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In this notebook, we'll go through how you can get started using PyTorch on the mnist dataset. This notebook is divided in the following parts: \n\n\n* [1. Data Preparation for training](#1)
\n * [1.1. Load data using pandas](#1.1.)
\n * [1.2. Create arrays from the data](#1.2.)
\n * [1.3. Create tensors from arrays](#1.3.)
\n * [1.4. Create dataloaders from tensors](#1.4.)
\n* [2. Define and train your model](#2)
\n * [2.1. Define your model](#2.1.)
\n * [2.2. Define your loss function and your optimizer](#2.2.)
\n * [2.3. Define your training and evaluating functions](#2.3.)
\n * [2.4. Train your model](#2.4.)
\n * [2.5. Get predictions and submit](#2.5.)
\n* [Appendix](#A)
\n * [Appendix A: save and load models](#A.A.)
","metadata":{}},{"cell_type":"markdown","source":"\n# **
1. Data Preparation For Training
**\n\n\n➡️ csvs or dataframes cannot be used right away by PyTorch. To train models, pytorch uses dataloaders. To go from csv to dataloader, we first need to create tensors from the data. The 4 step process to go from csv to training is given below. ","metadata":{}},{"cell_type":"markdown","source":"![image.png](attachment:68f96503-7feb-4a46-a29e-b7979e3bb1d6.png)","metadata":{},"attachments":{"68f96503-7feb-4a46-a29e-b7979e3bb1d6.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAABeIAAADMCAYAAAAf4dPTAAAMbWlDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnluSkJDQAghICb0jUgNICaEFkF4EGyEJJJQYE4KKvSwquHYRxYquiii2lWYBsSuLYu+LBRVlXdTFhsqbkICu+8r3zvfNvX/OnPlPuTO59wCg+YErkeShWgDkiwukCeHBjDFp6QzSU4ACElAHrsCCy5NJWHFx0QDK4P3v8u4GQBT3q84Krn/O/1fR4QtkPACQcRBn8mW8fIibAcA38CTSAgCICr3llAKJAs+BWFcKA4R4tQJnK/EuBc5U4qMDNkkJbIgvA6BG5XKl2QBo3IN6RiEvG/JofIbYVcwXiQHQdII4gCfk8iFWxO6Unz9JgcshtoP2EohhPICZ+R1n9t/4M4f4udzsIazMa0DUQkQySR532v9Zmv8t+XnyQR82cFCF0ogERf6whrdyJ0UpMBXibnFmTKyi1hB/EPGVdQcApQjlEclKe9SYJ2PD+gF9iF353JAoiI0hDhPnxUSr9JlZojAOxHC3oFNFBZwkiA0gXiSQhSaqbLZIJyWofKG1WVI2S6U/x5UO+FX4eiDPTWap+N8IBRwVP6ZRJExKhZgCsVWhKCUGYg2IXWS5iVEqm1FFQnbMoI1UnqCI3wriBIE4PFjJjxVmScMSVPYl+bLBfLEtQhEnRoUPFgiTIpT1wU7xuAPxw1ywywIxK3mQRyAbEz2YC18QEqrMHXsuECcnqng+SAqCE5RrcYokL05lj1sI8sIVeguIPWSFiaq1eEoB3JxKfjxLUhCXpIwTL8rhRsYp48GXg2jABiGAAeRwZIJJIAeI2rrruuEv5UwY4AIpyAYC4KzSDK5IHZgRw2siKAJ/QCQAsqF1wQOzAlAI9V+GtMqrM8gamC0cWJELnkKcD6JAHvwtH1glHvKWAp5Ajegf3rlw8GC8eXAo5v+9flD7TcOCmmiVRj7okaE5aEkMJYYQI4hhRHvcCA/A/fBoeA2Cww1n4j6DeXyzJzwltBMeEa4TOgi3J4rmSX+IcjTogPxhqlpkfl8L3AZyeuLBuD9kh8y4Pm4EnHEP6IeFB0LPnlDLVsWtqArjB+6/ZfDd01DZkV3JKHkYOYhs9+NKDQcNzyEWRa2/r48y1syherOHZn70z/6u+nx4j/rREluEHcLOYiew89hRrA4wsCasHmvFjinw0O56MrC7Br0lDMSTC3lE//DHVflUVFLmWu3a5fpZOVcgmFqgOHjsSZJpUlG2sIDBgm8HAYMj5rk4Mdxc3dwAULxrlH9fb+MH3iGIfus33fzfAfBv6u/vP/JNF9kEwAFvePwbvunsmABoqwNwroEnlxYqdbjiQoD/EprwpBkCU2AJ7GA+bsAL+IEgEAoiQSxIAmlgAqyyEO5zKZgCZoC5oBiUguVgDVgPNoNtYBfYCw6COnAUnABnwEVwGVwHd+Hu6QQvQQ94B/oQBCEhNISOGCJmiDXiiLghTCQACUWikQQkDclAshExIkdmIPORUmQlsh7ZilQhB5AG5ARyHmlHbiMPkS7kDfIJxVAqqouaoDboCJSJstAoNAkdj2ajk9EidAG6FC1HK9E9aC16Ar2IXkc70JdoLwYwdUwfM8ecMSbGxmKxdCwLk2KzsBKsDKvEarBG+JyvYh1YN/YRJ+J0nIE7wx0cgSfjPHwyPgtfgq/Hd+G1+Cn8Kv4Q78G/EmgEY4IjwZfAIYwhZBOmEIoJZYQdhMOE0/AsdRLeEYlEfaIt0RuexTRiDnE6cQlxI3EfsZnYTnxM7CWRSIYkR5I/KZbEJRWQiknrSHtITaQrpE7SBzV1NTM1N7UwtXQ1sdo8tTK13WrH1a6oPVPrI2uRrcm+5FgynzyNvIy8ndxIvkTuJPdRtCm2FH9KEiWHMpdSTqmhnKbco7xVV1e3UPdRj1cXqc9RL1ffr35O/aH6R6oO1YHKpo6jyqlLqTupzdTb1Lc0Gs2GFkRLpxXQltKqaCdpD2gfNOgaLhocDb7GbI0KjVqNKxqvNMma1poszQmaRZplmoc0L2l2a5G1bLTYWlytWVoVWg1aN7V6tenaI7VjtfO1l2jv1j6v/VyHpGOjE6rD11mgs03npM5jOka3pLPpPPp8+nb6aXqnLlHXVpejm6NbqrtXt023R09Hz0MvRW+qXoXeMb0OfUzfRp+jn6e/TP+g/g39T8NMhrGGCYYtHlYz7Mqw9wbDDYIMBAYlBvsMrht8MmQYhhrmGq4wrDO8b4QbORjFG00x2mR02qh7uO5wv+G84SXDDw6/Y4waOxgnGE833mbcatxrYmoSbiIxWWdy0qTbVN80yDTHdLXpcdMuM7pZgJnIbLVZk9kLhh6DxchjlDNOMXrMjc0jzOXmW83bzPssbC2SLeZZ7LO4b0mxZFpmWa62bLHssTKzGm01w6ra6o412ZppLbRea33W+r2NrU2qzUKbOpvntga2HNsi22rbe3Y0u0C7yXaVdtfsifZM+1z7jfaXHVAHTwehQ4XDJUfU0ctR5LjRsd2J4OTjJHaqdLrpTHVmORc6Vzs/dNF3iXaZ51Ln8mqE1Yj0EStGnB3x1dXTNc91u+vdkTojI0fOG9k48o2bgxvPrcLtmjvNPcx9tnu9+2sPRw+BxyaPW550z9GeCz1bPL94eXtJvWq8urytvDO8N3jfZOoy45hLmOd8CD7BPrN9jvp89PXyLfA96Punn7Nfrt9uv+ejbEcJRm0f9djfwp/rv9W/I4ARkBGwJaAj0DyQG1gZ+CjIMogftCPoGcuelcPaw3oV7BosDT4c/J7ty57Jbg7BQsJDSkLaQnVCk0PXhz4IswjLDqsO6wn3DJ8e3hxBiIiKWBFxk2PC4XGqOD2R3pEzI09FUaMSo9ZHPYp2iJZGN45GR0eOXjX6Xox1jDimLhbEcmJXxd6Ps42bHHcknhgfF18R/zRhZMKMhLOJ9MSJibsT3yUFJy1LuptslyxPbknRTBmXUpXyPjUkdWVqx5gRY2aOuZhmlCZKq08npaek70jvHRs6ds3YznGe44rH3RhvO37q+PMTjCbkTTg2UXMid+KhDEJGasbujM/cWG4ltzeTk7khs4fH5q3lveQH8VfzuwT+gpWCZ1n+WSuznmf7Z6/K7hIGCsuE3SK2aL3odU5Ezuac97mxuTtz+/NS8/blq+Vn5DeIdcS54lOTTCdNndQucZQUSzom+05eM7lHGiXdIUNk42X1Bbrwo75Vbif/Sf6wMKCwovDDlJQph6ZqTxVPbZ3mMG3xtGdFYUW/TMen86a3zDCfMXfGw5msmVtnIbMyZ7XMtpy9YHbnnPA5u+ZS5ubO/W2e67yV8/6anzq/cYHJgjkLHv8U/lN1sUaxtPjmQr+Fmxfhi0SL2ha7L163+GsJv+RCqWtpWennJbwlF34e+XP5z/1Ls5a2LfNatmk5cbl4+Y0VgSt2rdReWbTy8arRq2pXM1aXrP5rzcQ158s8yjavpayVr+0ojy6vX2e1bvm6z+uF669XBFfs22C8YfGG9xv5G69sCtpUs9lkc+nmT1tEW25tDd9aW2lTWbaNuK1w29PtKdvP/sL8pWqH0Y7SHV92ind27ErYdarKu6pqt/HuZdVotby6a8+4PZf3huytr3Gu2bpPf1/pfrBfvv/FgYwDNw5GHWw5xDxU86v1rxsO0w+X1CK102p76oR1HfVp9e0NkQ0tjX6Nh4+4HNl51PxoxTG9Y8uOU44vON7fVNTU2yxp7j6RfeJxy8SWuyfHnLx2Kv5U2+mo0+fOhJ05eZZ1tumc/7mj533PN1xgXqi76HWxttWz9fBvnr8dbvNqq73kfan+ss/lxvZR7cevBF45cTXk6plrnGsXr8dcb7+RfOPWzXE3O27xbz2/nXf79Z3CO31359wj3Cu5r3W/7IHxg8rf7X/f1+HVcexhyMPWR4mP7j7mPX75RPbkc+eCp7SnZc/MnlU9d3t+tCus6/KLsS86X0pe9nUX/6H9x4ZXdq9+/TPoz9aeMT2dr6Wv+98seWv4dudfHn+19Mb1PniX/67vfckHww+7PjI/nv2U+ulZ35TPpM/lX+y/NH6N+nqvP7+/X8KVcgc+BTA40KwsAN7sBICWBgAd9m2UscpecEAQZf86gMB/wsp+cUC8AKiB3+/x3fDr5iYA+7fD9gvya8JeNY4GQJIPQN3dh4ZKZFnubkouKuxTCA/6+9/Cno20CoAvy/v7+yr7+79sg8HC3rFZrOxBFUKEPcOWuC+Z+Zng34iyP/0uxx/vQBGBB/jx/i8oI5DWRgDObQAAAJZlWElmTU0AKgAAAAgABQESAAMAAAABAAEAAAEaAAUAAAABAAAASgEbAAUAAAABAAAAUgEoAAMAAAABAAIAAIdpAAQAAAABAAAAWgAAAAAAAACQAAAAAQAAAJAAAAABAAOShgAHAAAAEgAAAISgAgAEAAAAAQAABeKgAwAEAAAAAQAAAMwAAAAAQVNDSUkAAABTY3JlZW5zaG905aLbpAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAAtxpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDYuMC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6ZXhpZj0iaHR0cDovL25zLmFkb2JlLmNvbS9leGlmLzEuMC8iCiAgICAgICAgICAgIHhtbG5zOnRpZmY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vdGlmZi8xLjAvIj4KICAgICAgICAgPGV4aWY6VXNlckNvbW1lbnQ+U2NyZWVuc2hvdDwvZXhpZjpVc2VyQ29tbWVudD4KICAgICAgICAgPGV4aWY6UGl4ZWxYRGltZW5zaW9uPjE1MDY8L2V4aWY6UGl4ZWxYRGltZW5zaW9uPgogICAgICAgICA8ZXhpZjpQaXhlbFlEaW1lbnNpb24+MjA0PC9leGlmOlBpeGVsWURpbWVuc2lvbj4KICAgICAgICAgPHRpZmY6UmVzb2x1dGlvblVuaXQ+MjwvdGlmZjpSZXNvbHV0aW9uVW5pdD4KICAgICAgICAgPHRpZmY6WFJlc29sdXRpb24+MTQ0LzE8L3RpZmY6WFJlc29sdXRpb24+CiAgICAgICAgIDx0aWZmOllSZXNvbHV0aW9uPjE0NC8xPC90aWZmOllSZXNvbHV0aW9uPgogICAgICAgICA8dGlmZjpPcmllbnRhdGlvbj4xPC90aWZmOk9yaWVudGF0aW9uPgogICAgICA8L3JkZjpEZXNjcmlwdGlvbj4KICAgPC9yZGY6UkRGPgo8L3g6eG1wbWV0YT4K0lt02wAAPLlJREFUeAHt3QecHFX9APCXBAgkkEIiCSEkhN6E0EJTCUUUEQVBUfRvQQQBFaSoIAqI2Cj6V1EU/gqKgCiCFOlNkBZBauiEhBAIPWiABDD/fRPeZu9yl9ze3e7O7nyHT7jZ3ZlXvm/e/m5/NzvTZ15pCRYCBAgQIECAAAECBAgQIECAAAECBAgQIECgJgJ9a1KqQgkQIECAAAECBAgQIECAAAECBAgQIECAAIFMQCLegUCAAAECBAgQIECAAAECBAgQIECAAAECBGooIBFfQ1xFEyBAgAABAgQIECBAgAABAgQIECBAgAABiXjHAAECBAgQIECAAAECBAgQIECAAAECBAgQqKGARHwNcRVNgAABAgQIECBAgAABAgQIECBAgAABAgQk4h0DBAgQIECAAAECBAgQIECAAAECBAgQIECghgIS8TXEVTQBAgQIECBAgAABAgQIECBAgAABAgQIEJCIdwwQIECAAAECBAgQIECAAAECBAgQIECAAIEaCkjE1xBX0QQIECBAgAABAgQIECBAgAABAgQIECBAQCLeMUCAAAECBAgQIECAAAECBAgQIECAAAECBGooIBFfQ1xFEyBAgAABAgQIECBAgAABAgQIECBAgAABiXjHAAECBAgQIECAAAECBAgQIECAAAECBAgQqKGARHwNcRVNgAABAgQIECBAgAABAgQIECBAgAABAgQk4h0DBAgQIECAAAECBAgQIECAAAECBAgQIECghgIS8TXEVTQBAgQIECBAgAABAgQIECBAgAABAgQIEJCIdwwQIECAAAECBAgQIECAAAECBAgQIECAAIEaCkjE1xBX0QQIECBAgAABAgQIECBAgAABAgQIECBAQCLeMUCAAAECBAgQIECAAAECBAgQIECAAAECBGooIBFfQ1xFEyBAgAABAgQIECBAgAABAgQIECBAgAABiXjHAAECBAgQIECAAAECBAgQIECAAAECBAgQqKGARHwNcRVNgAABAgQIECBAgAABAgQIECBAgAABAgQk4h0DBAgQIECAAAECBAgQIECAAAECBAgQIECghgIS8TXEVTQBAgQIECBAgAABAgQIECBAgAABAgQIEJCIdwwQIECAAAECBAgQIECAAAECBAgQIECAAIEaCkjE1xBX0QQIECBAgAABAgQIECBAgAABAgQIECBAQCLeMUCAAAECBAgQIECAAAECBAgQIECAAAECBGooIBFfQ1xFEyBAgAABAgQIECBAgAABAgQIECBAgAABiXjHAAECBAgQIECAAAECBAgQIECAAAECBAgQqKGARHwNcRVNgAABAgQIECBAgAABAgQIECBAgAABAgQk4h0DBAgQIECAAAECBAgQIECAAAECBAgQIECghgIS8TXEVTQBAgQIECBAgAABAgQIECBAgAABAgQIEJCIdwwQIECAAAECBAgQIECAAAECBAgQIECAAIEaCkjE1xBX0QQIECBAgAABAgQIECBAgAABAgQIECBAQCLeMUCAAAECBAgQIECAAAECBAgQIECAAAECBGooIBFfQ1xFEyBAgAABAgQIECBAgAABAgQIECBAgAABiXjHAAECBAgQIECAAAECBAgQIECAAAECBAgQqKGARHwNcRVNgAABAgQIECBAgAABAgQIECBAgAABAgQk4h0DBAgQIECAAAECBAgQIECAAAECBAgQIECghgJL1LDsuhU9feYr4dFpL4Vpz8wKM1+cHWb9Z06YM/etMG/evLq1QUUEml2gT58+of9S/cLgZfuHEcsPDGNGDg6rjxkaRo8Y1Oxd034CnQqIH53SeIFAlwXEjy5T2bCFBMSPFhpMXWmYgPjRMHoVN1BA/GggvqpbRqCZ40efUrK6KbPVc994K9x81/QwafKMMPOF2S1zMOkIgbwJjBg2MGy27qiw1fjRYakl++WtedpDoGoB8aNqMjsQ6JaA+NEtNjvlWED8yPHgaFpLCYgfLTWcOlMSED8cBgTqI9AM8aMpE/HXT5oarr59Snh9zpvZSA4c0D+MHbV8WHGFwWHYkIFhuYFLlxKGS4TSCb4WAgS6KBD/JDf3jTfDv2e/Hl54eXZ4+tlZYeqMF8PsV+dkJSzdf4mww4RxYeJmY7tYos0I5E9A/MjfmGhR8wuIH80/hnqweAHxY/FGtiBQrYD4Ua2Y7ZtRQPxoxlHT5rwLNHP8aKpEfPwKzwXXPhSmPj0rOybGlJLv660xKowbPSzvx4j2EWhagSnTXwj3PzIjTCsl5eMydsXBYbft1nLJmqYd0WI2XPwo5rjrdWMFxI/G+qu9dwTEj95xVAqBagTEj2q0bJtXAfEjryOjXa0s0Azxo2kS8Xc++Ew457L7s+u+Dx08IGwxflUJ+FaePfqWO4H4hnbrXY+Hl2a9Wvq2SZ/wiZ3WCxuvPTJ37dQgAu0FxI/2Ih4TqK+A+FFfb7X1noD40XuWSiLQHQHxoztq9smDgPiRh1HQhiIL5Dl+NEUi/pa7p4fzr3kwO4bWXX3FsM2ENV12psgzSt8bJhC//nPD7Q+HyY8+nbVh9+3XDltuOLph7VExgcUJiB+LE/I6gfoIiB/1cVZL7wmIH71nqSQCPREQP3qiZ99GCIgfjVBXJ4GFBfIaP/odU1oWbm5+nol/STzvyslZgzbfcJWw1carScLnZ3i0pGAC8b4Lq5QuBdW3b5/w1MyXwwNTng/Dhw4IKw5ftmASutsMAuJHM4ySNhZFQPwoyki3Rj/Fj9YYR71oDQHxozXGsSi9ED+KMtL62QwCeY0fuU7Ex2tqnX7BXdn4xiT8Juu7SWQzHOza2PoCo1YYUk7G3/foc2HdVYeHQcv2b/2O62HTCIgfTTNUGlowAfGjYAPehN0VP5pw0DS5EALiRyGGuak7KX409fBpfAsL5C1+9M2zdbwx67zSdwni5Wgk4fM8UtpWRIE4J+PcjHM0zlULgTwJiB95Gg1tIdBWQPxo6+FRvgTEj3yNh9YQqBQQPyo1rOdNQPzI24hoD4EFAnmKH7lNxF8/aWqY+vSsEG/MGq8JbyFAIH8CcW7GORrnapyzFgJ5EBA/8jAK2kBg0QLix6J9vNoYAfGjMe5qJVCNgPhRjZZt6yUgftRLWj0Eui+Ql/iRy0T83DfeClffPiXT3WL8qq4J3/3jzJ4EaioQr7kV52hc4pyNc9dCoJEC4kcj9dVNoOsC4kfXrWxZHwHxoz7OaiHQUwHxo6eC9u9tAfGjt0WVR6A2AnmJH7lMxN981/Tw+pw3w5hRy4dxpRtDWggQyK9AnKNxrsY5G+euhUAjBcSPRuqrm0B1AuJHdV62rq2A+FFbX6UT6E0B8aM3NZXVUwHxo6eC9idQP4E8xI9cJuInTZ6RjcJ6a4yq32ioiQCBbgukuZrmbrcLsiOBHgqkYzAdkz0szu4ECNRYIM3VNHdrXJ3iCXQqkI7BdEx2uqEXCBDIhUCaq2nu5qJRGlFIgXQMpmOykAg6TaCJBNJcTXO33k3PXSI+3ml65guzw8AB/Z0NX++jQX0EuikQ/6oY52ycu3EOWwg0QkD8aIS6Ogn0TED86JmfvXtHQPzoHUelEKingPhRT211dSYgfnQm43kC+RVodPzIXSL+0WkvZaM1tnSpCwsBAs0jkOZsmsPN03ItbRWBdOylY7FV+qUfBFpdIM3ZNIdbvb/6lz+BdOylYzF/LdQiAgQ6EkhzNs3hjrbxHIFaCqRjLx2LtaxL2QQI9J5AmrNpDvdeyYsvKXeJ+GnPzMpaveIKgxffelsQIJAbgTRn0xzOTcM0pDAC6dhLx2JhOq6jBJpcIM3ZNIebvDua34QC6dhLx2ITdkGTCRRSIM3ZNIcLiaDTDRVIx146FhvaGJUTINBlgTRn0xzu8o69sGHuEvEzX5yddWvYkIG90D1FECBQL4E0Z9Mcrle96iGQBNKxl47F9LyfBAjkWyDN2TSH891arWtFgXTspWOxFfuoTwRaUSDN2TSHW7GP+pRvgXTspWMx363VOgIEkkCas2kOp+fr8TN3ifhZ/5mT9Xu5gUvXo//qIECglwTSnE1zuJeKVQyBLgukYy8di13e0YYECDRUIM3ZNIcb2hiVF1IgHXvpWCwkgk4TaEKBNGfTHG7CLmhykwukYy8di03eHc0nUBiBNGfTHK5nx3OXiJ8z962s/0stuUQ9HdRFgEAPBdKcTXO4h8XZnUDVAunYS8di1QXYgQCBhgikOZvmcEMaodJCC6RjLx2LhcbQeQJNJJDmbJrDTdR0TW0RgXTspWOxRbqlGwRaXiDN2TSH69nh3CXi582bl/W/T596MqiLAIGeCqQ5m+ZwT8uzP4FqBdKxl47Fave3PQECjRFIczbN4ca0Qq1FFkjHXjoWi2yh7wSaSSDN2TSHm6nt2toaAunYS8dia/RKLwi0vkCas2kO17PHuUvE17Pz6iJAgAABAgQIECBAgAABAgQIECBAgAABArUWkIivtbDyCRAgQIAAAQIECBAgQIAAAQIECBAgQKDQAhLxhR5+nSdAgAABAgQIECBAgAABAgQIECBAgACBWgtIxNdaWPkECBAgQIAAAQIECBAgQIAAAQIECBAgUGgBifhCD7/OEyBAgAABAgQIECBAgAABAgQIECBAgECtBSTiay2sfAIECBAgQIAAAQIECBAgQIAAAQIECBAotIBEfKGHX+cJECBAgAABAgQIECBAgAABAgQIECBAoNYCEvG1FlY+AQIECBAgQIAAAQIECBAgQIAAAQIECBRaQCK+0MOv8wQIECBAgAABAgQIECBAgAABAgQIECBQawGJ+FoLK58AAQIECBAgQIAAAQIECBAgQIAAAQIECi0gEV/o4dd5AgQIECBAgAABAgQIECBAgAABAgQIEKi1gER8rYWVT4AAAQIECBAgQIAAAQIECBAgQIAAAQKFFpCIL/Tw6zwBAgQIECBAgAABAgQIECBAgAABAgQI1FpAIr7WwsonQIAAAQIECBAgQIAAAQIECBAgQIAAgUILSMQXevh1ngABAgQIECBAgAABAgQIECBAgAABAgRqLSARX2th5RMgQIAAAQIECBAgQIAAAQIECBAgQIBAoQUk4gs9/DpPgAABAgQIECBAgAABAgQIECBAgAABArUWkIivtbDyCRAgQIAAAQIECBAgQIAAAQIECBAgQKDQAhLxhR5+nSdAgAABAgQIECBAgAABAgQIECBAgACBWgtIxNdaWPkECBAgQIAAAQIECBAgQIAAAQIECBAgUGgBifhCD7/OEyBAgAABAgQIECBAgAABAgQIECBAgECtBSTiay2sfAIECBAgQIAAAQIECBAgQIAAAQIECBAotIBEfKGHX+cJECBAgAABAgQIECBAgAABAgQIECBAoNYCEvG1FlY+AQIECBAgQIAAAQIECBAgQIAAAQIECBRaQCK+0MOv8wQIECBAgAABAgQIECBAgAABAgQIECBQawGJ+FoLK58AAQIECBAgQIAAAQIECBAgQIAAAQIECi0gEV/o4dd5AgQIECBAgAABAgQIECBAgAABAgQIEKi1gER8rYWVT4AAAQIECBAgQIAAAQIECBAgQIAAAQKFFpCIL/Tw6zwBAgQIECBAgAABAgQIECBAgAABAgQI1FpAIr7WwsonQIAAAQIECBAgQIAAAQIECBAgQIAAgUILSMQXevh1ngABAgQIECBAgAABAgQIECBAgAABAgRqLSARX2th5RMgQIDAIgUee/KlRb7uRQIECBAg0JHA9ddf39HTnutlgX+/Miuc/MNjsn8vv/xiL5fes+LunHRLOO0XJ4Xz//i7nhVUo70fnHxv5vbrU06sUQ2KJUCgOwLiR3fU7JMELjjvrCz23HbL39NTfhLossASXd7ShgQIECBAoEYCV97yeFbyjluuWqMaFNtTgadnTA+T77s79O3bN2y7w049Lc7+BAgQ6BWBY445Jisn/eyVQhXSRuCVUiL+lB9/L3vuIx/9VBgyZPk2r1fz4LXXXg0333hdtstmW2wdBg0aUs3uC237j79fG35ywrFh3GprhN33/PRCrzf6iYceuC+zGzZ8hbDvgYc1ujnqJ0CgQiDFjfSz4iWrTSZQ788pp/78hPDoww+ELxxwaNh8y/c0mZbmNlpAIr7RI6B+AgQIFFxgtZWHhitvfTxUnhkvIZ+/g+KmG64O3/jqvlnDHntmbv4aqEUECBROYOLEieHYY48NlWc2Sqjk+zB4+qnpYd9P75Y18vy/3RTGbzwh3w3WOgIEWlJA/GitYfU5pbXGs9V7IxHf6iOsfwQIEGgCgR23WDX88sk7QjozPjVZQj5J+EmAAAECHQkcffTRWSI+JuQrFwn5Sg3rBAgQINBeQPxoL+JxVwV+c/bFYc7rr4ehy3f/G2Jdrct2rScgEd96Y1q3HsWvlj737MwwbNjwMHDZ5RZZ73//+9/Sts+EZQYM6PFXUBdZkRcJEGhKgXhWfPyXzoqXkG/KYSw3es6c10OfPn3CUkv1Lz/XlZV58+aFV155OQwePHSxm3cnrrz5xhvh6aefCssvP2yxcauyAW+9+WaYNeul0i/bw7N+Vb5mnQCBxgrEsxrjv3RWvIR874zHW2+9Ffr161dVYbP/8+8we/Z/Sp8N3hH6LdHzj5m9VV6MLc8+MyNr0/B3jOhyn7pTf4xNMf7Ff9Us3Y2bMWYuueRSYZllBlRTnW0JECgJiB/FPAwW9Xt9jH0vPP9s9p663KDBnQKtNHpMp6+1fyF+hlhuucHZ5T3bv+ZxMQX6FrPbet0Tgb9dfH7YebtNwvrjhoRtN18rbLD6sDCx9DPesCL+olu5PPLQ5LDfZ3cPa4xaOmw1fpWw0ZorhAnrjw7fPGz/8J9/v1Le9DN7fiArc/+9P1Z+rv3KgfvsmW3ztYP2af+SxwQItIBAPCu+/RIT8pX/2r/uce0FXnrx+ey994TjjypXFmPAbu/fqvw4rsQ/zv7wuCOzbdcdOyisM2a58LEPTQw/P/n47LU2G5cefGXfvbJtL7/0gvDA/fdklyrYcI3hYeO1RmRx4qsHfDq89NIL7XcLXY0rlTted/VlWXvXWnlgmDhhzXLc+v1vfxniL9yVy3333Jm160M7bh7mzp0TfvCdI8KaoweEzdZbKbz4wnOVm1onQCAnAvGsxvZLTMjHf/HM+PjPsniBf91xWzjoi5/K3oPXXGmZ7D38ZycdH14tJdc7W55/bmaI8SH+fh8/E2y54djsPfN979kwnHHaz8Ibcxdcyiz+nr/3J3cpF3Xw/v+Tvd8+/OD95eeqKa+8Uycrjz/6cNh7rw+FGFu22mhc2PydK2ft/M5Rh7T5HFK5e3frv/DPZ2d1jV/zHVl98fI7fz73zIU+G1XWVW3cjJ+fYvyNn7nuuvP2EI3jZ6uTvv/tymKtEyBQhYD4UQVWDjdd3OeUrv5eHz8r7PWR94YY+2Ici+/lq41cKhz25c+FyffetVDP4/tx3P6c351Wfu3/Tv1J9h4dY2L8DHPEIfuFrUuxJ362ieXFnFdsj4VAz09VYFgoge9++7Dw21//dKE+Pzl1SjjsK3tnbzh773dQ9nq8ecX7txm/0LbxL4znnvV/4R83XhsuuPzmMHTosDB23GohXtfrwcn3hmlPPB7GrNI2ITd1ymPh8ksuyMrarXSTKAsBAq0n0P6s+MoeOkO+UqO+63NLSZT43ly5tH/8xOOPZn90je/7lcsdt98c4r/rr7k8nHrGn0PlmYgPlRIvcfvf/d8vwm0331C5W3YmykV/OTc89uhD4dwLrgkDBi6bvV5NXEkFnvi9b4Vf/vSH6WH5Z4xbxxxxULj0r38KZ/7xb6F//6Wz1+IfiVP/Ysz7wxm/Ku9jhQCBfAq0P6uxspXOkK/U6Hz96ssvzt7HK7dI7+Hxj5YdLf8u3cT1k7vvmL2Xt389vl8f961Dw52Tbgk//fXZ2csP3H93iO+9aUnr8Sz6uFRbXiqno59XXXZR+OLn9ljopfg55MzTfx4uufC8cN5F14dVVl29vE136o8nIZ38g6PDL/73B+Vy4so1V16a/Ys3ae1o6U7cjDExxqd/3HhN9rmro3I9R4BAdQLiR3Veedt6cZ9TuvJ7/Y3XXxX2+dSHO+zaBX/6Q4j/LrzilvDODTcpb/PP0uebGOc2GL9p+blnSt+6je/R8RtOl170pzbxLn7DKua74h+8z7/0xrDGWuuW97NSPAFnxBdvzLvd4/gGlZLwm2+1TfjjX68L9z3+UrjoytvC2uu+Myv3+KMPD/FMkrj84DvfyH7GX0DjtpOnvhLufezFcNLPz8iej798X3Hphdn6rrvvlf2M/7vq8ovK62nlskvOT6thl932LK9bIUCgtQQ6Oiu+soeVZ8e3T85Xbme99wTescLIcNVN94WDD19wxml8fO0t85Pu8Wv48ez1+MtoXI4/4Rfh5rueyPb5xrfnJybiL50HfL7j9+6UhP/aUceH6257KFx81e1hm+3en5V1/z3/yhIZ2YPS/6qJK3Gfm0t/8E1J+Hdts0O4/Ia7wsPTXw1X3nhP2Osz8288O+nWm8Jpp5yUqmjzMybhYwzb/ytfD9876dSwqK+ottnRAwIE6i7Q0VmNlY1whnylRtv1maXLtsRvsMYlXm7yxJ/9NnuPP+fCa8JGm2ye/XG07R7zH8Uzvivf++Pv+fdNeTn86eIbQvysEJdLL/pzmP7k1Gz9D+dfFU4/66/Zevzf/556VhYr1nvn/BN3qi2vXFC7lXiG5KGlsxjjEt/DYz/uf2JWuPlfU8Kx359/QlFMyB9ZOqOx8tu83an/misuKSfhV19znXDGuZeG6259MPz89HOzumM97Zeexs2YFIrLDu/fJXzruJPCzh/+aPsqPCZAoAoB8aMKrJxturjPKZXN7ej3+tdenR0O3Ofj2WabTNgqXHnTveGRGa+Hv//z0ex3/7T/2Wf+Oq0u9ueUxx7JkvC7ffST4Yq/353Fuf2+dHi2X0zIn1H6Y7Cl2ALOiC/2+He59/EXxpQAGbniSuGU08/JrpUbC1hvg43C0d/7SfjErttn5d1x+y3hfTvvGm4vJTfi8tFPfDZsuvnW2XooXS541z32CpNuvTE7Kz6ejfLxT30+bLTpFmHlseOyN6z43Oe/ePD87d/+f/qFc9sddgojRo5q85oHBAi0jsCizoqv7GX7JLybulbq9O563759w6qrrxlGjlqpXHB8nJa/lZIs99z1z+zhb86+qJxEj0+suvohof/SS4djjzw4OzP+sUceDKutsXbatfzzD+dfGbbYemL58U9O/V32dfv4xOT77i7/AbaauJLFreOOyMqMyZFfnXl+WHrpZbLHsQ0xGfPyiy+EeLm1H//o2Cwxv3zpusaVS4xLf77k723O5K983ToBAvkRWNRZjZWtdIZ8pcb89TNPP6X8ZEyir7XO+tnj+I3Vsy+4Onxg241DTCy0X9J7f/z9/OP/s0/55Y032zIc/s3vhj12fnf23GOPPBBGrzw2xGvqxpvbpWX0mFWy+JIeV1te2q/9z1N/dkKIyY64xPfw9E3bpUufYT71uS+GeaX/4jei4h+C4ze2YvvjUm39MYn/05O/m+0b48WFpW/6LjNgYPY41rnOuhuE7bda+KzH3oibRx7zo4U+L2UV+x8BAlULiB9Vk+Vmh8V9TqlsaEe/109+6IFyvDjqOyeG1VZfK9slxqs9P7l3duLoDddeXv62bGV5i1qPJzB9+dBvljeJJxzdctN1WZy59+47ys9bKaaARHwxx73qXqev2cQdP7rX58pJ+FTQhC3eHT6371fCy6VrYcU3w7jEhEf8JfhP55wR3j1xhzBhy/eUX4sJkG8ee0Lo8/a28YZGMWEfv9oZfwl+avq07Jf1WE68HnA62+ajn5h/dkt83pJfgcNOvjq/jdOylhJon5Bvqc41SWcuvvCPWUvjVzPTmeyVTd9l149lifj43JWX/TXs3y4RH8+arEzCx+0GDRoSxq22Rpb4iWc2pqWauBLjVjyjPi4xPqUkfCorxqoP7rpnloiPzz1a+iPBhHaJ+MOOOE4SPoHV6We1NzisU7NU04ICbRLyg97Vgj3sepfOOuPUbON49l5Kwqe94023DzzoiA4vhRLP8IuJitFjxqXNyz9fKd2cLi3//e+8tLrIn71VXjypJy4f+sjHy0n4yop3+uBHskR8fG7yvf8qJ+Krrf/xxx4ux5kDDz6inIRPdcXL3uy+56fD+X/8XXoq+9nTuBnj42e/8OU2ZXoQqr5BLjMC3RUQP7or19j9Ovq9Pv7BOZ4QFH//rLz0TGxpvD/KW2+9mTX6rf+2vafU4nqyz/5tTyyN24/feEKW63pmxlOL293rLS4gEd/iA9xb3Zv+5BPlotZce73yeuVK/Ati5RLPjDnlx9/Lvs4arx8Zvxq6/Y47h63fs33Y6t3bhvZnHu6y28ezRHws45orLg6f/vyBWXHp2vDxq7ITd5h/uYLKeqwTIEDgsekLPvDTqK/AQw/cl1UYb0q0uBvGpcsTVLZw9TUXPkM+vr7S6LFZIv7NN98ob15NXHly2pTyflu8fYmE8hNvr6TLqsWH8Xq98Y/KlUvldR8rn7dOgEDrCNxwww1hk12Km4iP10VPZ49vOmHrDgd2nfU37PD59B76yiuly9GUTryJJ85MK116ctoTj1V99mCsoDfKizfZjn+IjUv8zNHREu9XEj+XxMvGxGR6Wqqtf3pFnImX8OloiXGkfSK+p3Fz4023DP369euoOs8RIFBHgaLHjzpS90pVHf1eH3NM8YSgGAuv/NtfS3FscnY5taml+xbGG6um+FhNA+KZ9+nbUZX7jVjRlR0qPYq8LhFf5NGvou9PTZt/bce4y+rtzmbsrJiDD/926eZ3/cOvfn5i9gYWf9k97+zfZv/iPh/YZffwre+eHFYYsWJWxJjSG9ZmW7yrdNmam8IlpZvnpUT8+efNP4vkY6Uz8dPN9Dqr0/P5EDjxkB3y0RCtaEqBdB34rjQ+XsomXlc+/vRNjK6I9e42b735ZvlGRPG+H+1vVte+tnhmSftlSOmG3V1dqokr06c9US52hREjy+uVKyuMnB9/4nMvvvBc5UvZ+oCB8y8xsNALnqiZQOX1mmtWiYJbVuCYY44Jbc5UXERP46UI4nWB488ix48ZTz1ZVorJ6Y6WYcPbXrYrbfNk6X3216ecGDq6dm5MblSbwOiN8p5+anpq3iIvZxnPVo+fTZ5/bsE13Kutf3rF56PhK4wo11u50v753oibyy43qLIK628LiB8OhZ4IiB890WuOfTv7vf6s354afnT8N6uOWZ31engnsbSz7T1fPAGJ+OKNebd6vMSSS5b3i3ee7soSv/Yfv6a5z/5fDf+87R+la2JdH/5+/ZXlr3DG6/JOnfp4+OsVt5a/ShgvTxMT8XeU7kI9s3Q2ywvPP1dO8uy2xye7Uq1tCBBocoGuXG6mMgHf5N1t6ub3rTgjL55JuM/+hyyyP2NL18xtv6TLmbV/vqPH1cSVyj/cvv76a2HAwGUXKjJe+iwtK45aOa36SYBAkwp0JQlfmYBv0m72arMrL9s1Z86C67dXVvJK6UzB9kvcdp9Pfbh8+ch4mbGJ278vrLHWemHcqmuE5YcPL9/ro/2+HT3urfLavPe/9lpHVYWYDI+fNeISr10fl+7Uv/Qy8+87Evd//dVSXUPiWttl1sttv7HXG3GzbQ0eESDQGwLiR28oNl8ZF5x3Vjj6iK9kDY9/jI73Otxks61CPEl0jbXWzU4q/eVPf1hVxyrf56va0caFEZCIL8xQ96yj6SZHsZRJt90U4k2YKpd4U7xvfHXfMKt0PcgP775X2GbbHcPUKY9l14BfZ70Nsq+Gxq+HHnbkcWHmMzPCoV/6XHazinj93nj2SXyji8uOO304fO2gfbL1q6+4JDw9Y/5ZLfFGe/GmsBYCBFpbYHFJeAn4fI1/vJ5ifG+O7+Uvv/RidjPujq7x/fCD94c333gjrLjS6G53IJ5ZWVVcqUj6x8vOtL8cWmzI3f+aVG7PyqWbBloIEGhegXg246IWCfiOdeIN6dLy2KMPpdU2P6eVvqLffrn3rjvKSfh4ecp4L47KJd3fqfK5Ra33VnmV33R6/LGO+/NI6RI6aYmXEIhLd+ofu8pqqZjSJW4eanNT8/RCjD+VSz3jZmW91gkQ6FxA/OjcptVf+ePZv8m6OLJ0M+9Lrv1nGNrum7qVl7psdQv9q59A3/pVpaZmFog3BUrLzTdel1bLP6+98tLs+odXX35xWLZ01uF9pRsf7fLeCeGD228aHn90wbUX4w4jRo4Kh3z9mPK+8QavaVlu0OAskRMf//Uv54QL/nRW9tInP7Nv2sRPAgRaWKCzRHxMwO//0U2yf3Hdkh+B9dYfnzUmXpM3JtzbL/H6ijtN3CiLCbfe/Pf2L3f5cbVxJd58KS3x2sUdLZNuuTF7Ol5CobP7n3S0n+cIEMifQGdnM8YE/HXXXZf9i+uWtgLxW6/p2uh/OONX2R9N224Rwu9/88v2T5W/sRpf2PJd2y70+vXXXL7Qc4t6Il7eLC09KS9+cypdB/ic359eutHewjfYu3PSLamqkK6L3536Kz8fdRRnXnt1dvjDmb8q15VW6hU3U31+EiCwaAHxY9E+rfxqylVtsNFmCyXh43v4dVdf1srd17cGCfRtUL2qbTKBQYOGhP2/8vWs1TfdcHX5Ou/xiXiG+wnfO6rco822fHeIZ8GnJX6Vp/KrrvEX4iv+dmF6OfvKT/lBaWXXty9BE78ymm629IEP7VG5iXUCBFpQoKMkvAR8fga68vIFD9x/T7lhXzjg0PL6CaXrK8abHaUl/nIbvwEVl+yG29t3/4bb1caVwYOHhr33Oyiru/L+JNkTpf/9+dwzw6UX/Tl7+IXSJXVcczfJ+Emg+QQ6OptRAr7r4/ilQ76ZbRyvmX74QZ8Pb8ydW975tF+cFG64duGk+qjRCy7nFd9P47dj4xJvlhoT+t8/dv7nhvhcZXmVl3O59+474svZ0t3y0v6VPw846BvZw5hcP+rwA9r8cWHyvXeFHxx3RPZ6vDfVppvPv0Ftd+qPlzH49N4HZGVd9Jdzw+9+84tyM+Il0eK3fDu6Tn694ma5MVYIEOhUQPzolKapXujsc8riOrHq6mtmm1xZyk89NX1aefP4TbC9PvLe8nv4nNc7vnRbeQcrBKoQkIivAqvom37hwENC+vrmEYfsFyasPzrssfO7w1bjVyl/NfXnp58blllmQIiJ+y9++WsZ2V/O+33Ybot1wlf23Svs99ndw0ZrrRBO/+WPs9e+fOhR2faVtluVzqqpvFnUjh/YNQx/x4jKTawTINCCApWJeAn4/A3w+E02LzcqftspxoC4xF9gDz786Gw9njWy9carhk/t8b4sPrz3XeuX48Mpb8eHbMNu/K87ceWAg75ejicxbk3cfK2w914fCltvNC58/eAvZK2Ilz779D7zEyndaJZdCBDIgUDl2YwS8NUPyPt33i28a5sdsh1jQnmTdVfMEhDxvfIH3zkixK/st1/i2YPpc8Fvf/3TsMUGY8LHd90urDNmufDtb3w52yf+ATYuR33twPCzk47P1kettHK5vGOOOChssPqwEJPj3S0vK7Td/3Z4/y7hPaXLZMYl/iF243VGhs/s+YGw83abZN/Oisnx2LYjj/lRec/u1v/lQ79ZjjPHHnlwFhtjDIwxMt4PKxmUKyqt1CtuVtZpnQCBjgXEj45dmu3Zzj6nLK4f8R6FaXnPpqtncSx+Xth2i7XDPXf9s/yNsXi5tRjjOrpUW9rfTwJdFZCI76qU7UI8u/Ciq24LMTEel3jWzL/uuC1bj7+g/+y0c8JOH/xI9jj+70uHHJndqDWuxzPb45mH8dI16cyQePbNl0o3c22/9FtiibDHxz9TfrpyvfykFQIEWkogJeEl4PM7rPEa6un9P7YyxoC0xETEyaecmSUc4nv8LTddV44PMdF92u8vDO+e+N60efZzidJ7fVz69un4V5F0E9e+fftl28X/VRtXhi4/PFx2/Z1hm+3mn4kfz46MZ3amb1vtXPq21V/+dlMW31IlfUqXNUhLakN67CcBAvkTSGczSsB3f2zidctP+/0FYa+3LwUZ38dvu/mG7L0yvofH98m0pPuAxBNvfn3mX0J8PS4xJky6df52MRF+yTWTwreOOymLC/G1++69MxUR9vvSYeX1WNec0ln03S2v1PRs6ddvfkyJD2IbY9v2+9Lh2WuxjviN3gcn35s9jpfiufiq28uXsIlPdrf+eP+Ri6++PWwyYaus7NjXGANjnR/c9WPhtN/9JXu+/f+6EzdTPOxXcaP09uV6TIBA1wXEj65b5X3Lzj6nLO73+o987H9Kny/mfyss9jHGsfh5IZ4Y+r+nnhXOOv/KEL89lV5L3/wtf46p4nODzxUZo/+VBPrMKy15kjjs5Kuz5hzwyW3y1CxtaSfwyisvZ7/MxutmjR4zLowdu2qI15jsaIk3XI1/QYxf9Ym/GI9eeWx2Ld53rDCyo80918QCv/jDDVnrTzxkhybuhaY3QuCXf7oj7LjFqqEn138XP+ozcs89+0x2Y9Zhw9+x0A1QX3vt1ew68fE9P35FNJ4t+c4NN8ne+3uzddXGlfirzoynnsza9uzMp8Oqq60Z1lh73TBkyPK92Sxl9UBA/OgBXsF33XbbbcPRRx8dYiK+u4v4sUBuZunkmXtKN2KNl5WM799jSje+Tsn3BVstWIuXnYn38Ig3046JkHXfOb7Nt11nzXopPPbwg1k5ld9wjcmMeHnLeMZ4PKEn1dHd8ha0qO3aSy8+n733TyndNHXFUSuVPoOs36a+tlvPv4xOd/oTL80z5fFHspu+Dh4ytGS3cZe+0VuvuNm+n630WPxopdGsb1/Ej/p616O2RX1OWVT98bPF5PvuLse+GM8ql3hZzhin1ij98TmeOGppDYFGxQ+J+NY4fvSCQC4EGvVGlovOa0TDBSRSGj4EGkCg2wLiR7fp7NgLAuJHLyAqgkCDBMSPBsGrNhMQPxwIBJpXoFHxY8H3r5vXTssJECBAgAABAgQIECBAgAABAgQIECBAgEBuBSTiczs0GkaAAAECBAgQIECAAAECBAgQIECAAAECrSAgEd8Ko6gPBAgQIECAAAECBAgQIECAAAECBAgQIJBbAYn43A6NhhEgQIAAAQIECBAgQIAAAQIECBAgQIBAKwhIxLfCKOoDAQIECBAgQIAAAQIECBAgQIAAAQIECORWQCI+t0OjYQQIECBAgAABAgQIECBAgAABAgQIECDQCgIS8a0wivpAgAABAgQIECBAgAABAgQIECBAgAABArkVkIjP7dBoGAECBAgQIECAAAECBAgQIECAAAECBAi0goBEfCuMoj4QIECAAAECBAgQIECAAAECBAgQIECAQG4FJOJzOzQaRoAAAQIECBAgQIAAAQIECBAgQIAAAQKtICAR3wqjqA8ECBAgQIAAAQIECBAgQIAAAQIECBAgkFsBifjcDo2GESBAgAABAgQIECBAgAABAgQIECBAgEArCEjEt8Io6gMBAgQIECBAgAABAgQIECBAgAABAgQI5FZAIj63Q6NhBAgQIECAAAECBAgQIECAAAECBAgQINAKAhLxrTCK+kCAAAECBAgQIECAAAECBAgQIECAAAECuRWQiM/t0GgYAQIECBAgQIAAAQIECBAgQIAAAQIECLSCgER8K4yiPhAgQIAAAQIECBAgQIAAAQIECBAgQIBAbgUk4nM7NBpGgAABAgQIECBAgAABAgQIECBAgAABAq0gIBHfCqOoDwQIECBAgAABAgQIECBAgAABAgQIECCQWwGJ+NwOjYYRIECAAAECBAgQIECAAAECBAgQIECAQCsISMS3wijqAwECBAgQIECAAAECBAgQIECAAAECBAjkVkAiPrdDo2EECBAgQIAAAQIECBAgQIAAAQIECBAg0AoCEvGtMIr6QIAAAQIECBAgQIAAAQIECBAgQIAAAQK5FZCIz+3QaBgBAgQIECBAgAABAgQIECBAgAABAgQItIKARHwrjKI+ECBAgAABAgQIECBAgAABAgQIECBAgEBuBSTiczs0GkaAAAECBAgQIECAAAECBAgQIECAAAECrSAgEd8Ko6gPBAgQIECAAAECBAgQIECAAAECBAgQIJBbAYn43A6NhhEgQIAAAQIECBAgQIAAAQIECBAgQIBAKwhIxLfCKOoDAQIECBAgQIAAAQIECBAgQIAAAQIECORWQCI+t0OjYQQIECBAgAABAgQIECBAgAABAgQIECDQCgIS8a0wivpAgAABAgQIECBAgAABAgQIECBAgAABArkVkIjP7dBoGAECBAgQIECAAAECBAgQIECAAAECBAi0gkDuEvF9+vTJXOfNawVefSBQHIE0Z9McLk7P9TQvAunYS8diXtqlHQQILFogzdk0hxe9tVcJ9L5AOvbSsdj7NSiRAIFaCKQ5m+ZwLepQJoFFCaRjLx2Li9rWawQI5Ecgzdk0h+vZstwl4vsv1S/r/9w33qyng7oIEOihQJqzaQ73sDi7E6haIB176VisugA7ECDQEIE0Z9McbkgjVFpogXTspWOx0Bg6T6CJBNKcTXO4iZquqS0ikI69dCy2SLd0g0DLC6Q5m+ZwPTucu0T84GX7Z/3/9+zX6+mgLgIEeiiQ5myawz0szu4EqhZIx146FqsuwA4ECDREIM3ZNIcb0giVFlogHXvpWCw0hs4TaCKBNGfTHG6ipmtqiwikYy8diy3SLd0g0PICac6mOVzPDucuET9i+YFZ/194eXY9HdRFgEAPBdKcTXO4h8XZnUDVAunYS8di1QXYgQCBhgikOZvmcEMaodJCC6RjLx2LhcbQeQJNJJDmbJrDTdR0TW0RgXTspWOxRbqlGwRaXiDN2TSH69nh3CXix4wcnPX/6Wdn1dNBXQQI9FAgzdk0h3tYnN0JVC2Qjr10LFZdgB0IEGiIQJqzaQ43pBEqLbRAOvbSsVhoDJ0n0EQCac6mOdxETdfUFhFIx146FlukW7pBoOUF0pxNc7ieHc5dIn71MUOz/k+d8WI9HdRFgEAPBdKcTXO4h8XZnUDVAunYS8di1QXYgQCBhgikOZvmcEMaodJCC6RjLx2LhcbQeQJNJJDmbJrDTdR0TW0RgXTspWOxRbqlGwRaXiDN2TSH69nh3CXiR48YFEYMGxhmvzonTJn+Qj0t1EWAQDcF4lyNczbO3TiHLQQaISB+NEJdnQR6JiB+9MzP3r0jIH70jqNSCNRTQPyop7a6OhMQPzqT8TyB/Ao0On7kLhEfh2qzdUdlI3b/IzPyO3JaRoBAWSDN1TR3yy9YIVBngXQMpmOyztWrjgCBKgXSXE1zt8rdbU6g1wTSMZiOyV4rWEEECNREIM3VNHdrUolCCXRBIB2D6Zjswi42IUCggQJprqa5W++m5DIRv9X40WHp/kuEaaXL08S/VFgIEMivQJyjca7GORvnroVAIwXEj0bqq5tAdQLiR3Vetq6tgPhRW1+lE+hNAfGjNzWV1VMB8aOngvYnUD+BPMSPXCbil1qyX9hhwrhsJG696/Ewb179BkVNBAh0XSDOzThH4xLnbJy7FgKNFBA/GqmvbgJdFxA/um5ly/oIiB/1cVYLgZ4KiB89FbR/bwuIH70tqjwCtRHIS/zIZSI+kk/cbGwYu+Lg8NKsV8MNtz9cm1FQKgECPRKIczPO0ThX45y1EMiDgPiRh1HQBgKLFhA/Fu3j1cYIiB+NcVcrgWoExI9qtGxbLwHxo17S6iHQfYG8xI/cJuIj7W7brRX69OkTJj/6dLjjvqnd17YnAQK9LhDnZJybcY7GuWohkCcB8SNPo6EtBNoKiB9tPTzKl4D4ka/x0BoClQLiR6WG9bwJiB95GxHtIbBAIE/xo98xpWVB0/K1NmjZ/mH40AHh3keeDU/NfDn07dsnjFphSL4aqTUECigQ38Ruu/uJrOd7fWD9sNYqwwqooMt5FhA/8jw62lZkAfGjyKPfHH0XP5pjnLSyeALiR/HGvNl6LH4024hpb1EE8hY/cp2IjwfFisOXDcsNWCo8MOX5LBk/+7W5YexKw0pn4RblkNFPAvkRiNfUil/nueuB6Vmjdt9+7TBh/VH5aaCWEKgQED8qMKwSaLCA+NHgAVB9VQLiR1VcNiZQUwHxo6a8Cu9lAfGjl0EVR6AHAnmNH33mlZYe9Ktuu9754DPhnMvuL924dV4YOnhA2GL8qmHcaGfh1m0AVFR4gXh36Xhj1nhN+Hg5mk/stF7YeO2RhXcBkH8B8SP/Y6SFrS0gfrT2+LZy78SPVh5dfWsGAfGjGUZJGzsSED86UvEcgfoJ5Dl+NE0iPg7X9JmvhAuufShMfXpWNnpjRi0f1ltjlIR8/Y5lNRVQIL6B3f/IjDBtxotZ7+ONWeP170aPGFRADV1uVgHxo1lHTrubWUD8aObR0/YkIH4kCT8J1E9A/KiftZpqJyB+1M5WyQQ6E2iG+NFUifgEff2kqeHq26eE1+e8mT01cED/MLaUlF9xhcFh2JCBYbmBS4elllzC5WsSmJ8EuiAQvxsz9403w79nvx5eeHl2ePrZWWFqKfk++9U52d5L918i7DBhXIh3hLcQaFYB8aNZR0678ywgfuR5dLSttwTEj96SVA6BBQLixwILa60rIH607tjqWeMEmjl+NGUiPg713DfeCjffNT1MmjwjzHxhduNGX80EWlxgxLCBYbN1R4Wtxo8u/YGrX4v3VveKICB+FGGU9TEPAuJHHkZBG3pTQPzoTU1lEehcQPzo3MYrzSkgfjTnuGl18wk0Q/xo2kR85eEQv/Lz6LSXwrRnZoWZL84Os/4zJ8yZ+1Z2PfnK7awTINC5QLzue/+l+oXBy/YPI5YfGMaMHBxWHzPUJWg6J/NKCwiIHy0wiLrQcAHxo+FDoAENEBA/GoCuypYTED9abkh1qAsC4kcXkGxCYDECzRw/WiIRv5jx8TIBAgQIECBAgAABAgQIECBAgAABAgQIEGiYQN+G1axiAgQIECBAgAABAgQIECBAgAABAgQIECBQAAGJ+AIMsi4SIECAAAECBAgQIECAAAECBAgQIECAQOMEJOIbZ69mAgQIECBAgAABAgQIECBAgAABAgQIECiAgER8AQZZFwkQIECAAAECBAgQIECAAAECBAgQIECgcQIS8Y2zVzMBAgQIECBAgAABAgQIECBAgAABAgQIFEBAIr4Ag6yLBAgQIECAAAECBAgQIECAAAECBAgQINA4AYn4xtmrmQABAgQIECBAgAABAgQIECBAgAABAgQKICARX4BB1kUCBAgQIECAAAECBAgQIECAAAECBAgQaJyARHzj7NVMgAABAgQIECBAgAABAgQIECBAgAABAgUQkIgvwCDrIgECBAgQIECAAAECBAgQIECAAAECBAg0TkAivnH2aiZAgAABAgQIECBAgAABAgQIECBAgACBAghIxBdgkHWRAAECBAgQIECAAAECBAgQIECAAAECBBonIBHfOHs1EyBAgAABAgQIECBAgAABAgQIECBAgEABBCTiCzDIukiAAAECBAgQIECAAAECBAgQIECAAAECjROQiG+cvZoJECBAgAABAgQIECBAgAABAgQIECBAoAACEvEFGGRdJECAAAECBAgQIECAAAECBAgQIECAAIHGCUjEN85ezQQIECBAgAABAgQIECBAgAABAgQIECBQAAGJ+AIMsi4SIECAAAECBAgQIECAAAECBAgQIECAQOMEJOIbZ69mAgQIECBAgAABAgQIECBAgAABAgQIECiAgER8AQZZFwkQIECAAAECBAgQIECAAAECBAgQIECgcQIS8Y2zVzMBAgQIECBAgAABAgQIECBAgAABAgQIFEBAIr4Ag6yLBAgQIECAAAECBAgQIECAAAECBAgQINA4AYn4xtmrmQABAgQIECBAgAABAgQIECBAgAABAgQKICARX4BB1kUCBAgQIECAAAECBAgQIECAAAECBAgQaJyARHzj7NVMgAABAgQIECBAgAABAgQIECBAgAABAgUQkIgvwCDrIgECBAgQIECAAAECBAgQIECAAAECBAg0TkAivnH2aiZAgAABAgQIECBAgAABAgQIECBAgACBAghIxBdgkHWRAAECBAgQIECAAAECBAgQIECAAAECBBonIBHfOHs1EyBAgAABAgQIECBAgAABAgQIECBAgEABBCTiCzDIukiAAAECBAgQIECAAAECBAgQIECAAAECjROQiG+cvZoJECBAgAABAgQIECBAgAABAgQIECBAoAACEvEFGGRdJECAAAECBAgQIECAAAECBAgQIECAAIHGCUjEN85ezQQIECBAgAABAgQIECBAgAABAgQIECBQAAGJ+AIMsi4SIECAAAECBAgQIECAAAECBAgQIECAQOMEJOIbZ69mAgQIECBAgAABAgQIECBAgAABAgQIECiAgER8AQZZFwkQIECAAAECBAgQIECAAAECBAgQIECgcQIS8Y2zVzMBAgQIECBAgAABAgQIECBAgAABAgQIFEBAIr4Ag6yLBAgQIECAAAECBAgQIECAAAECBAgQINA4AYn4xtmrmQABAgQIECBAgAABAgQIECBAgAABAgQKICARX4BB1kUCBAgQIECAAAECBAgQIECAAAECBAgQaJyARHzj7NVMgAABAgQIECBAgAABAgQIECBAgAABAgUQkIgvwCDrIgECBAgQIECAAAECBAgQIECAAAECBAg0TuD/AW0dfxrLfhFQAAAAAElFTkSuQmCC"}}},{"cell_type":"markdown","source":"\n## **
1.1. Load data using pandas
**","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest_data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\nprint(train_data.shape)\nprint(test_data.shape)\ntrain_data.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## **
1.2. Create arrays from the data
**\n\nThen, let's create numpy arrays from the data: \n- we remove label from X\n- we normalize the data by dividing by 255 (the pixel value is a number between 0 and 255)\n- we reshape from 1*784 to 28*28 to have the right image format\n- we convert to numpy.float32 which is a format supported by Pytorch (as opposed to double or other float formats)","metadata":{}},{"cell_type":"code","source":"X_train, X_test = train_data \\\n .drop('label', axis=1) \\\n .values \\\n .astype(np.float32) \\\n .reshape(-1,28,28)/255 \\\n , test_data \\\n .values \\\n .astype(np.float32) \\\n .reshape(-1,28,28)/255 \ny_train = train_data['label']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## **
1.3. Create tensors from arrays
**\n\nWe then create tensors from the arrays. ","metadata":{}},{"cell_type":"code","source":"X_train_tensor = torch.tensor(X_train)\nX_test_tensor = torch.tensor(X_test)\ny_train_tensor = torch.tensor(y_train)\ntrain_tensor = TensorDataset(X_train_tensor, y_train_tensor)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## **
1.4. Create dataloaders from tensors
**\n\nFinally, we define a batch size and create dataloaders from the tensors which are then going to be used for training. ","metadata":{}},{"cell_type":"code","source":"batch_size = 64\n\n# Create data loaders.\ntrain_dataloader = DataLoader(train_tensor, batch_size=batch_size)\ntest_dataloader = DataLoader(X_test_tensor, batch_size=batch_size)\n\nfor X, y in train_dataloader:\n print(f\"Shape of X [N, C, H, W]: {X.shape}\")\n print(f\"Shape of y: {y.shape} {y.dtype}\")\n break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can visualize a few images using makegrid function for torchvision","metadata":{}},{"cell_type":"code","source":"for batch_idx, (data, target) in enumerate(train_dataloader):\n img_grid = make_grid(data[0:8,].unsqueeze(1), nrow=8)\n img_target_labels = target[0:8,].numpy()\n break\n \nplt.imshow(img_grid.numpy().transpose((1,2,0)))\nplt.rcParams['figure.figsize'] = (10, 2)\nplt.title(img_target_labels, size=16)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# **
2. Define and train your model
**","metadata":{}},{"cell_type":"markdown","source":"\n## **
2.1. Define your model
**","metadata":{}},{"cell_type":"markdown","source":"We define a Neural Network from the mother class nn.Module. The details to get this NN structure are not discussed here but in general, having one (or more) convolutional block and one linear block is one of the basic structures.","metadata":{}},{"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(f\"Using {device} device\")\n\nclass NeuralNetwork(nn.Module):\n def __init__(self):\n super(NeuralNetwork, self).__init__()\n self.conv_block = nn.Sequential(\n nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1),\n nn.BatchNorm2d(32),\n nn.ReLU(inplace=True),\n nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),\n nn.BatchNorm2d(64),\n nn.ReLU(inplace=True),\n nn.Dropout(p=0.25),\n nn.MaxPool2d(kernel_size=2, stride=2),\n nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),\n nn.BatchNorm2d(128),\n nn.ReLU(inplace=True),\n nn.Dropout(p=0.25),\n nn.MaxPool2d(kernel_size=2, stride=2) \n )\n \n self.linear_block = nn.Sequential(\n nn.Dropout(p=0.3),\n nn.Linear(128*7*7, 128),\n nn.BatchNorm1d(128),\n nn.ReLU(inplace=True),\n nn.Linear(128, 64),\n nn.BatchNorm1d(64),\n nn.ReLU(inplace=True),\n nn.Dropout(0.3),\n nn.Linear(64, 10)\n )\n \n def forward(self, x):\n x = self.conv_block(x)\n x = x.view(x.size(0), -1)\n x = self.linear_block(x)\n \n return x\n\nmodel = NeuralNetwork().to(device)\nprint(model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## **
2.2. Define your loss function and your optimizer
**","metadata":{}},{"cell_type":"markdown","source":"We then define: \n- loss\n- optimizer\n\nThese will subsequently be used for training. ","metadata":{}},{"cell_type":"code","source":"loss_fn = nn.CrossEntropyLoss()\noptimizer = torch.optim.SGD(model.parameters(), lr=1e-3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## **
2.3. Define your training and evaluating functions
**","metadata":{}},{"cell_type":"markdown","source":"We then define 3 functions that are going to be used for training, evaluation and prediction: \n- train\n- evaluate \n- predict","metadata":{}},{"cell_type":"code","source":"def train(dataloader, model, loss_fn, optimizer):\n size = len(dataloader.dataset)\n model.train()\n for batch, (X, y) in enumerate(dataloader):\n X, y = X.to(device), y.to(device)\n X = X.unsqueeze(1)\n\n # Compute prediction error\n pred = model(X)\n loss = loss_fn(pred, y)\n\n # Backpropagation\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n\n if batch % 100 == 0:\n loss, current = loss.item(), batch * len(X)\n print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n \ndef evaluate(dataloader, model, loss_fn):\n size = len(dataloader.dataset)\n num_batches = len(dataloader)\n model.eval()\n test_loss, correct = 0, 0\n with torch.no_grad():\n for X, y in dataloader:\n X, y = X.to(device), y.to(device)\n pred = model(X)\n test_loss += loss_fn(pred, y).item()\n correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n test_loss /= num_batches\n correct /= size\n print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n \ndef predict(model, data_loader):\n model.eval()\n test_preds = torch.LongTensor()\n \n for i, data in enumerate(data_loader):\n data = data.unsqueeze(1)\n \n if torch.cuda.is_available():\n data = data.cuda()\n \n output = model(data)\n \n preds = output.cpu().data.max(1, keepdim=True)[1]\n test_preds = torch.cat((test_preds, preds), dim=0)\n \n return test_preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## **
2.4. Train your model
**","metadata":{}},{"cell_type":"code","source":"epochs = 5\nfor t in range(epochs):\n print(f\"Epoch {t+1}\\n-------------------------------\")\n train(train_dataloader, model, loss_fn, optimizer)\n# test(test_dataloader, model, loss_fn)\nprint(\"Done!\")","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## **
2.5. Get predictions and submit
**","metadata":{}},{"cell_type":"markdown","source":"Finally, we get the predictions and create a csv in the right format for submission. ","metadata":{}},{"cell_type":"code","source":"for batch_idx, data in enumerate(test_dataloader):\n img_grid = make_grid(data[0:8,].unsqueeze(1), nrow=8)\n break\n \nplt.imshow(img_grid.numpy().transpose((1,2,0)))\nplt.rcParams['figure.figsize'] = (10, 2)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions =predict(model, test_dataloader)\ny_test = predictions.squeeze().numpy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),pd.Series(y_test, name ='Label')],axis = 1)\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"__I hope you found this notebook. If you liked it or learned something new, do not hesitate to upvote it :)__","metadata":{}},{"cell_type":"markdown","source":"\n# **
Appendix
**","metadata":{}},{"cell_type":"markdown","source":"\n## **
Appendix A: Save and load models
**","metadata":{}},{"cell_type":"code","source":"torch.save(model.state_dict(), \"model.pth\")\nprint(\"Saved PyTorch Model State to model.pth\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = NeuralNetwork()\nmodel.load_state_dict(torch.load(\"model.pth\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Further improvements to be added : \n- cross validation \n- evaluation ","metadata":{}},{"cell_type":"code","source":"# # read and implement this https://github.com/christianversloot/machine-learning-articles/blob/main/how-to-use-k-fold-cross-validation-with-pytorch.md\n# # K-fold Cross Validation model evaluation\n# k_folds = 5\n# kfold = KFold(n_splits=k_folds, shuffle=True)\n\n# for fold, (train_ids, test_ids) in enumerate(kfold.split(dataset)):\n \n# # Print\n# print(f'FOLD {fold}')\n# print('--------------------------------')\n \n# # Sample elements randomly from a given list of ids, no replacement.\n# train_subsampler = torch.utils.data.SubsetRandomSampler(train_ids)\n# test_subsampler = torch.utils.data.SubsetRandomSampler(test_ids)\n \n# # Define data loaders for training and testing data in this fold\n# trainloader = torch.utils.data.DataLoader(\n# dataset, \n# batch_size=10, sampler=train_subsampler)\n# testloader = torch.utils.data.DataLoader(\n# dataset,\n# batch_size=10, sampler=test_subsampler)\n \n# # Init the neural network\n# network = SimpleConvNet()\n \n# # Initialize optimizer\n# optimizer = torch.optim.Adam(network.parameters(), lr=1e-4)\n \n# # Run the training loop for defined number of epochs\n# for epoch in range(0, num_epochs):\n\n# # Print epoch\n# print(f'Starting epoch {epoch+1}')\n\n# # Set current loss value\n# current_loss = 0.0\n\n# # Iterate over the DataLoader for training data\n# for i, data in enumerate(trainloader, 0):\n \n# # Get inputs\n# inputs, targets = data\n \n# # Zero the gradients\n# optimizer.zero_grad()\n \n# # Perform forward pass\n# outputs = network(inputs)\n \n# # Compute loss\n# loss = loss_function(outputs, targets)\n \n# # Perform backward pass\n# loss.backward()\n \n# # Perform optimization\n# optimizer.step()\n \n# # Print statistics\n# current_loss += loss.item()\n# if i % 500 == 499:\n# print('Loss after mini-batch %5d: %.3f' %\n# (i + 1, current_loss / 500))\n# current_loss = 0.0\n \n# # Process is complete.\n# print('Training process has finished. Saving trained model.')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]}]}