{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!nvidia-smi","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-28T01:28:01.199251Z","iopub.execute_input":"2023-03-28T01:28:01.199766Z","iopub.status.idle":"2023-03-28T01:28:02.208513Z","shell.execute_reply.started":"2023-03-28T01:28:01.199719Z","shell.execute_reply":"2023-03-28T01:28:02.207176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

0    REFERENCES

","metadata":{}},{"cell_type":"markdown","source":"Processed Data: [GISLR Feature Data: On the Shoulders](https://www.kaggle.com/code/roberthatch/gislr-feature-data-on-the-shoulders)
\nFeatureGEN class: [GISLR Feature Data: On the Shoulders](https://www.kaggle.com/code/roberthatch/gislr-feature-data-on-the-shoulders)
\nChapter html format: [🤟 GISLR 🤟 - 📚Learn – 🔭EDA – 🤖Baseline\n](https://www.kaggle.com/code/dschettler8845/gislr-learn-eda-baseline)
\nTime check: [[LB 0.67] one pytorch transformer solution](https://www.kaggle.com/code/hengck23/lb-0-67-one-pytorch-transformer-solution)\n\n\nIt's my first time participating in Kaggle competition. Please let me know if there is something I need to know","metadata":{}},{"cell_type":"markdown","source":"### Data tokenizing\nI Simply resize input data of [GISLR Feature Data: On the Shoulders](https://www.kaggle.com/code/roberthatch/gislr-feature-data-on-the-shoulders) into (bs, 23, 84*3). 23 tokens and 84*3 feature dimension.\n\nI'll try to find optimal configuration for ensembling(small size with fine LB score).
\nI uploaded several checkpoints on my dataset page. You can try those checkpoints. ","metadata":{}},{"cell_type":"markdown","source":"### Several submission results","metadata":{}},{"cell_type":"markdown","source":"![exp.png](attachment:ab93ce82-126c-4f80-b160-8af43be162be.png)","metadata":{},"attachments":{"ab93ce82-126c-4f80-b160-8af43be162be.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAABLkAAADqCAYAAABObpeHAAAABHNCSVQICAgIfAhkiAAAABl0RVh0U29mdHdhcmUAZ25vbWUtc2NyZWVuc2hvdO8Dvz4AAABEaVRYdENyZWF0aW9uIFRpbWUAAAAAADIwMjPrhYQgMDPsm5QgMjfsnbwgKOyblCkg7Jik7KCEIDA47IucIDQz67aEIDI27LSI1mIUvQAAIABJREFUeJzs3W+MJHd97/v3XNmaih/sVGRs1wnedcUgu8AEFwTLhUGmOQhc5yh7XT7imNLxJbQcDGVMoMH40neT4DZJnEYccF/fgDtAoAnyTcexcDuOThoQchnZ3o5McCXC0Eb+U147SrELcu0+sGvlK/V90D07s7Ozs57dnenqmc9Lamm6q6v61/Wdqvr2t371q7k4jkdnzJ+JiIjILDF3LJAfOjjtZoiIiIiISEmcccb8mbxj10+n3Q5ZZt+/v8iuyz477WbIMvv27WPXrl3TboYso5iUj2JSPopJ+ex77Avseu1vTrsZsozyrvLRvqt8FJPyUUzKRzEph/9t2g0QERERERERERE5VSpyiYiIiIiIiIjIzFORS0REREREREREZp6KXCIiIiIiIiIiMvNU5BIRERERERERkZl3EkWuV0juuRv/vXuw3Fsw31DDurxJ9evPkJ/mxhX334nx3j7DFX8vtmP49S/hXP8YKcD+n1K/7nNYb6hhunuo3JqMX1/VSwy+/lVct4b5hltw3v939J4/tfatlD96N+57/454Ju9uPyCy5pibW+3h0zndgV4mjiysKN64D9gOhk1co0I7W31yLzSOxNMwDAzTwvVr9NJNbeUWk9P1jeNsM3PMuc1V9xNriasWdm2wIa2dSXmP0DQJe8Wx04qYqmXid9fYOaUtPKNyGvZfGXGnf+T4ojjJRundeBNzr/0oc6/9KMaFN2G84Rbc604uX1m/X9G5/nNUvp5RANmDfYL33oL5hpsw3c/h3/pTVjvEZPd/Feu1txD9y3EWW2R061/CuXwP9uV7sHd/nfYTL62/ec//EO/CL9FZLcfa/xjhlU2aT7yy/uVOjfKumaa8awqUd2045V3bw2mLU7msu8hVPHg3/q0ZfuuPyZIvkv+8RfJVl/TLd1Ltn0SicpKKJx4g/MYOmndchk1G6w/uorfzWpLHW+QPfQj30Q7h11c/2mT3fIPgOws0v/9F8p//Me1LXqD5jSdZZRM+aeYV19K+4gmqt/70tBf/Nsc8V9/3MqPRaMWjT9WcdtvkVJ330QcZjUYURUGR9omMLmG1veqPFnk1TMJ+cWQ72fvJC5i/+m95eXG7Seo461xipT0kaXob0tqZZPrUQpN+u3vM/2neb9EjoBZsws4p69Ood48kW4qTbKTzPvhpRv/+VxTPfIVi8Ami+R8TfuqhDd9Xp3/zbeqHfTo3WBjPP0T4sUcxb/4s2c+/QvbAbsz776J6z4oK0/7HiG5/AWPHmcddbvLlrxI95dJ76HbSf76d+IMvU7/+PuLTmYCdexnt2yzaN36Xwelc7oZT3rWVKe863ZR3bTjlXTLD1l3kyp7PyM+/BP+Ss468Zv2uT+/7f0ynchbwPPUrbyH8yx9Svf5OKlfuwX5/n/jRh8bP3zt+PljMjZ5/jOj9415hlnsL7vU/XJp2XAfp3v4jjA9fQ7AA7H+S3hM7iW56E5YBLFxM4+MXMbwnWaWK/yu633kGb881+OeeCSxQue1mBrddjAEU/a9i7P47Wrd/Hf/9TezL/5Tq/c/Qmzx3Lv9Twvt/tWx5v6Z/6524l9+C+YY9VI6c3TwT79O7cb93H+2n1ruWZ8Ck6ltrVfErFTzHxvGbDBYresWQblTBsW1s28Z2Q1pLE0naVTzbwrIsbK9KZ7gsEy2GtEMPxzIxLZews95zMbIupksYupAO1+j9KKeq6AYYXo1W6GDZETFA2iWq2FjWeFtwg9aRbSiOHNz6YPK3hRO1aVUD/IqHY2/H7cLAq0XYcZtuuvz1jF67jxVG+CR0qh62dZx9y6JhE9dY3juioBsYuM3JOi2Os5yiR+TVeOiXPapOhWZydJzW3O9lbSqGT7NTI/ArVFwbp1Inns2zIDINCzsJr94Jz2dH9tXZg9/Fv/KWSQ71Ofzbl3pY5f/yQ4L3LuZXe3BvXMqv8n/5IeHuz417U13+pwR/+eTSCbmDCfUvHyTccwU2wPxOanfcQMt/DQZg7HQJ3w7DJ369rHG/ovOp+yhuuoZgx/G+wCukT/0a660X4RjjV+y3X4J9IGN4CNj/EJUL76T+N3cTXPcl3Mv34N2aMLhn/Ny7cg9e/egeZMP776Zy5R6sN9RwrlvKLc13/x71hb3UVxbiZpnyrq1DedemUN51qpR3bSfDbkTFsbFM86hjSz5oEbiL24yNGy5tM2W27iKX/e63473wAMF1f0e7/yTD/ePu4ObO12AaAGdizh8i/skOmt/8BPGPPkzw/H0EX4bGNz9B/INPEB18gEb8EnCQTr1D//UfYph8kWzwCfznv0v1Oyc4r7E/obN3J9WrXzN54dizhuaOs+Dp548tchUvED91Ntb+PsF792C7e3Cv/y79/ePJBsATvyC7ukr/3jrJzQt0P9WhX7lu/Pw2i/6XfkiyuLyfJQze+j9I/vmLZN+/Cu75NvUHJ13kF1yqlV/TuWdTri3YfIcfIs5q9OKYwTCmVjQJG+M1kzQCosSnO0xJ05S4XtAMIvo5FHFE0MipDTKyLKUXDInC1pFY5f0eNGKGWc6wZdOvN8cHJtkYeUKnk2AHIe6027KFGQaQ9BmGMVnapkJGJ6rSdzoMs4ws7eOndart1ZOoJ3sxdrNHPx6QtF3i7bhdOFVqXkK7nSy9lnZpxy5RzWPYqhIlFXppRpYN6bgxUdRZ95ny4y7HCGh3Qy44L6AzjKmv2GDW2u8BcDiml1Xp9mPiJCYqWtQ76SmsENlWDj5P557nsf3Lxvvq4qfUPrYX4+Zxz/rsgfdQfOPbNB4F+BWdxnfJP/jp8bTkszTOfYLu3oNwMCH6gz75Bz/B8J9vJ/3+tVj33EV4/7gglMeP0t/hUr1k8rnnXkjg72R8vv4Vskd/SGvv2QRX7TzStPRvvk1j/ho6v3/2Gl/gTDz/YvIHH50Uo15h2E/ILnGpnDt5y+EnGXAVvbtvJrn7SopvfJ1o/5V0776Zwb1XYdzzj3QXU6rDz9D72aV0f3Q72eM3E+x/gPDLixNfQ/DfdzL4+x9vrSKC8q6tQXnXplDedRoo79oeDg/oJQHdYUqWxQRZY3JsSenU6uRRTJZlZNmAhtWnG5e/D+r6x+Ta+S7ih26mfsnL9L7ybTzv45iXf4nqX/50WSJxBq5/CRYAZ+PsPAPnqkvGZwQ5G/tcyPa/BCxQ/esWw8bF4+TJ2In/u2eRPfWrlZ96lOKJX5Cccz7eYlJ07kUEl2S0vzQ5w3fwGZrfeIKDvEKxsph8+CXyQy/Qjy2aD9xOOvg0jR0/JvzYQ0vtP+cigkvGhTPz/LMxd1xIeMW455qx82ysA4eWNt5zXKJJsc3YeRnRW18iPhL4M/HebpH+5JkZ7I58mPuv+Y0V17gbmGF/6S3zlxJW3XFhEBs/cMgGMSkJ3V6KF0W4i2drgwi/iOkmMOj0ySsRgQVg4NZjssFSt2KjElGdnOa1XA87z8hmoGI8S375V/54XAjDYO4330IjC2jWFmMpG8b0CMf/+IBFtZczbFUm+z8X3zPJhumqsy5UqviTWQ3HwdqW24VFUAvIuu0jlzcl7TbDSkRog1MfkA+ak/2OieePz5Svd/97cstZe78HwLxDGC7tMx3bIEtn7+ggm+eX37lzPB7XhTcx98Y/o7HfpfnhneP/IeNNdB+/ne7VC+M377wY/3UvkR54CTgL65wzGX7vx/T+5VfkLBDc9gla/gLF3kfpz7+NxrXjnlksXEzt2rOJ7/8FOTD8yQvw1ouO/fH9YAfztR/nP/0fP8K4qUrjiskJxucfovqV36DZvAxr5TwrWNf+D1qv/1cqb6lhuTXcr5xJvXnl0mVF8zsJ3j05gXm+hT1/Nv67J9/33LOx5w+R7l9879kEH37T+DONnUTX7iT7yS+O5HLW7/429lPPkMzMJYvKu7Yy5V1TorzrFCnv2hbmbYKaPzmeukRVd3JsMbEsg2GvS2+QkmMRtPq0ghMd7afvjJOa69wLqe65kCpA8RLDvXup1+/CP3Azw9smxaAVYzIYC8uezy/9mf3kh9S/nJAcHPd+Kg68BJW1P744cIji3ItYugr4NdT++sNkn7oPz/1/MXb+NtF/v4SLn2DSu2yZ+TMx5nfg33TFpLv8awg+fiXW+/6VwcF3Tbrmn3n0QWfl8+XOOXtZUncW5o4zKQ4ujU1mnrsD9v+aHE6Y/JXLPFffl9ML1jr8mljLLsU2TRPyjJycPDcwl080TEwzJ81z8jzHtMxl69Q4Kk6GuWyaYYwvIz0N30iWnPfRPlm7MnmWM+zWCDyfbBAT2VNs2FZnWsv2W5ANWtQbPZJ8/B9eZDn4q89qmMZR28x23S5Mv0aIT7vfpOIntLs5QTvEAoq0T6Peoj9c7GOdcpjqutfTyS1nrf3e4pzm+MzykenrbJhsO+d98BNkzYsnz15ieP89BLvvJHvgZqKdLzG45z4af/8MWQHwCtnTTIpTZxHeUYfv9GnVv0D4wpm4V11F67Z3Ye9/mYMv7CVwf7zsk17BeP1L5EC2/yDWzoVjG/PuKvm/X0f+1BM0bryTyoGbSfacSetj38O67bOE5x47y9FeIa7fSf3w7zH82RXYBmQP3k3luq9jff9jhACcibGYIx65OmBx/jOPyh9hAWfZpZHmwllw8ODSZZfnnI11+FmyQ8zItqa8aytT3jUlyrtOmfKu7cDCWfXYYhJ2BtBu0oo8wtTADeq0WhFeyceKXGdPrldIH32M/lPL7lhjnIXz7vfQ/KBF+sTz66vc7n+U6h/8CG66kcEP/oTkB39Cu3LWiedbzblvonn3n5AmtzN84AZC49dkr//tY89EGhbu+a+QH1h+152XgDNWJE+v0qFlCRUvkR96BfPck/wOMyc/6oxGnmeTg4mFaRbkyycWOVluYpompmWS59myHVdOOsxmdID+rcDECWsE5mAmup9uGVmHatCGeo9BkpAkCW2/5EeMMjA8apFNv90j7bfpERL5BjCkHYT0rSbxZH0OmpXVd+vGyiynoDiyA1rHco6y1n5PWZWcDmfhXP0egh3P0N17kOLBuwm+fJDwq3WSH/wJyQ9uoHr+srcvWIQfrxL/4IvkD32Y8IX7CP7iScxzf4OF111JP/ni5FLGL5IlLfJ7Jyf6Viie+imd/vOTY/aZmK93qX9wJ8MHf0H6VELnqZeI6382GfvrLtov/JrOH+yhcszNf56n9+Cv8a69DHuySVjvfhsVfkHviZNZHy+NC1gT+cGXYGGBrb0XVd61NSjvmgrlXSdHedc2sPLYki8ViE2HsN4hTjLyYZcwrRPU4ym189Vb9+WK6T/cR3D93XSW3fI5f+oxmvdk2G+9cH29lQ6/THZ4B+7rF8bV8SceorX3JYpD/9+alVvjnB0Y+5cfnA/SuX4Plb8cJ2HF849R+1KG/+FJ1/n9z9C9/8lJAW4n1WvPpv+VByZjQhyk95c/Jnu7S+VktocX/pV2fzKw6fMJnZ+chVdZWgv5/kNw7tlbM+k6PKTbHozjUCR0u0PsSgUbhzCwGbTbLI49mHZbxIZP6IEX+Bj9Np20AAqGrQA3aG2tcTNmSjE+aKUWrjNb/Q1nWpGTFeN1bgBF0qYV5xR5sS3PFK6HU41wB22iVh+7VmN8j52cNAfLtcf723xAqzOAYpX1aVpYpCSLw3CkXbrJ4cmTEy3HgCInP2aha+/3RE7dK6QP/ojeCwu4r1ugOPAy+fzZOOecOZ52/w/pHYDi4CtQPElt9500/2WcqxnnWjjnT3rav/0KKod+TGsxd+EgvVu/Tu3B8XPr3AWy/csGbD/4rzQ+1qH+6OS1g8/Q+fvnMS85D+v1PsnPW8uKZTcSnX821b++nfgGC/gV8T0Jg/0AZ+O+/kwG/SeWDY6fEB86G29peK9X73BG9zvPTHKQjO4/PI/99ouWCnUHfk02vwPruAPhzyDlXVuE8q6pUN510pR3bXFHHVuGdLvJ+NhSxNQ8n+ZkpHnDcnDs2ahqrLPIdSaV5qfpXfsK7RtvxbzwJswLazjX/4ji2huJ96wzS9n5NpofhNbuW3De+6f43zmLRvNK7L0dvNufOe5sxiUX4R54YZI0ASwQ7vk97O/diXXhTVi7+3DTJ+j4kx5VT/yI6FM/YnIfBpyP30jnrc8SvquG+YY/o37gbXTvuGLdlxMWAG+9FG/vt/Heuwdr9z9SfPDDtBbHqeAVBntPovhXCquNDTHH3JzJkeEh5j0qdofAdbBtn47ZpDMZEdBt9Gi7fQJnfLeLStui0W/jG2D4LfpNk3bFwjQtKl2bVq+hwTdPp8MPceN/Ojp2ht85Uhj+5V+9+8h4H4Zh4dVT/E4f3ZF3E9khzQhanoXjuvhtk0Y7wo6reIt3jZHVWSE1f8j3Bh5RaE9e9Kg1Q7K6i+24eGEXt9kmMLqEldbRNyExAxpNm37o4FV8/Cb4/nmTs4onWI4bENgxoWVT7R+dca213xM5Gb/8zpeZe+1HmXvtTRgX3oJ3+6/x7/gEzd8F86rfo7HzXwm8PTjvvZPGwStp37ST5C++RLR3J7WbzqP/qVsnPaz+jPqhK+l++mJYcOn89XsovvIF7Mv3YLlfoLn/IqpvHV+i6Lz1fPjJL47cYMf43WvpNy8k+dSt4/HBvG/QO383vdvedOIrP4pnadbvpvM8wALVOz5MeOgBPHcP9uW34H7ql/hfvZHo9SexcuYvxn/dXoL3fg7b+xKdc/4bnZuW8tDsiWdJX3/hkbFayk9510xT3lV+yrtOnvKurW2+gu8sHlsqdKzJscWoUKs79KvO5O6KLvU8otuoTLvFJzT38N5HRu/Y9dNpt2OdDtK57lbalT9mcMNrTvz2aTn4GIHXx3vgT6ivI4Hb9+8vsuuyz25cu06HtIXn9IiymOpsFHRPyb59+9i1a9e0myHLKCblo5iUj2JSPvse+wK7Xvub027G8R1MCN/1Xey7P0/zkhO/vZx+Rfv9f0b3f7+N+PdXGV9sBeVd5aN9V/koJuWjmJSPYlIO67+7YiksEO65kuIb99E/eOJ3T8crDL78AMlV15zcGUoRERGRzbbg0rjpLDq3PzqDd4Yeyx/8R5r7307z2hMXuERERGRrmdEiFxiX7Kbz4UPUPvVYKccUyB+9h+jRS+jc9qatOR6XiIiIbEnODR+mOd8n/Ho2e2PV7H+M6NaM6K7/hqdLVkRERLadM6bdgJN3Ju4NNzO8YdrtWJ15xXUkP5h2KzaQXWNQ1KbdChERETntXkP1m5+nOu1mnIxzL6P7o8um3YrTT3mXiIjIqzKzPblEREREREREREQWzT2895HRzjMennY7ZKVzw2m3QESk1Obm5hiNRtNuhsj67O9OuwWyGuVdIiIiW8IZALve9n9Oux3bUqPR4Lbbbjvm9eeee053ZSgZ3SmjfBST8lFMykcxKZ99hIrJlCjvmh3ad5WPYlI+ikn5KCbloMsVp6jRaHDrrbdOuxkiIiIiW57yLhERka1PRa4pU8IlIiIisjmUd4mIiGxtKnKVgBIuERERkc2hvEtERGTrUpGrJJRwiYiIiGwO5V0iIiJb0xnTboAsaTQa026CiIiIyLagvEtERGTrmZ2eXHmP0DQJe8Wx04qYqmXid3MA0m5ExTYxTRPL8an3s01u7Mmb2YQri2kFNobh0z0qRDGRNcfc3LKHYRL2p9XQ7SPr1/Eda2k7iJdvBwXDboRrzuHUk6m1UURESkp5V7kdN+8CHeOn4/h5l3LhjXZocBcf+a+X8eY3v5k3v/MaPnPv0xxenHj4Z9z7mZB3XvZm3nzxm3nnNX/I3T87vOpyDtwdYtv2UY+Lr/kmT2/aN9ki+iGmseJ/fm6OuUqHfPKWtX+nrC7rBlhzFtFgY5u/tZ14f5QnbULHYM5rkR41b0HSqeLZJoZhYNoe1c6QVbKEqZudIpfpUwtN+u0uKzeBvN+iR0AtMGHYJIgSvG5KnuckbZteWKUzO/nW7Ek7+JU6Q8fDXjmtyMmKS7nj2RGj0eRR5HT9KbRzOxm2CKoxbmdInucMGhZxo8M41S2Iax5B16bizk+5oSIiUkrKu8prrbxLx/jpWCvvUi68sQ48wB9+5G7mb7yPx/7t33j4a7t54fMf4QuPHwYOM/j8R/j8C+/jaz/8N/7tyYf5f7wX+PxHvsDjqyzq0KFD7Nj9bdI0PfJ48r7red1mf6dZ53fJi2X/76NnueuqC7g68jHhBL9TjiPrEtUTjAVjc77DVnWC/VHeC/GqMW7FYeURpIgj/FpK0E0pioK04zGIQlrDTf8WJzQ7RS4MvFqEHbfppstfz+i1+1hhhG9A0umQVuo0PBMAq9Kg5sZ0esq2Nozh0owHtKsux+x2ipy8MDG1P9pUSbtNFjSPbAd22GEQ13En0+1ql6RXp2JNr40iIlJmyrtKa628Cx3jp2HNvEu58IY6/PgDDHa8jxt3n888sOON11F73yEeuHdcxjrnfZ/la//zet64A2AHb3n/+3jdgad5+tCxyzp04BA7duzYzOZvC2knokmDdjjeKZ3od8oqS6BTrVPUmwTmpjR56zrR/sip0x90qXqrrGgrpNltU5tMMyshvjUkUZHrFDlVal5Cu72szpt2accuUc0DCtJhiuU4yw76Fo5tMizj2t8qLBf3eIlUnpOT0ok8bMvEtF2Cem9F10c5vTIGSYptDqn7LrZlYXshrcFiB2ED23VWTYxFRESOUN5VTmvlXTrGT8EJ8i7lwhtsnnlYujyReXbsmOfQ0y9wiHle967deOdPJh16mgfuupcD3m68VWpZhw4c4tDjLcL3vJPL3nwZ7wk/w90/W6UaJq9e3qPeyIlaVca7rRP9TjlW2q7SMJp0InuTGr2FnWB/ZDou9nEOIIbjU/WXji9F0qOfV/C9TWj3Os1WkQuLoBaQddvEk4s/k3abYSUitAEK8gJM6+jIGKZBkZfxatFtwLDxw4Cw1mOY5WRxHbMXEjQ1RsTGycnzwwx6AyrtAWmW0o8Kmn6V3vGPHyIiIiso7xI5sRPkXcqFN9S89z68Qw9w170vcBg49LN7uev7Bzh8+PCywtcLfPOai7Hf/B7+6Gcef/7n7+f8VZZ1jreb3buv48/ve5jH/u1/8T/f9QJfCD/D91XnOmnDdoPYa1BzFl9Z5++UtE21adJsh6hz6mlwmvZHxbBNGHRx222qJQzMjBW5wPRrhHRp93MoYtrdnKC2+E9vYBqQZ0cnVkVeYKiP8HRYAc1Oi6hiYQCGHdKseQy7fXSOd6MYGMY8TrVOYBuAgVOtERgxPQ3UKCIi66C8S+RETpB3KRfeWDt28z+/dj18M+Sd73wPH2o9jfe+NzK/Y8eyMYXO5/r7niR98mG+fd0BPn/NR3jgwLGLeuN1f86f3/g+XrcD4BzecmON3fMDHhisPlC9nEAR02rnhDV/We/S9fxOGdIKm1itNmEJCykz6TTsj9JehOd3sNsDuuMzXqUzc0UuDI9aZNNv90j7bXqERP7iZmPgeA7ZcPko/ynJMMfxnNWXJxsrHzJI0qPuulAUBRiGutJvGBvXMcnTladDDLTSRURkXZR3iZzACfIu5cIbbod3I1/7Xw/z2MM/5L6v1TjnwNO8znsjOzjA4w88wGCxoDV/Pm+57nrexYDv/2zlUg7zws8eP3qsrsPjyyDn53UTh5My6NIrfIKjLmdbx++UYY/OMCeOXCzLwrIC2s89RyewqZRxtPNZcIr7o7QTUKnn1PsxLb+8lcfZK3IBTjXCHbSJWn3sWo3l240TVnHiJo3Jdb1Zr0Er9YmC8gZhKyuGbUIvoDG5LWyR9mi0EtzQX+WOQHK6eFEIvQbtYQEUDDstelQIjz+io4iIyKqUd4msba28S7nwBjv0EJ9553/l84NDwGEOPPR5Wg+9ket3j++J+PTdf8Qf/tG9PH0Y4DAvfP9eHjr0Ot7yOoAXGNz7fR4/AHCIwRc+RPiZxfce4vFvtvg+72P3W6bz1WbdcDAgd71jBpRf+3dKStzpMcgAp06S52RZNnn0iC64gGovJa7pRMrJOJX9UZHUCeoFjX6X0Cl5if7hvY+MZs+Lo/s+sDBi/qrRt/7j2KnP3vfJ0bsuWBjNzy+Mzrv06tFf7H1x85t4Cp577rlpN2Fdfn7H5aN5GLH8MX/e6KMPjkaj0cujn3/rQ6PLL1gYzc/PjxbOu3R09Wf/abRK2Ept1mKyfL0vLCyMzrv0A6M7jmwH/zT6wML80fGC0fy77pqpuMxeTLY+xaR8FJPymc2YKO8qk7XzLh3jp2OtvEu58EZ7/nt7Rh94x++MLrrod0Zv+883jL669+DSxP17R1+94b+M3vY7F02m//7otu89P55W/MPo9y9622jPTxbfG4+aN/znI+99xwduHv39E8Wmf59Xq8wxGY1Gowc/dN5o4UP/tMqUNbaXl/92dNX8eaOP7l1tiXtHn7zgguNMK4eyx2Tt/dGLo29ddezxg/mrR/e9PBrt/eh5x05jfnT5HT+f8nc61tzDex8ZvcO7YjPranIC+/btY9euXdNuhiyjmJSPYlI+ikn5KCblo5iUj2JSPopJ+Sgm5aOYlI9iUg4zebmiiIiIiIiIiIjIcipyiYiIiIiIiIjIzFORS0REREREREREZp6KXCIiIiIiIiIiMvPmHt77yGjnb50/7XaIiIisy9zcHKPRaNrNEBERERGRkjgD0B0ASkZ3ZSgfxaR8FJPyUUzKRzEpH8VkehqNBrfddtsxrz/33HOKScloOykfxaR8FJPyUUzKQZcrioiIiMiW12g0uPXWW6fdDBEREdlAKnKJiIiIyLagQpeIiMjWpiKXiIiIiGwbKnSJiIhsXSpyiYiIiMi2okKXiIjI1nTGtBsgIiIiIrLZGo3GtJsgIiIip9ns9OTKe4SmSdgrjp1WxFQtE7+bA5B2Iyq2iWmaWI5PvZ9tcmO3mSKhU61gWyamYWJ7Ie2kWDbN+ODEAAAgAElEQVS5Q9WzMU0T03IJWgPyKTZ3u8j6dXzHWtoO4qXtIB+0CF0L0zAwTJtKrUc6vaaKiEjZrCPvokjp1ytYcybV/uY281TNZKFLeVcpHT/viomsOebmlj0Mk3DGthUR2RqO/zuwoBcYR++r5gzs2mBp5rRLVFk6voQlPb7MTpHL9KmFJv12l5Ulq7zfokdALTBh2CSIErxuSp7nJG2bXlilozrXBimIawG1NKA3zMmLlG4lpRbUGQAUA+pBnazaJ8tz8kEdoxVQ65dxc9hChi2CaozbGZLnOYOGRdzokADkXap+kyKKyYqCYtjC6oVU29pIRERk4tXmXSQ0Kz7NwsM7bxoN3W6Ud5XSWnlXkZMVl3LHsyNGo8mjyOn60260iGw7a/4OzMlyuOpbLy7tq0YFacsbz1sMqPk10rBHmudkcQSdJr10ml9odbNT5MLAq0XYcZtuuvz1jF67jxVG+AYknQ5ppU7DMwGwKg1qbkynpx/wG8UKmvQ6NVwTwMSrBjjZkGEODDp0i4B65GAA2CGN0KTXjlnl3LCcJkm7TRY0j2wHdthhENdxAXCotru0FmNiBVQ9gzRJp9ZeEREpm1eXd4GJ3x4QtwKsqbRz+1HeVT5r5l1FTl6YmMZUmygiwtq/A3Py3MA8zs6qiNt0jSqtyMUEDCeim/So2pvW+FdthopcgFOl5iW028nSa2mXduwS1TygIB2mWM4kaABYOLbJMBlufnu3BQPHD6nYk6f5kG6zQ1YJqZiQDYfkjouzbA7bdSiGQ10et2EyBkmKbQ6p+y62ZWF7Ia3B5Cyu6RKEFezFt+d9ugMDL3COszwREdmWTph3Adi442qLbArlXeVzgrwrz8lJ6UTe+BJT2yWoa5gIEZmCNX8H5mR5wbAd4NoWpmXjhU0Wr7xOk4TCNomrFRzbwnIqVDtJKU+gzFaRC4ugFpB128STtZm02wwrEaENUJAXYFpHVx8N06DIy7j6t5KUlmcw95tvIEoqtNtVbKDIczAMlkfEMA2MIivl9btbQ06eH2bQG1BpD0izlH5U0PSr9Fau9HxAM4gY+B3avn6kiIjIcifKu2R6lHeVxwnyLsPGDwPCWo9hlpPFdcxeSNBMTrhkEZENc8zvQBMvqOKHDfppRj7sUS1aBGGbFMiznIP9HsNql2GaMez6pLWAWly+OsuMFbnA9GuEdGn3cyhi2t2coBZOusgbmAbk2dErusgLDPUR3mA2tUHB6OVn6UcZNS+gm4FhmlAUR1V4i7ygMC1UUtkoBoYxj1OtE9gGYOBUawRGTG/ZuIHFsEPohcR+l0HbVzxEROQYa+ddMj3Ku8rjBHmXFdDstIgqFgZg2CHNmsew20fXmYjINKz+O9AhbLVpVr3xMd50iZpVrEGXOAfDMJj3IuqVcQZgujVqlZx+r3x7spkrcmF41CKbfrtH2m/TIyTyFwtYBo7nkA2Hyw7uKckwx/F0KdbGyBh0u0e6MWLYeFENn5heApbjYA6Tow7iw2SI4ThL3STlNLNxHZM8XXnO1mDx1G6RtPD9FmZrQL/uKfEVEZHVrZl3yeZT3lU+J8i78iGDJD268FgUx/S4ExHZDMf9HVhkJIPk6JvNFADjfZXtOhj5sb2CjRLuyGavyAU41Qh30CZq9bFrNbzl08IqTtykMbkOPus1aKU+UaBzjhtl2I4Iow7DAqAg7XXo5w6eA1QiqmaPRmt8vW4x7NDoFISRrwP7BvKiEHoN2sMCKBh2WvSoELpAEVMLWtjtmLav7UJERNa2Vt4lm095V/mslXcVwzahF9CYVCaLdBwfN/RVeBSRzbXW78AioRlUCFvJuJCVD2k3lsZ8NP0qftam0R0X7fOkRSs28cs4rvPDex8ZzZ4XR/d9YGHE/FWjb/3HsVOfve+To3ddsDCan18YnXfp1aO/2Pvi5jfxFDz33HPTbsL6/MeDo7+4+tLReQvz43V+8VWjT9737JHJL//8b0cfuvyC0cL8/GjhvEtHH7jr8dHL02vtSZm5mIxeHv38Wx8aXX7BwmhhYWF03qUfGN2xuB3c94HRPIxY8Zi/6lujWdpSZi8mW59iUj6KSfnMZkzWyLv2fnJ0wfzKY8r86NK/eHwqLT0ZMxcT5V0ltEbetWza/CQmV3/2n0ar/IQptdmLydanmJRP6WNygt+BL+69Y/SBS88bLczPj+YXLhhd/oE7RstLKS8+ftd4+sLCaOGCy0cf+tbPS3l8mXt47yOjd3hXTKnEJqvZt28fu3btmnYzZBnFpHwUk/JRTMpHMSkfxaR8FJPyUUzKRzEpH8WkfBSTcpjJyxVFRERERERERESWU5FLRERERERERERmnopcIiIiIiIiIiIy81TkEhERERERERGRmTf38N5HRjt/6/xpt0NERGRd5ubmGI1G026GiIiIiIiUxBmA7gBQMrorQ/koJuWjmJSPYlI+ikn5KCblo5iUj2JSPopJ+Sgm5aOYlIMuVxQRERERERERkZmnIpeIiIiIiIiIiMw8FblERERERERERGTmqcglIiIiIiIiIiIzT0UuERERERERERGZeSpyiYiIiIiIiIjIzFORS0REREREREREZp6KXCIiIiIiIiIiMvNU5BIRERERERERkZmnIpeIiIiIiIiIiMw8FblERERERERERGTmqcglIiIiIiIiIiIzT0UuERERERERERGZeSpyiYiIiIiIiIjIzFORS0REREREREREZp6KXCIiIiIiIiIiMvNU5BIRERERERERkZmnIpeIiIiIiIiIiMw8FblERERERERERGTmzT2895HRzt86f9rtEBERWZe5uTlGo9G0myEiIiIiIiVxBsCuXbum3Q5ZZt++fYpJySgm5aOYlI9iUj6KSfkoJuWjmJSPYlI+ikn5KCblo5iUgy5XFBERERERERGRmacil4iIiIiIiIiIzDwVuUREREREREREZOapyCUiIiIiIiIiIjNPRS4REREREREREZl5KnKJiIiIiIiIiMjMU5FLRERERERERERmnopcIiIiIiIiIiIy81TkEhERERERERGRmacil4iIiIiIiIiIzDwVuUREREREREREZOapyCUiIiIiIiIiIjNPRS4REREREREREZl5KnKJiIiIiIiIiMjMU5FLRERERERERERmnopcIiIiIiIiIiIy81TkEhERERERERGRmacil4iIiIiIiIiIzDwVuUREREREREREZObNPbz3kdHO3zp/2u0QERFZl7m5OUaj0bSbISIiIiIiJXEGwK5du6bdDllm3759iknJKCblo5iUj2JSPopJ+Sgm5aOYlI9iUj6KSfkoJuWjmJSDLlcUEREREREREZGZpyKXiIiIiIiIiIjMPBW5RERERERERERk5qnIJSIiIiIiIiIiM09FLhERERERERERmXkqcomIiIiIiIiIyMxTkUtERERERERERGaeilwiIiIiIiIiIjLzVOQSEREREREREZGZpyKXiIiIiIiIiIjMPBW5RERERERERERk5qnIJSIiIiIiIiIiM09FLhERERERERERmXkqcomIiIiIiIiIyMxTkUtERERERERERGaeilwiIiIiIiIiIjLzVOQSEREREREREZGZpyKXiIiIiIiIiIjMPBW5RERERERERERk5s09vPeR0c7fOn/a7RAREVmXubk5RqPRtJshIiIiIiIlcQbArl27pt0OkVLbt2+ftpOSUUzKRzEpH8WkfBQTkRPTdlI+ikn5KCblo5iUgy5XFBERERERERGRmacil4iIiIiIiIiIzDwVuUREREREREREZOapyCUiIiIiIiIiIjNPRS4REREREREREZl5KnKJiIiIiIiIiMjMU5FLRERERERERERmnopcIiIiIiIiIiIy81TkEhERERERERGRmbfNilwFSbOCZZrY1R75qSwqH9DpDU+5RXHVwq4NTnk5J60Y0o0qOLaNbdvYXkg7WWXNFAl112Cu0jl6vfVD3Hqy7IWcQSvEs01Mw8S0K0TdIcUGfw0REREpG+Vdx1DeJSIisqG2WZErpd8d4LZS0k6AeQpLKuIWtU5yyklEpT0kaXqnuJSTlzQComFAb5iSpilxlFMP6sRHfbGCQaNKNzeZX3wp61GLWnSHBUY2oFOPaA0K8l4Vv1kQ9TPyImfYdoirIa1Tz0tFRERkpijvWkl5l4iIyMbaRkWulE4Q0BweJq57uFGPnIJhJ6LiTM6mOT61frpsli5RxcayLCzLwg1aDHJgUMeLehzs13DdGvGJPjqLqfsO9mQ5tlelMxxnM3Hk4NYHMGxRmUy3LAvLNJibc1g8WZcPWoSeM26n7RI041M7IwpAQTpMsbwKjjF+xa742NmQ4bKFF4M6Ub9Cu+4uvWhViEKTuNPnn/s9Mq9K4BpgV2n32lQnC7T8KhUzJUlPubEiIiIyM5R3HUt5l4iIyEbbRkUum2qvR92Zx28nJO0AI67h1xL87pA0TRl2KwzCkFYKkNGJqvSdDsMsI0v7+GmdansIXpNuzWHeb5EkLSon+OSkFdExGiRZRpal9CPodgZHn410asRZRpZlZFlCJ7C44OoGNRfIe0RBkzzqM0xT0qSF1QkIu9mxH5Z18E0Dw1j5MAm6K9MzAy+okPc74ySSgmGvR+YGVKzJW4qYWjWm0m7iHXUK1sQmIbGafCvIGeQWtgGmGxB6k5mLjKTdpI9POL2TpiIiIrLplHcp7xIREdl8Z0y7AdM06PbI/Q41d3z2y3AjIrdBu59Si2yqvZwQA2M8Ed8zaQ9TwFnX55iWSdHt0uk7hBUXp9qhf9x3FwxbIdUkpDcIsYAi7tA3QvpVe9wWs0KtauN2Y/IwPLr7v1Wln1dfddusaptWv0LF6mCaBTkVmv1o8g1z4lrEIOgy8AyKdEVLzQrNdkDFrGAlR09L6g5v+cKTcN67+Gy3S3Aq1yiIiIjIzFPepbxLRERko22jnlwrFeRZzsFe9Uh3dstyqCcGRTo+85YNWkS+h+u6uK5L1Du5jup2rU+/4TBohjimiVOJ6A5XH1Ui70cETZNmr4k36cqeZzkHn+sQLOtWX2mlGHl+il3nC+LIp140GOY5WVaQdmzafkg3g7xfIxpU6TRcjFXmNt2Aig2YLn7FPmqa2xwyGr3Isx2fQegRxRoCVUREZPtS3qW8S0REZONt4yKXgWmZnBd2J13VJ488J2m6kHWoBm2o9xgkCUmS0PZP9rSYiVdt0o2H5HlC2x0QhS2OGRN02CaoJvi9DlV72dyWycLFEf3l7cxy8jjCXrmMdXWbT+j1U7xqiD3Jpiw/pEJMLymIOz2yrIVvjxM8J4o5PKjh2FX6x8md0rhDd7D4OSa2X6PmZfR7yeoziIiIyDagvEt5l4iIyMbbxkUu8MIAo9+kk04yhzymGVbppkCRkxUWrmNhAEXSphXnFHlBAeMzbK/qjF5Or+oRtCe3czZsHNeGIj96bIg8JgoaGM0eraMHYcCoVKnkXVq9xbEgMnq1kFp/lbEhrCr9vKAoVj5yeuHKZNHGdQwGvT6LS8oHPeLcxrMNgm5OvizBG7YrzHsthmkHf7VTjEAxaFGt1uhN1mkx7NAeGDieAxQM+x16q90qW0RERLY05V3Ku0RERDbati5yGZUWvaZFuzK+e47l1hi4VXwbsEOaEbQ8C8d18dsmjXaEHVfx6gNsP8RNathWQGeVnGeJiV+PMDv+ke75Xsug0amz7J45ZN06f/VkzqDuLd3px7IJOhmYAZ1ejaLpjdtpeTSzCtXFgUZPmkW10yXMG3iWjW1buNUhfrdHtL7hL45w6j26fkbdszAMA6vSxqh26YQmkBM3IhqrJYkiIiKypSnvUt4lIiKy0eYe3vvI6B3eFdNuh2wTRT+imjXoVk81Udxc+/btY9euXdNuhiyjmJSPYlI+ikn5KCaymZR3yemimJSPYlI+ikk5bOu7K8rmy6hQC2Yr0Vq0b9++aTdBVlBMykcxKR/FRGT7Ut4lp5NiUj6KSfkoJtOnItcpG1D3Inr56iOCGm6dXrd67ECl25TthzO7LlSVLxedKSkfxaR8FJPyUfJ7qpR3rYfyLjlddDwpH8WkfBSTclCR65R5NAcJzWk3Q0RERGTLU94lIiIix7etB54XEREREREREZGtQUUuERERERERERGZeSpyiYiIiIiIiIjIzFORa5PEVQu7Nph2M06vIqVXq+CYBoZhYrkhrUE+npbUcebmmFv+MHw6k8nkA1qBi2WamKaNV+0wXH0MWREREZF1Ud6lvEtERLYnFbnkpCUNnzB2aQ8LiiKlH6bUgzoxQJ5TXPBJ9o5GjBYfRZ+qCZDTiwJaRp1BlpNnfapZnaA+QPmWiIiIyLGUd4mIiJzYlilyxZGFE7VpVQP8iodju4SdIQB5x8dwW6RL76Zq2UQDgIS6YxE2W1QDn4pjY1eaxHF7/NwdP188Uba2gqRdxbMtLMvC9qp0VjtNViR0qh62tcr7spi676w6LR+0CFwby7KwLBs3bC21K4tpBi6ObWPbNl7YJplMW3O+k1Zg+g26nQYVC8DEDQOcfEiaA0VOblqYq87ap9O3iBohtgEYDlE9IO922GLnXEVERLYk5V3Ku0RERMpoyxS5AJ7sxdjNHv14QNJ2ievN8dmtNRmYxi+JBxbNXp942CVI/y+CBjR6feKkT5Q3aPRPnJ0UcUTQyKkNMrIspRcMicIWwxXvG7aqREmFXpqRZUM6bkwUdciApBXRMRok2XgZ/Qi6nQEFKZ1anTyKybKMLBvQsPp04wxIaYcBbbNBnKak6YA6Dfxa/wTzrWh/L8Q0DIyVD7NCOz12vdmVkMBdSqfSXo/UDfBMyLOcIutRrzhYponl+ETd4fiM4TBhWNg49rLFOS52PmR4bLNERESkhJR3Ke8SEREpmy1V5FqoVPGt8d+G42DlGdmrOnM2jxv4jGe1cex5nMDHnjy3LchexYIGnT55JSKwAAzcekw2qOOseJ9TH5APmrgGgInnu5AOyQDTMimSLp1+QlYYONUO/WYFAxPLMhj2uvQGKTkWQatPK7Ag69Md2ET1YPIdLIJagNHvMlhrvhWMoEteFBQrH3lMZK/93dNuFb9l0uxE4+9r+1SDkKiTkOUZScthUPWpxQUUBYVhYBjLP9zApKBQv3kREZGZoLxLeZeIiEjZbKkil2EaLB2/x3+/2mO3YRrHf27wKhTkeY5pmke1wVxl3iLt06hWcF0X13XxajGHJ221a336DYdBM8QxTZxKRHdYACZhZ0CrMqQVeVimjVdtj7u/5ynZ4SHNijXpGm9hBT3yIifL15jvtMiI6x6VJjTjPpEz/sJmpUa7XcMf94vH8ps0/Ix+NwHTwChWJFZFQc6KBExERERKS3mX8i4REZGy2VJFrrUVS4nXygP9aWFgWiZ5ni1L8HLSYcbRec2QdhDSt5rESUKSJAyaFeaPTDfxqk268ZA8T2i7g6Wu96ZDWO8QJxn5sEuY1gnqMZg21rxLY5BNusaPH3neIzTXmG+F9XWbB8joVStUk5DeoENgL03J0wGDdPk3n6x/wwDbwzGGDJYts0gGpJaDe+yJThEREZk5yruUd4mIiGy+bVHkMiwLIxuSLo4x2u8Qb0D/bC/wMfptOmkBFAxbAW6wfOBVgJw0B8u1x4OD5gNancE4ASSnV/UI2pMxFAwbx7WhyCmKmJrn05ycCjQsB8eejMtg+YTekE4rniR2BcNORNiMydeab4X1dptP2yHRMKLfq00uAViSdSMqfo3eZF1k/QbNvk0QOmD4REFBp94dxyRPaDV7mGGEN14ycafLQONEiIiIzBzlXcq7REREpmV7FLn8Oq1KQuR6VCo+taGHb/Pq+9S/6s9p0W+atCsWpmlR6dq0eg3co97lUWuGZHUX23Hxwi5us01gdAkrHX67HmF2/CN3+fFaBo1OHdeoUKs79KvOpGu8Sz2P6DYqgE3U7eKnNVzLxrZsgg4EgYe55nynYki3/RC//OdP8YbfmGNubvIwbGoDcOo9un5K3bUwDBOnNqTS7dP0xt3o/VaPhtHEswwMO6DntOk1JmuqGNCKanRWjhwrIiIipae8S3mXiIjItMw9vPeR0Tu8K6bdDpGjDOoBcbVHfeXosVOyb98+du3aNe1myDKKSfkoJuWjmJSPYiJlpLxLTkQxKR/FpHwUk3LYFj25ZNZkFG6NakkSLREREZGtS3mXiIhsHWdMuwGzY0Ddi+jlq/e1N9w6vW51cvtrOTUWlVAjoYqIiGxfyrs2j/IuERHZOlTketU8moOE5rSbISIiIrLlKe8SERGR9dPliiIiIiIiIiIiMvNU5BIRERERERERkZmnuyuKvAr79u2bdhNERGSL0J2XRNamvEtERE6WxuQ6XYZNXHdAPe8RZi08p0eUxVTNzWpAQS8wueb+w8tem+eCT8akLW/ylpR+o0r1Cwn+P+V0/KV35oMWUdSkP8wpDAuv2qLTCjSg6zL6UVIuukVv+Sgm5aOYlI9+vJ8myru2PO27ykXHk/JRTMpHMSkHXa64EeyIftYj3LRECyAny+Gqb73IaDSaPIqlRIuEZsWnWXh4562ctUvVb1JEMVlRUAxbWL2QajvbzC8gIiIisn7Ku0RERGRCRa6TlcfUfRvLsnE9n1o/5chNrtM2vhXQzYGsTcXwqbcjAr+Ca9t4tR6Dzvi559h4UZ9TT2ty8tzANI3jTDfx2wPiVsCxN4l2qLa7tCIHA8AKqHoGaZKecqtERERETpnyLhEREXkVVOQ6KQVxPaRNg0GWkgw6VJI+Tx7v7YdjBtTp9WOSfkTxf4dEWUS3HzOI6xidBt302NkGNRvTMDBWPEy3TnLMu3OyvGDYDnBtC9Oy8cIm8ZEszsZ1j3OK03QJwspSF/m8T3dg4AXOOtaJiIiIyEZQ3iUiIiKvjopcJ2VIP87xqotjJ1gEkc/K3uhHzLsE/vid2A72vI3vu5Ozdza2kZGuckrRa6XkRUGx4pEnTdxj3m3iBVX8sEE/zciHPapFiyBsk67nq+UDmkHEwO/Q9je137+IiIjIKpR3iYiIyKujItdJWeyiviwZMU2On5oYGMaRPwGDpd7txuS1U+UQttr/f3t3DBwplt9x/NdVWyUuWhwd5bJVLzMXHc7wbkLIZTg6QkJCLmtnHbYznOGMy9oZmQnZqt1aLiPEGauIybiMicZBtzQajWYkTUtqJH0/VVSJoeE99C9Jv3nvNa1t4u+Xxdue0m0ip92pme53hbkvFfuxmnCntgi/cj8AAADPhdwFAADuh0Gub2LLsmZN07UUM066Z6a5twctm59HdW336TMmZkmy7pXl5i5XGOay81b12idoAQCAhSB3AQCA+2GQ65u4CgNbbVkdlqQP2hXVo4etBy2bnztto0Bx3u37MfUqNqXGIFZwV3KaG2VRLlM0KsLPH4+qsdWubB62/B4AAOBRkLsAAMD9MMj1TSyF21LJvJFnOzJeqjaIb3lewzOyQxXVRk4ZyliWLBOqtNaqd8l+GX2byVgrrVb/pv9+93f99U8rrVaWvG0n1YXK337TX//0D1qtVlebFZb74NYXytL8loeuAgAAPDVyFwAAuJ/Vz7/+8uFH/4dT9wOLNiiPcvlVLv/UXTmRi4sLnZ+fn7obuIaaLA81WR5qsjzUBHcjd/FzsjzUZHmoyfJQk2VgJRfuNo1y0uzNBi0AAIBnQ+4CAOCbfXfqDuAFsH3F4ak7AQAA8AaQuwAA+Gas5AIAAAAAAMCLxyAXAAAAAAAAXjwGuQAAAAAAAPDiMcgFAAAAAACAF49BrkdWxZbcdXf0deZup1037XfGQoEVqBiPvuwtDQ2qskCubcmybDlerLydnqAhAACAx0XuAgAA1zHItVBdsX6W0NNtQsWNp6KfNc+D6njQOlqrefKWAQAAloHcBQDA68Ag11Fm9WUiz3FkXE9BUujTfDSoykJ5xsgYIzdItevnw6FcvhUoyxOFQSDfNXLDrdpJ6vNAUfmbuk0gL6l0ecmhXit0jRzblgnWao7OYrPscKNduVHgSJItL47kTr0GJhUBAMCikLsAAMDXMch1jKFUmrYKqkFD36nOJjXN+6vD3TZS3LjKu0HD0KuOB6XRRleL6t//pGbMVDWN2r5RNm8Vbzq5WaWtfyZv06grI9mSpFZ1F6jsB41jo3jMtS6Hz/s0lgptS5Z1c7MV7W4mKEsmiBV59sdbqioNXiTfFgAAwHKQuwAAwB2+O3UHXrK5rdU6oXLfkiRZXqLU32grSeq1K3sF60aBLUmWTJIqXK+167bybElnf1SceNqfbRRGrrJdo0HJLa25itNQzr4hBZ6l3TBKMp++zElUT7edf7dhlyjMbW3rVO43XeF1u7i4OHUXcAM1WR5qsjzUBK8Fuett4XfX8lCT5aEmy0NNTo9BriNM0yTZziEsSZIt+2ombtQ0vVeTuXLWH8+ZJcWTtJ8mtOVcm7mzbVuaRt2+Yv36taVrjT6CUc06UlK7yptakXnMa78e5+fnp+4Crrm4uKAmC0NNloeaLA/h99uRu94WfnctC39PloeaLA81WQberngEy7ZuhKNJ49WOI2OfKSgGjeN4tU3TqCK47fXSNI2S7eioFesPWjYvSaOqJFDSxarakqAFAAAWidwFAADuwiDXEWw/lDfW2h2eejq1ucr28tkQrsLIVZvn6g7PPJ26QklcXO3rfa9d0e7D2txpt+tlgmC/EN6SpmnWgzmJ6mnWPN/cJlXx5zFuKGKlfaq6yuR9Nks5q69LVd3tc5wAAADPhdwFAADuwiDXMUyqIvdUR0aOcRXmRnH0+6vD3qZS6deKXSNjHHlxJTsJP4aaM1+BKRV5rowJVdpblWtPkq0gDjRtPDlBrv7JbqDXrvhJ7/72F/3hdyutVofNMspaSZrUbFJt6vHJegAAAHAv5C4AAHCH1c+//vLhR/+HU/fj7Rly+W6ldGyULPgTdRxfJrkAAAkXSURBVOY6VTJutEucU3flpHh/9fJQk+WhJstDTZaHmpwIuetF4edkeajJ8lCT5aEmy8BKLnzVqEBZ9LaDFgAAwHMgdwEAcBw+XRFfZcL45odlAwAA4AmQuwAAOA6DXKdiMrVzdupeAAAAvH7kLgAA3gTerggAAAAAAIAXj0EuAAAAAAAAvHgMcgEAAAAAAODFY5DrGzWpIydtnqGlVqnjKHmOpiRp7rT2LK2CUtMzNQkAAPA15C4AAHAfDHLhmlntJtFusnV26q4AAAC8auQuAAAeG4Ncx5h7FbEv17FlO57isr86NLW5Yt+VMUbGeIq2zccZurlTmfgyjiPHcWT8RGU/Hw4OqtL9MdfzPz1Ps/oykW/25zmOq3DdaHys22nXSutAxdp7pCsCAAA8EnIXAAC4A4NcR5jqSto06sdJfW5Ur7dqJGmqlEZbTWmtfhg0dLmcMlK828eiPk+UdoGqYdQ49iq9RmlaapQ07TIltVHej+q7RrnTqH532WClLOsUVIPGcdTYF/L6UvVws2e9tr4ty7JubLbMl5b6z42ypFFQbOXbj/+9AgAAOAa5CwAA3IVBriNYQarEtSRJjufLTKPGSZqbUrUVa5MYWZJkB8oSo2a3nx10162mditvf1B+6ElDr1FSVzdSkCiyJcmSiVOF31826MixBjVlpXaYJDvQtiqVmJs9c7VuJ83zfGObNBTBLXcyqclStVGprW899rcJAADgaOQuAABwFwa5jmDZtq6iiWXJkjRLmsZJf/+tVORcLm93FOSDrGnSJGkeam2SQJ7nyfM8+Vmj95JmzZqmWbZtX29E9mUjVqCi3SmeS6W+keV4irfHL5uf6kxpm6jceCJqAQCAJSJ3AQCAuzDI9QRsx9b3/5KqHsf98vZx1DhOmppURr2KKFbtbNV0nbquU7sNDg8ctWTZlqbp2ufrzJPGa7uWCZUVtbpx0lhnmvNIWTXf6MFDls3PaspK45grPDxzwk0bvW8zuSZRffPSAAAAC0LuAgAAlxjkegJWkCiYdsqry7m+UVUWK6tHSZOGSXI8I1uSplZ52UrzrFmSH/pSXWp/6qy+KPbPm5CkLlcQZKoPl7VdV669n4n81EOWzVuKdpOma8GwLwKd+bn6oVRoSRpb7cpGwyN/nwAAAI5F7gIAAJcY5HoKdqSyyjRvfRlj5Di+tmOgxHck+cq2sca1J+N68uOdvG2hyNopDnKNcaEi7JW6jozraz1His3hul6sjT9o7R+W45tEbVgqj574iaV9oSzN1T1tKwAAAA9H7gIAAAern3/95cOP/g+n7gcWbVAe5fKrXP6pu3IiFxcXp+4CAOCVOD8/P3UXsGjkLnIXAOBbfXfqDuAFmEY5afZmg9Yl/lOyLBcXF9RkYajJ8lCT5eE/77gTuUsSuWtp+HuyPNRkeajJMjDIhbvZvuLw1J0AAAB4A8hdAAB8M57JBQAAAAAAgBePQS4AAAAAAAC8eAxyAQAAAAAA4MVjkAsAAAAAAAAvHoNcDzXk8q1A5fT4l65iS+66k9Rp7dqKq8dv42vGItBqtfpks/xc/dUrZvW7VJ69OvTzmnlQlQVybUuWZcvxYuXtE3yTAADA20HuIncBAPAADHItkqdNO6iMnrfVaZr0/Z//Vx8+fLja5jaTK0ma1WS+op1R4J19dm63CRU3nop+1jwPquNB62it5nlvAQAA4IHIXQAAvBYMcknSWCiwQq2LVFEYyDNGflapLff7vmvkp7XGm+cdZhezPFEY7F/nhlvdbyJtVl8m8hxHxvUUJMW18zptfKOk2n+9dh3F21xJFCpwjUywVdMU+31vv/8Yk3fTOMm27S8eN8lOXbVW4Hx+L3a40a7cHI7Z8uJI7tRrYFIRAABcR+6SRO4CAOApMMh16X2jVmtVdaOuTjX/V6x0TLWrG7XNWla50W647byf1IyZqqZR2zfK5q3iTXfLC28YSqVpq6AaNPSd6mxS07y/5YWWbOudmtbRtqrV9DtFw38o2kibqlbT1UqnjTb156lmrmLZliXr5mYHKm65l2mcNLUbBa6RYztyg0RFd3ldS8ZzZd16M5ZMECvyPga1oao0eJH8L2c3AADwVpG7yF0AADwBBrkunXmKQrP/2rgyZ0Zh6O3DhWNkrFHDZ1OKks7+qDg5vE5GYeRqbBsNdzQ3t7VaJ1Ti78+0vESp//ly9EMj8qJQzqEN15zJjUKZw75xpHH8PGxZ0U7TPGu+uU2NUvN5K04QK45TFe2gcepUhoPWQaLqgbOCwy5RmNvalulhyT0AAMA15C5yFwAAT4BBriuWLOvqS+1n8j4e+8JUmiRbzrVZM9u2pWnUXflkmibJdq5d1tZXVqzLsq0v73+xbw/jpYWKdSTXliRH/nqj2Gq0a+Z7XmFUs/YVbKVtUyt1H6ljAADglSF3kbsAAHh8DHIdbdL1ybxpGiXb0V2rxS3buhHKPr3OY3jYsvlZQ9eqnz75J82SLOs+oWlUlQRKulhVWyoyj3QTAAAAV8hde+QuAABuwyDXsd732hXtPjTNnXa7XiYIDkvav8z2Q3ljrd3hyaVTm6tsb3s2xLd72LL5Sc06VJCU6uf9fptvVClS7N/d1lDESvtUdZXJYyIRAAA8BXKXJHIXAABfwiDXsc58BaZU5LkyJlRpb1WuvbvPM6mK3FMdGTnGVZgbxdHvn76/X+QoKXdKtFXgWLIso7g2yutCoS1JtWLb0mq10r//z3v933/+q1arlayg0Kheu+InvfvbX/SH3620Wh02yyhrT3hLAADgdSF3kbsAAPiK1c+//vLhR/+HU/fjZRpy+W6ldGyU8Gk2r9rFxYXOz89P3Q1cQ02Wh5osDzVZHmpyBHLXm8HPyfJQk+WhJstDTZaBlVwAAAAAAAB48b47dQdep1l16iv70qfjWIG2TaGIWUgAAIAjkbsAAMAeg1zHMJnaObvlgKWw6NQ/e4cAAABeKXIXAAC4A29XBAAAAAAAwIu3+vnXXz788z/+06n7AQDAg6xWK3348OHU3QAAAACwEN9J4hMAFoZPZVgearI81GR5qMnyUJPloSbLQ02Wh5osDzVZHmqyPNRkGXi7IgAAAAAAAF68/wf/hyOagq+eWAAAAABJRU5ErkJggg=="}}},{"cell_type":"code","source":"!pip install tflite_runtime\n!pip install tflite-support\n!pip install einops\n!pip install onnxsim\n!pip install onnx_tf","metadata":{"execution":{"iopub.status.busy":"2023-03-28T01:28:02.255277Z","iopub.execute_input":"2023-03-28T01:28:02.255997Z","iopub.status.idle":"2023-03-28T01:28:58.143121Z","shell.execute_reply.started":"2023-03-28T01:28:02.255956Z","shell.execute_reply":"2023-03-28T01:28:58.141840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nimport numpy as np\nimport pandas as pd\nfrom glob import glob\nfrom tqdm.notebook import tqdm\nfrom einops import rearrange\nfrom einops.layers.torch import Rearrange\n\nimport onnx\nimport onnxsim\nfrom onnx_tf.backend import prepare\nimport tensorflow as tf\n\nimport torch\nimport torch.nn as nn\nimport torch.optim.lr_scheduler as scheduler\nfrom torch.utils.data import Dataset, DataLoader, random_split","metadata":{"execution":{"iopub.status.busy":"2023-03-28T01:28:58.146631Z","iopub.execute_input":"2023-03-28T01:28:58.147055Z","iopub.status.idle":"2023-03-28T01:29:20.532941Z","shell.execute_reply.started":"2023-03-28T01:28:58.147007Z","shell.execute_reply":"2023-03-28T01:29:20.531825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

3    DATASET

","metadata":{}},{"cell_type":"code","source":"path = '/kaggle/input/gislr-feature-data-on-the-shoulders'","metadata":{"execution":{"iopub.status.busy":"2023-03-28T01:29:20.535078Z","iopub.execute_input":"2023-03-28T01:29:20.536298Z","iopub.status.idle":"2023-03-28T01:29:20.541390Z","shell.execute_reply.started":"2023-03-28T01:29:20.536246Z","shell.execute_reply":"2023-03-28T01:29:20.540299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROWS_PER_FRAME = 543\ndef load_relevant_data_subset(pq_path):\n data_columns = ['x', 'y', 'z']\n data = pd.read_parquet(pq_path, columns=data_columns)\n n_frames = int(len(data) / ROWS_PER_FRAME)\n data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T01:29:20.545887Z","iopub.execute_input":"2023-03-28T01:29:20.546559Z","iopub.status.idle":"2023-03-28T01:29:20.566126Z","shell.execute_reply.started":"2023-03-28T01:29:20.546520Z","shell.execute_reply":"2023-03-28T01:29:20.564956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PreprocessedGISLDataset(Dataset):\n def __init__(self, path) -> None:\n super().__init__()\n \n self.dataset = np.load(os.path.join(path, \"feature_data.npy\"))\n self.labels = np.load(os.path.join(path, \"feature_labels.npy\"))\n \n def __len__(self):\n return len(self.dataset)\n \n def __getitem__(self, idx):\n return self.dataset[idx, :], self.labels[idx]\n","metadata":{"execution":{"iopub.status.busy":"2023-03-28T01:29:20.567946Z","iopub.execute_input":"2023-03-28T01:29:20.568455Z","iopub.status.idle":"2023-03-28T01:29:20.577266Z","shell.execute_reply.started":"2023-03-28T01:29:20.568293Z","shell.execute_reply":"2023-03-28T01:29:20.576259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = PreprocessedGISLDataset(path)\n\ndataset_size = len(dataset)\n# train_size = int(dataset_size * 0.9)\n# validation_size = dataset_size - train_size\n\n# train_dataset, validation_dataset = random_split(dataset, [train_size, validation_size])\ntrain_dataloader = DataLoader(dataset, batch_size=256, shuffle=True, drop_last=True)\n# train_dataloader = DataLoader(train_dataset, batch_size=256, shuffle=True, drop_last=True)\n# validation_dataset = DataLoader(validation_dataset, batch_size=64, shuffle=True, drop_last=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T07:36:58.860930Z","iopub.execute_input":"2023-03-28T07:36:58.861560Z","iopub.status.idle":"2023-03-28T07:36:58.937092Z","shell.execute_reply.started":"2023-03-28T07:36:58.861521Z","shell.execute_reply":"2023-03-28T07:36:58.935567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4    MODEL

","metadata":{}},{"cell_type":"code","source":"### https://github.com/davda54/sam\n\nclass SAM(torch.optim.Optimizer):\n def __init__(self, params, base_optimizer, rho=0.05, **kwargs):\n assert rho >= 0.0, f\"Invalid rho, should be non-negative: {rho}\"\n\n defaults = dict(rho=rho, **kwargs)\n super(SAM, self).__init__(params, defaults)\n\n self.base_optimizer = base_optimizer(self.param_groups, **kwargs)\n self.param_groups = self.base_optimizer.param_groups\n\n @torch.no_grad()\n def first_step(self, zero_grad=False):\n grad_norm = self._grad_norm()\n for group in self.param_groups:\n scale = group[\"rho\"] / (grad_norm + 1e-12)\n\n for p in group[\"params\"]:\n if p.grad is None: continue\n e_w = p.grad * scale.to(p)\n p.add_(e_w) # climb to the local maximum \"w + e(w)\"\n self.state[p][\"e_w\"] = e_w\n\n if zero_grad: self.zero_grad()\n\n @torch.no_grad()\n def second_step(self, zero_grad=False):\n for group in self.param_groups:\n for p in group[\"params\"]:\n if p.grad is None: continue\n p.sub_(self.state[p][\"e_w\"]) # get back to \"w\" from \"w + e(w)\"\n\n self.base_optimizer.step() # do the actual \"sharpness-aware\" update\n\n if zero_grad: self.zero_grad()\n\n @torch.no_grad()\n def step(self, closure=None):\n assert closure is not None, \"Sharpness Aware Minimization requires closure, but it was not provided\"\n closure = torch.enable_grad()(closure) # the closure should do a full forward-backward pass\n\n self.first_step(zero_grad=True)\n closure()\n self.second_step()\n\n def _grad_norm(self):\n shared_device = self.param_groups[0][\"params\"][0].device # put everything on the same device, in case of model parallelism\n norm = torch.norm(\n torch.stack([\n p.grad.norm(p=2).to(shared_device)\n for group in self.param_groups for p in group[\"params\"]\n if p.grad is not None\n ]),\n p=2\n )\n return norm","metadata":{"execution":{"iopub.status.busy":"2023-03-28T01:29:53.890514Z","iopub.execute_input":"2023-03-28T01:29:53.890878Z","iopub.status.idle":"2023-03-28T01:29:53.905588Z","shell.execute_reply.started":"2023-03-28T01:29:53.890842Z","shell.execute_reply":"2023-03-28T01:29:53.903803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# original code: https://github.com/lucidrains/vit-pytorch/blob/main/vit_pytorch/simple_vit.py\nfrom einops import rearrange\nfrom einops.layers.torch import Rearrange\n\n\n# helpers\n\ndef pair(t):\n return t if isinstance(t, tuple) else (t, t)\n\nclass LearnablePositionalEncoding(nn.Module):\n def __init__(self, embedding_size, max_seq_len):\n super().__init__()\n self.embedding_size = embedding_size\n self.max_seq_len = max_seq_len\n \n self.position_embeddings = nn.Parameter(torch.zeros(max_seq_len, embedding_size))\n nn.init.normal_(self.position_embeddings, std=0.02)\n \n def forward(self, x):\n # x shape: (batch_size, seq_len, embedding_size)\n seq_len = x.size(1)\n position_ids = torch.arange(seq_len, dtype=torch.long, device=x.device)\n position_embeddings = self.position_embeddings[position_ids, :]\n position_embeddings = position_embeddings.unsqueeze(0)\n position_embeddings = position_embeddings.expand(x.size(0), -1, -1)\n x = x + position_embeddings\n return x\n\n# classes\n\nclass FeedForward(nn.Module):\n def __init__(self, dim, hidden_dim):\n super().__init__()\n self.net = nn.Sequential(\n nn.LayerNorm(dim),\n nn.Linear(dim, hidden_dim),\n # nn.GELU(), # GELU not supported in tflie 2.9.1\n nn.ReLU(),\n nn.Linear(hidden_dim, dim),\n )\n def forward(self, x):\n return self.net(x)\n\nclass Attention(nn.Module):\n def __init__(self, dim, heads = 8, dim_head = 64):\n super().__init__()\n inner_dim = dim_head * heads\n self.heads = heads\n self.scale = dim_head ** -0.5\n self.norm = nn.LayerNorm(dim)\n\n self.attend = nn.Softmax(dim = -1)\n\n self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False)\n self.to_out = nn.Linear(inner_dim, dim, bias = False)\n\n def forward(self, x):\n x = self.norm(x)\n\n qkv = self.to_qkv(x).chunk(3, dim = -1)\n q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = self.heads), qkv)\n\n dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale\n\n attn = self.attend(dots)\n\n out = torch.matmul(attn, v)\n out = rearrange(out, 'b h n d -> b n (h d)')\n return self.to_out(out)\n\nclass Transformer(nn.Module):\n def __init__(self, dim, depth, heads, dim_head, mlp_dim):\n super().__init__()\n self.layers = nn.ModuleList([])\n for _ in range(depth):\n self.layers.append(nn.ModuleList([\n Attention(dim, heads = heads, dim_head = dim_head),\n FeedForward(dim, mlp_dim)\n ]))\n def forward(self, x):\n for attn, ff in self.layers:\n x = attn(x) + x\n x = ff(x) + x\n return x\n\nclass SimpleViT(nn.Module):\n def __init__(self, *, feat_size=84*3, num_classes, dim, depth, heads, mlp_dim, channels = 3, dim_head = 64):\n super().__init__()\n \n self.feat_size = feat_size\n self.input_len = 15+8\n self.dim = dim\n# self.to_patch_embedding = nn.Sequential(\n# Rearrange('b c (h p1) (w p2) -> b h w (p1 p2 c)', p1 = patch_height, p2 = patch_width),\n# nn.LayerNorm(patch_dim),\n# nn.Linear(patch_dim, dim),\n# nn.LayerNorm(dim),\n# )\n self.pos_encoding = LearnablePositionalEncoding(dim, self.input_len)\n \n self.to_patch_embedding = nn.Sequential(\n nn.LayerNorm(feat_size),\n nn.Linear(feat_size, dim),\n nn.LayerNorm(dim),\n )\n\n self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim)\n\n self.to_latent = nn.Identity()\n self.linear_head = nn.Sequential(\n nn.LayerNorm(dim),\n nn.Linear(dim, num_classes)\n )\n \n def forward(self, input):\n input = input.reshape(input.shape[0], 23, 84*3)\n # *_, h, w, dtype = *img.shape, img.dtype\n # \n x = self.to_patch_embedding(input)\n x = rearrange(x, 'b ... d -> b (...) d')# + pe\n x = self.pos_encoding(x)\n \n x = self.transformer(x)\n x = x.mean(dim = 1)\n \n x = self.to_latent(x)\n return self.linear_head(x)\n \n \n# v = SimpleViT(\n# feat_size = 84*3,\n# num_classes = 250,\n# dim = 512,\n# depth = 3,\n# heads = 8,\n# mlp_dim = 1024\n# ).to('cuda')\n\n# data = torch.randn(1, 23, 84*3).cuda()\n\n# preds = v(data) \n# print(preds.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T01:29:53.907469Z","iopub.execute_input":"2023-03-28T01:29:53.908180Z","iopub.status.idle":"2023-03-28T01:29:53.934686Z","shell.execute_reply.started":"2023-03-28T01:29:53.908142Z","shell.execute_reply":"2023-03-28T01:29:53.933572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = SimpleViT(\n feat_size = 84*3,\n num_classes = 250,\n dim = 256,\n depth = 3,\n heads = 6,\n mlp_dim = 1024\n).to('cuda')","metadata":{"execution":{"iopub.status.busy":"2023-03-28T01:29:53.936607Z","iopub.execute_input":"2023-03-28T01:29:53.937147Z","iopub.status.idle":"2023-03-28T01:29:59.426709Z","shell.execute_reply.started":"2023-03-28T01:29:53.937012Z","shell.execute_reply":"2023-03-28T01:29:59.425647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

5    TRAINING

","metadata":{}},{"cell_type":"code","source":"epochs = 30\nloss_fn = nn.CrossEntropyLoss()\noptimizer = torch.optim.AdamW(model.parameters(), lr=3e-4, weight_decay=1e-6)\n# optimizer = SAM(model.parameters(), base_optimizer, lr=3e-4, weight_decay=1e-6)\n\ncosine_scheduler = scheduler.CosineAnnealingLR(optimizer=optimizer,\n T_max=10,\n last_epoch=-1,\n verbose=False)\n\nfor e in range(epochs):\n train_loss = 0\n train_acc = 0\n num_iter = 0\n model.train()\n for x, y in tqdm(train_dataloader):\n num_iter+=1\n pred = model(x.to(torch.float32).to(\"cuda\"))\n loss = loss_fn(pred.to(torch.float32), y.type(torch.LongTensor).to(\"cuda\"))\n \n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n \n# with torch.set_grad_enabled(True):\n# pred = model(x.to(torch.float32).to(\"cuda\"))\n# loss = loss_fn(pred.to(torch.float32), y.type(torch.LongTensor).to(\"cuda\"))\n\n# loss.backward()\n\n# # sam optimizer first_steop\n# optimizer.first_step(zero_grad=True)\n\n# # sam optimizer second_steop\n# pred = model(x.to(torch.float32).to(\"cuda\"))\n# loss_fn(pred.to(torch.float32), y.type(torch.LongTensor).to(\"cuda\")).backward()\n# optimizer.second_step(zero_grad=True)\n \n pred = pred.max(1).indices\n train_acc+=(torch.sum(pred == y.to(\"cuda\"))/x.shape[0])\n cosine_scheduler.step()\n\n print(f\"Training Acc @ epoch {e}: {train_acc/num_iter*100}%\")\n\n# val_loss = 0\n# val_acc = 0\n# num_iter = 0\n# model.eval()\n# for x, y in validation_dataset:\n# num_iter+=1\n# pred = model(x.to(\"cuda\").to(torch.float32))\n# loss = loss_fn(pred.to(torch.float32), y.type(torch.LongTensor).to(\"cuda\"))\n \n# pred = pred.max(1).indices\n# val_acc+=(torch.sum(pred == y.to(\"cuda\"))/x.shape[0])\n# print(f\"Validation Acc @ epoch {e}: {val_acc/num_iter*100}%\")\n\n# test_loss = 0\n# test_acc = 0\n# num_iter = 0\n# model.eval()\n# for x, y in validation_dataset:\n# num_iter+=1\n# pred = model(x.to(\"cuda\").to(torch.float32))\n# loss = loss_fn(pred.to(torch.float32), y.type(torch.LongTensor).to(\"cuda\"))\n\n# pred = pred.max(1).indices\n# test_acc+=(torch.sum(pred == y.to(\"cuda\"))/x.shape[0])\n# print(f\"Test Acc: {test_acc/num_iter}%\")\n","metadata":{"execution":{"iopub.status.busy":"2023-03-28T01:29:59.430073Z","iopub.execute_input":"2023-03-28T01:29:59.431732Z","iopub.status.idle":"2023-03-28T01:33:19.322969Z","shell.execute_reply.started":"2023-03-28T01:29:59.431686Z","shell.execute_reply":"2023-03-28T01:33:19.321775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

6    FEATURE PROCESSING MODULE

","metadata":{}},{"cell_type":"code","source":"DROP_Z = False\n\nNUM_FRAMES = 15\nSEGMENTS = 3\n\nLEFT_HAND_OFFSET = 468\nPOSE_OFFSET = LEFT_HAND_OFFSET + 21\nRIGHT_HAND_OFFSET = POSE_OFFSET + 33\n\n## average over the entire face, and the entire 'pose'\naveraging_sets = [[0, 468], [POSE_OFFSET, 33]]\n\nlip_landmarks = [\n 61,\n 185,\n 40,\n 39,\n 37,\n 0,\n 267,\n 269,\n 270,\n 409,\n 291,\n 146,\n 91,\n 181,\n 84,\n 17,\n 314,\n 405,\n 321,\n 375,\n 78,\n 191,\n 80,\n 81,\n 82,\n 13,\n 312,\n 311,\n 310,\n 415,\n 95,\n 88,\n 178,\n 87,\n 14,\n 317,\n 402,\n 318,\n 324,\n 308,\n]\nleft_hand_landmarks = list(range(LEFT_HAND_OFFSET, LEFT_HAND_OFFSET + 21))\nright_hand_landmarks = list(range(RIGHT_HAND_OFFSET, RIGHT_HAND_OFFSET + 21))\n\npoint_landmarks = [\n item\n for sublist in [lip_landmarks, left_hand_landmarks, right_hand_landmarks]\n for item in sublist\n]\n\nLANDMARKS = len(point_landmarks) + len(averaging_sets)\nprint(LANDMARKS)\nif DROP_Z:\n INPUT_SHAPE = (NUM_FRAMES, LANDMARKS * 2)\nelse:\n INPUT_SHAPE = (NUM_FRAMES, LANDMARKS * 3)\n\nFLAT_INPUT_SHAPE = (INPUT_SHAPE[0] + 2 * (SEGMENTS + 1)) * INPUT_SHAPE[1]","metadata":{"execution":{"iopub.status.busy":"2023-03-28T01:33:26.360160Z","iopub.execute_input":"2023-03-28T01:33:26.360575Z","iopub.status.idle":"2023-03-28T01:33:26.371338Z","shell.execute_reply.started":"2023-03-28T01:33:26.360541Z","shell.execute_reply":"2023-03-28T01:33:26.370085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tf_nan_mean(x, axis=0):\n return tf.reduce_sum(\n tf.where(tf.math.is_nan(x), tf.zeros_like(x), x), axis=axis\n ) / tf.reduce_sum(\n tf.where(tf.math.is_nan(x), tf.zeros_like(x), tf.ones_like(x)), axis=axis\n )\n\n\ndef tf_nan_std(x, axis=0):\n d = x - tf_nan_mean(x, axis=axis)\n return tf.math.sqrt(tf_nan_mean(d * d, axis=axis))\n\n\ndef flatten_means_and_stds(x, axis=0):\n # Get means and stds\n x_mean = tf_nan_mean(x, axis=0)\n x_std = tf_nan_std(x, axis=0)\n\n x_out = tf.concat([x_mean, x_std], axis=0)\n x_out = tf.reshape(x_out, (1, INPUT_SHAPE[1] * 2))\n x_out = tf.where(tf.math.is_finite(x_out), x_out, tf.zeros_like(x_out))\n return x_out","metadata":{"execution":{"iopub.status.busy":"2023-03-28T01:33:26.691274Z","iopub.execute_input":"2023-03-28T01:33:26.692028Z","iopub.status.idle":"2023-03-28T01:33:26.701429Z","shell.execute_reply.started":"2023-03-28T01:33:26.691980Z","shell.execute_reply":"2023-03-28T01:33:26.700290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeatureGen(tf.keras.layers.Layer):\n def __init__(self):\n super(FeatureGen, self).__init__()\n \n def call(self, x_in):\n# print(right_hand_percentage(x))\n x_list = [tf.expand_dims(tf_nan_mean(x_in[:, av_set[0]:av_set[0]+av_set[1], :], axis=1), axis=1) for av_set in averaging_sets]\n x_list.append(tf.gather(x_in, point_landmarks, axis=1))\n x = tf.concat(x_list, 1)\n\n x_padded = x\n for i in range(SEGMENTS):\n p0 = tf.where( ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) != 0) , 1, 0)\n p1 = tf.where( ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) == 0) , 1, 0)\n paddings = [[p0, p1], [0, 0], [0, 0]]\n x_padded = tf.pad(x_padded, paddings, mode=\"SYMMETRIC\")\n x_list = tf.split(x_padded, SEGMENTS)\n x_list = [flatten_means_and_stds(_x, axis=0) for _x in x_list]\n\n x_list.append(flatten_means_and_stds(x, axis=0))\n \n ## Resize only dimension 0. Resize can't handle nan, so replace nan with that dimension's avg value to reduce impact.\n x = tf.image.resize(tf.where(tf.math.is_finite(x), x, tf_nan_mean(x, axis=0)), [NUM_FRAMES, LANDMARKS])\n x = tf.reshape(x, (1, INPUT_SHAPE[0]*INPUT_SHAPE[1]))\n x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n x_list.append(x)\n x = tf.concat(x_list, axis=1)\n return x\n","metadata":{"execution":{"iopub.status.busy":"2023-03-28T01:33:26.992107Z","iopub.execute_input":"2023-03-28T01:33:26.993026Z","iopub.status.idle":"2023-03-28T01:33:27.005204Z","shell.execute_reply.started":"2023-03-28T01:33:26.992988Z","shell.execute_reply":"2023-03-28T01:33:27.003930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

7    TORCH TO ONNX

","metadata":{}},{"cell_type":"code","source":"x = torch.randn(2, 5796, requires_grad=True).to(\"cuda\")\n# torch_out = torch_model(x)\n\nmodel = model.to(\"cuda\")\n\ntorch.onnx.export(model, \n x, \n \"MLP.onnx\", \n export_params=True, \n opset_version=10, \n do_constant_folding=True, \n input_names = ['input'], \n output_names = ['output'],\n dynamic_axes={'input' : {0 : 'batch_size'}, \n 'output' : {0 : 'batch_size'}})","metadata":{"execution":{"iopub.status.busy":"2023-03-28T01:33:27.746660Z","iopub.execute_input":"2023-03-28T01:33:27.747036Z","iopub.status.idle":"2023-03-28T01:33:28.206740Z","shell.execute_reply.started":"2023-03-28T01:33:27.747003Z","shell.execute_reply":"2023-03-28T01:33:28.203701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

8    ONNX to TF

","metadata":{}},{"cell_type":"code","source":"tf_model_path = \"tf_MLP\"\nonnx_asl_module = onnx.load(\"MLP.onnx\")\ntf_rep = prepare(onnx_asl_module)\ntf_rep.export_graph(tf_model_path)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T01:33:28.460384Z","iopub.execute_input":"2023-03-28T01:33:28.461129Z","iopub.status.idle":"2023-03-28T01:33:39.406087Z","shell.execute_reply.started":"2023-03-28T01:33:28.461089Z","shell.execute_reply":"2023-03-28T01:33:39.404954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TFModel(tf.Module):\n def __init__(self, tf_model_path):\n super().__init__()\n\n self.feature_gen = FeatureGen()\n self.model = tf.saved_model.load(tf_model_path)\n self.feature_gen.trainable = False\n self.model.trainable = False\n\n @tf.function(\n input_signature=[\n tf.TensorSpec(shape=[None, 543, 3], dtype=tf.float32, name=\"inputs\")\n ]\n )\n def call(self, input):\n output_tensors = {}\n features = self.feature_gen(tf.cast(input, dtype=tf.float32))\n\n output_tensors[\"outputs\"] = self.model(**{\"input\": features})[\"output\"][0, :]\n\n return output_tensors\n\n\nmytfmodel = TFModel(\"./tf_MLP\")\ntf.saved_model.save(\n mytfmodel, \"tf_infer_model\", signatures={\"serving_default\": mytfmodel.call}\n)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T01:33:39.409403Z","iopub.execute_input":"2023-03-28T01:33:39.409753Z","iopub.status.idle":"2023-03-28T01:33:40.839598Z","shell.execute_reply.started":"2023-03-28T01:33:39.409722Z","shell.execute_reply":"2023-03-28T01:33:40.838499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

9    TF to TFlite

","metadata":{}},{"cell_type":"code","source":"tf_infer_model_path = \"./tf_infer_model\"\n# converter = tf.lite.TFLiteConverter.from_saved_model(tf_infer_model_path)\n# tflite_model = converter.convert()\nconverter = tf.lite.TFLiteConverter.from_saved_model(tf_infer_model_path)\nconverter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS, tf.lite.OpsSet.SELECT_TF_OPS]\nconverter.allow_custom_ops = True\n#converter.optimizations = [tf.lite.Optimize.DEFAULT] # https://www.kaggle.com/competitions/asl-signs/discussion/394371\ntflite_model = converter.convert()\n\ntflite_model_path = \"model.tflite\"\n\n# Save the model\nwith open(tflite_model_path, \"wb\") as f:\n f.write(tflite_model)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T01:33:40.841127Z","iopub.execute_input":"2023-03-28T01:33:40.843242Z","iopub.status.idle":"2023-03-28T01:33:42.291850Z","shell.execute_reply.started":"2023-03-28T01:33:40.843199Z","shell.execute_reply":"2023-03-28T01:33:42.290784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pq_path = \"/kaggle/input/asl-signs/train_landmark_files/53618/1001379621.parquet\"\n\nimport tflite_runtime.interpreter as tflite\n\ninterpreter = tflite.Interpreter(tflite_model_path)\ninterpreter.allocate_tensors()\n\nfound_signatures = list(interpreter.get_signature_list().keys())\n\n# if REQUIRED_SIGNATURE not in found_signatures:\n# raise KernelEvalException('Required input signature not found.')\n\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\noutput = prediction_fn(inputs=load_relevant_data_subset(pq_path))\nsign = np.argmax(output[\"outputs\"])\n\nprint(sign, output[\"outputs\"].shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T01:33:42.294238Z","iopub.execute_input":"2023-03-28T01:33:42.294554Z","iopub.status.idle":"2023-03-28T01:33:42.532794Z","shell.execute_reply.started":"2023-03-28T01:33:42.294526Z","shell.execute_reply":"2023-03-28T01:33:42.531664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip $tflite_model_path","metadata":{"execution":{"iopub.status.busy":"2023-03-28T01:33:42.534381Z","iopub.execute_input":"2023-03-28T01:33:42.535325Z","iopub.status.idle":"2023-03-28T01:33:43.831182Z","shell.execute_reply.started":"2023-03-28T01:33:42.535282Z","shell.execute_reply":"2023-03-28T01:33:43.829900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

10    TIME CHECK

","metadata":{}},{"cell_type":"code","source":"# # https://www.kaggle.com/code/hengck23/lb-0-67-one-pytorch-transformer-solution\n# import pandas as pd\n# import numpy as np\n# import os\n# import shutil\n# from datetime import datetime\n# from timeit import default_timer as timer\n\n# mode = 'debug' #debug #submit\n\n# if mode in ['debug']: \n# try:\n# import tflite_runtime\n# except:\n# !pip install tflite-runtime\n\n# import tflite_runtime.interpreter as tflite \n# import tflite_runtime\n# print(tflite_runtime.__version__)\n# #'2.11.0'\n \n# #import tensorflow as tf\n# #print(tf.__version__)\n# # 2.11.0\n\n# print('import ok')\n# '''\n# Your model must also require less than 40 MB in memory and \n# perform inference with less than 100 milliseconds of latency per video. \n# Expect to see approximately 40,000 videos in the test set. \n# We allow an additional 10 minute buffer for loading the data and miscellaneous overhead.\n\n# '''\n# def time_to_str(t, mode='min'):\n# if mode=='min':\n# t = int(t)/60\n# hr = t//60\n# min = t%60\n# return '%2d hr %02d min'%(hr,min)\n\n# elif mode=='sec':\n# t = int(t)\n# min = t//60\n# sec = t%60\n# return '%2d min %02d sec'%(min,sec)\n\n# else:\n# raise NotImplementedError\n\n \n# ROWS_PER_FRAME = 543\n# def load_relevant_data_subset(pq_path):\n# data_columns = ['x', 'y', 'z']\n# data = pd.read_parquet(pq_path, columns=data_columns)\n# n_frames = int(len(data) / ROWS_PER_FRAME)\n# data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n# return data.astype(np.float32)\n\n# if mode in ['debug']: \n \n# interpreter = tflite.Interpreter(tflite_model_path)\n# prediction_fn = interpreter.get_signature_runner('serving_default')\n\n# valid_df = pd.read_csv('/kaggle/input/asl-demo/train_prepared.csv') \n# valid_df = valid_df[valid_df.fold==2].reset_index(drop=True)\n# valid_df = valid_df[:4_000]\n# valid_num = len(valid_df)\n# valid = {\n# 'sign':[],\n# }\n\n# start_timer = timer()\n# for t, d in valid_df.iterrows():\n\n# pq_file = f'/kaggle/input/asl-signs/{d.path}'\n# #print(pq_file)\n# xyz = load_relevant_data_subset(pq_file)\n\n# output = prediction_fn(inputs=xyz)\n# p = output['outputs'].reshape(-1)\n\n# valid['sign'].append(p)\n\n# #---\n# if t%100==0:\n# time_taken = timer() - start_timer\n# print('\\r %8d / %d %s'%(t,valid_num,time_to_str(time_taken,'sec')),end='',flush=True)\n\n# print('\\n')\n\n\n# truth = valid_df.label.values\n# sign = np.stack(valid['sign'])\n# predict = np.argsort(-sign, -1)\n# correct = predict==truth.reshape(valid_num,1)\n# topk = correct.cumsum(-1).mean(0)[:5]\n\n\n# print(f'time_taken = {time_to_str(time_taken,\"sec\")}')\n# print(f'time_taken for LB = {time_taken*1000/valid_num:05f} msec\\n')\n# for i in range(5):\n# print(f'topk[{i}] = {topk[i]}') \n# print('----- end -----\\n')\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-03-27T07:09:58.537841Z","iopub.execute_input":"2023-03-27T07:09:58.538239Z","iopub.status.idle":"2023-03-27T07:09:58.556717Z","shell.execute_reply.started":"2023-03-27T07:09:58.538198Z","shell.execute_reply":"2023-03-27T07:09:58.555291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}