{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Hello\n### This is a QUICK START Notebook.\n\n![](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAScAAACrCAMAAAATgapkAAAA5FBMVEX///8nquBszvU0ZK4dqN8thaNmndIAqrgAvvH6/v/v+f3r+Pxqw+g2YK1Es+PR7Pjd8frA5PWur7Ho6Ol5e374+PiAyuyy3vOBg4ZzdXmf1vBNueXNzs+QkpS7vL43bLOOzu3e3t8AoN2goqSb3fgucrI2m8Bbks0gWaootMsLpLhasOBJg8JRptpQfLpNo8sArtDH1+klgrQPfJ1MlL8hkq0ArtgugacgjLYArrRFvt1Tm7Wx0Op8rdm+5uqswt5ewc1htdUtcqqZxdd6y9aQrdMjjsuh2+Gx0uB5rsR0mMl7u9TB6YPWAAAYaUlEQVR4nO2dCV/bONPAbcdJ8G3ncrBzEkIghBtSKNvS+9jn+3+fdw5JtgN02X0LqYH57baOLKvRn5nRaCQZTXsUsbYNI36cpp+VOIZhbK/7S5RAXjk9TF453S92GN3cnNh0nePk/Pj810lor/GL/VESgOc2/N3dw7/wU7gvOFmxsXN2Xqu9uwnX+/3+DIkMkrfzqbd7Ap/DGqCx4GJkGDvVKnya1d6drPtbrlvCd4bgVK/vzGpWxik4OzqqAqed+c6i4n192Tr1V+1IcKoDp9oM9CZATuCTflZRzudwY16p7B6+YJWyr8+r1Y9IyUUcR7XajaadMCf7gjgtj+FG/dQDUjfr/rrrkiA9IxY74IYuSJ/Q4ASnkO4tNzb24M4lcKrsfl33F16PBBsb1UxAoeYEiDiFWnBZrV4Cpo0UOE0rKMMXqVEhMDhXmM5QoWoHvgMuCyTUog0p9fkpYaoc1l4gKAcR1GqS00f37ds9KAm1G+QUaHGO09QjTrNa7a91f+0nFwMIHNRqrFE7NORtEKd3tcVweJLpk7Q7b4EAXxqobcYkFMpQnBzt3cLDsS3UBSf04xdeZUiYajVn3d/8SSU8XurcbzI6Q4IybOt8CLrzVQtnPmGiuKB+oB9w9drMWvd3f0KxPmAQoDi5glO6YVh2FYzM+xvD8gP/+HhOmJY6CFEa7r4ky3tfrx/PFouF8E+Ck1u9/J8WViGo9A41uwa3r4hS/dhETrPZYoij3gvKH3zAEBtwVGYUE+Dcxd13d6pnn7UfZ9XT6fTSsg89zzvNcfKnHg17Lygufy8ibJA3GI7DcLdPNgicfnLM+d6aQQ0G9dZAddKHFRFGvRgP9WFvzxOcphdsdOSrzoGTiM2/2DPANAWMF8fguHTTTAWmivdSPBRmLFMRO1aEa2JOIuisc46gcipiUK5zUBEaNVt3B55IYuo2d3oohjridMlcjiBkmsOoN5XBOlfZx+GOPNTLyEVZotukUFPBCcMjfUlUftjO20sgKNVJKBR5sFkFDPaFJA4CgQY5efuC2W5lgVHlcrlMIeKOdLw5lJwol/eOVK5yALVfRmiwLTiR3bnKBj1TTFSAU2wSxFOlT+nBPgXksxlW94N19+EJRJid4VaGw4pH/aahzPsmOIW3OBkHEEtNydMz15ewrh5Kzz2cDoHGgVKtrZw+bVQwCkW/fl6r7bwljqfn5yJ0n72EEe+v4XBmuG6KZqYPPW+4b+xdeoegLL7Sp+2NdDFcpMYRYgJQxPGTzMAc7A5fwIj3t0cGJZQHPROnfEE2NlBbHOQkjZPKj4aZFe64bs3zXsDiC6ZN3uACAYmJ1K4EJ9+QnAwpzOkC/ZNwVufozZ+/gwp3MWp6o7K6OP5LfXJvc6KoCRRuR2XSqea7dXfj0eUHcBpOp4oT+vL6BffeuM1p76hWo1TdPFMnlGc/F/55OpySKxLyaVoZ7lD/z6sZJ+mfjPpV/YqzUMucOtVmz92RW9fcVcUJEUmT2lnlhMsI89NLxITB+s5yKdO/zyBnYIVRdO+P22GVmMlgKfM7asLr8CoDrytsbCy83Xn9Svr9DcHpXekNzyLnsn1PP4Iz4ATTfo9BLSWhCwOCpQKn+jGDmXq7F/W50j9WqEX5p3jCsdwD6vP19eevOOR5U1M/EImU6unw1DXcTxVvwZxiXGdJBZlvl/Wcne7h0HjoeWV3UDIbYNw9V7Vsyz7ktORhTS0HY7K8do6J3sU+7umJcdVOoTnO+zO4cYCpzrJnyWPJaTUUDD98ZxWjCAqDqCyB+QkRcSntfXovPDfLEnNT8gOGCIe0clVGcb58+PCFlmpV6LOyZ9f6UK+/p6sTwcmbLqrV+Xx+Ua1WlMDkOGROV8cSzVEtm/whwCvMGR8+eR9/g7zn9SP0GdF9dveB74M+eWpFQKw7ncoSrzL0DqHWCXjuowNpdvNjJLW/IReHqX4Zp8JOXYidpZeM1QHJ+c7qpNGKEyuPeGwuFgimny4uPRzJTqY7QodSwUZMX1iWQHRYwqnwd9mB75rKL/1iv0Q43IV57e7MuxTrvVdCoaZXV3PmNBRhUgozF64DtnYq/xmYPe+XMVf3QXbgC36y4+3t6JfhTfj18HB2o50oBeFVT2+IV8RpJjjtK04XnjLTlPIvJTyz8KXA6SFih+HJzc1ekRPo03xY5OTjfum6XM/zduBiL93gPNVj9uhx5L3s7/tcoXXfzALLbdo1d1Qv2J23eLvwKnlOiEOCQh+2EK79TgdYAhGG90EVOBRGbefHJJv6ZeF4uB3wIpP0zHMx2h0YxnR3ZuO6FCUFeNEqV2ea51TGAY9AfVC+WwUHsVKq8N077Nh2lqWUoPb2hsLs8NZi38GtwPX56ZDXGISDItPM69O3UqYM3n//nhmdwpSb5t3McLdulCUpWY6W8Nmlfc9izfOtTZzqc49me0KfLir5JT4AtV/+/T2hkRMeliyb9ckoqpPM9e7jKhRjMjo27pkGNMClhpvHLiBmv/RQpDpBWHXxaffv0qRW7Du/qbWd50TuNjbSbTpHRyVunhOXeJWFjhepsY+ZpRDtEed8w+EbMQP8putCm7YW33ScOs/aT9vb/yxfco47J04BkxFpvMeZxvFVs5NrB6ZKzOk3wMme169ETCXXhFWibgGKVYGCYWlSUHU5YytKUOS0zXvFaZ2X3XiBEykU3PUPxHyuA9Xs6qUI0RUnBYpW1HHN87QsA949+hStcLIEJ/zxO7fsjie4G7rKCKDe2Rdqaqxywpd8m3ay0NrwWVk4aWHeP1nya69yohM/wu7Yxxc5UWopS8Wh47evl4ui2VWzFWSPFkqBUwlnwtA1NeMKi5zAP6EpqulYGIQ3BU7Y+YMip/b1xsY3CMClF88Mb29vuagMOb/34FnSnyRIh9XLGhU4UfBZnMSERzv5CGp5i5NFlvomT4k5HePuX5cODld3SsnJilVmruDIC9P6gHF9pzOJLKQa9En4cWqFOFWrtzhB9M7NHlVrtVJyyks+gMorksGDHgwAVztiprug4KhCu8A4B05uDhemVtb2qpz1le26tWdwGk8tJmznYxwn3dj4+MM6mQ0vcZ/qgs5h0GoLr74c7y030mOqGqUQLCyLnHTKbMqW958DJy1klYoKbsleQtfPznGrnEqI415Vmt0N5/NczoGyx2kB03mtnucEg+az2PZrhbde+OFcC+XJLa+8EatSKlf+netuM6hs6epyNsMsHbhxIcezci5N/bMQpzwlsUP8DWrYJbjo4+VxXcDlscCsiaXQ81NM003nyo8bb+vz57oJyr5eVacp68qbT1PAtJRRAQoZ3n6F9Qkd/qFIoAtQbwtpweclyGla0KdsVnLBmFJlq6hQlL87nXMMQQujO5i023v7du9fZeNLJshpWMDkZZEkT9+irPa2OAIzrdN6cG043JeK9G9XLUomv+T0LW91KGB5FDBcXrE+ea5h5EHtlXEF72Hy82yV05GMJYff1FRZinV9Blb6SWxyXczkQLcnMBnRPf9M6cVY1lfCAszroiOfetOiNqG0f9I7VnAH67k4DKQUau6WcsHlYbK9h8u6OXXCJF366RNto7/jcI/1Ix9oKk4X8/pFNf3l0ny5JajXCwrFhxSnED3tDm/uzHY7n8/OJCepULNdPPiyU5w6PiuxxIYKOWdJZb//vn8tzv58DaQ+fpS7gQ0Xn8Xj18kTfvMnFtqIcEpb5zxvIQawdzfhLzXDCn5+NIyjN6cH9ACuI+A4WZ60778X3ig1P50Oh594sdeIH9JdK9g+2t3dRTs1d6cw+L2pnl0/+rddo4gdG4ZBfvkftv8UJfw6o4VONtbq9bP1TiTOF8K0c339Obp7mfR+aYd/Galw5tHzxgTiBGF4O+fyULEcfPoZu6ZXeZVXeZVXeUlixaMRD/1WDEKXUUyXcMEz5jCxVeVErKAGySjmBeiYsi5OgrsYE3xOJt4TqEx3sLWk3HPpSN/S4T/samiCbPnQS9/U4TK1tESnuCAymZeVmqm+hduBHH8L6+DEz/ZdqrMFbYyoCXrIcrdM3zRdC+5gqVnqN5BEZgpTviDFXjhmYtmRCRhS10LRtMREbACTO0ksYqzq67GlOR1zhJwQnBZsQflIDy0r0IGw5ZqghHaSIqdAs+7du10O8YkD9Bv645B6dKAkdcXtRNdTLdOnxBTGM9rigo4ZrnKip0ItNjnJYCHd0gergSkyujH0Jc+J1UlLfNIvySncSulCmBrw1UdFTr5DTTha6qt5AOvTk/XpMSTWhYKEWzFwGoXBCGGl5FAc5GQHYFuSkxb5JpIKdbmwAJpX1Kc4ikZbiWbprvpHImpt9ITd+u0Sm8IkQvA8Dr7qEB0vKEOcJAn6J9CMxIwVJxi7dAAZ6iJzZ/krnEwdfDc+6afqH4nMETRXajcebokOo+dxzE4QELfMP6EddcyOnvXS9nXH8n3+EAA0Nd6R3YVWQuxdXTmlZ+CfNJcdFHkhx5T53SInqCQ40cbQBJQrZr42DQMdkytZ7J9sUqXApAVnO3genELfHEVRh4b/HCc/AgkkJ8sXnGAQCwNkY3VMN44SnygHph9HLjkg8uMJRg8YUiRYw4EfQoLNlWUP+t0CMdDWloknODIjBD++BeJjnEl+PhScHBdu8KpWosNjKStKhNcj+kUwWN/C+Al9PraBQf0WSdnXSi2Vi1NZT8tGsfCCCxzrVmUtnwEMxfoE11dPBTyaUmvlVqdXeZVXeXqJ6BgoJ0g0ucDnKJcbJpxHsSIoshJ2QhHfdhKVY0Fp9gaDsbgcDAZNuuo1xT1cqh9DhfZKhWYPpS0vemOuO+5lj7f5Wvy1JnFpGubTQGfJ2chIzM2skYnJExfigBSGMFunwBuu8S8IzPGedOuDbqvb2pxAH9uNFlx2B1ja6BOYwSZ0eNDqdjcnVIoVWg2+02q1NvtNuIAiLOy1mvK59iY20uvyP9Bf5yFA5iSyJ2IO5ugmA3O3OhA2YgxlwcwYAkgKsCwXOY0QkR3LrS+DzUYbfuRd6ONkcwCXA0LSaPXxLvadOt1DpWjkKgxa7XZ73IVabW3Shz80aiPPabCJD7X73bVzCk2KF12Tw/BETzkAMkl/MDUiOPkIkDiFNA/kk/8gzU3SDupniw1ksAlW2Oh28QZyam4Kw+mJC6oInKgu/jlhpqucJi0EOm61mo/M4ldCnCIdJyqhmfr0wgO/ExG3TjZFY076COcsxCkppie5oySTCf/d7jewv2NSI9Snbp/d16QravYnklOjdT+ndn/CJd11OijilKQRzF1H9CdOTgKe7fu8JdhS+gT+yYV5CHLq6Ooeiuq7prFjAmmAs2r0QX0G7HOa3c1+T/DjCl3kBL570KJn7ubU7I7hCfhTNbwOQU6W69p+YvtxSGlynP5TIoVSbVaqQ6niZKe+Q5xw7mu5ukiwTFqqxa7EMCFObUAxJk7gvvponlLhECLc7Pe7jOkOTnhj3G0Ouu1BP3tuHYKcbH8EE97YdBzsdWS6SZKgaxohLCvpgFNSnMDJuzjexeTq45E+ombIGbFMxLjUxp8/9bfRbXTZt7QnrTF8lBUa7McHwvPcrU8Aqd3tQWONdQ54yMmBeCA0dc64gY6kPgjoS8jDIN7OOGmBaQIny/RpUUUfUTPgfKgTY3S4rFATdFnUt3a/hX3HCk3w3U1RoQHMgBNctVhTcpyYe4+Ggj4NB02+sS5BTpSJpESU68MHiglsjBFiU0+CyC3oE5bS0oKO2ZKOTOeOIWAa9yab1PV+b9zr07An9KLfxfGu0WySS+5tUgUa9ojTgIfIHKd2F5obED/09mO8GnfH2toEOdEqSoROO9HBQbNrxrUAKDRN04cgClyYRfaJklCcGeBvetFVnDmeQGzJQ1Kv3+12J3QpbAV8DARMUJqr0KfLAQ4A7T4RynHSmtgcRa1dGvN66NDX6MidEN8QpYlUJVyFYmHB4kxJGAS8cIXFMpEiQgK4l1/jbY6b7eySL9qihO60x+PbFZq5P9vZdVaFHm2qq1d5lVd5lVcpCO5VfZQNN/+fHbR/oLg48D9Cf2xcKu/8/nbXJa6p6/4jKJSNS/DPi5P5KJzMV04PkVdOD5NXTg+TV04Pk1dOD5NXTg+TV04Pk5fDyQmiO37ZEBYGt+N3i8pze3pXOd3dWnmkwKmjc2Ic9xX4tEnX9Ec5iI4o1dMggKpq06bT0bncDWNswl7hZMWpaK20hzdynCw6dEFL48iDftE59D97KUQgS03a1avLvedRVg7w4NopcgpT0RhWKOmbWnKcRtgbXhg3uVPUfbmHVQtkb+km/s+cYtMslq9wCiQjesYs57sQFCergz3j1fCELn38DajYN1pNx90ZrEqix5JTIKBm5UVOgU630tHI9amFUoKSnBiTLiwJOtQJbcuyA+wa82COncBxwpgNictTenAE5UHim7c4WT6W+LSJyiGV1cu4415y6uQMCT2K7EtERxSwSBdWiWKNVPXIzJmm7d6yu4g0Uo50pKnZiY7yCHGyhbLctXOZeq6JHmYm05F65po5vqBdK5xsUsiw0JxZwkGP9algdHmxbFI0C89hgGT9jQUeB8zKzI6yEM48p2AljqLPJRzziFMnbztKwjjppORdgJO1YjCSU7jimVc5Ub0Eg0whWH/0FD37vUJ2YBZ1BSXo+KYav4BTaBb7JzlFRbO7xYn9Xk5Kmg92VQiYH64xwM4FS5JTrorkhH/nNfFOTitSak55r+H4Mr4WGnU/JxrOcqp4NydTYBehVlk5CXOQamGlHDSmozjgkOfX+vQru6PH/RUpYaRJ/skP2ZOL8ZpDooTP/ghOzsq4VfBPufHrTj8ea1ZRnrKHv0do/S5kS8MN9bJM6YjgpKHCZa+6LY53OX6rnCg8LWNguSIiziS7kh2mMllBchoVDSwRH3Gbaz5wHK1wsv0VwyypqHlwlM1RMaRUnGScSQamMlU8ueVjm+LsNUl8a34XFUNYKyih0Wn5vMpI+XLqOv+aODsR4x3MZ8kjRzSf5QkvcbJpCHNx950Vju6ZB4O3ozOQVuCaSSlBZZwsV2qMQxDcJI47vowLhMJh8N5J5ShPWpLwiJl2XP/OvErI+Rjd7XQ6lN0bra+3/13y+Uz6yafCF4koWlecGIiM3jNOMpTMoveVPF0kq8u0VQmnd4X9KgGbkEpoZsKmMjKLxYKTdSvmdor7VQI/X4HPbJdNcP+TjJswg2uaeOhcpMpBXKxgCpcS+apYjnckiZrBjeR45+T3P1mxetBMy5ilo73NQVj4xHud7XjkuqPE0WwsUhWCxHXdThyuxOFWhOX43iwZP2n4WKY4VpB04MEkLqMu/YPcEzZz8ep8Rb7gKTGzuP5h7T1vIU53mBBxeoZK819F2deKFGL5FypBZmc0HUmle8/0KvTLmV76neLkIp8kl96Mcgkod9VtvTwJKPYhIg5nc9nsInrvIb+uhuKux9hmXSLhjJ6ejjouz0RGXE5rnqbvjsR8JqdO7XZTHphSR53aubvarwtZiqX526pxWfWPOE/luGreYsoJDpXznFhOSzLv1B70uy1xzK7XF4fFx311arwhDxJrg748qdmeZIUTkj69wkAe01aPc+N8yA9Kue46DxDnJMlN7cyOioZg3pIrVzndZnezO2lM+B0ZvU1xxlBd4JFNSaehThc3u6qvjVafhDiJk7Hq8TY3vsnv1+jBjwPlD+EkF6two1PBV0eumLj4rhr62n18Q4bUkDs4DVotdYK/dRcngCdtaSDeMiIfb/db1PiYKFLpn2J2LGGUgMSrv7XDCvH9iEl+w1xDqgB9/duc2nj2tS3r3skpOzY9aHX5CLF4HPRLnFkvNlo+yVkVym1OcDGQbxL5Z06bgz595Mfb3eI7RErMqdcqOAvoSZNEdamPryUQ7vkeTj18QLzvp9fs8lF1fHzMb7fJtd5oY91H6stjSq/VKH7sChFOiU7aS/99Nyd+hD6j5vXQ1pjTSuOq9T/IQT1UbnGaNEgm8h0hiKbJw9V9nOgR9kdoofgiJMmJarUbEAqQNbb61HgJOa2YhjK3MV8AIYgS6TUi2oP8E3qyyeaAHx+zwbYnoJ5jrdT+CXxP/u0Vq358ICyFte6BnIALq2ObX7ahSdAl5oQkxNB9R1yArwUho2Kf8kBOWrsl3JtqvPyc4Efd7YFpge+4HY8P1Efqd6Mlhzbg1BSXYrxr8vvGxARFvQJJNC7trqGqlk4AFEwn8IV+tzi11dDEV9nQ1hSj4lgV0uukVKQ1kDG8aJxeBJSNpk/dx98jPfCz/QZZz3giXxmJU9dmY5DVwaiIx0IkyFeNrLDRyz+uqSfHDSQ7IN5NUbXxf6qOQsKsZsD3AAAAAElFTkSuQmCC)\n### Working latest hyperparameter tool: *TPOT (ML GENETIC algorithm)*\n\n\n### Contents\n\n* Data Loading\n* Exploratory Aata Analysis with **PRACTICAL REASONING**\n* Data pre-processing\n* Model traning\n* Model testing\n* Submit Predictions\n\n### <p style=\"text-align:center;\">Do UPVOTE, if you get a KICK start for your work :-)</p>","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set_style(\"whitegrid\")\n        \n\nimport plotly.express as px\nimport plotly.figure_factory as ff\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\n\nimport os\n\nfrom sklearn.preprocessing import StandardScaler\nfrom tpot import TPOTRegressor\nimport sklearn\nfrom sklearn.ensemble import RandomForestRegressor\n\nfrom sklearn.model_selection import cross_val_score, ShuffleSplit, train_test_split, RandomizedSearchCV\nfrom sklearn.metrics import mean_squared_error","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df= pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<hr>\n\n\n# Exploratory Aata Analysis with PRACTICAL REASONING","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"a=df.groupby('Patient').Sex.unique().values\nSex_labels, Sex_counts=np.unique(a, return_counts=True)\n\nb=df.groupby('Patient').SmokingStatus.unique().values\nSmoke_labels, Smoke_counts=np.unique(b, return_counts=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#PLOT\n# Create subplots: use 'domain' type for Pie subplot\nfig = make_subplots(rows=1, cols=2, specs=[[{'type':'domain'}, {'type':'domain'}]])\nfig.add_trace(go.Pie(labels= Sex_labels, values= Sex_counts, name=\"SEX\"),\n              1, 1)\nfig.add_trace(go.Pie(labels= Smoke_labels, values= Smoke_counts, name=\"Status\"),\n              1, 2)\n\n# Use `hole` to create a donut-like pie chart\nfig.update_traces(hole=.4, hoverinfo=\"label+percent+name\")\n\nfig.update_layout(\n    title_text=\"Totall unique patient in Traning data:{}\".format(len(df.Patient.value_counts())),\n    # Add annotations in the center of the donut pies.\n    annotations=[dict(text='Sex ratio', x=0.17, y=0.5, font_size=20, showarrow=False),\n                 dict(text='Smoke counts', x=0.85, y=0.5, font_size=17, showarrow=False)])\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"smk_stats= pd.DataFrame(df.groupby(['SmokingStatus','Sex']).Patient.unique())\nsmk_stats= smk_stats.reset_index()\nsmk_stats['Patient']= smk_stats.Patient.apply(lambda x:len(x))\n\n#PLOT\nfig = px.bar(smk_stats, x='SmokingStatus', y='Patient', color='Sex',\n             barmode='group', title='Smoking Status estimation.', height=500)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"###  Why **Men** smoke more than **Women** ?\n#### There are a few theories.\n* Cigarettes are an aggression outlet and men are naturally more aggressive.\n* Cigarettes are an adult form of breast-feeding and men take longer, much longer to be weaned.\n* Smoking is an exercise in inhaling power and men are naturally more power hungry.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"a=df.groupby('Sex').FVC.unique()\na\n# Add histogram data\nx1 = a['Male']\nx2 = a['Female']\n\n# Group data together\nhist_data = [x1, x2]\ngroup_labels = ['Male', 'Female']\n\n# Create distplot with custom bin_size\nfig = ff.create_distplot(hist_data, group_labels, show_hist=False)\n# Add title\nfig.update_layout(title_text='FVC distribution')\nfig.update_xaxes(title_text='FVC')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Ideal value for FVC acc. to respective Sex and Age of patient \n* Normal values in healthy **males** aged 20-60 range from **4500 to 3500 milliliters**.\n* Normal values for **females** aged 20-60 range from **3250 to 2500 milliliters**.\n* **Above graph reflect those facts**.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"a=df.groupby('SmokingStatus').FVC.unique()\na\n# Add histogram data\nx1 = a['Currently smokes']\nx2 = a['Ex-smoker']\nx3= a['Never smoked']\n\n# Group data together\nhist_data = [x1, x2, x3]\ngroup_labels = ['Currently smokes', 'Ex-smoker', 'Never smoked']\n\n# Create distplot with custom bin_size\nfig = ff.create_distplot(hist_data, group_labels, show_hist=False)\n\n# Add title\nfig.update_layout(title_text='FCV distribution')\nfig.update_xaxes(title_text='FCV')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Smoking effect on lung capacity\n* Smoking hurts your lung capacity. The tar in cigarette smoke coats your lungs and makes the air sacs less elastic. Smoking also produces phlegm that can make your lungs congested. Smoking even a few cigarettes a day can decrease your body’s ability to use oxygen effectivel","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"a=df.groupby('Sex').Weeks.unique()\na\n# Add histogram data\nx1 = a['Male']\nx2 = a['Female']\n\n# Group data together\nhist_data = [x1, x2]\ngroup_labels = ['Male', 'Female']\n\n# Create distplot with custom bin_size\nfig = ff.create_distplot(hist_data, group_labels, show_hist=False)\n\n# Add title\nfig.update_layout(title_text='CT scan--->General check-up Week-gap distribution')\nfig.update_xaxes(title_text='Week')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Female are more regular in check-ups then male patient","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"f, ax= plt.subplots(1, 2, figsize=(20,6))\n\na=df.groupby('SmokingStatus').Age.unique()\na\n# Add histogram data\nx1 = a['Currently smokes']\nx2 = a['Ex-smoker']\nx3= a['Never smoked']\nsns.kdeplot(x1, label='Currently smokes', ax=ax[0]); sns.kdeplot(x2, label='Ex-smoker', ax=ax[0])\nsns.kdeplot(x3, label='Never smoked', ax=ax[0])\n\nax[0].set_xlabel('Age'); ax[1].set_xlabel('Age')\nax[0].set_ylabel('Prob. Distribution'); ax[1].set_ylabel('Prob. Distribution')\n\na=df.groupby('Sex').Age.unique()\na\n# Add histogram data\nx1 = a['Male']\nx2 = a['Female']\nsns.kdeplot(x1, label='Male', ax=ax[1]); sns.kdeplot(x2, label='Female', ax=ax[1])\n\nplt.suptitle(\"AGE Distribution\", size=16)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Above 2 graphs are Normally Distributed","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"f, ax= plt.subplots(1, 2, figsize=(20,6))\n\nsns.scatterplot(df.FVC, df.Percent, hue=df.Sex, ax=ax[0])\nsns.scatterplot(df.Weeks, df.FVC, hue=df.Sex, ax=ax[1])\n\nplt.suptitle(\"Sex Distribution\", size=16)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# View the correlation heat map\ncorr= df.corr()\nplt.figure(figsize=(8,5))\nsns.heatmap(corr, annot=True, cmap=\"YlGnBu\",linewidth = 3, linecolor = \"white\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### MY another Notebook: *LUNG segmentation [2D+3D] Techniques -OSIC*\n\n## https://www.kaggle.com/akhileshdkapse/lung-segmentation-2d-3d-techniques-osic\n\n\n![](https://epos.myesr.org/posterimage/esr/ecr2016/132743/media/638205.png)\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"<hr>\n\n\n\n# Data pre-processing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df=pd.concat([df.drop(['Sex'], axis=1), pd.get_dummies(df['Sex'])], axis=1)\ndf=pd.concat([df.drop(['SmokingStatus'], axis=1), pd.get_dummies(df['SmokingStatus'])], axis=1)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.drop(['Female', 'Currently smokes', 'Patient'], inplace=True, axis=1)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.values.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<hr>\n\n\n\n# Model traning\n\n\n\n#### <p style=\"text-align:left;\">GENETIC algorithm ---> TPOT</p>\n**TPOT** is a Python Automated *Machine Learning tool* that optimizes machine learning pipelines using **genetic programming**.\n\n* TPOT will automate the most tedious part of machine learning by intelligently exploring thousands of possible pipelines to find the best one for your data.\n\n![](https://raw.githubusercontent.com/EpistasisLab/tpot/master/images/tpot-ml-pipeline.png)\n\n* Once TPOT is finished searching (or you get tired of waiting), it provides you with the Python code for the best pipeline it found so you can tinker with the pipeline from there.<br>\n* TPOT is built on top of scikit-learn, so all of the code it generates should look familiar... if you're familiar with scikit-learn, anyway.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"xtrain, xtest, ytrain, ytest= train_test_split(df.drop(['FVC'], axis=1).values, df.FVC.values, test_size=0.12, random_state=45)\nxtrain.shape, xtest.shape, ytrain.shape, ytest.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_estimators = [11,180,200,210,166,73,89,107]\nmax_features = ['auto', 'sqrt','log2']\nmax_depth = [21,22,24,27,30,32,43]\nmin_samples_split = [2, 5, 9,14,11]\nmin_samples_leaf = [1, 2, 4,6,8]\n\nparam = {'n_estimators': n_estimators,\n               'max_features': max_features,\n               'max_depth': max_depth,\n               'min_samples_split': min_samples_split,\n               'min_samples_leaf': min_samples_leaf}\n\n\nmodel = TPOTRegressor(generations= 8, population_size= 30, offspring_size= 16,\n                                 verbosity= 2, early_stop= 12,\n                                 config_dict={'sklearn.ensemble.RandomForestRegressor': param}, \n                                 cv = 4, scoring = 'neg_mean_squared_error')\nmodel.fit(xtrain,ytrain)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred=model.predict(xtest)\n\nprint('---------------------------')\nprint(mean_squared_error(ytest,y_pred))\nprint('---------------------------')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,5))\nplt.plot(y_pred, label='Train')\nplt.plot(ytest, label='Predictions')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,8))\nplt.plot(ytrain, label='Train')\nplt.plot(model.predict(xtrain), label='Predictions')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<hr>\n\n\n\n# Model testing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df2= pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\ndf2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df2=pd.concat([df2.drop(['Sex'], axis=1), pd.get_dummies(df2['Sex'])], axis=1)\ndf2=pd.concat([df2.drop(['SmokingStatus'], axis=1), pd.get_dummies(df2['SmokingStatus'])], axis=1)\n\nid_test= df2.Patient\ndf2.drop(['Patient'], inplace=True, axis=1)\ndf2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df2.FVC= scaler_Fcv.transform(df2.FVC.values.reshape(-1,1))\n# df2.Percent= scaler_Percent.transform(df2.Percent.values.reshape(-1,1))\n# df2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,5))\nplt.plot(df2.FVC.values, label='test')\nplt.plot(model.predict(df2.drop(['FVC'], axis=1).values), label='Prediction')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<hr>\n\n\n\n# Submitting Predictions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")\nsubmission[['Patient','Weeks']] = submission.Patient_Week.str.split(\"_\",expand=True)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.drop(['FVC', 'Confidence'], axis=1, inplace=True)\ndf2['Patient'] = id_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission=pd.merge(submission,df2.drop('Weeks', 1),on='Patient',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result = submission.iloc[:, 2:]\nresult= result.drop('FVC', axis=1)\nsubmission['FVC'] = model.predict(result).astype('int32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['tot'] =submission.groupby(['Ex-smoker','Male','Age'])['FVC'].transform('mean')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['Confidence'] = 100*submission['FVC']/submission['tot']\nsubmission['Confidence']= submission['Confidence'].astype('int32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_final = submission[['Patient_Week','FVC','Confidence']]\nsub_final","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_final.to_csv(\"/kaggle/working/submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"![](https://i.pinimg.com/originals/25/7d/3a/257d3afd123f2b88a6832067819596ef.gif)","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}