{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<h1 style=\"color:#189AB4;font-size:60px;\"><strong>Air Quality Demographics EDA <strong style=\"color:black\"></strong></strong></h1>\n\n<p style=\"font-size:120%\">Description of the data:\n\nThis dataset represents daily air quality measurements in the United States for 2019 and 2020 in EPA’s Air Quality System (AQS, https://www.epa.gov/aqs) database in which both PM2.5 and ozone are measured concurrently.  These PM2.5 and ozone concentration data are joined with locational, meteorological, demographic information, and concentrations of other major air quality pollutants when available.</p>\n","metadata":{}},{"cell_type":"markdown","source":"![](https://www2.iqair.com/sites/default/files/styles/wide_hero_2x/public/blog/2021-01/CostofAir21_Desk_a.jpg)","metadata":{}},{"cell_type":"markdown","source":"<a id=\"2\"></a>\n# **<center><span style=\"color:#00BFC4;\">Importing the libraries  </span></center>**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n!pip install openpyxl\nimport squarify\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)\npd.set_option('display.expand_frame_repr', False)\npd.set_option('max_colwidth', -1)\nsns.set_style('darkgrid')","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:15:06.302125Z","iopub.execute_input":"2022-03-12T05:15:06.302379Z","iopub.status.idle":"2022-03-12T05:15:06.311941Z","shell.execute_reply.started":"2022-03-12T05:15:06.302349Z","shell.execute_reply":"2022-03-12T05:15:06.311351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h1 style=\"color:#189AB4;font-size:60px;\"><strong>Our</strong> <strong style=\"color:black\">Data in Numbers:</strong></h1>","metadata":{}},{"cell_type":"code","source":"df = pd.read_excel(r\"../input/phase-ii-widsdatathon2022/epa/epa/Datathon_EPA_Air_Quality_Demographics_Meteorology_2019.xlsx\",engine='openpyxl')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:15:06.312914Z","iopub.execute_input":"2022-03-12T05:15:06.313153Z","iopub.status.idle":"2022-03-12T05:15:54.311339Z","shell.execute_reply.started":"2022-03-12T05:15:06.313122Z","shell.execute_reply":"2022-03-12T05:15:54.30975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:39:06.865821Z","iopub.execute_input":"2022-03-12T05:39:06.866121Z","iopub.status.idle":"2022-03-12T05:39:06.873052Z","shell.execute_reply.started":"2022-03-12T05:39:06.866093Z","shell.execute_reply":"2022-03-12T05:39:06.872155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe().T","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:39:20.112486Z","iopub.execute_input":"2022-03-12T05:39:20.113088Z","iopub.status.idle":"2022-03-12T05:39:20.204652Z","shell.execute_reply.started":"2022-03-12T05:39:20.113059Z","shell.execute_reply":"2022-03-12T05:39:20.203827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h1 style=\"color:#189AB4;font-size:60px;\"><strong>Data </strong> <strong style=\"color:black\">Demographics:</strong></h1>","metadata":{}},{"cell_type":"markdown","source":"<h1 style=\"color:#189AB4\"><strong>The First Question!</strong> What is the Null Distribution Column Wise?</h1>","metadata":{}},{"cell_type":"code","source":"df1 = pd.DataFrame(df.isna().sum().sort_values(ascending=False))\ndf1['null']=df1.index\ndf1['count']=df1.iloc[:,:-1]\ndf1.reset_index(drop=True, inplace=True)\ndf1 = df1.drop(df1.columns[[0]],axis = 1)\nplt.title('Null Distribution Column-Wise')\nax = sns.barplot(y='null',x='count',data=df1.head(14))\nfor i in ax.containers:\n    ax.bar_label(i)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:40:10.357519Z","iopub.execute_input":"2022-03-12T05:40:10.357968Z","iopub.status.idle":"2022-03-12T05:40:10.707429Z","shell.execute_reply.started":"2022-03-12T05:40:10.357938Z","shell.execute_reply":"2022-03-12T05:40:10.70597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Out of 129470 rows 128811 values in column **LEAD_UG_PER_CUBIC_METER** & 126163 values in column **BENZENE_PPBC** are missing\n- It is better to drop these columns as they contain mostly null values","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(24,12))\nsns.scatterplot(x=\"PM25_UG_PER_CUBIC_METER\",y=\"STATE\",data=df)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:16:06.654686Z","iopub.execute_input":"2022-03-12T05:16:06.655429Z","iopub.status.idle":"2022-03-12T05:16:08.075783Z","shell.execute_reply.started":"2022-03-12T05:16:06.655394Z","shell.execute_reply":"2022-03-12T05:16:08.074694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\ngroupby=pd.DataFrame(df.groupby(['STATE']).sum())\n#.plot(kind='pie', autopct='%1.0f%%',y='PEOPLE_OF_COLOR_FRACTION')\ngroupby['State']=groupby.index\ngroupby.reset_index(drop=True, inplace=True)\ngroupby = groupby.drop(groupby.columns[[0]],axis = 1)\ngroupby.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:47:30.598767Z","iopub.execute_input":"2022-03-12T05:47:30.599163Z","iopub.status.idle":"2022-03-12T05:47:30.661037Z","shell.execute_reply.started":"2022-03-12T05:47:30.599138Z","shell.execute_reply":"2022-03-12T05:47:30.660379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shapes = groupby[['PM25_UG_PER_CUBIC_METER','State']].sort_values(by='PM25_UG_PER_CUBIC_METER')\nshapes['PM25_UG_PER_CUBIC_METER'].head(10).unique()\n\nshapes = groupby[['PM25_UG_PER_CUBIC_METER','State']].sort_values(by='PM25_UG_PER_CUBIC_METER')\nshapes['State'].head(10).unique()","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:16:08.139495Z","iopub.execute_input":"2022-03-12T05:16:08.140254Z","iopub.status.idle":"2022-03-12T05:16:08.15235Z","shell.execute_reply.started":"2022-03-12T05:16:08.140221Z","shell.execute_reply":"2022-03-12T05:16:08.151131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"2\"></a>\n# **<center><span style=\"color:#00BFC4;\">States with the highest PM25_UG_PER_CUBIC_METER </span></center>**","metadata":{}},{"cell_type":"code","source":"import squarify\nplt.figure(figsize=(18,10))\nsquarify.plot(sizes=[567.7,  568. , 1692.7, 1938.8, 2149.1, 2363.2, 2541.5, 3437.4,4114.3, 4578.1], \n              label=['Idaho', 'Puerto Rico', 'Oregon', 'Hawaii', 'Alaska', 'Nebraska','Washington', 'Rhode Island', 'Maine', 'West Virginia'], alpha=.7,color = sns.color_palette('Set1',10),\n                  pad=0.8,text_kwargs={'fontsize':9})\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:16:08.153883Z","iopub.execute_input":"2022-03-12T05:16:08.15411Z","iopub.status.idle":"2022-03-12T05:16:08.362641Z","shell.execute_reply.started":"2022-03-12T05:16:08.154083Z","shell.execute_reply":"2022-03-12T05:16:08.362077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"2\"></a>\n# **<center><span style=\"color:#00BFC4;\">States with the highest PM25_UG_PER_CUBIC_METER </span></center>**","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(18,10))\nsquarify.plot(sizes=[567.7,  568. , 1692.7, 1938.8, 2149.1, 2363.2, 2541.5, 3437.4, 4114.3, 4578.1], \n              label=['Alabama', 'Alaska', 'Arizona', 'Arkansas', 'California','Colorado', 'Connecticut', 'Delaware', 'District Of Columbia',\n       'Florida'], alpha=.7,color = sns.color_palette('bright',10),\n                  pad=0.8,text_kwargs={'fontsize':9})\nplt.axis('off')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:16:08.363993Z","iopub.execute_input":"2022-03-12T05:16:08.365009Z","iopub.status.idle":"2022-03-12T05:16:08.573244Z","shell.execute_reply.started":"2022-03-12T05:16:08.364974Z","shell.execute_reply":"2022-03-12T05:16:08.571932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"2\"></a>\n# **<center><span style=\"color:#00BFC4;\">Representation of PEOPLE_OF_COLOR_FRACTION by STATES </span></center>**","metadata":{}},{"cell_type":"code","source":"import squarify\nplt.figure(figsize=(18,10))\nsquarify.plot(sizes=[25.92,34.56,118.9,421.26,434.97,549.02,554.63,571.27,579.36,581.04,583.23,585.55,643.75,709.69,820.32,1048.53,1186.39,1817.47,2265.89], \n              label=['Idaho', 'Maine', 'Alaska', 'Arkansas', 'Iowa', 'Delaware',\n       'Kentucky', 'Illinois', 'Colorado', 'Hawaii',\n       'District Of Columbia', 'Louisiana', 'Kansas', 'Alabama',\n       'Connecticut', 'Georgia', 'Indiana', 'Florida', 'Arizona'], alpha=.7,color = sns.color_palette('husl'),\n                  pad=0.8,text_kwargs={'fontsize':9})\nplt.axis('off')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:16:08.575337Z","iopub.execute_input":"2022-03-12T05:16:08.575604Z","iopub.status.idle":"2022-03-12T05:16:09.511805Z","shell.execute_reply.started":"2022-03-12T05:16:08.57557Z","shell.execute_reply":"2022-03-12T05:16:09.510714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"2\"></a>\n# **<center><span style=\"color:#00BFC4;\">Barplot Representation of PEOPLE_OF_COLOR_FRACTION by STATES </span></center>**","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20,6))\nax = sns.boxplot(x='STATE',y='PEOPLE_OF_COLOR_FRACTION',data=df)\nplt.xticks(rotation=90)\nax.set_title(\"People of colour fraction \")","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:16:09.708338Z","iopub.execute_input":"2022-03-12T05:16:09.708582Z","iopub.status.idle":"2022-03-12T05:16:11.818867Z","shell.execute_reply.started":"2022-03-12T05:16:09.708545Z","shell.execute_reply":"2022-03-12T05:16:11.817377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"2\"></a>\n# **<center><span style=\"color:#00BFC4;\">Barplot Representation of LOW_INCOME_FRACTION in different STATES </span></center>**","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20,6))\nax = sns.boxplot(x='STATE',y='LOW_INCOME_FRACTION',data=df)\nplt.xticks(rotation=90)\nax.set_title(\"Low Income Fraction Distribution in various States\")","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:16:11.821966Z","iopub.execute_input":"2022-03-12T05:16:11.822181Z","iopub.status.idle":"2022-03-12T05:16:14.142939Z","shell.execute_reply.started":"2022-03-12T05:16:11.822158Z","shell.execute_reply":"2022-03-12T05:16:14.141792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"2\"></a>\n# **<center><span style=\"color:#00BFC4;\">Relative_humidity_Representation in different STATES </span></center>**","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20,6))\nax = sns.boxplot(x='STATE',y='RELATIVE_HUMIDITY',data=df)\nplt.xticks(rotation=90)\nax.set_title(\"Relative Humidity Distribution in various States \")","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:16:14.144047Z","iopub.execute_input":"2022-03-12T05:16:14.144243Z","iopub.status.idle":"2022-03-12T05:16:16.443894Z","shell.execute_reply.started":"2022-03-12T05:16:14.144219Z","shell.execute_reply":"2022-03-12T05:16:16.4432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"2\"></a>\n# **<center><span style=\"color:#00BFC4;\">States having highest Wind_Speeds </span></center>**","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(18,8))\ndf1 = df[['WIND_SPEED_METERS_PER_SECOND','STATE']].sort_values(by='WIND_SPEED_METERS_PER_SECOND',ascending=False)\nplt.xticks(rotation=90)\nsns.barplot(x='STATE',y='WIND_SPEED_METERS_PER_SECOND',data=df1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:16:16.445157Z","iopub.execute_input":"2022-03-12T05:16:16.445578Z","iopub.status.idle":"2022-03-12T05:16:20.900046Z","shell.execute_reply.started":"2022-03-12T05:16:16.445523Z","shell.execute_reply":"2022-03-12T05:16:20.899462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfa=groupby.sort_values(by=['PEOPLE_OF_COLOR_FRACTION'], ascending=False)\nplt.figure(figsize=(12,6))\nplt.xticks(rotation=90)\nsns.barplot(x='State',y='PEOPLE_OF_COLOR_FRACTION',data=dfa)\nplt.xlabel('STATES')\nplt.ylabel('PEOPLE of Colour')\nplt.title('Distribution of people with different skintones across States')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:16:20.901353Z","iopub.execute_input":"2022-03-12T05:16:20.901762Z","iopub.status.idle":"2022-03-12T05:16:22.326938Z","shell.execute_reply.started":"2022-03-12T05:16:20.901722Z","shell.execute_reply":"2022-03-12T05:16:22.325866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfa=groupby.sort_values(by=['LOW_INCOME_FRACTION'], ascending=False)\nplt.figure(figsize=(12,8))\nsns.barplot(x='State',y='LOW_INCOME_FRACTION',data=dfa)\nplt.xticks(rotation=90)\nplt.xlabel('COUNT')\nplt.ylabel('PEOPLE of Colour')\nplt.title('States having the highest LOW_INCOME_FRACTION')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:46:09.482264Z","iopub.execute_input":"2022-03-12T05:46:09.482529Z","iopub.status.idle":"2022-03-12T05:46:10.841752Z","shell.execute_reply.started":"2022-03-12T05:46:09.482506Z","shell.execute_reply":"2022-03-12T05:46:10.840793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(18,10))\nsns.scatterplot(x=\"PEOPLE_OF_COLOR_FRACTION\",y=\"STATE\",data=df,hue='LOW_INCOME_FRACTION')","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:46:52.140996Z","iopub.execute_input":"2022-03-12T05:46:52.141302Z","iopub.status.idle":"2022-03-12T05:47:04.301344Z","shell.execute_reply.started":"2022-03-12T05:46:52.141271Z","shell.execute_reply":"2022-03-12T05:47:04.300475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"California seems to be the clear winner in this as well, followed by Utah","metadata":{}},{"cell_type":"code","source":"dfa=groupby.sort_values(by=['TEMPERATURE_CELSIUS'], ascending=False)\nplt.figure(figsize=(12,8))\nsns.barplot(x='State',y='TEMPERATURE_CELSIUS',data=dfa)\nplt.xticks(rotation=90)\nplt.xlabel('States-->')\nplt.ylabel('Humidity Count')\nplt.title('States with the highest Relative humidity')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:16:23.791222Z","iopub.execute_input":"2022-03-12T05:16:23.791548Z","iopub.status.idle":"2022-03-12T05:16:25.280977Z","shell.execute_reply.started":"2022-03-12T05:16:23.79151Z","shell.execute_reply":"2022-03-12T05:16:25.280086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfa=groupby.sort_values(by=['LOW_INCOME_FRACTION'], ascending=False)\nplt.figure(figsize=(12,8))\nsns.barplot(x='State',y='LOW_INCOME_FRACTION',data=dfa)\nplt.xticks(rotation=90)\nplt.xlabel('COUNT')\nplt.ylabel('PEOPLE of Colour')\nplt.title('States with the hottest Temperatures')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:16:25.282683Z","iopub.execute_input":"2022-03-12T05:16:25.283528Z","iopub.status.idle":"2022-03-12T05:16:26.666465Z","shell.execute_reply.started":"2022-03-12T05:16:25.283493Z","shell.execute_reply":"2022-03-12T05:16:26.66538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,12))\ndataplot = sns.heatmap(df.corr(), cmap=\"YlGnBu\", annot=True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-12T05:16:27.130285Z","iopub.execute_input":"2022-03-12T05:16:27.130532Z","iopub.status.idle":"2022-03-12T05:16:28.595355Z","shell.execute_reply.started":"2022-03-12T05:16:27.130504Z","shell.execute_reply":"2022-03-12T05:16:28.594202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}