{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Energy Efficient Buildings Detailed π EDA π \n## If you find this notebook useful, support with an upvoteπ","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"**The competition is organised by `Kaggle` and is in the `WiDS Datathon 2022` series.**\n\n\n**In this competition, you are supposed to predict the Site EUI for each row, given the characteristics of the building and the weather data for the location of the building..**\n","metadata":{}},{"cell_type":"markdown","source":"\n# **Importing the libraries **","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nsns.set_style(\"whitegrid\")\nimport os","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:17.588018Z","iopub.execute_input":"2022-03-03T05:20:17.588484Z","iopub.status.idle":"2022-03-03T05:20:17.593157Z","shell.execute_reply.started":"2022-03-03T05:20:17.588427Z","shell.execute_reply":"2022-03-03T05:20:17.592321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# **Column Description **\n- Variable_Name, Describtion,\n- \"id\", \"building id\",\n- \"Year_Factor\", \"anonymized year in which the weather and energy usage factors were observed\",\n- \"State_Factor\", \"anonymized state in which the building is located\",\n- \"building_class\", \"building classification\",\n- \"facility_type\", \"building usage type\",\n- \"floor_area\", \"floor area (in square feet) of the building\",\n- \"year_built\", \"year in which the building was constructed\",\n- \"energy_star_rating\", \"the energy star rating of the building\",\n- \"ELEVATION\", \"elevation of the building location\",\n- \"january_min_temp\", \"minimum temperature in January (in Fahrenheit) at the location of the building\",\n- \"january_avg_temp\", \"average temperature in January (in Fahrenheit) at the location of the building\",\n- \"january_max_temp\", \"maximum temperature in January (in Fahrenheit) at the location of the building\",\n- \"cooling_degree_days\", \"cooling degree day for a given day is the number of degrees where the daily average temperature exceeds 65 degrees Fahrenheit. Each month is summed to produce an annual total at the location of the building.\",\n- \"heating_degree_days\", \"heating degree day for a given day is the number of degrees where the daily average temperature falls under 65 degrees Fahrenheit. Each month is summed to produce an annual total at the location of the building.\",\n- \"precipitation_inches\", \"annual precipitation in inches at the location of the building\",\n- \"snowfall_inches\", \"annual snowfall in inches at the location of the building\",\n- \"snowdepth_inches\", \"annual snow depth in inches at the location of the building\",\n- \"avg_temp\", \"average temperature over a year at the location of the building\",\n- \"days_below_30F\", \"total number of days below 30 degrees Fahrenheit at the location of the building\",\n- \"days_below_20F\", \"total number of days below 20 degrees Fahrenheit at the location of the building\",\n- \"days_below_10F\", \"total number of days below 10 degrees Fahrenheit at the location of the building\",\n- \"days_below_0F\", \"total number of days below 0 degrees Fahrenheit at the location of the building\",\n- \"days_above_80F\", \"total number of days above 80 degrees Fahrenheit at the location of the building\",\n- \"days_above_90F\", \"total number of days above 90 degrees Fahrenheit at the location of the building\",\n- \"days_above_100F\", \"total number of days above 100 degrees Fahrenheit at the location of the building\",\n- \"days_above_110F\", \"total number of days above 110 degrees Fahrenheit at the location of the building\",\n- \"direction_max_wind_speed\", \"wind direction for maximum wind speed at the location of the building. Given in 360-degree compass point directions (e.g. 360 = north, 180 = south, etc.).\",\n- \"direction_peak_wind_speed\", \"wind direction for peak wind gust speed at the location of the building. Given in 360-degree compass point directions (e.g. 360 = north, 180 = south, etc.).\",\n- \"max_wind_speed\", \"maximum wind speed at the location of the building\",\n- \"days_with_fog\", \"number of days with fog at the location of the building\",\n- \"site_eui\", \"Target Site Energy Usage Intensity is the amount of heat and electricity consumed by a building as reflected in utility bills\",)\n\n\n","metadata":{}},{"cell_type":"markdown","source":"\n# **Data Loading and Preparation **","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(r\"../input/widsdatathon2022/train.csv\")\ntest = pd.read_csv(r\"../input/widsdatathon2022/test.csv\")\nsubmission = pd.read_csv(r\"../input/widsdatathon2022/sample_solution.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:17.594699Z","iopub.execute_input":"2022-03-03T05:20:17.594928Z","iopub.status.idle":"2022-03-03T05:20:18.442562Z","shell.execute_reply.started":"2022-03-03T05:20:17.5949Z","shell.execute_reply":"2022-03-03T05:20:18.441804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:18.443621Z","iopub.execute_input":"2022-03-03T05:20:18.44489Z","iopub.status.idle":"2022-03-03T05:20:18.8936Z","shell.execute_reply.started":"2022-03-03T05:20:18.444835Z","shell.execute_reply":"2022-03-03T05:20:18.892692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(columns=['id'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:18.895143Z","iopub.execute_input":"2022-03-03T05:20:18.895351Z","iopub.status.idle":"2022-03-03T05:20:18.913197Z","shell.execute_reply.started":"2022-03-03T05:20:18.895326Z","shell.execute_reply":"2022-03-03T05:20:18.912077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:18.967332Z","iopub.execute_input":"2022-03-03T05:20:18.968186Z","iopub.status.idle":"2022-03-03T05:20:18.974581Z","shell.execute_reply.started":"2022-03-03T05:20:18.968144Z","shell.execute_reply":"2022-03-03T05:20:18.973587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:18.97594Z","iopub.execute_input":"2022-03-03T05:20:18.976253Z","iopub.status.idle":"2022-03-03T05:20:19.022219Z","shell.execute_reply.started":"2022-03-03T05:20:18.97621Z","shell.execute_reply":"2022-03-03T05:20:19.021328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n π Observations in Train Data:
\n \n* There are total of 63 columns and 75757 rows in train data.
\n* All 62 feature columns have missing values in them with days_with_fog having highest missing values
\n* site_eui is the target variable which is only available in the train dataset.
\n
","metadata":{}},{"cell_type":"markdown","source":"### Quick view of Train Data : ","metadata":{}},{"cell_type":"markdown","source":"Below are the first 5 rows of train dataset:","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:26:29.580757Z","iopub.execute_input":"2022-03-03T05:26:29.581414Z","iopub.status.idle":"2022-03-03T05:26:29.608575Z","shell.execute_reply.started":"2022-03-03T05:26:29.581371Z","shell.execute_reply":"2022-03-03T05:26:29.607727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**A brief statistical overview of train dataset**","metadata":{}},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:54:32.058086Z","iopub.execute_input":"2022-03-03T05:54:32.058365Z","iopub.status.idle":"2022-03-03T05:54:32.277332Z","shell.execute_reply.started":"2022-03-03T05:54:32.058336Z","shell.execute_reply":"2022-03-03T05:54:32.276723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'\\033[94mNumber of rows in train data: {train.shape[0]}')\nprint(f'\\033[94mNumber of columns in train data: {train.shape[1]}')\nprint(f'\\033[94mNumber of values in train data: {train.count().sum()}')\nprint(f'\\033[94mNumber missing values in train data: {sum(train.isna().sum())}')","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:44:14.321525Z","iopub.execute_input":"2022-03-03T05:44:14.321842Z","iopub.status.idle":"2022-03-03T05:44:14.368298Z","shell.execute_reply.started":"2022-03-03T05:44:14.321804Z","shell.execute_reply":"2022-03-03T05:44:14.367412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# ** EDA **","metadata":{}},{"cell_type":"code","source":"sns.countplot(x='State_Factor',hue='building_class',data=train,order = train['State_Factor'].value_counts().index)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:19.052976Z","iopub.execute_input":"2022-03-03T05:20:19.053464Z","iopub.status.idle":"2022-03-03T05:20:19.450363Z","shell.execute_reply.started":"2022-03-03T05:20:19.053419Z","shell.execute_reply":"2022-03-03T05:20:19.44973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## Percentage of data belonging to both Train and Test data ","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(18,6))\nplt.subplot(1, 2, 1) \ntrain['State_Factor'].value_counts().plot(kind='pie',autopct='%1.1f%%')\n\nplt.subplot(1, 2, 2) \ntest['State_Factor'].value_counts().plot(kind='pie',autopct='%1.1f%%')","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:19.451451Z","iopub.execute_input":"2022-03-03T05:20:19.451788Z","iopub.status.idle":"2022-03-03T05:20:19.78733Z","shell.execute_reply.started":"2022-03-03T05:20:19.451759Z","shell.execute_reply":"2022-03-03T05:20:19.786351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We find that State_11 didnt even exist in Train dataset however it exists in Test dataset only","metadata":{}},{"cell_type":"markdown","source":"\n## Correlation amongst numerical columns ","metadata":{}},{"cell_type":"code","source":"x = train.corr()\nplt.figure(figsize=(8,6))\nsns.heatmap(x)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:19.788444Z","iopub.execute_input":"2022-03-03T05:20:19.789183Z","iopub.status.idle":"2022-03-03T05:20:21.472575Z","shell.execute_reply.started":"2022-03-03T05:20:19.789137Z","shell.execute_reply":"2022-03-03T05:20:21.47173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# **Distribution of Buildings Constructed w.r.t each year **","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(24,6))\nsns.countplot(x='year_built',data=train[(train.year_built>1920)])\nplt.xticks(rotation=90)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:21.473709Z","iopub.execute_input":"2022-03-03T05:20:21.473928Z","iopub.status.idle":"2022-03-03T05:20:23.688406Z","shell.execute_reply.started":"2022-03-03T05:20:21.4739Z","shell.execute_reply":"2022-03-03T05:20:23.687426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## We find that construction of buildings was at an alltime high in 1927, after which it started to shrink. Some of the reasons which can be attributed are \n- Great Economic Depression 1929 \n- in 1940's construction again reduced (https://www.encyclopedia.com/social-sciences/culture-magazines/1940s-business-and-economy-topics-news)\n\n- Recession in 1990's (https://en.wikipedia.org/wiki/Early_1990s_recession#:~:text=Overall%20real%20GDP%20growth%20for,dropping%20to%2010.3%25%20in%201994.)","metadata":{}},{"cell_type":"markdown","source":"\n# **Distribution of Top 10 Facilities in Buildings in %**","metadata":{}},{"cell_type":"code","source":"df = pd.DataFrame(train['facility_type'].value_counts().head(10))\ndf['facility'] = df.index\ndf.rename(columns = {'facility_type':'count'}, inplace = True)\ndf['%share'] = df['count']/df['count'].sum()\ndf.drop(columns=['count'],inplace=True)\ndf.reset_index(drop=True)\nplt.figure(figsize=(12,6))\nax = sns.barplot(y='facility',x='%share',data=df)\nfor i in ax.containers:\n ax.bar_label(i,)\nplt.title(\"Distribution of Top 10 Facilities in Buildings in %\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:24.03032Z","iopub.execute_input":"2022-03-03T05:20:24.030602Z","iopub.status.idle":"2022-03-03T05:20:24.371469Z","shell.execute_reply.started":"2022-03-03T05:20:24.030563Z","shell.execute_reply":"2022-03-03T05:20:24.370579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# ** Commercial Buildings Characterestics which have a high site_eui**","metadata":{}},{"cell_type":"code","source":"df = pd.DataFrame(train[train['building_class']=='Commercial'][['site_eui','floor_area','year_built','ELEVATION','State_Factor','facility_type']].sort_values(by=['site_eui']).tail(200))\ndf.dropna()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:24.372556Z","iopub.execute_input":"2022-03-03T05:20:24.372865Z","iopub.status.idle":"2022-03-03T05:20:24.411556Z","shell.execute_reply.started":"2022-03-03T05:20:24.372833Z","shell.execute_reply":"2022-03-03T05:20:24.410686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,8))\nsns.histplot(data = train, x = \"site_eui\",hue='State_Factor')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:24.412781Z","iopub.execute_input":"2022-03-03T05:20:24.412986Z","iopub.status.idle":"2022-03-03T05:20:31.493278Z","shell.execute_reply.started":"2022-03-03T05:20:24.412962Z","shell.execute_reply":"2022-03-03T05:20:31.492544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train[train['building_class']=='Residential'])","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:31.494336Z","iopub.execute_input":"2022-03-03T05:20:31.495032Z","iopub.status.idle":"2022-03-03T05:20:31.515135Z","shell.execute_reply.started":"2022-03-03T05:20:31.49499Z","shell.execute_reply":"2022-03-03T05:20:31.514278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# **Building Wise Top 10 Distributions**","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(24,6))\n\nplt.subplot(1, 2, 1)\ntrain[train['building_class']=='Commercial'][['facility_type']].value_counts().plot(kind='bar')\n\nplt.subplot(1, 2, 2)\ntrain[train['building_class']=='Residential'][['facility_type']].value_counts().plot(kind='bar')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:31.516116Z","iopub.execute_input":"2022-03-03T05:20:31.516769Z","iopub.status.idle":"2022-03-03T05:20:34.11793Z","shell.execute_reply.started":"2022-03-03T05:20:31.516738Z","shell.execute_reply":"2022-03-03T05:20:34.1173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[train['building_class']=='Residential'][['facility_type']].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:34.118818Z","iopub.execute_input":"2022-03-03T05:20:34.119416Z","iopub.status.idle":"2022-03-03T05:20:34.146391Z","shell.execute_reply.started":"2022-03-03T05:20:34.119382Z","shell.execute_reply":"2022-03-03T05:20:34.145618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n\n# ** Distribution of Top 100 Site_EUI wrt to States**","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nsns.boxplot(x=\"State_Factor\",\n y=\"site_eui\",\n data=df)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:34.147592Z","iopub.execute_input":"2022-03-03T05:20:34.147853Z","iopub.status.idle":"2022-03-03T05:20:34.405374Z","shell.execute_reply.started":"2022-03-03T05:20:34.147825Z","shell.execute_reply":"2022-03-03T05:20:34.404787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# ** Distribution of Site_eui between 1889 to 2015 **","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20,6))\nsns.boxplot(x=\"year_built\",\n y=\"site_eui\",\n data=df)\nplt.xticks(rotation=90)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:34.406469Z","iopub.execute_input":"2022-03-03T05:20:34.407143Z","iopub.status.idle":"2022-03-03T05:20:36.61834Z","shell.execute_reply.started":"2022-03-03T05:20:34.407108Z","shell.execute_reply":"2022-03-03T05:20:36.617785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Observations\n- The floor area varies between 900 square feet to 63K square feet\n- Floor area of 50K square feet and above tend to have high median site_eui","metadata":{}},{"cell_type":"code","source":"train['floor_area'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:36.619289Z","iopub.execute_input":"2022-03-03T05:20:36.619642Z","iopub.status.idle":"2022-03-03T05:20:36.631926Z","shell.execute_reply.started":"2022-03-03T05:20:36.619602Z","shell.execute_reply":"2022-03-03T05:20:36.631013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# **Correlation between Floor area and building class**","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\nsns.scatterplot(data=train,x='floor_area',hue='building_class',y='site_eui')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:36.634607Z","iopub.execute_input":"2022-03-03T05:20:36.63531Z","iopub.status.idle":"2022-03-03T05:20:38.702351Z","shell.execute_reply.started":"2022-03-03T05:20:36.635262Z","shell.execute_reply":"2022-03-03T05:20:38.701411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# ** Distribution of Floor_area wrt energy_star_rating on the basis of States**","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20,6))\nplt.subplot(1,2,1)\nsns.scatterplot(data=train,y='floor_area',hue='State_Factor',x='energy_star_rating')\n\nplt.subplot(1,2,2)\nsns.scatterplot(data=test,y='floor_area',hue='State_Factor',x='energy_star_rating')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:38.70373Z","iopub.execute_input":"2022-03-03T05:20:38.704037Z","iopub.status.idle":"2022-03-03T05:20:41.872843Z","shell.execute_reply.started":"2022-03-03T05:20:38.703996Z","shell.execute_reply":"2022-03-03T05:20:41.872159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# ** Distribution of site_eui wrt ELEVATION**","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\n\nsns.scatterplot(data=train, x=\"ELEVATION\", y=\"site_eui\",hue='building_class')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T05:20:41.873815Z","iopub.execute_input":"2022-03-03T05:20:41.874473Z","iopub.status.idle":"2022-03-03T05:20:43.872487Z","shell.execute_reply.started":"2022-03-03T05:20:41.874437Z","shell.execute_reply":"2022-03-03T05:20:43.871909Z"},"trusted":true},"execution_count":null,"outputs":[]}]}