{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"raw","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-07T15:49:27.688131Z","iopub.execute_input":"2022-08-07T15:49:27.688522Z","iopub.status.idle":"2022-08-07T15:49:27.726405Z","shell.execute_reply.started":"2022-08-07T15:49:27.688449Z","shell.execute_reply":"2022-08-07T15:49:27.724479Z"}}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nimport datetime as dt\nimport gc #garbage collect to free up memory","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:04.467646Z","iopub.execute_input":"2022-08-08T15:59:04.468203Z","iopub.status.idle":"2022-08-08T15:59:05.564437Z","shell.execute_reply.started":"2022-08-08T15:59:04.468160Z","shell.execute_reply":"2022-08-08T15:59:05.563248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"energy=pd.read_csv('/kaggle/input/ashrae-energy-prediction/train.csv',parse_dates=['timestamp'])\nweather=pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_train.csv',parse_dates=['timestamp'])\nbuilding_meta=pd.read_csv('/kaggle/input/ashrae-energy-prediction/building_metadata.csv',parse_dates=['year_built'])","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:05.566287Z","iopub.execute_input":"2022-08-08T15:59:05.566678Z","iopub.status.idle":"2022-08-08T15:59:22.966423Z","shell.execute_reply.started":"2022-08-08T15:59:05.566644Z","shell.execute_reply":"2022-08-08T15:59:22.965114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1) timestamp from all the 3 datasets were converted to datetime format","metadata":{}},{"cell_type":"code","source":"energy.sample(3)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:22.967894Z","iopub.execute_input":"2022-08-08T15:59:22.968282Z","iopub.status.idle":"2022-08-08T15:59:24.145988Z","shell.execute_reply.started":"2022-08-08T15:59:22.968246Z","shell.execute_reply":"2022-08-08T15:59:24.144581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"energy.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:24.149490Z","iopub.execute_input":"2022-08-08T15:59:24.150055Z","iopub.status.idle":"2022-08-08T15:59:24.159453Z","shell.execute_reply.started":"2022-08-08T15:59:24.150015Z","shell.execute_reply":"2022-08-08T15:59:24.158090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"energy=energy.astype({'building_id':'int16','meter':'int8','meter_reading':'float32'})","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:24.160859Z","iopub.execute_input":"2022-08-08T15:59:24.161348Z","iopub.status.idle":"2022-08-08T15:59:24.322960Z","shell.execute_reply.started":"2022-08-08T15:59:24.161314Z","shell.execute_reply":"2022-08-08T15:59:24.321661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather.sample(3)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:24.324884Z","iopub.execute_input":"2022-08-08T15:59:24.325331Z","iopub.status.idle":"2022-08-08T15:59:24.347855Z","shell.execute_reply.started":"2022-08-08T15:59:24.325291Z","shell.execute_reply":"2022-08-08T15:59:24.346513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:24.349863Z","iopub.execute_input":"2022-08-08T15:59:24.350304Z","iopub.status.idle":"2022-08-08T15:59:24.365308Z","shell.execute_reply.started":"2022-08-08T15:59:24.350264Z","shell.execute_reply":"2022-08-08T15:59:24.363728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather=weather.astype({'site_id':'int8','air_temperature':'float16',\n                       'cloud_coverage':'float16','dew_temperature':'float16',\n                       'precip_depth_1_hr':'float16','sea_level_pressure':'float16'})\nweather=weather.astype({'wind_speed':'float16'})\nweather=weather.astype({'wind_direction':'float16'})","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:24.367235Z","iopub.execute_input":"2022-08-08T15:59:24.367911Z","iopub.status.idle":"2022-08-08T15:59:24.396120Z","shell.execute_reply.started":"2022-08-08T15:59:24.367867Z","shell.execute_reply":"2022-08-08T15:59:24.394845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building_meta.sample(3)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:24.397516Z","iopub.execute_input":"2022-08-08T15:59:24.399389Z","iopub.status.idle":"2022-08-08T15:59:24.414539Z","shell.execute_reply.started":"2022-08-08T15:59:24.399344Z","shell.execute_reply":"2022-08-08T15:59:24.413102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building_meta.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:24.420510Z","iopub.execute_input":"2022-08-08T15:59:24.420980Z","iopub.status.idle":"2022-08-08T15:59:24.430625Z","shell.execute_reply.started":"2022-08-08T15:59:24.420940Z","shell.execute_reply":"2022-08-08T15:59:24.429457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Overview","metadata":{}},{"cell_type":"markdown","source":"## Train Dataset","metadata":{}},{"cell_type":"code","source":"energy.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:24.432135Z","iopub.execute_input":"2022-08-08T15:59:24.433445Z","iopub.status.idle":"2022-08-08T15:59:24.456138Z","shell.execute_reply.started":"2022-08-08T15:59:24.433396Z","shell.execute_reply":"2022-08-08T15:59:24.455076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Weather Data","metadata":{}},{"cell_type":"code","source":"weather.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:24.457566Z","iopub.execute_input":"2022-08-08T15:59:24.457924Z","iopub.status.idle":"2022-08-08T15:59:24.484347Z","shell.execute_reply.started":"2022-08-08T15:59:24.457893Z","shell.execute_reply":"2022-08-08T15:59:24.483010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Building meta data","metadata":{}},{"cell_type":"code","source":"building_meta.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:24.486148Z","iopub.execute_input":"2022-08-08T15:59:24.487088Z","iopub.status.idle":"2022-08-08T15:59:24.502994Z","shell.execute_reply.started":"2022-08-08T15:59:24.487007Z","shell.execute_reply":"2022-08-08T15:59:24.501582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Timestamp alignment  \n\nThe weather data is not in the local time format, and hence need to be aligned with the local time format so that it matches with the energy dataframe.","metadata":{}},{"cell_type":"markdown","source":"## Checking for date discrepency","metadata":{}},{"cell_type":"code","source":"temp_df=weather[['site_id','timestamp','air_temperature']]\ntemp_df['temp_rank']=temp_df.groupby(['site_id', temp_df.timestamp.dt.date],)['air_temperature'].rank('average')\ndf_2d=temp_df.groupby(['site_id', temp_df.timestamp.dt.hour])['temp_rank'].mean().unstack(level=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:24.504554Z","iopub.execute_input":"2022-08-08T15:59:24.504959Z","iopub.status.idle":"2022-08-08T15:59:24.645366Z","shell.execute_reply.started":"2022-08-08T15:59:24.504921Z","shell.execute_reply":"2022-08-08T15:59:24.644223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 8))\nsns.heatmap(df_2d,cmap='Reds');\nplt.xlabel('Hour')\nplt.title('Mean temperature rank by hour (initial timestamps)')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:24.646815Z","iopub.execute_input":"2022-08-08T15:59:24.647447Z","iopub.status.idle":"2022-08-08T15:59:25.165653Z","shell.execute_reply.started":"2022-08-08T15:59:24.647409Z","shell.execute_reply":"2022-08-08T15:59:25.164388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It is expected it should be hotter at noon ie between 1000 to 1500 hrs and cooler at night and at morning.  \nSite_id : 1,5,12 seem to be following this trend, while other sites are somewhat offset. We need to realign them.  \n## create a custom-made function for realignment.\n(Credits: https://www.kaggle.com/code/frednavruzov/aligning-temperature-timestamp/notebook)","metadata":{}},{"cell_type":"code","source":"def time_alignment(df):\n    temp_df=df[['site_id','timestamp','air_temperature']]\n    \n    # calculate ranks of hourly temperatures within date/site_id chunks\n    temp_df['temp_rank']=temp_df.groupby(['site_id', temp_df.timestamp.dt.date],)['air_temperature'].rank('average')\n    \n    # create 2D dataframe of site_ids (0-16) x mean hour rank of temperature within day (0-23)\n    df_2d=temp_df.groupby(['site_id', temp_df.timestamp.dt.hour])['temp_rank'].mean().unstack(level=1)\n    \n    # align scale, so each value within row is in [0,1] range\n    df_2d = df_2d / df_2d.max(axis=1).values.reshape((-1,1))\n    \n    # sort by 'closeness' of hour with the highest temperature\n    site_ids_argmax_maxtemp=pd.Series(np.argmax(df_2d.values,axis=1)).sort_values().index\n    \n    # assuming (1,5,12) tuple has the most correct temp peaks at 14:00\n    site_ids_offsets= pd.Series(df_2d.values.argmax(axis=1) - 14)\n    \n    # align rows so that site_id's with similar temperature hour's peaks are near each other\n    df_2d=df_2d.iloc[site_ids_argmax_maxtemp]\n    temp_df['offset'] = temp_df.site_id.map(site_ids_offsets)\n    \n    # add offset\n    temp_df['timestamp_aligned'] = (temp_df.timestamp - pd.to_timedelta(temp_df.offset, unit='H'))\n    \n    # replace the timestamp with aligned timestamps in the original dataframe\n    df['timestamp']=temp_df['timestamp_aligned']\n    return df ","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:25.167291Z","iopub.execute_input":"2022-08-08T15:59:25.168548Z","iopub.status.idle":"2022-08-08T15:59:25.180581Z","shell.execute_reply.started":"2022-08-08T15:59:25.168493Z","shell.execute_reply":"2022-08-08T15:59:25.179148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather=time_alignment(weather)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:25.182878Z","iopub.execute_input":"2022-08-08T15:59:25.183500Z","iopub.status.idle":"2022-08-08T15:59:25.329692Z","shell.execute_reply.started":"2022-08-08T15:59:25.183447Z","shell.execute_reply":"2022-08-08T15:59:25.328492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## After timestamp alignment","metadata":{}},{"cell_type":"code","source":"temp_df=weather[['site_id','timestamp','air_temperature']]\ntemp_df['temp_rank']=temp_df.groupby(['site_id', temp_df.timestamp.dt.date],)['air_temperature'].rank('average')\ndf_2d=temp_df.groupby(['site_id', temp_df.timestamp.dt.hour])['temp_rank'].mean().unstack(level=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:25.331293Z","iopub.execute_input":"2022-08-08T15:59:25.332014Z","iopub.status.idle":"2022-08-08T15:59:25.455404Z","shell.execute_reply.started":"2022-08-08T15:59:25.331965Z","shell.execute_reply":"2022-08-08T15:59:25.454006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 8))\nsns.heatmap(df_2d,cmap='Reds');\nplt.xlabel('Hour')\nplt.title('Mean temperature rank by hour (realigned timestamps)')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:25.457201Z","iopub.execute_input":"2022-08-08T15:59:25.457924Z","iopub.status.idle":"2022-08-08T15:59:25.924512Z","shell.execute_reply.started":"2022-08-08T15:59:25.457873Z","shell.execute_reply":"2022-08-08T15:59:25.923078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### This seems far more plausible","metadata":{}},{"cell_type":"markdown","source":"## Merging into one single dataframe","metadata":{}},{"cell_type":"code","source":"# Merge train data with building meta data\nenergy=energy.merge(building_meta,on='building_id',how='left')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:25.926229Z","iopub.execute_input":"2022-08-08T15:59:25.926628Z","iopub.status.idle":"2022-08-08T15:59:31.480442Z","shell.execute_reply.started":"2022-08-08T15:59:25.926595Z","shell.execute_reply":"2022-08-08T15:59:31.478804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge weather data\ndf=energy.merge(weather,on=['site_id','timestamp'],how='left')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:31.483428Z","iopub.execute_input":"2022-08-08T15:59:31.484057Z","iopub.status.idle":"2022-08-08T15:59:38.155879Z","shell.execute_reply.started":"2022-08-08T15:59:31.483996Z","shell.execute_reply":"2022-08-08T15:59:38.151613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:38.158198Z","iopub.execute_input":"2022-08-08T15:59:38.158950Z","iopub.status.idle":"2022-08-08T15:59:38.184222Z","shell.execute_reply.started":"2022-08-08T15:59:38.158892Z","shell.execute_reply":"2022-08-08T15:59:38.182923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Removing the parent dataframes for better memory management","metadata":{}},{"cell_type":"code","source":"del energy,weather,building_meta\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:38.186163Z","iopub.execute_input":"2022-08-08T15:59:38.187043Z","iopub.status.idle":"2022-08-08T15:59:38.411096Z","shell.execute_reply.started":"2022-08-08T15:59:38.186991Z","shell.execute_reply":"2022-08-08T15:59:38.409621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Data Analysis","metadata":{}},{"cell_type":"markdown","source":"## Data dictionary","metadata":{}},{"cell_type":"markdown","source":"1) building_id - Foreign key for the building metadata.  \n2) meter - The meter id code. Read as {0: electricity, 1: chilledwater, 2: steam, 3: hotwater}. Not every building has all meter types.  \n3) timestamp - When the measurement was taken  \n4) meter_reading - The target variable. Energy consumption in kWh (or equivalent). Note that this is real data with measurement error, which we expect will impose a baseline level of modeling error. UPDATE: as discussed here, the site 0 electric meter readings are in kBTU.","metadata":{}},{"cell_type":"markdown","source":"site_id - Foreign key for the weather files.  \nbuilding_id - Foreign key for training.csv  \nprimary_use - Indicator of the primary category of activities for the building   \nsquare_feet - Gross floor area of the building  \nyear_built - Year building was opened  \nfloor_count - Number of floors of the building  ","metadata":{}},{"cell_type":"markdown","source":"Weather data from a meteorological station as close as possible to the site.  \n  \nsite_id  \nair_temperature - Degrees Celsius  \ncloud_coverage - Portion of the sky covered in clouds, in oktas  \ndew_temperature - Degrees Celsius  \nprecip_depth_1_hr - Millimeters  \nsea_level_pressure - Millibar/hectopascals  \nwind_direction - Compass direction (0-360)  \nwind_speed - Meters per second  ","metadata":{}},{"cell_type":"code","source":"df.isna().sum()*100/len(df)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:38.412469Z","iopub.execute_input":"2022-08-08T15:59:38.412972Z","iopub.status.idle":"2022-08-08T15:59:40.212433Z","shell.execute_reply.started":"2022-08-08T15:59:38.412920Z","shell.execute_reply":"2022-08-08T15:59:40.211109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dropping columns with high percent of missing values\n#df.drop(columns=['year_built','floor_count','cloud_coverage','precip_depth_1_hr',\n                # 'sea_level_pressure','wind_direction'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:40.214502Z","iopub.execute_input":"2022-08-08T15:59:40.214999Z","iopub.status.idle":"2022-08-08T15:59:40.219988Z","shell.execute_reply.started":"2022-08-08T15:59:40.214950Z","shell.execute_reply":"2022-08-08T15:59:40.218742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Deleting instead of dropping to free up memory\n\ndel df['year_built']\ndel df['floor_count']\ndel df['cloud_coverage']\ndel df['precip_depth_1_hr']\ndel df['sea_level_pressure']\ndel df['wind_direction']\ngc.collect()    ","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:40.221969Z","iopub.execute_input":"2022-08-08T15:59:40.222448Z","iopub.status.idle":"2022-08-08T15:59:40.366962Z","shell.execute_reply.started":"2022-08-08T15:59:40.222404Z","shell.execute_reply":"2022-08-08T15:59:40.365508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:40.369157Z","iopub.execute_input":"2022-08-08T15:59:40.369667Z","iopub.status.idle":"2022-08-08T15:59:40.382670Z","shell.execute_reply.started":"2022-08-08T15:59:40.369619Z","shell.execute_reply":"2022-08-08T15:59:40.381469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Missing values after dropping columns\ndf.isna().sum()*100/len(df)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:40.391155Z","iopub.execute_input":"2022-08-08T15:59:40.391564Z","iopub.status.idle":"2022-08-08T15:59:41.711423Z","shell.execute_reply.started":"2022-08-08T15:59:40.391529Z","shell.execute_reply":"2022-08-08T15:59:41.710008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df['site_id']==0]['meter_reading'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:41.713007Z","iopub.execute_input":"2022-08-08T15:59:41.713399Z","iopub.status.idle":"2022-08-08T15:59:42.251112Z","shell.execute_reply.started":"2022-08-08T15:59:41.713365Z","shell.execute_reply":"2022-08-08T15:59:42.249734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert the KBTU to KWh in site 0\ndf.loc[(df['site_id'] == 0),'meter_reading'] *= 0.2931","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:42.252753Z","iopub.execute_input":"2022-08-08T15:59:42.253399Z","iopub.status.idle":"2022-08-08T15:59:42.418024Z","shell.execute_reply.started":"2022-08-08T15:59:42.253347Z","shell.execute_reply":"2022-08-08T15:59:42.416991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df['site_id']==0]['meter_reading'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:42.419240Z","iopub.execute_input":"2022-08-08T15:59:42.420115Z","iopub.status.idle":"2022-08-08T15:59:42.604638Z","shell.execute_reply.started":"2022-08-08T15:59:42.420075Z","shell.execute_reply":"2022-08-08T15:59:42.603426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def meter(x):\n    if x==0:\n        return 'electricity'\n    elif x==1:\n        return 'chill_water'\n    elif x==2:\n        return 'steam'\n    else:\n        return 'hotwater'","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:42.607453Z","iopub.execute_input":"2022-08-08T15:59:42.608265Z","iopub.status.idle":"2022-08-08T15:59:42.615086Z","shell.execute_reply.started":"2022-08-08T15:59:42.608211Z","shell.execute_reply":"2022-08-08T15:59:42.613972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['meter']=df['meter'].apply(lambda x:meter(x))","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:42.616794Z","iopub.execute_input":"2022-08-08T15:59:42.617425Z","iopub.status.idle":"2022-08-08T15:59:49.317021Z","shell.execute_reply.started":"2022-08-08T15:59:42.617390Z","shell.execute_reply":"2022-08-08T15:59:49.315856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_date_df,min_date_df=df['timestamp'].agg(['max','min'])","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:49.320228Z","iopub.execute_input":"2022-08-08T15:59:49.321285Z","iopub.status.idle":"2022-08-08T15:59:49.463005Z","shell.execute_reply.started":"2022-08-08T15:59:49.321237Z","shell.execute_reply":"2022-08-08T15:59:49.461717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(min_date_df)\nprint(max_date_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:49.464673Z","iopub.execute_input":"2022-08-08T15:59:49.465112Z","iopub.status.idle":"2022-08-08T15:59:49.472833Z","shell.execute_reply.started":"2022-08-08T15:59:49.465021Z","shell.execute_reply":"2022-08-08T15:59:49.470302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Obs:  \n1) So the date ranges from 1st Jan to 31st Dec 2016.","metadata":{}},{"cell_type":"code","source":"df['meter'].value_counts().index","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:49.474192Z","iopub.execute_input":"2022-08-08T15:59:49.475495Z","iopub.status.idle":"2022-08-08T15:59:50.330805Z","shell.execute_reply.started":"2022-08-08T15:59:49.475445Z","shell.execute_reply":"2022-08-08T15:59:50.329447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,6))\nplt.pie(df['meter'].value_counts().values,explode=[0.1,0,0,0],labels=df['meter'].value_counts().index,autopct='%.1f%%')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:50.333759Z","iopub.execute_input":"2022-08-08T15:59:50.334342Z","iopub.status.idle":"2022-08-08T15:59:52.179725Z","shell.execute_reply.started":"2022-08-08T15:59:50.334288Z","shell.execute_reply":"2022-08-08T15:59:52.178091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.impute import SimpleImputer\nimputer=SimpleImputer(strategy='median')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:52.181966Z","iopub.execute_input":"2022-08-08T15:59:52.187654Z","iopub.status.idle":"2022-08-08T15:59:52.534264Z","shell.execute_reply.started":"2022-08-08T15:59:52.187483Z","shell.execute_reply":"2022-08-08T15:59:52.532784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imputer=imputer.fit(df[['air_temperature','dew_temperature','wind_speed']])","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:59:52.536035Z","iopub.execute_input":"2022-08-08T15:59:52.537003Z","iopub.status.idle":"2022-08-08T16:00:05.385483Z","shell.execute_reply.started":"2022-08-08T15:59:52.536950Z","shell.execute_reply":"2022-08-08T16:00:05.383708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[['air_temperature','dew_temperature','wind_speed']]=imputer.transform(df[['air_temperature','dew_temperature','wind_speed']])","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:00:05.387398Z","iopub.execute_input":"2022-08-08T16:00:05.387826Z","iopub.status.idle":"2022-08-08T16:00:06.585861Z","shell.execute_reply.started":"2022-08-08T16:00:05.387785Z","shell.execute_reply":"2022-08-08T16:00:06.584548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isna().sum()*100/len(df)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:00:06.587417Z","iopub.execute_input":"2022-08-08T16:00:06.587894Z","iopub.status.idle":"2022-08-08T16:00:08.744610Z","shell.execute_reply.started":"2022-08-08T16:00:06.587851Z","shell.execute_reply":"2022-08-08T16:00:08.743294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Statistical Summary","metadata":{}},{"cell_type":"code","source":"sns.countplot(x='site_id',data=df)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:00:08.746215Z","iopub.execute_input":"2022-08-08T16:00:08.746640Z","iopub.status.idle":"2022-08-08T16:00:11.070116Z","shell.execute_reply.started":"2022-08-08T16:00:08.746604Z","shell.execute_reply":"2022-08-08T16:00:11.068932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,5))\nsns.countplot(x='primary_use',data=df)\nplt.xticks(rotation=60)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:00:11.071725Z","iopub.execute_input":"2022-08-08T16:00:11.072259Z","iopub.status.idle":"2022-08-08T16:00:25.085032Z","shell.execute_reply.started":"2022-08-08T16:00:11.072209Z","shell.execute_reply":"2022-08-08T16:00:25.083739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nsns.barplot(x='primary_use',y='meter_reading',data=df)\nplt.xticks(rotation='vertical')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:00:25.086722Z","iopub.execute_input":"2022-08-08T16:00:25.087145Z","iopub.status.idle":"2022-08-08T16:10:47.816402Z","shell.execute_reply.started":"2022-08-08T16:00:25.087106Z","shell.execute_reply":"2022-08-08T16:10:47.814835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,4))\nsns.histplot(df['square_feet'],kde=True,color='red',bins=100)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:10:47.818619Z","iopub.execute_input":"2022-08-08T16:10:47.820053Z","iopub.status.idle":"2022-08-08T16:12:17.655972Z","shell.execute_reply.started":"2022-08-08T16:10:47.819991Z","shell.execute_reply":"2022-08-08T16:12:17.654545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:12:17.657638Z","iopub.execute_input":"2022-08-08T16:12:17.658127Z","iopub.status.idle":"2022-08-08T16:12:17.851332Z","shell.execute_reply.started":"2022-08-08T16:12:17.658079Z","shell.execute_reply":"2022-08-08T16:12:17.850001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So, the area of buildings is heavily skewed with some unusually large buildings","metadata":{}},{"cell_type":"code","source":"fig,ax=plt.subplots(1,1,figsize=(10,6))\ndf[['timestamp', 'meter_reading']].set_index('timestamp').resample('D').mean()['meter_reading'].plot(ax=ax).set_ylabel('Meter reading')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:12:17.853215Z","iopub.execute_input":"2022-08-08T16:12:17.853602Z","iopub.status.idle":"2022-08-08T16:12:19.151913Z","shell.execute_reply.started":"2022-08-08T16:12:17.853567Z","shell.execute_reply":"2022-08-08T16:12:19.150677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,ax=plt.subplots(1,1,figsize=(10,6))\ndf[['timestamp', 'air_temperature']].set_index('timestamp').resample('D').mean()['air_temperature'].plot(ax=ax).set_ylabel('Temperature')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:12:19.153524Z","iopub.execute_input":"2022-08-08T16:12:19.153940Z","iopub.status.idle":"2022-08-08T16:12:20.421682Z","shell.execute_reply.started":"2022-08-08T16:12:19.153903Z","shell.execute_reply":"2022-08-08T16:12:20.420430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,ax=plt.subplots(8,2,figsize=(15,30))\nfor i in range(16):\n    df[df['site_id']==i][['timestamp', 'air_temperature']].set_index('timestamp').resample('D').mean()['air_temperature'].plot(ax=ax[i%8][i//8]).set_ylabel('Temperature')\n    ax[i%8][i//8].set_title('site_id {}'.format(i))\n    plt.subplots_adjust(hspace=0.45)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:12:20.423911Z","iopub.execute_input":"2022-08-08T16:12:20.425511Z","iopub.status.idle":"2022-08-08T16:12:28.628040Z","shell.execute_reply.started":"2022-08-08T16:12:20.425440Z","shell.execute_reply":"2022-08-08T16:12:28.626914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Meter reading vs site_id","metadata":{}},{"cell_type":"code","source":"fig,ax=plt.subplots(8,2,figsize=(15,30))\nfor i in range(16):\n    df[df['site_id']==i][['timestamp', 'meter_reading']].set_index('timestamp').resample('D').mean()['meter_reading'].plot(ax=ax[i%8][i//8]).set_ylabel('meter_reading')\n    ax[i%8][i//8].set_title('site_id {}'.format(i))\n    plt.subplots_adjust(hspace=0.45)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:12:28.629613Z","iopub.execute_input":"2022-08-08T16:12:28.630232Z","iopub.status.idle":"2022-08-08T16:12:37.009566Z","shell.execute_reply.started":"2022-08-08T16:12:28.630189Z","shell.execute_reply":"2022-08-08T16:12:37.008429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We observe that the meter_reading of site_id 13 is extremely high and is contributing almost exclusively to the anamalous meter reading behaviour of the entire dataset.","metadata":{}},{"cell_type":"markdown","source":"## meter reading vs building_id","metadata":{}},{"cell_type":"code","source":"sns.scatterplot(x='building_id',y='meter_reading',data=df,hue='site_id')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:12:37.011156Z","iopub.execute_input":"2022-08-08T16:12:37.011973Z","iopub.status.idle":"2022-08-08T16:20:34.591809Z","shell.execute_reply.started":"2022-08-08T16:12:37.011931Z","shell.execute_reply":"2022-08-08T16:20:34.590378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df['meter_reading']==np.max(df['meter_reading'].values)]","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:20:34.593287Z","iopub.execute_input":"2022-08-08T16:20:34.594338Z","iopub.status.idle":"2022-08-08T16:20:34.641928Z","shell.execute_reply.started":"2022-08-08T16:20:34.594298Z","shell.execute_reply":"2022-08-08T16:20:34.640761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So we see that building 1099 belonging to site_id 13 is the anomaly in this dataset. We will remove this to prevent overfitting and generalize our model.","metadata":{}},{"cell_type":"code","source":"df_copy=df[df['building_id']!=1099]","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:20:34.643904Z","iopub.execute_input":"2022-08-08T16:20:34.644420Z","iopub.status.idle":"2022-08-08T16:20:36.458343Z","shell.execute_reply.started":"2022-08-08T16:20:34.644370Z","shell.execute_reply":"2022-08-08T16:20:36.457075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:20:36.460390Z","iopub.execute_input":"2022-08-08T16:20:36.460943Z","iopub.status.idle":"2022-08-08T16:20:36.821985Z","shell.execute_reply.started":"2022-08-08T16:20:36.460890Z","shell.execute_reply":"2022-08-08T16:20:36.820741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=df_copy","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:20:36.823684Z","iopub.execute_input":"2022-08-08T16:20:36.824728Z","iopub.status.idle":"2022-08-08T16:20:36.832403Z","shell.execute_reply.started":"2022-08-08T16:20:36.824677Z","shell.execute_reply":"2022-08-08T16:20:36.831094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_copy\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:20:36.833885Z","iopub.execute_input":"2022-08-08T16:20:36.834508Z","iopub.status.idle":"2022-08-08T16:20:37.039661Z","shell.execute_reply.started":"2022-08-08T16:20:36.834471Z","shell.execute_reply":"2022-08-08T16:20:37.038626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets also remove the meter reading 0 in some of the sites as they are erratic and hence cannot be used for consistent predictions.","metadata":{}},{"cell_type":"code","source":"df_copy=df[df['meter_reading']!=0]","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:20:37.041262Z","iopub.execute_input":"2022-08-08T16:20:37.041943Z","iopub.status.idle":"2022-08-08T16:20:38.852417Z","shell.execute_reply.started":"2022-08-08T16:20:37.041904Z","shell.execute_reply":"2022-08-08T16:20:38.850957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:20:38.854231Z","iopub.execute_input":"2022-08-08T16:20:38.854605Z","iopub.status.idle":"2022-08-08T16:20:39.254033Z","shell.execute_reply.started":"2022-08-08T16:20:38.854572Z","shell.execute_reply":"2022-08-08T16:20:39.252681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=df_copy","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:20:39.256287Z","iopub.execute_input":"2022-08-08T16:20:39.257745Z","iopub.status.idle":"2022-08-08T16:20:39.265923Z","shell.execute_reply.started":"2022-08-08T16:20:39.257654Z","shell.execute_reply":"2022-08-08T16:20:39.263679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_copy\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:20:39.268226Z","iopub.execute_input":"2022-08-08T16:20:39.269237Z","iopub.status.idle":"2022-08-08T16:20:39.499565Z","shell.execute_reply.started":"2022-08-08T16:20:39.269184Z","shell.execute_reply":"2022-08-08T16:20:39.498220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lets plot the meter_reading again\nfig,ax=plt.subplots(1,1,figsize=(10,6))\ndf[['timestamp', 'meter_reading']].set_index('timestamp').resample('D').mean()['meter_reading'].plot(ax=ax).set_ylabel('Meter reading')\nax.set_title('Meter_reading after anomaly removal')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:20:39.501275Z","iopub.execute_input":"2022-08-08T16:20:39.501955Z","iopub.status.idle":"2022-08-08T16:20:40.587145Z","shell.execute_reply.started":"2022-08-08T16:20:39.501897Z","shell.execute_reply":"2022-08-08T16:20:40.585956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"## Conversion of timestamp to other features: day_of_month, month,day_of_week,hour","metadata":{}},{"cell_type":"code","source":"def time_features(df):\n    df['hour']= np.uint8(df['timestamp'].dt.hour)\n    df['day']= np.uint8(df['timestamp'].dt.day)   #day of month\n    df['weekday']= np.uint8(df['timestamp'].dt.weekday)   #day of week\n    df['month']= np.uint8(df['timestamp'].dt.month)  \n    \n    df.drop(['timestamp'],axis=1,inplace=True)\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:20:40.588700Z","iopub.execute_input":"2022-08-08T16:20:40.589100Z","iopub.status.idle":"2022-08-08T16:20:40.596462Z","shell.execute_reply.started":"2022-08-08T16:20:40.589038Z","shell.execute_reply":"2022-08-08T16:20:40.595378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_copy=time_features(df)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:20:40.598052Z","iopub.execute_input":"2022-08-08T16:20:40.598742Z","iopub.status.idle":"2022-08-08T16:20:48.263263Z","shell.execute_reply.started":"2022-08-08T16:20:40.598701Z","shell.execute_reply":"2022-08-08T16:20:48.262076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering of primary use column","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nax=sns.barplot(x='primary_use',y='meter_reading',data=df_copy)\nplt.xticks(rotation='vertical')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:20:48.264791Z","iopub.execute_input":"2022-08-08T16:20:48.265197Z","iopub.status.idle":"2022-08-08T16:29:34.122390Z","shell.execute_reply.started":"2022-08-08T16:20:48.265160Z","shell.execute_reply":"2022-08-08T16:29:34.121025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nax=sns.countplot(x='primary_use',data=df_copy)\nplt.xticks(rotation='vertical')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:29:34.124243Z","iopub.execute_input":"2022-08-08T16:29:34.125083Z","iopub.status.idle":"2022-08-08T16:29:47.073707Z","shell.execute_reply.started":"2022-08-08T16:29:34.125012Z","shell.execute_reply":"2022-08-08T16:29:47.072486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_copy['primary_use'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:29:47.075265Z","iopub.execute_input":"2022-08-08T16:29:47.075627Z","iopub.status.idle":"2022-08-08T16:29:48.590269Z","shell.execute_reply.started":"2022-08-08T16:29:47.075595Z","shell.execute_reply":"2022-08-08T16:29:48.588289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reduce the nos of categories to:  education, office, residential, entertainment, public_services and all other clubbed into 'others'\ndef primary_use(use):\n    if use=='Education':\n        return 'Education'\n    elif use=='Office':\n        return 'Office'\n    elif use=='Lodging/residential':\n        return 'Residential'\n    elif use=='Entertainment/public assembly':\n        return 'Entertainment'\n    elif use=='Public services':\n        return 'Public Services'\n    else:\n        return 'Others'","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:29:48.591825Z","iopub.execute_input":"2022-08-08T16:29:48.592244Z","iopub.status.idle":"2022-08-08T16:29:48.598056Z","shell.execute_reply.started":"2022-08-08T16:29:48.592208Z","shell.execute_reply":"2022-08-08T16:29:48.597113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_copy['primary_use']=df_copy['primary_use'].apply(lambda x:primary_use(x))","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:29:48.599466Z","iopub.execute_input":"2022-08-08T16:29:48.600280Z","iopub.status.idle":"2022-08-08T16:29:55.390856Z","shell.execute_reply.started":"2022-08-08T16:29:48.600244Z","shell.execute_reply":"2022-08-08T16:29:55.389893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=df_copy","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:29:55.392317Z","iopub.execute_input":"2022-08-08T16:29:55.392699Z","iopub.status.idle":"2022-08-08T16:29:55.397827Z","shell.execute_reply.started":"2022-08-08T16:29:55.392663Z","shell.execute_reply":"2022-08-08T16:29:55.396425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_copy\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:29:55.399203Z","iopub.execute_input":"2022-08-08T16:29:55.399557Z","iopub.status.idle":"2022-08-08T16:29:55.685281Z","shell.execute_reply.started":"2022-08-08T16:29:55.399524Z","shell.execute_reply":"2022-08-08T16:29:55.684104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nax=sns.countplot(x='primary_use',data=df)\nplt.xticks(rotation='vertical')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:29:55.687156Z","iopub.execute_input":"2022-08-08T16:29:55.687558Z","iopub.status.idle":"2022-08-08T16:30:07.636760Z","shell.execute_reply.started":"2022-08-08T16:29:55.687523Z","shell.execute_reply":"2022-08-08T16:30:07.635775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transformation of target Columns\nInstead of taking meter_reading, let's take the meter/sq_ft as target variable","metadata":{}},{"cell_type":"code","source":"df['meter_per_area']=df['meter_reading']/df['square_feet']","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:30:07.638321Z","iopub.execute_input":"2022-08-08T16:30:07.639613Z","iopub.status.idle":"2022-08-08T16:30:07.689845Z","shell.execute_reply.started":"2022-08-08T16:30:07.639561Z","shell.execute_reply":"2022-08-08T16:30:07.688584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:30:07.695440Z","iopub.execute_input":"2022-08-08T16:30:07.696529Z","iopub.status.idle":"2022-08-08T16:30:07.734971Z","shell.execute_reply.started":"2022-08-08T16:30:07.696477Z","shell.execute_reply":"2022-08-08T16:30:07.732926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,4))\nsns.histplot(df['meter_per_area'],kde=True,color='red',bins=100)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:30:07.737463Z","iopub.execute_input":"2022-08-08T16:30:07.738008Z","iopub.status.idle":"2022-08-08T16:31:26.839485Z","shell.execute_reply.started":"2022-08-08T16:30:07.737957Z","shell.execute_reply":"2022-08-08T16:31:26.838027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(df['meter_reading'])","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:36:45.623842Z","iopub.execute_input":"2022-08-08T16:36:45.624306Z","iopub.status.idle":"2022-08-08T16:36:46.135056Z","shell.execute_reply.started":"2022-08-08T16:36:45.624269Z","shell.execute_reply":"2022-08-08T16:36:46.133971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(np.log1p(df['meter_reading']))","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:38:05.715929Z","iopub.execute_input":"2022-08-08T16:38:05.717422Z","iopub.status.idle":"2022-08-08T16:38:06.555001Z","shell.execute_reply.started":"2022-08-08T16:38:05.717359Z","shell.execute_reply":"2022-08-08T16:38:06.553774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['log_meter']=np.log1p(df['meter_reading'])","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:40:20.883563Z","iopub.execute_input":"2022-08-08T16:40:20.887638Z","iopub.status.idle":"2022-08-08T16:40:21.285822Z","shell.execute_reply.started":"2022-08-08T16:40:20.887572Z","shell.execute_reply":"2022-08-08T16:40:21.284781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv('cleaned_train_ashrae.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:42:31.699590Z","iopub.execute_input":"2022-08-08T16:42:31.700121Z","iopub.status.idle":"2022-08-08T16:44:47.031298Z","shell.execute_reply.started":"2022-08-08T16:42:31.700079Z","shell.execute_reply":"2022-08-08T16:44:47.029981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# # Encode the primary_use column","metadata":{}},{"cell_type":"code","source":"#rename the column\ndf.rename(columns={'primary_use':'use'},inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T16:31:26.842357Z","iopub.execute_input":"2022-08-08T16:31:26.842933Z","iopub.status.idle":"2022-08-08T16:31:26.850578Z","shell.execute_reply.started":"2022-08-08T16:31:26.842878Z","shell.execute_reply":"2022-08-08T16:31:26.849099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:41:49.946434Z","iopub.execute_input":"2022-08-08T15:41:49.946878Z","iopub.status.idle":"2022-08-08T15:41:49.953135Z","shell.execute_reply.started":"2022-08-08T15:41:49.946840Z","shell.execute_reply":"2022-08-08T15:41:49.951923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ohe=OneHotEncoder(drop='first')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T15:44:52.892897Z","iopub.execute_input":"2022-08-08T15:44:52.893369Z","iopub.status.idle":"2022-08-08T15:44:52.899757Z","shell.execute_reply.started":"2022-08-08T15:44:52.893328Z","shell.execute_reply":"2022-08-08T15:44:52.898418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ohe.fit_transform(df['use'],y=df['meter_reading'])","metadata":{},"execution_count":null,"outputs":[]}]}