{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**In this project, you are tasked to model the shortfall between the energy generated by means of fossil fuels and various renewable sources - for the country of Spain.**\n\nIn the context of this problem, the three hourly load shortfall (target) is the difference between the energy generated by means of fossil fuels and renewable sources.\n\nWe might have wind speed data for Barcelona but not rainfall data whereas we have both rainfall and wind speed information for Valencia.\n\n**If you have any questions, feel free to drop a comment**\n","metadata":{}},{"cell_type":"markdown","source":"**Notebook Structure**\n1. <a href='#di'>Data Import</a><br>\n2. <a href='#eda'>EDA and Data Cleaning</a><br>\n3. <a href='#encode'>Encoding</a><br>\n4. <a href='#mc'>Testing Multicollinearity</a><br>\n5. <a href='#params'>Model Fitting and Parameter Tuning</a><br>\n6. <a href='#ps'>Prediction and Submission</a><br>","metadata":{}},{"cell_type":"markdown","source":"# Data Import <span id=\"di\">  </span>","metadata":{}},{"cell_type":"code","source":"# Importing data - train, test and submission\nimport pandas as pd\npd.set_option('display.max_columns', None)\ntrain = pd.read_csv('../input/edsa-individual-electricity-shortfall-challenge/df_train.csv')\ntest = pd.read_csv('../input/edsa-individual-electricity-shortfall-challenge/df_test.csv')\nsubmission = pd.read_csv('../input/edsa-individual-electricity-shortfall-challenge/sample_submission_load_shortfall (1).csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-13T17:52:24.681486Z","iopub.execute_input":"2022-07-13T17:52:24.681977Z","iopub.status.idle":"2022-07-13T17:52:24.905710Z","shell.execute_reply.started":"2022-07-13T17:52:24.681881Z","shell.execute_reply":"2022-07-13T17:52:24.904496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:24.910053Z","iopub.execute_input":"2022-07-13T17:52:24.910411Z","iopub.status.idle":"2022-07-13T17:52:24.994979Z","shell.execute_reply.started":"2022-07-13T17:52:24.910379Z","shell.execute_reply":"2022-07-13T17:52:24.993874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:24.996411Z","iopub.execute_input":"2022-07-13T17:52:24.997002Z","iopub.status.idle":"2022-07-13T17:52:25.067849Z","shell.execute_reply.started":"2022-07-13T17:52:24.996969Z","shell.execute_reply":"2022-07-13T17:52:25.066748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:25.070978Z","iopub.execute_input":"2022-07-13T17:52:25.071480Z","iopub.status.idle":"2022-07-13T17:52:25.085646Z","shell.execute_reply.started":"2022-07-13T17:52:25.071407Z","shell.execute_reply":"2022-07-13T17:52:25.084044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Getting columns\ny = train.iloc[:,-1].values\ny","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:25.087617Z","iopub.execute_input":"2022-07-13T17:52:25.087939Z","iopub.status.idle":"2022-07-13T17:52:25.096570Z","shell.execute_reply.started":"2022-07-13T17:52:25.087910Z","shell.execute_reply":"2022-07-13T17:52:25.095398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Getting column for submussion\ntime = test.iloc[:,1].values\ntime","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:25.098459Z","iopub.execute_input":"2022-07-13T17:52:25.098909Z","iopub.status.idle":"2022-07-13T17:52:25.109613Z","shell.execute_reply.started":"2022-07-13T17:52:25.098864Z","shell.execute_reply":"2022-07-13T17:52:25.108364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA and Data Cleaning <span id=\"eda\">  </span>","metadata":{}},{"cell_type":"code","source":"# Dropping identification columns\ntrain.drop(columns = ['Unnamed: 0', 'time'], inplace = True)\ntest.drop(columns = ['Unnamed: 0', 'time'], inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:25.111091Z","iopub.execute_input":"2022-07-13T17:52:25.111410Z","iopub.status.idle":"2022-07-13T17:52:25.133180Z","shell.execute_reply.started":"2022-07-13T17:52:25.111377Z","shell.execute_reply":"2022-07-13T17:52:25.131809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Correcting types\ntrain.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:25.135317Z","iopub.execute_input":"2022-07-13T17:52:25.135817Z","iopub.status.idle":"2022-07-13T17:52:25.149112Z","shell.execute_reply.started":"2022-07-13T17:52:25.135771Z","shell.execute_reply":"2022-07-13T17:52:25.147914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:25.150546Z","iopub.execute_input":"2022-07-13T17:52:25.151226Z","iopub.status.idle":"2022-07-13T17:52:25.170202Z","shell.execute_reply.started":"2022-07-13T17:52:25.151179Z","shell.execute_reply":"2022-07-13T17:52:25.168898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Getting null precentages where there is null\ntrain.isnull().sum()[train.isnull().sum() != 0].sort_values(ascending=False) *100 / train.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:25.175692Z","iopub.execute_input":"2022-07-13T17:52:25.177394Z","iopub.status.idle":"2022-07-13T17:52:25.198250Z","shell.execute_reply.started":"2022-07-13T17:52:25.177204Z","shell.execute_reply":"2022-07-13T17:52:25.196485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.isnull().sum()[test.isnull().sum() != 0].sort_values(ascending=False) *100 / test.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:25.200368Z","iopub.execute_input":"2022-07-13T17:52:25.201103Z","iopub.status.idle":"2022-07-13T17:52:25.219921Z","shell.execute_reply.started":"2022-07-13T17:52:25.201054Z","shell.execute_reply":"2022-07-13T17:52:25.218032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Shape of data\ncounts = train.iloc[:,:-1].nunique()\ncounts","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:25.221390Z","iopub.execute_input":"2022-07-13T17:52:25.222108Z","iopub.status.idle":"2022-07-13T17:52:25.261793Z","shell.execute_reply.started":"2022-07-13T17:52:25.222065Z","shell.execute_reply":"2022-07-13T17:52:25.260510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Shape of data\ncounts = test.iloc[:,:-1].nunique()\ncounts","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:25.263670Z","iopub.execute_input":"2022-07-13T17:52:25.264069Z","iopub.status.idle":"2022-07-13T17:52:25.284136Z","shell.execute_reply.started":"2022-07-13T17:52:25.264037Z","shell.execute_reply":"2022-07-13T17:52:25.282747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.duplicated().sum() # duplicate rows","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:25.285829Z","iopub.execute_input":"2022-07-13T17:52:25.286190Z","iopub.status.idle":"2022-07-13T17:52:25.322954Z","shell.execute_reply.started":"2022-07-13T17:52:25.286160Z","shell.execute_reply":"2022-07-13T17:52:25.321873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.duplicated().sum() # duplicate rows","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:25.324257Z","iopub.execute_input":"2022-07-13T17:52:25.325141Z","iopub.status.idle":"2022-07-13T17:52:25.346132Z","shell.execute_reply.started":"2022-07-13T17:52:25.325091Z","shell.execute_reply":"2022-07-13T17:52:25.344828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Getting mean and standrad deviation\ntrain.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:25.347793Z","iopub.execute_input":"2022-07-13T17:52:25.348355Z","iopub.status.idle":"2022-07-13T17:52:25.525568Z","shell.execute_reply.started":"2022-07-13T17:52:25.348304Z","shell.execute_reply":"2022-07-13T17:52:25.521887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Getting mean and standrad deviation\ntest.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:25.527575Z","iopub.execute_input":"2022-07-13T17:52:25.528260Z","iopub.status.idle":"2022-07-13T17:52:25.703254Z","shell.execute_reply.started":"2022-07-13T17:52:25.528202Z","shell.execute_reply":"2022-07-13T17:52:25.701878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Frequency distribution of all columns - visual check if any value is more than 85%\nimport matplotlib.pyplot as plt\ncount = 0\nfig, ax = plt.subplots(10, 5, figsize = (18, 20))\nfor i in range(train.shape[1]):\n    plt.subplot(10, 5, count + 1)\n    plt.hist(train.iloc[:, i].dropna(axis = 0), rwidth = 0.9, color = 'green')\n    plt.xlabel(train.columns[i], fontsize = 15)\n    count += 1\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:25.704586Z","iopub.execute_input":"2022-07-13T17:52:25.704927Z","iopub.status.idle":"2022-07-13T17:52:32.978843Z","shell.execute_reply.started":"2022-07-13T17:52:25.704883Z","shell.execute_reply":"2022-07-13T17:52:32.977513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Encoding <span id=\"encode\">  </span>","metadata":{}},{"cell_type":"code","source":"# Label Encoding\nfrom sklearn.preprocessing import LabelEncoder\nlabelencoder = LabelEncoder()\ntrain['Valencia_wind_deg'] = labelencoder.fit_transform(train['Valencia_wind_deg'])\ntrain['Seville_pressure'] = labelencoder.fit_transform(train['Seville_pressure'])","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:33.020486Z","iopub.execute_input":"2022-07-13T17:52:33.020791Z","iopub.status.idle":"2022-07-13T17:52:33.436134Z","shell.execute_reply.started":"2022-07-13T17:52:33.020764Z","shell.execute_reply":"2022-07-13T17:52:33.435042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['Valencia_wind_deg'] = labelencoder.fit_transform(test['Valencia_wind_deg'])\ntest['Seville_pressure'] = labelencoder.fit_transform(test['Seville_pressure'])","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:33.440870Z","iopub.execute_input":"2022-07-13T17:52:33.441220Z","iopub.status.idle":"2022-07-13T17:52:33.451339Z","shell.execute_reply.started":"2022-07-13T17:52:33.441190Z","shell.execute_reply":"2022-07-13T17:52:33.449777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(columns = ['load_shortfall_3h'], inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:33.453112Z","iopub.execute_input":"2022-07-13T17:52:33.453740Z","iopub.status.idle":"2022-07-13T17:52:33.464715Z","shell.execute_reply.started":"2022-07-13T17:52:33.453700Z","shell.execute_reply":"2022-07-13T17:52:33.463484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testing Multicollinearity <span id=\"mc\">  </span>","metadata":{}},{"cell_type":"code","source":"# VIF\nimport numpy as np\ninv_corr_matrix = np.linalg.inv(train.corr())\ninv_corr_matrix = pd.DataFrame(data = inv_corr_matrix, index = train.columns, columns = train.columns)\n# corr = X_df.astype(float).corr()\nvif_coefficients =  np.diag(np.array(inv_corr_matrix))\nmutlicollinear_column_indices = [i for i in range(len(vif_coefficients)) if vif_coefficients[i] > 10]","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:33.465826Z","iopub.execute_input":"2022-07-13T17:52:33.466793Z","iopub.status.idle":"2022-07-13T17:52:33.543259Z","shell.execute_reply.started":"2022-07-13T17:52:33.466752Z","shell.execute_reply":"2022-07-13T17:52:33.541910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop data with non collinear variables\ntrain.drop(train.columns[mutlicollinear_column_indices], axis = 1, inplace = True)\ntest.drop(test.columns[mutlicollinear_column_indices], axis = 1, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:33.545008Z","iopub.execute_input":"2022-07-13T17:52:33.546382Z","iopub.status.idle":"2022-07-13T17:52:33.559158Z","shell.execute_reply.started":"2022-07-13T17:52:33.546324Z","shell.execute_reply":"2022-07-13T17:52:33.557703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filling series with mean or mode\ntrain['Valencia_pressure'].fillna(train['Valencia_pressure'].mean(), inplace = True)\ntest['Valencia_pressure'].fillna(test['Valencia_pressure'].mean(), inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:33.561007Z","iopub.execute_input":"2022-07-13T17:52:33.562330Z","iopub.status.idle":"2022-07-13T17:52:33.576331Z","shell.execute_reply.started":"2022-07-13T17:52:33.562270Z","shell.execute_reply":"2022-07-13T17:52:33.574677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Fitting and Parameter Tuning <span id=\"params\">  </span>","metadata":{}},{"cell_type":"code","source":"# XGB Regressor\nfrom xgboost import XGBRegressor\nregressor = XGBRegressor(learning_rate= 0.05, max_depth= 4) #paramters added after tuning in next step\nregressor.fit(train, y)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:51.294095Z","iopub.execute_input":"2022-07-13T17:52:51.294500Z","iopub.status.idle":"2022-07-13T17:52:52.312503Z","shell.execute_reply.started":"2022-07-13T17:52:51.294464Z","shell.execute_reply":"2022-07-13T17:52:52.311507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Parameter Tuning\nfrom sklearn.model_selection import GridSearchCV\nparameters = [{'max_depth': [4, 5, 6, 7, 8], 'learning_rate': [0.01, 0.05, 0.1, 0.15]}]\ngrid_search = GridSearchCV(estimator = regressor, param_grid = parameters, scoring = 'neg_mean_squared_error', cv = 10, n_jobs= -1)\ngrid_search = grid_search.fit(X_opt, y)\nbest_accuracy =  grid_search.best_score_\nbest_parameters = grid_search.best_params_\nprint('best_accuracy: ', best_accuracy, 'best_parameters: ', best_parameters)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:37:20.116533Z","iopub.execute_input":"2022-07-13T16:37:20.116903Z","iopub.status.idle":"2022-07-13T16:39:36.908641Z","shell.execute_reply.started":"2022-07-13T16:37:20.116874Z","shell.execute_reply":"2022-07-13T16:39:36.907649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction and Submission <span id=\"ps\">  </span>","metadata":{}},{"cell_type":"code","source":"# Prediction\ny_pred = regressor.predict(test)\ny_pred","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:57.573252Z","iopub.execute_input":"2022-07-13T17:52:57.573685Z","iopub.status.idle":"2022-07-13T17:52:57.593639Z","shell.execute_reply.started":"2022-07-13T17:52:57.573648Z","shell.execute_reply":"2022-07-13T17:52:57.590798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Output / Submitting / Submission\noutput = pd.DataFrame({'time': time, 'load_shortfall_3h': y_pred})\noutput.to_csv('submission.csv', index=False)\nprint(\"Submitted successfully!\")","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:52:59.774140Z","iopub.execute_input":"2022-07-13T17:52:59.774561Z","iopub.status.idle":"2022-07-13T17:52:59.794515Z","shell.execute_reply.started":"2022-07-13T17:52:59.774527Z","shell.execute_reply":"2022-07-13T17:52:59.793116Z"},"trusted":true},"execution_count":null,"outputs":[]}]}