{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-14T11:16:35.137349Z","iopub.execute_input":"2022-07-14T11:16:35.137789Z","iopub.status.idle":"2022-07-14T11:16:35.147742Z","shell.execute_reply.started":"2022-07-14T11:16:35.137752Z","shell.execute_reply":"2022-07-14T11:16:35.146914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nnp.random.seed(7)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:35.190683Z","iopub.execute_input":"2022-07-14T11:16:35.191766Z","iopub.status.idle":"2022-07-14T11:16:35.196806Z","shell.execute_reply.started":"2022-07-14T11:16:35.191728Z","shell.execute_reply":"2022-07-14T11:16:35.195743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=pd.read_csv(\"../input/acea-water-prediction/Aquifer_Petrignano.csv\")\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:35.198254Z","iopub.execute_input":"2022-07-14T11:16:35.199225Z","iopub.status.idle":"2022-07-14T11:16:35.227581Z","shell.execute_reply.started":"2022-07-14T11:16:35.199193Z","shell.execute_reply":"2022-07-14T11:16:35.226821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.tail()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:35.229138Z","iopub.execute_input":"2022-07-14T11:16:35.229437Z","iopub.status.idle":"2022-07-14T11:16:35.246097Z","shell.execute_reply.started":"2022-07-14T11:16:35.229411Z","shell.execute_reply":"2022-07-14T11:16:35.244972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:35.257760Z","iopub.execute_input":"2022-07-14T11:16:35.258264Z","iopub.status.idle":"2022-07-14T11:16:35.269416Z","shell.execute_reply.started":"2022-07-14T11:16:35.258234Z","shell.execute_reply":"2022-07-14T11:16:35.268532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:35.284364Z","iopub.execute_input":"2022-07-14T11:16:35.284980Z","iopub.status.idle":"2022-07-14T11:16:35.316978Z","shell.execute_reply.started":"2022-07-14T11:16:35.284949Z","shell.execute_reply":"2022-07-14T11:16:35.315895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:35.321020Z","iopub.execute_input":"2022-07-14T11:16:35.321326Z","iopub.status.idle":"2022-07-14T11:16:35.331405Z","shell.execute_reply.started":"2022-07-14T11:16:35.321299Z","shell.execute_reply":"2022-07-14T11:16:35.330403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_value_percentage=data.isnull().sum()/len(data)*100","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:35.356557Z","iopub.execute_input":"2022-07-14T11:16:35.357368Z","iopub.status.idle":"2022-07-14T11:16:35.364044Z","shell.execute_reply.started":"2022-07-14T11:16:35.357331Z","shell.execute_reply":"2022-07-14T11:16:35.363001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_value_percentage","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:35.399016Z","iopub.execute_input":"2022-07-14T11:16:35.399783Z","iopub.status.idle":"2022-07-14T11:16:35.406804Z","shell.execute_reply.started":"2022-07-14T11:16:35.399731Z","shell.execute_reply":"2022-07-14T11:16:35.405953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime,date","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:35.450230Z","iopub.execute_input":"2022-07-14T11:16:35.451428Z","iopub.status.idle":"2022-07-14T11:16:35.455835Z","shell.execute_reply.started":"2022-07-14T11:16:35.451389Z","shell.execute_reply":"2022-07-14T11:16:35.454802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['Date']=pd.to_datetime(data['Date'])","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:35.473302Z","iopub.execute_input":"2022-07-14T11:16:35.474258Z","iopub.status.idle":"2022-07-14T11:16:35.488183Z","shell.execute_reply.started":"2022-07-14T11:16:35.474214Z","shell.execute_reply":"2022-07-14T11:16:35.487377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:35.503240Z","iopub.execute_input":"2022-07-14T11:16:35.503661Z","iopub.status.idle":"2022-07-14T11:16:35.517519Z","shell.execute_reply.started":"2022-07-14T11:16:35.503626Z","shell.execute_reply":"2022-07-14T11:16:35.516623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:35.524276Z","iopub.execute_input":"2022-07-14T11:16:35.524828Z","iopub.status.idle":"2022-07-14T11:16:35.543242Z","shell.execute_reply.started":"2022-07-14T11:16:35.524794Z","shell.execute_reply":"2022-07-14T11:16:35.542484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(nrows=7, ncols=1, figsize=(15, 25))\n\nfor i, column in enumerate(data.drop('Date', axis=1).columns):\n    sns.lineplot(x=data['Date'], y=data[column].fillna(method='ffill'), ax=ax[i], color='dodgerblue')\n    ax[i].set_title('Feature: {}'.format(column), fontsize=14)\n    ax[i].set_ylabel(ylabel=column, fontsize=14)\n                      \n    ax[i].set_xlim([date(2009, 1, 1), date(2020, 6, 30)]) ","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:35.556974Z","iopub.execute_input":"2022-07-14T11:16:35.557644Z","iopub.status.idle":"2022-07-14T11:16:38.326371Z","shell.execute_reply.started":"2022-07-14T11:16:35.557595Z","shell.execute_reply":"2022-07-14T11:16:38.325303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=data[data['Rainfall_Bastia_Umbra'].notna()].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:38.328751Z","iopub.execute_input":"2022-07-14T11:16:38.329216Z","iopub.status.idle":"2022-07-14T11:16:38.337270Z","shell.execute_reply.started":"2022-07-14T11:16:38.329177Z","shell.execute_reply":"2022-07-14T11:16:38.336129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:38.338768Z","iopub.execute_input":"2022-07-14T11:16:38.339377Z","iopub.status.idle":"2022-07-14T11:16:38.356039Z","shell.execute_reply.started":"2022-07-14T11:16:38.339338Z","shell.execute_reply":"2022-07-14T11:16:38.355049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(nrows=7, ncols=1, figsize=(15, 25))\n\nfor i, column in enumerate(data.drop('Date', axis=1).columns):\n    sns.lineplot(x=data['Date'], y=data[column].fillna(method='ffill'), ax=ax[i], color='dodgerblue')\n    ax[i].set_title('Feature: {}'.format(column), fontsize=14)\n    ax[i].set_ylabel(ylabel=column, fontsize=14)\n                      \n    ax[i].set_xlim([date(2009, 1, 1), date(2020, 6, 30)]) \n    plt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:38.358053Z","iopub.execute_input":"2022-07-14T11:16:38.358522Z","iopub.status.idle":"2022-07-14T11:16:41.768737Z","shell.execute_reply.started":"2022-07-14T11:16:38.358476Z","shell.execute_reply":"2022-07-14T11:16:41.767887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> From above diagram we can see that few features are having 0 values which seems to be either outliers or null values. We will replaces these values with Null.\n> we will modify Volume_C10_Petrignano and Hydrometry_Fiume_Chiascio_Petrignano","metadata":{}},{"cell_type":"code","source":"f,ax=plt.subplots(nrows=4,ncols=1,figsize=(15,12))\nold_Hydrometry=data['Hydrometry_Fiume_Chiascio_Petrignano'].copy()\ndata['Hydrometry_Fiume_Chiascio_Petrignano']=data['Hydrometry_Fiume_Chiascio_Petrignano'].replace(0,np.nan)\n\n\n\nsns.lineplot(x=data['Date'],y=old_Hydrometry,ax=ax[0],color='darkorange',label='original')\nsns.lineplot(x=data['Date'],y=data['Hydrometry_Fiume_Chiascio_Petrignano'].fillna(np.inf),ax=ax[1],color='dodgerblue',label='modified')\nax[0].set_title(\"Old Hydrometry_Fiume_Chiascio_Petrignano\")\nax[0].set_ylabel('feature:Hydrometer')\nax[0].set_xlim([date(2009,1,1),date(2020,6,30)])\nax[1].set_title(\"Hydrometry_Fiume_Chiascio_Petrignano\")\nax[1].set_ylabel('feature:Hydrometer')\nax[1].set_xlim([date(2009,1,1),date(2020,6,30)])\n\nold_Volume=data['Volume_C10_Petrignano'].copy()\ndata['Volume_C10_Petrignano']=data['Volume_C10_Petrignano'].replace(0,np.nan)\n\n\nsns.lineplot(x=data['Date'],y=old_Volume,ax=ax[2],color='darkorange',label='original')\nsns.lineplot(x=data['Date'],y=data['Volume_C10_Petrignano'].fillna(np.inf),color='dodgerblue',ax=ax[3],label='modified')\nax[2].set_title(\"Volume_C10_Petrignano\")\nax[2].set_ylabel(\"Feature:Volume_C10_Petrignano\")\nax[2].set_xlim([date(2009,1,1),date(2020,6,30)])\n\nax[3].set_title(\"Volume_C10_Petrignano\")\nax[3].set_ylabel(\"Feature:Volume_C10_Petrignano\")\nax[3].set_xlim([date(2009,1,1),date(2020,6,30)])\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:41.770018Z","iopub.execute_input":"2022-07-14T11:16:41.770512Z","iopub.status.idle":"2022-07-14T11:16:43.545508Z","shell.execute_reply.started":"2022-07-14T11:16:41.770481Z","shell.execute_reply":"2022-07-14T11:16:43.544398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:43.547287Z","iopub.execute_input":"2022-07-14T11:16:43.547975Z","iopub.status.idle":"2022-07-14T11:16:43.557725Z","shell.execute_reply.started":"2022-07-14T11:16:43.547920Z","shell.execute_reply":"2022-07-14T11:16:43.556719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,8))\nsns.heatmap(data.isnull().T)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:43.558989Z","iopub.execute_input":"2022-07-14T11:16:43.559783Z","iopub.status.idle":"2022-07-14T11:16:44.345317Z","shell.execute_reply.started":"2022-07-14T11:16:43.559741Z","shell.execute_reply":"2022-07-14T11:16:44.344153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:44.346695Z","iopub.execute_input":"2022-07-14T11:16:44.346983Z","iopub.status.idle":"2022-07-14T11:16:44.353749Z","shell.execute_reply.started":"2022-07-14T11:16:44.346957Z","shell.execute_reply":"2022-07-14T11:16:44.352748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f,ax=plt.subplots(nrows=4,ncols=1,figsize=(15,12))\nsns.lineplot(x=data['Date'],y=data['Volume_C10_Petrignano'].fillna(0),ax=ax[0],color='darkorange',label='modified')\nsns.lineplot(x=data['Date'],y=data['Volume_C10_Petrignano'].fillna(np.inf),ax=ax[0],color='dodgerblue',label='original')\nax[0].set_title('Fill NaN with 0')\nax[0].set_ylabel(ylabel='Volume_C10_Petrignano')\n\nmean_drainage=data['Volume_C10_Petrignano'].mean()\nsns.lineplot(x=data['Date'],y=data['Volume_C10_Petrignano'].fillna(mean_drainage),ax=ax[1],color='darkorange',label='modified')\nsns.lineplot(x=data['Date'],y=data['Volume_C10_Petrignano'].fillna(np.inf),ax=ax[1],color='dodgerblue',label='original')\nax[1].set_title('Fill NaN with mean')\nax[1].set_ylabel(ylabel='Volume_C10_Petrignano')\n\nsns.lineplot(x=data['Date'],y=data['Volume_C10_Petrignano'].ffill(),ax=ax[2],color='darkorange',label='modified')\nsns.lineplot(x=data['Date'],y=data['Volume_C10_Petrignano'].fillna(np.inf),ax=ax[2],color='dodgerblue',label='modified')\nax[2].set_title(f'ffill')\nax[2].set_ylabel(ylabel='Volume_C10_Petrignano')\n\n\nsns.lineplot(x=data['Date'],y=data['Volume_C10_Petrignano'].interpolate(),ax=ax[3],color='darkorange',label='modified')\nsns.lineplot(x=data['Date'],y=data['Volume_C10_Petrignano'].fillna(np.inf),ax=ax[3],color='dodgerblue',label='original')\nax[3].set_title('Fill Nan with Interpolate')\nax[3].set_ylabel(ylabel='Volume_C10_Petrignano')\n\n\nfor i in range(4):\n    ax[i].set_xlim([date(2019, 5, 1), date(2019, 10, 1)])\n    \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:44.355288Z","iopub.execute_input":"2022-07-14T11:16:44.355644Z","iopub.status.idle":"2022-07-14T11:16:46.551660Z","shell.execute_reply.started":"2022-07-14T11:16:44.355615Z","shell.execute_reply":"2022-07-14T11:16:46.550837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# With interpolate we can see a better replacement of NAN values","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:46.555194Z","iopub.execute_input":"2022-07-14T11:16:46.555777Z","iopub.status.idle":"2022-07-14T11:16:46.560557Z","shell.execute_reply.started":"2022-07-14T11:16:46.555742Z","shell.execute_reply":"2022-07-14T11:16:46.559344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:46.561958Z","iopub.execute_input":"2022-07-14T11:16:46.562362Z","iopub.status.idle":"2022-07-14T11:16:46.581851Z","shell.execute_reply.started":"2022-07-14T11:16:46.562321Z","shell.execute_reply":"2022-07-14T11:16:46.579859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['Depth_to_Groundwater_P24']=data['Depth_to_Groundwater_P24'].interpolate()\ndata['Depth_to_Groundwater_P25']=data['Depth_to_Groundwater_P25'].interpolate()\ndata['Volume_C10_Petrignano']=data['Volume_C10_Petrignano'].interpolate()\ndata['Hydrometry_Fiume_Chiascio_Petrignano']=data['Hydrometry_Fiume_Chiascio_Petrignano'].interpolate()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:46.583859Z","iopub.execute_input":"2022-07-14T11:16:46.584766Z","iopub.status.idle":"2022-07-14T11:16:46.596804Z","shell.execute_reply.started":"2022-07-14T11:16:46.584721Z","shell.execute_reply":"2022-07-14T11:16:46.595756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:46.598052Z","iopub.execute_input":"2022-07-14T11:16:46.599245Z","iopub.status.idle":"2022-07-14T11:16:46.609031Z","shell.execute_reply.started":"2022-07-14T11:16:46.599210Z","shell.execute_reply":"2022-07-14T11:16:46.607602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,ax=plt.subplots(nrows=3,ncols=2,figsize=(15,15))\nsns.lineplot(x=data['Date'],y=data['Volume_C10_Petrignano'],color='dodgerblue',ax=ax[0,0])\nax[0,0].set_title(\"Volume_C10_Petrignano\")\n\n\nresampled_data=data[['Date','Volume_C10_Petrignano']].resample('7D',on='Date').sum().reset_index(drop=False)\nsns.lineplot(x=resampled_data['Date'],y=resampled_data['Volume_C10_Petrignano'],color='dodgerblue',ax=ax[1,0])\nax[1,0].set_title(\"Weekly Volume_C10_Petrignano\")\n\nresampled_data=data[['Date','Volume_C10_Petrignano']].resample('M',on='Date').sum().reset_index(drop=False)\nsns.lineplot(x=resampled_data['Date'],y=resampled_data['Volume_C10_Petrignano'],color='dodgerblue',ax=ax[2,0])\nax[2,0].set_title(\"Monthly Volume_C10_Petrignano\")\n\nfor i in range(3):\n    ax[i,0].set_xlim([date(2009,1,1),date(2020,6,30)])\n\n    \nsns.lineplot(x=data['Date'],y=data['Temperature_Bastia_Umbra'],color='dodgerblue',ax=ax[0,1])\nax[0,1].set_title(\"Temperature_Bastia_Umbra\")\n\n\nresampled_data=data[['Date','Temperature_Bastia_Umbra']].resample('7D',on='Date').sum().reset_index(drop=False)\nsns.lineplot(x=resampled_data['Date'],y=resampled_data['Temperature_Bastia_Umbra'],color='dodgerblue',ax=ax[1,1])\nax[1,1].set_title(\"Weekly Temperature_Bastia_Umbra\")\n\nresampled_data=data[['Date','Temperature_Bastia_Umbra']].resample('M',on='Date').sum().reset_index(drop=False)\nsns.lineplot(x=resampled_data['Date'],y=resampled_data['Temperature_Bastia_Umbra'],color='dodgerblue',ax=ax[2,1])\nax[2,1].set_title(\"Monthly Temperature_Bastia_Umbra\")\n\nfor i in range(3):\n    ax[i,1].set_xlim([date(2009,1,1),date(2020,6,30)])","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:46.611270Z","iopub.execute_input":"2022-07-14T11:16:46.611764Z","iopub.status.idle":"2022-07-14T11:16:48.218526Z","shell.execute_reply.started":"2022-07-14T11:16:46.611723Z","shell.execute_reply":"2022-07-14T11:16:48.217513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:48.219709Z","iopub.execute_input":"2022-07-14T11:16:48.220012Z","iopub.status.idle":"2022-07-14T11:16:48.226810Z","shell.execute_reply.started":"2022-07-14T11:16:48.219984Z","shell.execute_reply":"2022-07-14T11:16:48.225779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"downsample=data[['Date', 'Rainfall_Bastia_Umbra', 'Depth_to_Groundwater_P24',\n       'Depth_to_Groundwater_P25', 'Temperature_Bastia_Umbra',\n       'Temperature_Petrignano', 'Volume_C10_Petrignano',\n       'Hydrometry_Fiume_Chiascio_Petrignano']].resample('7D',on='Date').mean().reset_index(drop=False)\n\ndf=downsample.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:48.228278Z","iopub.execute_input":"2022-07-14T11:16:48.228609Z","iopub.status.idle":"2022-07-14T11:16:48.245121Z","shell.execute_reply.started":"2022-07-14T11:16:48.228570Z","shell.execute_reply":"2022-07-14T11:16:48.243949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:48.246075Z","iopub.execute_input":"2022-07-14T11:16:48.246352Z","iopub.status.idle":"2022-07-14T11:16:48.260718Z","shell.execute_reply.started":"2022-07-14T11:16:48.246327Z","shell.execute_reply":"2022-07-14T11:16:48.259933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Removing Unwanted Columns\ndf=df.drop(['Depth_to_Groundwater_P24','Temperature_Petrignano'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:48.262156Z","iopub.execute_input":"2022-07-14T11:16:48.262621Z","iopub.status.idle":"2022-07-14T11:16:48.271774Z","shell.execute_reply.started":"2022-07-14T11:16:48.262585Z","shell.execute_reply":"2022-07-14T11:16:48.270910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:48.273266Z","iopub.execute_input":"2022-07-14T11:16:48.273970Z","iopub.status.idle":"2022-07-14T11:16:48.295126Z","shell.execute_reply.started":"2022-07-14T11:16:48.273927Z","shell.execute_reply":"2022-07-14T11:16:48.293309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rolling_window=52\nf,ax=plt.subplots(nrows=2,ncols=1,figsize=(15,15))\nsns.lineplot(x=df['Date'],y=df['Temperature_Bastia_Umbra'],ax=ax[0],color='dodgerblue')\nsns.lineplot(x=df['Date'],y=df['Temperature_Bastia_Umbra'].rolling(rolling_window).mean(),ax=ax[0],color='darkorange',label='rolling mean')\nsns.lineplot(x=df['Date'],y=df['Temperature_Bastia_Umbra'].rolling(rolling_window).std(),ax=ax[0],color='black',label='rolling std')\nax[0].set_title(\"Stationarity Check for Temperature\")\nax[0].set_ylabel(\"Temperature\")\n\n\n\nsns.lineplot(x=df['Date'],y=df['Volume_C10_Petrignano'],ax=ax[1],color='dodgerblue')\nsns.lineplot(x=df['Date'],y=df['Volume_C10_Petrignano'].rolling(rolling_window).mean(),ax=ax[1],color='darkorange',label='rolling mean')\nsns.lineplot(x=df['Date'],y=df['Volume_C10_Petrignano'].rolling(rolling_window).std(),ax=ax[1],color='black',label='rolling std')\n\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:48.296700Z","iopub.execute_input":"2022-07-14T11:16:48.297368Z","iopub.status.idle":"2022-07-14T11:16:49.325246Z","shell.execute_reply.started":"2022-07-14T11:16:48.297326Z","shell.execute_reply":"2022-07-14T11:16:49.324228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from statsmodels.tsa.stattools import adfuller","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:49.327048Z","iopub.execute_input":"2022-07-14T11:16:49.327775Z","iopub.status.idle":"2022-07-14T11:16:49.332491Z","shell.execute_reply.started":"2022-07-14T11:16:49.327726Z","shell.execute_reply":"2022-07-14T11:16:49.331622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:49.334098Z","iopub.execute_input":"2022-07-14T11:16:49.334765Z","iopub.status.idle":"2022-07-14T11:16:49.353312Z","shell.execute_reply.started":"2022-07-14T11:16:49.334723Z","shell.execute_reply":"2022-07-14T11:16:49.352198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=df.dropna()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:49.355275Z","iopub.execute_input":"2022-07-14T11:16:49.355802Z","iopub.status.idle":"2022-07-14T11:16:49.364080Z","shell.execute_reply.started":"2022-07-14T11:16:49.355761Z","shell.execute_reply":"2022-07-14T11:16:49.362929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results=adfuller(df['Depth_to_Groundwater_P25'].values)\nresults","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:49.365304Z","iopub.execute_input":"2022-07-14T11:16:49.365654Z","iopub.status.idle":"2022-07-14T11:16:49.404605Z","shell.execute_reply.started":"2022-07-14T11:16:49.365624Z","shell.execute_reply":"2022-07-14T11:16:49.403372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f,ax=plt.subplots(nrows=3,ncols=2,figsize=(15,9))\ndef visualize_adfuller_test(series,title,ax):\n    result=adfuller(series)\n    significance_level=0.05,\n    adf_stat=result[0]\n    p_value=result[1]\n    crit_val_1=result[4]['1%']\n    crit_val_2=result[4]['5%']\n    crit_val_3=result[4]['10%']\n    \n    if (p_value<significance_level) & (adf_stat<crit_val_1):\n        linecolor='green'\n    elif (p_value<significance_level) & (adf_stat<crit_val_2):\n        linecolor='orange'\n    elif (p_value<significance_level) & (adf_stat<crit_val_3):\n        linecolor='red'\n    else:\n        linecolor='purple'\n        \n    sns.lineplot(x=df['Date'],y=series,ax=ax,color=linecolor)\n    \n    ax.set_title(f'ADF stats:{adf_stat:0.3f},p_value: {p_value: 0.3f}\\nCritical values 1%: {crit_val_1:0.3f},Critical values 5%: {crit_val_2:0.3f},critical values 10% {crit_val_3:0.3f}')\n    ax.set_ylabel(ylabel=title)\n\n\nvisualize_adfuller_test(df['Rainfall_Bastia_Umbra'].values,'Rainfall',ax[0,0])    \nvisualize_adfuller_test(df['Depth_to_Groundwater_P25'].values,'Groundwater',ax[1,0])        \nvisualize_adfuller_test(df['Temperature_Bastia_Umbra'].values,'Temperature',ax[0,1])   \nvisualize_adfuller_test(df['Volume_C10_Petrignano'].values,'Volume',ax[1,1])\nvisualize_adfuller_test(df['Hydrometry_Fiume_Chiascio_Petrignano'].values,'Hydrometer',ax[2,0])\n\nf.delaxes(ax[2, 1])\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:49.406495Z","iopub.execute_input":"2022-07-14T11:16:49.407100Z","iopub.status.idle":"2022-07-14T11:16:50.922445Z","shell.execute_reply.started":"2022-07-14T11:16:49.407054Z","shell.execute_reply":"2022-07-14T11:16:50.921612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts_diff=np.diff(df['Depth_to_Groundwater_P25'])\ndf['Depth_to_Groundwater_P25_first_diff']=np.append([0],ts_diff)\n\nf,ax=plt.subplots(nrows=1,ncols=1,figsize=(10,5))\nvisualize_adfuller_test(df['Depth_to_Groundwater_P25_first_diff'],'first_difference',ax)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:50.923630Z","iopub.execute_input":"2022-07-14T11:16:50.924344Z","iopub.status.idle":"2022-07-14T11:16:51.235314Z","shell.execute_reply.started":"2022-07-14T11:16:50.924311Z","shell.execute_reply":"2022-07-14T11:16:51.234508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from statsmodels.tsa.seasonal import seasonal_decompose","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:51.236339Z","iopub.execute_input":"2022-07-14T11:16:51.238107Z","iopub.status.idle":"2022-07-14T11:16:51.242744Z","shell.execute_reply.started":"2022-07-14T11:16:51.238074Z","shell.execute_reply":"2022-07-14T11:16:51.241800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"core_columns=['Rainfall_Bastia_Umbra', 'Depth_to_Groundwater_P25',\n       'Temperature_Bastia_Umbra', 'Volume_C10_Petrignano',\n       'Hydrometry_Fiume_Chiascio_Petrignano']\n\nfor columns in core_columns:\n    decomp=seasonal_decompose(df[columns],period=52,model='additive',extrapolate_trend='freq')\n    df[f\"{columns}_trend\"]=decomp.trend\n    df[f\"{columns}_seasonal\"]=decomp.seasonal\n","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:51.247513Z","iopub.execute_input":"2022-07-14T11:16:51.248007Z","iopub.status.idle":"2022-07-14T11:16:51.282971Z","shell.execute_reply.started":"2022-07-14T11:16:51.247977Z","shell.execute_reply":"2022-07-14T11:16:51.282141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,ax=plt.subplots(nrows=4,ncols=2,figsize=(16,8))\nfor i,column in enumerate(['Temperature_Bastia_Umbra','Depth_to_Groundwater_P25']):\n    res=seasonal_decompose(df[column],period=52,model='additive',extrapolate_trend='freq')\n    \n    ax[0,i].set_title('Decomposition of {}'.format(column), fontsize=16)\n    res.observed.plot(ax=ax[0,i], legend=False, color='dodgerblue')\n    ax[0,i].set_ylabel('Observed', fontsize=14)\n\n    res.trend.plot(ax=ax[1,i], legend=False, color='dodgerblue')\n    ax[1,i].set_ylabel('Trend', fontsize=14)\n\n    res.seasonal.plot(ax=ax[2,i], legend=False, color='dodgerblue')\n    ax[2,i].set_ylabel('Seasonal', fontsize=14)\n    \n    res.resid.plot(ax=ax[3,i], legend=False, color='dodgerblue')\n    ax[3,i].set_ylabel('Residual', fontsize=14)\n\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:51.284288Z","iopub.execute_input":"2022-07-14T11:16:51.284632Z","iopub.status.idle":"2022-07-14T11:16:52.127646Z","shell.execute_reply.started":"2022-07-14T11:16:51.284601Z","shell.execute_reply":"2022-07-14T11:16:52.126525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weeks_in_month = 4\n\nfor column in core_columns:\n    df[f'{column}_seasonal_shift_b_2m'] = df[f'{column}_seasonal'].shift(-2 * weeks_in_month)\n    df[f'{column}_seasonal_shift_b_1m'] = df[f'{column}_seasonal'].shift(-1 * weeks_in_month)\n    df[f'{column}_seasonal_shift_1m'] = df[f'{column}_seasonal'].shift(1 * weeks_in_month)\n    df[f'{column}_seasonal_shift_2m'] = df[f'{column}_seasonal'].shift(2 * weeks_in_month)\n    df[f'{column}_seasonal_shift_3m'] = df[f'{column}_seasonal'].shift(3 * weeks_in_month)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:52.129064Z","iopub.execute_input":"2022-07-14T11:16:52.131425Z","iopub.status.idle":"2022-07-14T11:16:52.153563Z","shell.execute_reply.started":"2022-07-14T11:16:52.131387Z","shell.execute_reply":"2022-07-14T11:16:52.152489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f,ax=plt.subplots(nrows=5,ncols=1,figsize=(15,12))\nf.suptitle(\"Seasonal Components of Features\")\n\n\nfor i,column in enumerate(core_columns):\n    sns.lineplot(x=df['Date'],y=df[column+'_seasonal'],ax=ax[i],color='dodgerblue',label='P25')\n    ax[i].set_ylabel(ylabel=column)\n    \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:16:52.155027Z","iopub.execute_input":"2022-07-14T11:16:52.155742Z","iopub.status.idle":"2022-07-14T11:16:53.332005Z","shell.execute_reply.started":"2022-07-14T11:16:52.155709Z","shell.execute_reply":"2022-07-14T11:16:53.330881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f,ax=plt.subplots(ncols=1,nrows=1,figsize=(15,8))\ncorr=df[core_columns].corr()\nsns.heatmap(corr,annot=True,cmap='Greens',ax=ax)\n\nshift_columns=[['']]\n","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:19:18.456163Z","iopub.execute_input":"2022-07-14T11:19:18.456614Z","iopub.status.idle":"2022-07-14T11:19:18.873990Z","shell.execute_reply.started":"2022-07-14T11:19:18.456574Z","shell.execute_reply":"2022-07-14T11:19:18.872642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from statsmodels.graphics.tsaplots import plot_acf\nfrom statsmodels.graphics.tsaplots import plot_pacf","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:21:35.659578Z","iopub.execute_input":"2022-07-14T11:21:35.660328Z","iopub.status.idle":"2022-07-14T11:21:35.666386Z","shell.execute_reply.started":"2022-07-14T11:21:35.660290Z","shell.execute_reply":"2022-07-14T11:21:35.665509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:24:48.302645Z","iopub.execute_input":"2022-07-14T11:24:48.303033Z","iopub.status.idle":"2022-07-14T11:24:48.311187Z","shell.execute_reply.started":"2022-07-14T11:24:48.303004Z","shell.execute_reply":"2022-07-14T11:24:48.310349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f,ax=plt.subplots(nrows=2,ncols=1,figsize=(15,12))\nplot_acf(df['Depth_to_Groundwater_P25_first_diff'],lags=100,ax=ax[0])\nplot_pacf(df['Depth_to_Groundwater_P25_first_diff'],lags=100,ax=ax[1])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:26:08.046056Z","iopub.execute_input":"2022-07-14T11:26:08.046442Z","iopub.status.idle":"2022-07-14T11:26:08.480291Z","shell.execute_reply.started":"2022-07-14T11:26:08.046412Z","shell.execute_reply":"2022-07-14T11:26:08.479258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import TimeSeriesSplit","metadata":{"execution":{"iopub.status.busy":"2022-07-14T11:28:55.495815Z","iopub.execute_input":"2022-07-14T11:28:55.496303Z","iopub.status.idle":"2022-07-14T11:28:55.588241Z","shell.execute_reply.started":"2022-07-14T11:28:55.496263Z","shell.execute_reply":"2022-07-14T11:28:55.587241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import TimeSeriesSplit\n\nN_SPLITS = 3\n\nX = data['Date']\ny = data['Depth_to_Groundwater_P25']\n\nfolds = TimeSeriesSplit(n_splits=N_SPLITS)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T12:40:59.159073Z","iopub.execute_input":"2022-07-14T12:40:59.159449Z","iopub.status.idle":"2022-07-14T12:40:59.165463Z","shell.execute_reply.started":"2022-07-14T12:40:59.159421Z","shell.execute_reply":"2022-07-14T12:40:59.164307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(nrows=3, ncols=2, figsize=(16, 9))\n\nfor i, (train_index, valid_index) in enumerate(folds.split(X)):\n    X_train, X_valid = X[train_index], X[valid_index]\n    y_train, y_valid = y[train_index], y[valid_index]\n\n    sns.lineplot(\n        x=X_train, \n        y=y_train, \n        ax=ax[i,0], \n        color='dodgerblue', \n        label='train'\n    )\n    sns.lineplot(\n        x=X_train[len(X_train) - len(X_valid):(len(X_train) - len(X_valid) + len(X_valid))], \n        y=y_train[len(X_train) - len(X_valid):(len(X_train) - len(X_valid) + len(X_valid))], \n        ax=ax[i,1], \n        color='dodgerblue', \n        label='train'\n    )\n\n    for j in range(2):\n        sns.lineplot(x= X_valid, y= y_valid, ax=ax[i, j], color='darkorange', label='validation')\n    ax[i, 0].set_title(f\"Rolling Window with Adjusting Training Size (Split {i+1})\", fontsize=16)\n    ax[i, 1].set_title(f\"Rolling Window with Constant Training Size (Split {i+1})\", fontsize=16)\n\nfor i in range(N_SPLITS):\n    ax[i, 0].set_xlim([date(2009, 1, 1), date(2020, 6, 30)])\n    ax[i, 1].set_xlim([date(2009, 1, 1), date(2020, 6, 30)])\n    \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T12:41:05.158183Z","iopub.execute_input":"2022-07-14T12:41:05.158615Z","iopub.status.idle":"2022-07-14T12:41:07.195222Z","shell.execute_reply.started":"2022-07-14T12:41:05.158585Z","shell.execute_reply":"2022-07-14T12:41:07.194117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install fbprophet","metadata":{"execution":{"iopub.status.busy":"2022-07-14T12:43:29.195673Z","iopub.execute_input":"2022-07-14T12:43:29.196384Z","iopub.status.idle":"2022-07-14T12:45:40.645229Z","shell.execute_reply.started":"2022-07-14T12:43:29.196344Z","shell.execute_reply":"2022-07-14T12:45:40.643792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_size = int(0.85 * len(data))\ntest_size = len(data) - train_size\n\nunivariate_df = data[['Date', 'Depth_to_Groundwater_P25']].copy()\nunivariate_df.columns = ['ds', 'y']\n\ntrain = univariate_df.iloc[:train_size, :]\n\nx_train, y_train = pd.DataFrame(univariate_df.iloc[:train_size, 0]), pd.DataFrame(univariate_df.iloc[:train_size, 1])\nx_valid, y_valid = pd.DataFrame(univariate_df.iloc[train_size:, 0]), pd.DataFrame(univariate_df.iloc[train_size:, 1])\n\nprint(len(train), len(x_valid))","metadata":{"execution":{"iopub.status.busy":"2022-07-14T12:42:33.354290Z","iopub.execute_input":"2022-07-14T12:42:33.354700Z","iopub.status.idle":"2022-07-14T12:42:33.368807Z","shell.execute_reply.started":"2022-07-14T12:42:33.354666Z","shell.execute_reply":"2022-07-14T12:42:33.367589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error, mean_squared_error\nimport math\nfrom colorama import Fore\nfrom fbprophet import Prophet\n\n\n# Train the model\nmodel = Prophet()\nmodel.fit(train)\n\n# x_valid = model.make_future_dataframe(periods=test_size, freq='w')\n\n# Predict on valid set\ny_pred = model.predict(x_valid)\n\n# Calcuate metrics\nscore_mae = mean_absolute_error(y_valid, y_pred.tail(test_size)['yhat'])\nscore_rmse = math.sqrt(mean_squared_error(y_valid, y_pred.tail(test_size)['yhat']))\n\nprint(Fore.GREEN + 'RMSE: {}'.format(score_rmse))","metadata":{"execution":{"iopub.status.busy":"2022-07-14T12:47:22.138535Z","iopub.execute_input":"2022-07-14T12:47:22.138951Z","iopub.status.idle":"2022-07-14T12:47:29.798304Z","shell.execute_reply.started":"2022-07-14T12:47:22.138921Z","shell.execute_reply":"2022-07-14T12:47:29.797199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the forecast\nf, ax = plt.subplots(1)\nf.set_figheight(6)\nf.set_figwidth(15)\n\nmodel.plot(y_pred, ax=ax)\nsns.lineplot(x=x_valid['ds'], y=y_valid['y'], ax=ax, color='orange', label='Ground truth') #navajowhite\n\nax.set_title(f'Prediction \\n MAE: {score_mae:.2f}, RMSE: {score_rmse:.2f}', fontsize=14)\nax.set_xlabel(xlabel='Date', fontsize=14)\nax.set_ylabel(ylabel='Depth to Groundwater', fontsize=14)\n\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-14T12:48:08.290885Z","iopub.execute_input":"2022-07-14T12:48:08.291293Z","iopub.status.idle":"2022-07-14T12:48:08.718079Z","shell.execute_reply.started":"2022-07-14T12:48:08.291263Z","shell.execute_reply":"2022-07-14T12:48:08.716898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}