{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Introduction:\nThis notebook is created to analyze Housing statistics for the Canadian region.\n\n### About the data:\n\nDataset contains housing statistics for the Canadian region. \n\n### Objective:\n\nTo understand how the housing and rental prices are influenced by the various housing statistics. \n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-17T13:03:29.094824Z","iopub.execute_input":"2022-07-17T13:03:29.095249Z","iopub.status.idle":"2022-07-17T13:03:29.113641Z","shell.execute_reply.started":"2022-07-17T13:03:29.095214Z","shell.execute_reply":"2022-07-17T13:03:29.112645Z"}}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-07-30T15:51:04.365550Z","iopub.execute_input":"2022-07-30T15:51:04.366998Z","iopub.status.idle":"2022-07-30T15:51:04.422192Z","shell.execute_reply.started":"2022-07-30T15:51:04.366886Z","shell.execute_reply":"2022-07-30T15:51:04.421039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Importing master file ","metadata":{}},{"cell_type":"code","source":"df_master = pd.read_csv(\"/kaggle/input/housing-affordability-in-canada/housing-supply-price-rental.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-30T15:51:04.424664Z","iopub.execute_input":"2022-07-30T15:51:04.425532Z","iopub.status.idle":"2022-07-30T15:51:04.456065Z","shell.execute_reply.started":"2022-07-30T15:51:04.425484Z","shell.execute_reply":"2022-07-30T15:51:04.454934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_master.year.unique()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T15:51:04.457722Z","iopub.execute_input":"2022-07-30T15:51:04.458435Z","iopub.status.idle":"2022-07-30T15:51:04.479379Z","shell.execute_reply.started":"2022-07-30T15:51:04.458400Z","shell.execute_reply":"2022-07-30T15:51:04.478384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#converting years into integer\n\nfrom datetime import datetime\n\n\ndf_master = df_master[df_master['year'] != 1993.1].astype({'year': int})\n\n#converting year into datetime\ndf_master['year'] = df_master['year'].transform(lambda x : datetime.strptime(str(x), '%Y'))\n\ndf_master.year.unique()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T15:51:04.481862Z","iopub.execute_input":"2022-07-30T15:51:04.482965Z","iopub.status.idle":"2022-07-30T15:51:04.517695Z","shell.execute_reply.started":"2022-07-30T15:51:04.482923Z","shell.execute_reply":"2022-07-30T15:51:04.516676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pulling out statistics for the Toronto region\ndf_master[df_master.region == 'toronto'].head(5)","metadata":{"execution":{"iopub.status.busy":"2022-07-30T15:51:04.519265Z","iopub.execute_input":"2022-07-30T15:51:04.521668Z","iopub.status.idle":"2022-07-30T15:51:04.559365Z","shell.execute_reply.started":"2022-07-30T15:51:04.521634Z","shell.execute_reply":"2022-07-30T15:51:04.558025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plotting graphs and high-level EDA","metadata":{}},{"cell_type":"code","source":"#plotting all variables into a single graph to see the trends over the time\nimport plotly.express as px\nimport plotly.graph_objects as go\n\nfig = px.bar(df_master, x = df_master['year'])\nbuttonlist1 = []\n\n\nfor col in df_master.columns:\n    buttonlist1.append(\n        dict(\n        args = ['y', [df_master[str(col)]]], \n        label = str(col),\n        method ='restyle'\n        ))\n        \nfig.update_layout(\n        title = \"Housing features by year\", \n        yaxis_title = \"value\", \n        xaxis_title = \"labels\", \n        updatemenus = [\n            go.layout.Updatemenu(\n                buttons = buttonlist1, \n                direction = \"down\",\n                pad = {\"r\":10, \"t\":10}, \n                showactive = True, \n                x = 0.1, \n                xanchor = \"left\", \n                y = 1.1, \n                yanchor = \"top\"\n            )\n        ],\n    autosize = True\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-30T15:51:04.561268Z","iopub.execute_input":"2022-07-30T15:51:04.561729Z","iopub.status.idle":"2022-07-30T15:51:08.297837Z","shell.execute_reply.started":"2022-07-30T15:51:04.561685Z","shell.execute_reply":"2022-07-30T15:51:08.295758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Observations\n\n","metadata":{}},{"cell_type":"markdown","source":"# Correlation analysis","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n\nf = plt.figure(figsize=(19, 15))\nplt.matshow(df_master.corr(), fignum=f.number)\nplt.xticks(range(df_master.select_dtypes(['number']).shape[1]), df_master.select_dtypes(['number']).columns, fontsize=14, rotation=90)\nplt.yticks(range(df_master\n                 .select_dtypes(['number']).shape[1]), df_master.select_dtypes(['number']).columns, fontsize=14)\ncb = plt.colorbar()\ncb.ax.tick_params(labelsize=14, labelrotation = 90)\nplt.title('Correlation Matrix', fontsize=16);\n","metadata":{"execution":{"iopub.status.busy":"2022-07-30T15:51:08.299519Z","iopub.execute_input":"2022-07-30T15:51:08.300686Z","iopub.status.idle":"2022-07-30T15:51:09.521202Z","shell.execute_reply.started":"2022-07-30T15:51:08.300638Z","shell.execute_reply":"2022-07-30T15:51:09.519982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pulling out correlation for the Toronto region\ndf_city = df_master[(df_master.region == 'toronto') & (df_master.year > '2003') ]\n\nf = plt.figure(figsize=(19, 15))\nplt.matshow(df_city.corr(), fignum=f.number)\nplt.xticks(range(df_city.select_dtypes(['number']).shape[1]), df_city.select_dtypes(['number']).columns, fontsize=14, rotation=90)\nplt.yticks(range(df_city\n                 .select_dtypes(['number']).shape[1]), df_city.select_dtypes(['number']).columns, fontsize=14)\ncb = plt.colorbar()\ncb.ax.tick_params(labelsize=14, labelrotation = 90)\nplt.title('Correlation Matrix', fontsize=16);","metadata":{"execution":{"iopub.status.busy":"2022-07-30T15:51:09.522752Z","iopub.execute_input":"2022-07-30T15:51:09.523120Z","iopub.status.idle":"2022-07-30T15:51:10.719537Z","shell.execute_reply.started":"2022-07-30T15:51:09.523074Z","shell.execute_reply":"2022-07-30T15:51:10.718120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Which region HPI is highly correlated to the new total dwellings?","metadata":{}},{"cell_type":"code","source":"for region in df_master.region.unique():\n    print(region)\n    print(df_master[df_master.region == region][['total_dwelling', 'HPI_change']].corr()['total_dwelling'])","metadata":{"execution":{"iopub.status.busy":"2022-07-30T15:51:10.721457Z","iopub.execute_input":"2022-07-30T15:51:10.721809Z","iopub.status.idle":"2022-07-30T15:51:10.810905Z","shell.execute_reply.started":"2022-07-30T15:51:10.721779Z","shell.execute_reply":"2022-07-30T15:51:10.809684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Observation\n1. Toronto region has high correlation with the total dwellings. It means that the growth in the housing prices motivates builder to construct new housing projects. \n2. Manitoba region has negative correlation with the toal dwellings. It might be because add up housing inventory is outpacing the supply with demand","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"### Are new completed homes matching with the rise in the population?","metadata":{}},{"cell_type":"code","source":"for region in df_master.region.unique():\n    print(region)\n    print(df_master[df_master.region == region][['completed', 'population']].corr()['completed'])","metadata":{"execution":{"iopub.status.busy":"2022-07-30T15:51:10.813726Z","iopub.execute_input":"2022-07-30T15:51:10.814066Z","iopub.status.idle":"2022-07-30T15:51:10.925427Z","shell.execute_reply.started":"2022-07-30T15:51:10.814035Z","shell.execute_reply":"2022-07-30T15:51:10.923869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# average HPI increase by region\ndf_mean_hpi =  df_master.groupby(['region']).agg({'HPI_change' : 'mean'}).reset_index().dropna().sort_values(by = 'HPI_change')\ndf_mean_hpi","metadata":{"execution":{"iopub.status.busy":"2022-07-30T15:51:10.927042Z","iopub.execute_input":"2022-07-30T15:51:10.927422Z","iopub.status.idle":"2022-07-30T15:51:10.953223Z","shell.execute_reply.started":"2022-07-30T15:51:10.927386Z","shell.execute_reply":"2022-07-30T15:51:10.952078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Observation\n\nPrairie regions have high correlation between the completed house and the population. However, the price increase in those provinces have outpaced with regions which have lower correlation.","metadata":{}},{"cell_type":"markdown","source":"# Plotting HPI","metadata":{}},{"cell_type":"code","source":"hpi_df = pd.read_csv(\"/kaggle/input/housing-affordability-in-canada/HPI 1981-2022 by regions.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-30T15:56:15.211498Z","iopub.execute_input":"2022-07-30T15:56:15.211904Z","iopub.status.idle":"2022-07-30T15:56:15.236581Z","shell.execute_reply.started":"2022-07-30T15:56:15.211873Z","shell.execute_reply":"2022-07-30T15:56:15.235305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hpi_df.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-30T15:57:25.732270Z","iopub.execute_input":"2022-07-30T15:57:25.732809Z","iopub.status.idle":"2022-07-30T15:57:25.744042Z","shell.execute_reply.started":"2022-07-30T15:57:25.732737Z","shell.execute_reply":"2022-07-30T15:57:25.742664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\n\ndf_hpi_plot = hpi_df[hpi_df.Type == 'House and Land'].groupby('year').agg({'Canada' : 'mean'}).reset_index()\nfig = px.line(df, x=\"year\", y=\"Canada\", title='Canada HPI over the years')\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-30T15:56:17.032074Z","iopub.execute_input":"2022-07-30T15:56:17.032857Z","iopub.status.idle":"2022-07-30T15:56:17.140410Z","shell.execute_reply.started":"2022-07-30T15:56:17.032795Z","shell.execute_reply":"2022-07-30T15:56:17.138317Z"},"trusted":true},"execution_count":null,"outputs":[]}]}