{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-30T23:25:06.898643Z","iopub.execute_input":"2022-07-30T23:25:06.899113Z","iopub.status.idle":"2022-07-30T23:25:06.938942Z","shell.execute_reply.started":"2022-07-30T23:25:06.899023Z","shell.execute_reply":"2022-07-30T23:25:06.937545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# getting data\n\nbase_path = '../input/house-prices-advanced-regression-techniques'\ndf_train = pd.read_csv(os.path.join(base_path,'train.csv'), index_col='Id')\ndf_test  = pd.read_csv(os.path.join(base_path,'test.csv'), index_col='Id')","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:06.941209Z","iopub.execute_input":"2022-07-30T23:25:06.942037Z","iopub.status.idle":"2022-07-30T23:25:07.043880Z","shell.execute_reply.started":"2022-07-30T23:25:06.941985Z","shell.execute_reply":"2022-07-30T23:25:07.042382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:07.046506Z","iopub.execute_input":"2022-07-30T23:25:07.047103Z","iopub.status.idle":"2022-07-30T23:25:07.118381Z","shell.execute_reply.started":"2022-07-30T23:25:07.047038Z","shell.execute_reply":"2022-07-30T23:25:07.116678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:07.120498Z","iopub.execute_input":"2022-07-30T23:25:07.121038Z","iopub.status.idle":"2022-07-30T23:25:07.161265Z","shell.execute_reply.started":"2022-07-30T23:25:07.120988Z","shell.execute_reply":"2022-07-30T23:25:07.159569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train.columns)","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:07.164754Z","iopub.execute_input":"2022-07-30T23:25:07.165731Z","iopub.status.idle":"2022-07-30T23:25:07.173505Z","shell.execute_reply.started":"2022-07-30T23:25:07.165661Z","shell.execute_reply":"2022-07-30T23:25:07.172075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = ['MSZoning', 'LotArea', 'Street', 'Utilities', 'LotConfig', 'LandSlope',\n       'Neighborhood', 'YearBuilt', 'YearRemodAdd', 'RoofStyle', 'RoofMatl', 'ExterQual', \n            'Foundation',  'Electrical', 'FullBath', 'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr',\n        'GarageType', 'GarageCars', 'GarageArea', 'MoSold', 'YrSold',\n            'SaleType', 'SaleCondition', 'SalePrice'\n           ]","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:07.175974Z","iopub.execute_input":"2022-07-30T23:25:07.176952Z","iopub.status.idle":"2022-07-30T23:25:07.193507Z","shell.execute_reply.started":"2022-07-30T23:25:07.176898Z","shell.execute_reply":"2022-07-30T23:25:07.191752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualizing NaN data\ndata = df_train[features]\ndata.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:07.195302Z","iopub.execute_input":"2022-07-30T23:25:07.196449Z","iopub.status.idle":"2022-07-30T23:25:07.214262Z","shell.execute_reply.started":"2022-07-30T23:25:07.196413Z","shell.execute_reply":"2022-07-30T23:25:07.213011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:07.215961Z","iopub.execute_input":"2022-07-30T23:25:07.216370Z","iopub.status.idle":"2022-07-30T23:25:07.248109Z","shell.execute_reply.started":"2022-07-30T23:25:07.216338Z","shell.execute_reply":"2022-07-30T23:25:07.246849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# another imports\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom scipy.stats import norm\nfrom scipy import stats","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:55:05.387496Z","iopub.execute_input":"2022-07-30T23:55:05.387904Z","iopub.status.idle":"2022-07-30T23:55:05.395113Z","shell.execute_reply.started":"2022-07-30T23:55:05.387872Z","shell.execute_reply":"2022-07-30T23:55:05.393655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualizing all sales by year","metadata":{}},{"cell_type":"code","source":"# sales by year\ncolumns = ['YrSold', 'SaleCondition', 'SalePrice']\ndf_pivot = data[columns]\nyear_sales = df_pivot.groupby('YrSold').agg(['sum', 'min', 'mean', 'max'])\nyear_sales","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:07.265073Z","iopub.execute_input":"2022-07-30T23:25:07.266633Z","iopub.status.idle":"2022-07-30T23:25:07.309453Z","shell.execute_reply.started":"2022-07-30T23:25:07.266477Z","shell.execute_reply":"2022-07-30T23:25:07.308095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"year_sales.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:07.314548Z","iopub.execute_input":"2022-07-30T23:25:07.314883Z","iopub.status.idle":"2022-07-30T23:25:07.324585Z","shell.execute_reply.started":"2022-07-30T23:25:07.314854Z","shell.execute_reply":"2022-07-30T23:25:07.323356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"year_sales.plot(kind='line', y=('SalePrice',  'sum'), color='g', title='SalePrice')","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:07.325921Z","iopub.execute_input":"2022-07-30T23:25:07.328016Z","iopub.status.idle":"2022-07-30T23:25:07.640085Z","shell.execute_reply.started":"2022-07-30T23:25:07.327948Z","shell.execute_reply":"2022-07-30T23:25:07.638471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualizing sales by year in each Neighborhood","metadata":{}},{"cell_type":"code","source":"# sales by year\ncolumns = ['Neighborhood', 'YrSold', 'SaleCondition', 'SalePrice']\ndf_pivot = data[columns]\nyear_sales = pd.DataFrame(df_pivot.groupby(['Neighborhood', 'YrSold']).SalePrice.sum())\nyear_sales","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:07.642331Z","iopub.execute_input":"2022-07-30T23:25:07.642758Z","iopub.status.idle":"2022-07-30T23:25:07.668484Z","shell.execute_reply.started":"2022-07-30T23:25:07.642723Z","shell.execute_reply":"2022-07-30T23:25:07.667023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"year_sales.reset_index(level='Neighborhood', inplace=True)\nyear_sales","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:07.670837Z","iopub.execute_input":"2022-07-30T23:25:07.671335Z","iopub.status.idle":"2022-07-30T23:25:07.691763Z","shell.execute_reply.started":"2022-07-30T23:25:07.671302Z","shell.execute_reply":"2022-07-30T23:25:07.690307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pivot = pd.pivot_table(year_sales, index=year_sales.index, columns=['Neighborhood'], aggfunc='first')\ndf_pivot","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:07.693349Z","iopub.execute_input":"2022-07-30T23:25:07.693862Z","iopub.status.idle":"2022-07-30T23:25:07.766966Z","shell.execute_reply.started":"2022-07-30T23:25:07.693817Z","shell.execute_reply":"2022-07-30T23:25:07.765764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pivot.plot(kind='bar',figsize=(15,10))","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:07.768115Z","iopub.execute_input":"2022-07-30T23:25:07.768487Z","iopub.status.idle":"2022-07-30T23:25:08.540666Z","shell.execute_reply.started":"2022-07-30T23:25:07.768455Z","shell.execute_reply":"2022-07-30T23:25:08.538772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualizing the 10 Neighborhoods that had the biggest sales in all data","metadata":{}},{"cell_type":"code","source":"# getting the bigger sales\ncolumns = ['Neighborhood','SalePrice']\ndf_ = data[columns]\ndf_ = pd.DataFrame(df_.groupby('Neighborhood').SalePrice.sum())\ndf_ = df_.sort_values(by='SalePrice', ascending=False)\ndf_ = df_[:10]","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:08.542528Z","iopub.execute_input":"2022-07-30T23:25:08.543184Z","iopub.status.idle":"2022-07-30T23:25:08.558998Z","shell.execute_reply.started":"2022-07-30T23:25:08.543132Z","shell.execute_reply":"2022-07-30T23:25:08.557617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = df_.plot(kind='bar', title='\\nNeighborhood Sales', rot=0, figsize=(9,5), color='b')\nax.set_ylabel('Price ($)')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:08.561178Z","iopub.execute_input":"2022-07-30T23:25:08.562124Z","iopub.status.idle":"2022-07-30T23:25:09.007760Z","shell.execute_reply.started":"2022-07-30T23:25:08.562067Z","shell.execute_reply":"2022-07-30T23:25:09.006337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Analising the features ","metadata":{}},{"cell_type":"code","source":"import seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:09.010620Z","iopub.execute_input":"2022-07-30T23:25:09.011684Z","iopub.status.idle":"2022-07-30T23:25:10.331326Z","shell.execute_reply.started":"2022-07-30T23:25:09.011628Z","shell.execute_reply":"2022-07-30T23:25:10.329692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr = data.corr()\nfig, ax = plt.subplots(figsize=(10,10)) \nsns.heatmap(corr, annot=True, ax=ax)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:10.333843Z","iopub.execute_input":"2022-07-30T23:25:10.334927Z","iopub.status.idle":"2022-07-30T23:25:11.338028Z","shell.execute_reply.started":"2022-07-30T23:25:10.334871Z","shell.execute_reply":"2022-07-30T23:25:11.337065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualizing Prices for each feature with high correlation with SalePrice","metadata":{}},{"cell_type":"code","source":"feats = []\nline = corr.loc[['SalePrice']]\nfor c in corr.columns:\n    if line[c][0] > 0.5 and line[c][0] < 1:\n        feats.append(c)\nfeats.append('SalePrice')","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:36.260533Z","iopub.execute_input":"2022-07-30T23:25:36.260972Z","iopub.status.idle":"2022-07-30T23:25:36.270188Z","shell.execute_reply.started":"2022-07-30T23:25:36.260932Z","shell.execute_reply":"2022-07-30T23:25:36.268863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ = data[feats]\ndf_.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:25:38.985993Z","iopub.execute_input":"2022-07-30T23:25:38.986415Z","iopub.status.idle":"2022-07-30T23:25:39.002777Z","shell.execute_reply.started":"2022-07-30T23:25:38.986382Z","shell.execute_reply":"2022-07-30T23:25:39.001321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for f in feats[2:-1]:\n    plt.close()\n    df_pivot = pd.DataFrame(df_.groupby(f).SalePrice.sum())\n    df_pivot = pd.pivot_table(df_pivot, index=df_pivot.index, columns=[f], aggfunc='first')\n    ax = df_pivot.plot(kind='bar', title=f'{f} Sales', rot=0, figsize=(9,5), color='b')\n    ax.set_ylabel('Price ($)')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:45:08.276555Z","iopub.status.idle":"2022-07-30T23:45:08.277194Z","shell.execute_reply.started":"2022-07-30T23:45:08.276987Z","shell.execute_reply":"2022-07-30T23:45:08.277008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualizing the data distribuition","metadata":{}},{"cell_type":"code","source":"data.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:48:01.931312Z","iopub.execute_input":"2022-07-30T23:48:01.932747Z","iopub.status.idle":"2022-07-30T23:48:01.941764Z","shell.execute_reply.started":"2022-07-30T23:48:01.932699Z","shell.execute_reply":"2022-07-30T23:48:01.940799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in data.columns:\n    # selecting the numerical data\n    if data[k].dtypes == 'int64':\n        sns.distplot(data[k], fit=norm);\n        fig = plt.figure()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T23:54:19.221406Z","iopub.execute_input":"2022-07-30T23:54:19.221800Z","iopub.status.idle":"2022-07-30T23:54:22.292496Z","shell.execute_reply.started":"2022-07-30T23:54:19.221769Z","shell.execute_reply":"2022-07-30T23:54:22.291375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Looking for outliers","metadata":{}},{"cell_type":"code","source":"len(data.columns)","metadata":{"execution":{"iopub.status.busy":"2022-07-31T00:02:52.159512Z","iopub.execute_input":"2022-07-31T00:02:52.159981Z","iopub.status.idle":"2022-07-31T00:02:52.167855Z","shell.execute_reply.started":"2022-07-31T00:02:52.159946Z","shell.execute_reply":"2022-07-31T00:02:52.166502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(nrows=3, ncols=4, figsize=(9, 6))\nfig.suptitle(\"Outliers at numerical columns\", fontsize=20, y=0.95)\nplt.subplots_adjust(hspace=0.5)\ni = 0\nfor k in data.columns:\n    if data[k].dtypes == 'int64':\n        data.plot(y=k, kind='box', ax=axs.ravel()[i], figsize=(22,25))\n        \n        i += 1\n        \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-31T00:31:08.039444Z","iopub.execute_input":"2022-07-31T00:31:08.039825Z","iopub.status.idle":"2022-07-31T00:31:09.558303Z","shell.execute_reply.started":"2022-07-31T00:31:08.039795Z","shell.execute_reply":"2022-07-31T00:31:09.557429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}