{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#import libraries\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.stats import norm\nfrom sklearn.preprocessing import StandardScaler\nfrom scipy import stats\nimport warnings\nwarnings.filterwarnings('ignore')\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-04T23:20:50.185369Z","iopub.execute_input":"2022-08-04T23:20:50.186216Z","iopub.status.idle":"2022-08-04T23:20:50.197772Z","shell.execute_reply.started":"2022-08-04T23:20:50.186157Z","shell.execute_reply":"2022-08-04T23:20:50.196214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:20:50.205696Z","iopub.execute_input":"2022-08-04T23:20:50.206387Z","iopub.status.idle":"2022-08-04T23:20:50.230397Z","shell.execute_reply.started":"2022-08-04T23:20:50.206359Z","shell.execute_reply":"2022-08-04T23:20:50.229531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:20:50.231821Z","iopub.execute_input":"2022-08-04T23:20:50.232773Z","iopub.status.idle":"2022-08-04T23:20:50.258570Z","shell.execute_reply.started":"2022-08-04T23:20:50.232731Z","shell.execute_reply":"2022-08-04T23:20:50.257654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.tail()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:20:50.260514Z","iopub.execute_input":"2022-08-04T23:20:50.261449Z","iopub.status.idle":"2022-08-04T23:20:50.285463Z","shell.execute_reply.started":"2022-08-04T23:20:50.261415Z","shell.execute_reply":"2022-08-04T23:20:50.284519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.columns","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:20:50.286914Z","iopub.execute_input":"2022-08-04T23:20:50.287492Z","iopub.status.idle":"2022-08-04T23:20:50.294714Z","shell.execute_reply.started":"2022-08-04T23:20:50.287458Z","shell.execute_reply":"2022-08-04T23:20:50.293529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_train.columns)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:20:50.297178Z","iopub.execute_input":"2022-08-04T23:20:50.297634Z","iopub.status.idle":"2022-08-04T23:20:50.305232Z","shell.execute_reply.started":"2022-08-04T23:20:50.297576Z","shell.execute_reply":"2022-08-04T23:20:50.304171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['SalePrice'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:20:50.306457Z","iopub.execute_input":"2022-08-04T23:20:50.307328Z","iopub.status.idle":"2022-08-04T23:20:50.319904Z","shell.execute_reply.started":"2022-08-04T23:20:50.307293Z","shell.execute_reply":"2022-08-04T23:20:50.318827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(df_train['SalePrice'], color='red')\nplt.title('Distribution of sale price')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:20:50.322418Z","iopub.execute_input":"2022-08-04T23:20:50.323037Z","iopub.status.idle":"2022-08-04T23:20:50.641528Z","shell.execute_reply.started":"2022-08-04T23:20:50.323004Z","shell.execute_reply":"2022-08-04T23:20:50.640563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Skewness: %f' % df_train['SalePrice'].skew())\nprint('Kurtosis: %f' % df_train['SalePrice'].kurt())","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:20:50.642826Z","iopub.execute_input":"2022-08-04T23:20:50.643671Z","iopub.status.idle":"2022-08-04T23:20:50.650462Z","shell.execute_reply.started":"2022-08-04T23:20:50.643632Z","shell.execute_reply":"2022-08-04T23:20:50.649515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = 'GrLivArea'\ndt = pd.concat([df_train['SalePrice'], df_train[var]], axis=1)\ndt.plot.scatter(x=var, y='SalePrice', ylim=(0,900000),figsize=(7, 4), color='g')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:20:50.651709Z","iopub.execute_input":"2022-08-04T23:20:50.652712Z","iopub.status.idle":"2022-08-04T23:20:50.895497Z","shell.execute_reply.started":"2022-08-04T23:20:50.652676Z","shell.execute_reply":"2022-08-04T23:20:50.894633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = 'TotalBsmtSF'\ndt = pd.concat([df_train['SalePrice'], df_train[var]], axis=1)\ndt.plot.scatter(x=var, y='SalePrice', ylim=(0,900000),figsize=(7, 4), color='g')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:20:50.898888Z","iopub.execute_input":"2022-08-04T23:20:50.899159Z","iopub.status.idle":"2022-08-04T23:20:51.153568Z","shell.execute_reply.started":"2022-08-04T23:20:50.899133Z","shell.execute_reply":"2022-08-04T23:20:51.152607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = 'OverallQual'\ndt = pd.concat([df_train['SalePrice'], df_train[var]], axis=1)\nf, ax = plt.subplots(figsize=(7, 5))\nfig = sns.boxplot(x=var, y=\"SalePrice\", data=dt)\nfig.axis(ymin=0, ymax=900000);","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:20:51.155114Z","iopub.execute_input":"2022-08-04T23:20:51.155733Z","iopub.status.idle":"2022-08-04T23:20:51.481632Z","shell.execute_reply.started":"2022-08-04T23:20:51.155693Z","shell.execute_reply":"2022-08-04T23:20:51.480583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = 'YearBuilt'\ndt = pd.concat([df_train['SalePrice'], df_train[var]], axis=1)\nf, ax = plt.subplots(figsize=(12, 6))\nfig = sns.boxplot(x=var, y=\"SalePrice\", data=dt)\nfig.axis(ymin=0, ymax=900000);\nplt.xticks(rotation=90);","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:20:51.483046Z","iopub.execute_input":"2022-08-04T23:20:51.484134Z","iopub.status.idle":"2022-08-04T23:20:54.745456Z","shell.execute_reply.started":"2022-08-04T23:20:51.484096Z","shell.execute_reply":"2022-08-04T23:20:54.744523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#correlation matrix\ncorr_matrix = df_train.corr()\nf, ax = plt.subplots(figsize=(12, 9))\nsns.heatmap(corr_matrix, vmax=.9, square=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:20:54.747185Z","iopub.execute_input":"2022-08-04T23:20:54.747872Z","iopub.status.idle":"2022-08-04T23:20:55.626706Z","shell.execute_reply.started":"2022-08-04T23:20:54.747833Z","shell.execute_reply":"2022-08-04T23:20:55.625641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#correlation matrix for saleprice\nk = 10        #number of variables for heatmap\ncols = corr_matrix.nlargest(k, 'SalePrice')['SalePrice'].index\ncm = np.corrcoef(df_train[cols].values.T)\nsns.set(font_scale=1.2)\nhm = sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 10}, yticklabels=cols.values, xticklabels=cols.values)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:20:55.628428Z","iopub.execute_input":"2022-08-04T23:20:55.629090Z","iopub.status.idle":"2022-08-04T23:20:56.266432Z","shell.execute_reply.started":"2022-08-04T23:20:55.629051Z","shell.execute_reply":"2022-08-04T23:20:56.265485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#scatterplot\nsns.set()\ncols = ['SalePrice', 'OverallQual', 'GrLivArea', 'GarageCars', 'TotalBsmtSF', 'FullBath', 'YearBuilt']\nsns.pairplot(df_train[cols], size = 2.6)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:20:56.267898Z","iopub.execute_input":"2022-08-04T23:20:56.268458Z","iopub.status.idle":"2022-08-04T23:21:04.426284Z","shell.execute_reply.started":"2022-08-04T23:20:56.268414Z","shell.execute_reply":"2022-08-04T23:21:04.425425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#missing values\ntotal = df_train.isnull().sum().sort_values(ascending=False)\npercent = (df_train.isnull().mean()).sort_values(ascending=False)\nmissing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:04.427618Z","iopub.execute_input":"2022-08-04T23:21:04.428878Z","iopub.status.idle":"2022-08-04T23:21:04.457417Z","shell.execute_reply.started":"2022-08-04T23:21:04.428840Z","shell.execute_reply":"2022-08-04T23:21:04.456391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#standardizing data\nsaleprice_scaled = StandardScaler().fit_transform(df_train['SalePrice'][:,np.newaxis])\nlow_range = saleprice_scaled[saleprice_scaled[:,0].argsort()][:10]\nhigh_range= saleprice_scaled[saleprice_scaled[:,0].argsort()][-10:]\nprint('outer range (low) of the distribution:')\nprint(low_range)\nprint('\\nouter range (High) of the distribution:')\nprint(high_range)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:04.459345Z","iopub.execute_input":"2022-08-04T23:21:04.460022Z","iopub.status.idle":"2022-08-04T23:21:04.469760Z","shell.execute_reply.started":"2022-08-04T23:21:04.459985Z","shell.execute_reply":"2022-08-04T23:21:04.468678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#bivariate analysis saleprice/grlivarea\n\nvar = 'GrLivArea'\ndt = pd.concat([df_train['SalePrice'], df_train[var]], axis=1)\ndt.plot.scatter(x=var, y='SalePrice', ylim=(0,900000), color='g')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:04.475494Z","iopub.execute_input":"2022-08-04T23:21:04.476194Z","iopub.status.idle":"2022-08-04T23:21:04.726807Z","shell.execute_reply.started":"2022-08-04T23:21:04.476168Z","shell.execute_reply":"2022-08-04T23:21:04.725783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#deleting points\n\ndf_train.sort_values(by = 'GrLivArea', ascending = False)[:2]\ndf_train = df_train.drop(df_train[df_train['Id'] == 1299].index)\ndf_train = df_train.drop(df_train[df_train['Id'] == 524].index)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:04.728310Z","iopub.execute_input":"2022-08-04T23:21:04.728677Z","iopub.status.idle":"2022-08-04T23:21:04.741466Z","shell.execute_reply.started":"2022-08-04T23:21:04.728641Z","shell.execute_reply":"2022-08-04T23:21:04.740387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#bivariate analysis saleprice/grlivarea\n\nvar = 'TotalBsmtSF'\ndt = pd.concat([df_train['SalePrice'], df_train[var]], axis=1)\ndt.plot.scatter(x=var, y='SalePrice', ylim=(0,900000), color='g')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:04.742898Z","iopub.execute_input":"2022-08-04T23:21:04.743347Z","iopub.status.idle":"2022-08-04T23:21:05.002411Z","shell.execute_reply.started":"2022-08-04T23:21:04.743311Z","shell.execute_reply":"2022-08-04T23:21:05.001446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#histogram and normal probability plot\n\nsns.distplot(df_train['SalePrice'], fit=norm, color='g')\nfig = plt.figure()\nres = stats.probplot(df_train['SalePrice'], plot=plt)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:05.003800Z","iopub.execute_input":"2022-08-04T23:21:05.004891Z","iopub.status.idle":"2022-08-04T23:21:05.559682Z","shell.execute_reply.started":"2022-08-04T23:21:05.004847Z","shell.execute_reply":"2022-08-04T23:21:05.558643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#log transformation\ndf_train['SalePrice'] = np.log(df_train['SalePrice'])","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:05.561181Z","iopub.execute_input":"2022-08-04T23:21:05.561578Z","iopub.status.idle":"2022-08-04T23:21:05.568932Z","shell.execute_reply.started":"2022-08-04T23:21:05.561538Z","shell.execute_reply":"2022-08-04T23:21:05.567849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#transformed histogram and normal probability plot\nsns.distplot(df_train['SalePrice'], fit=norm, color='g')\nfig = plt.figure()\nres = stats.probplot(df_train['SalePrice'], plot=plt)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:05.570507Z","iopub.execute_input":"2022-08-04T23:21:05.570980Z","iopub.status.idle":"2022-08-04T23:21:06.121584Z","shell.execute_reply.started":"2022-08-04T23:21:05.570942Z","shell.execute_reply":"2022-08-04T23:21:06.120570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#histogram and normal probability plot\nsns.distplot(df_train['GrLivArea'], fit=norm, color='g')\nfig = plt.figure()\nres = stats.probplot(df_train['GrLivArea'], plot=plt)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:06.122984Z","iopub.execute_input":"2022-08-04T23:21:06.123580Z","iopub.status.idle":"2022-08-04T23:21:06.635187Z","shell.execute_reply.started":"2022-08-04T23:21:06.123541Z","shell.execute_reply":"2022-08-04T23:21:06.634271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['GrLivArea'] = np.log(df_train['GrLivArea'])","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:06.636550Z","iopub.execute_input":"2022-08-04T23:21:06.637028Z","iopub.status.idle":"2022-08-04T23:21:06.645288Z","shell.execute_reply.started":"2022-08-04T23:21:06.636991Z","shell.execute_reply":"2022-08-04T23:21:06.644250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#transformed histogram and normal probability plot\nsns.distplot(df_train['GrLivArea'], fit=norm, color='g')\nfig = plt.figure()\nres = stats.probplot(df_train['GrLivArea'], plot=plt)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:06.647111Z","iopub.execute_input":"2022-08-04T23:21:06.647813Z","iopub.status.idle":"2022-08-04T23:21:07.171401Z","shell.execute_reply.started":"2022-08-04T23:21:06.647779Z","shell.execute_reply":"2022-08-04T23:21:07.170497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#histogram and normal probability plot\nsns.distplot(df_train['TotalBsmtSF'], fit=norm, color='g');\nfig = plt.figure()\nres = stats.probplot(df_train['TotalBsmtSF'], plot=plt)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:07.172905Z","iopub.execute_input":"2022-08-04T23:21:07.173477Z","iopub.status.idle":"2022-08-04T23:21:07.717543Z","shell.execute_reply.started":"2022-08-04T23:21:07.173442Z","shell.execute_reply":"2022-08-04T23:21:07.716531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create column for new variable (one is enough because it's a binary categorical feature)\n# ( if area>0 it gets 1, for area==0 it gets 0 )\ndf_train['HasBsmt'] = pd.Series(len(df_train['TotalBsmtSF']), index=df_train.index)\ndf_train['HasBsmt'] = 0 \ndf_train.loc[df_train['TotalBsmtSF']>0,'HasBsmt'] = 1","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:07.719162Z","iopub.execute_input":"2022-08-04T23:21:07.719493Z","iopub.status.idle":"2022-08-04T23:21:07.728425Z","shell.execute_reply.started":"2022-08-04T23:21:07.719459Z","shell.execute_reply":"2022-08-04T23:21:07.727514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#transform data\ndf_train.loc[df_train['HasBsmt']==1,'TotalBsmtSF'] = np.log(df_train['TotalBsmtSF'])","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:07.730096Z","iopub.execute_input":"2022-08-04T23:21:07.730918Z","iopub.status.idle":"2022-08-04T23:21:07.737892Z","shell.execute_reply.started":"2022-08-04T23:21:07.730884Z","shell.execute_reply":"2022-08-04T23:21:07.736851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#histogram and normal probability plot\nsns.distplot(df_train[df_train['TotalBsmtSF']>0]['TotalBsmtSF'], fit=norm)\nfig = plt.figure()\nres = stats.probplot(df_train[df_train['TotalBsmtSF']>0]['TotalBsmtSF'], plot=plt)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:07.739511Z","iopub.execute_input":"2022-08-04T23:21:07.739971Z","iopub.status.idle":"2022-08-04T23:21:08.299514Z","shell.execute_reply.started":"2022-08-04T23:21:07.739933Z","shell.execute_reply":"2022-08-04T23:21:08.298637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#scatter plot\nplt.scatter(df_train['GrLivArea'], df_train['SalePrice'], color='g')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:08.301081Z","iopub.execute_input":"2022-08-04T23:21:08.301414Z","iopub.status.idle":"2022-08-04T23:21:08.537392Z","shell.execute_reply.started":"2022-08-04T23:21:08.301379Z","shell.execute_reply":"2022-08-04T23:21:08.536467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#scatter plot\nplt.scatter(df_train[df_train['TotalBsmtSF']>0]['TotalBsmtSF'], df_train[df_train['TotalBsmtSF']>0]['SalePrice'],color='g')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:08.538710Z","iopub.execute_input":"2022-08-04T23:21:08.539860Z","iopub.status.idle":"2022-08-04T23:21:08.787638Z","shell.execute_reply.started":"2022-08-04T23:21:08.539820Z","shell.execute_reply":"2022-08-04T23:21:08.786755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#convert categorical variable into dummy\ndf_train = pd.get_dummies(df_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:08.789153Z","iopub.execute_input":"2022-08-04T23:21:08.789474Z","iopub.status.idle":"2022-08-04T23:21:08.832158Z","shell.execute_reply.started":"2022-08-04T23:21:08.789441Z","shell.execute_reply":"2022-08-04T23:21:08.831324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.to_csv('submission.csv',index= False)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T23:21:15.257594Z","iopub.execute_input":"2022-08-04T23:21:15.258297Z","iopub.status.idle":"2022-08-04T23:21:15.336867Z","shell.execute_reply.started":"2022-08-04T23:21:15.258261Z","shell.execute_reply":"2022-08-04T23:21:15.335925Z"},"trusted":true},"execution_count":null,"outputs":[]}]}