{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\n\nfrom scipy.stats import norm\n\nfrom sklearn.preprocessing import StandardScaler\n\nfrom scipy import stats\n\nimport warnings\nwarnings.filterwarnings('ignore')\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2021-06-02T17:54:33.819644Z","iopub.execute_input":"2021-06-02T17:54:33.820185Z","iopub.status.idle":"2021-06-02T17:54:33.82643Z","shell.execute_reply.started":"2021-06-02T17:54:33.820132Z","shell.execute_reply":"2021-06-02T17:54:33.825739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-02T17:54:44.250332Z","iopub.execute_input":"2021-06-02T17:54:44.250854Z","iopub.status.idle":"2021-06-02T17:54:44.26013Z","shell.execute_reply.started":"2021-06-02T17:54:44.250818Z","shell.execute_reply":"2021-06-02T17:54:44.258753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-02T17:39:14.474915Z","iopub.execute_input":"2021-06-02T17:39:14.475311Z","iopub.status.idle":"2021-06-02T17:39:14.52353Z","shell.execute_reply.started":"2021-06-02T17:39:14.475281Z","shell.execute_reply":"2021-06-02T17:39:14.522722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-02T17:39:41.381613Z","iopub.execute_input":"2021-06-02T17:39:41.382164Z","iopub.status.idle":"2021-06-02T17:39:41.387146Z","shell.execute_reply.started":"2021-06-02T17:39:41.38213Z","shell.execute_reply":"2021-06-02T17:39:41.386372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe().T","metadata":{"execution":{"iopub.status.busy":"2021-06-02T17:40:06.837209Z","iopub.execute_input":"2021-06-02T17:40:06.83755Z","iopub.status.idle":"2021-06-02T17:40:06.965015Z","shell.execute_reply.started":"2021-06-02T17:40:06.837521Z","shell.execute_reply":"2021-06-02T17:40:06.964113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-02T17:41:30.962786Z","iopub.execute_input":"2021-06-02T17:41:30.963176Z","iopub.status.idle":"2021-06-02T17:41:30.994021Z","shell.execute_reply.started":"2021-06-02T17:41:30.963145Z","shell.execute_reply":"2021-06-02T17:41:30.992792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.columns","metadata":{"execution":{"iopub.status.busy":"2021-06-02T17:55:54.283408Z","iopub.execute_input":"2021-06-02T17:55:54.283793Z","iopub.status.idle":"2021-06-02T17:55:54.290761Z","shell.execute_reply.started":"2021-06-02T17:55:54.283757Z","shell.execute_reply":"2021-06-02T17:55:54.289984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['SalePrice'].describe().T","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:05:45.56157Z","iopub.execute_input":"2021-06-02T18:05:45.56227Z","iopub.status.idle":"2021-06-02T18:05:45.57354Z","shell.execute_reply.started":"2021-06-02T18:05:45.562208Z","shell.execute_reply":"2021-06-02T18:05:45.57235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(train['SalePrice']);","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:06:24.173858Z","iopub.execute_input":"2021-06-02T18:06:24.174256Z","iopub.status.idle":"2021-06-02T18:06:24.507868Z","shell.execute_reply.started":"2021-06-02T18:06:24.174215Z","shell.execute_reply":"2021-06-02T18:06:24.507109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(train['SalePrice']);","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:07:27.075881Z","iopub.execute_input":"2021-06-02T18:07:27.076468Z","iopub.status.idle":"2021-06-02T18:07:27.209578Z","shell.execute_reply.started":"2021-06-02T18:07:27.076433Z","shell.execute_reply":"2021-06-02T18:07:27.208883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Skewness: %f\" % train['SalePrice'].skew())\nprint(\"Kurtosis: %f\" % train['SalePrice'].kurt())","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:07:50.765296Z","iopub.execute_input":"2021-06-02T18:07:50.765671Z","iopub.status.idle":"2021-06-02T18:07:50.772339Z","shell.execute_reply.started":"2021-06-02T18:07:50.765638Z","shell.execute_reply":"2021-06-02T18:07:50.771333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = 'GrLivArea'\ndata = pd.concat([train['SalePrice'], train[var]], axis=1)\ndata.plot.scatter(x=var, y='SalePrice', ylim=(0,800000));","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:10:18.884977Z","iopub.execute_input":"2021-06-02T18:10:18.885337Z","iopub.status.idle":"2021-06-02T18:10:19.095738Z","shell.execute_reply.started":"2021-06-02T18:10:18.885308Z","shell.execute_reply":"2021-06-02T18:10:19.094834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = 'TotalBsmtSF'\ndata = pd.concat([train['SalePrice'], train[var]], axis=1)\ndata.plot.scatter(x=var, y='SalePrice', ylim=(0,800000));","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:10:48.34817Z","iopub.execute_input":"2021-06-02T18:10:48.348565Z","iopub.status.idle":"2021-06-02T18:10:48.54081Z","shell.execute_reply.started":"2021-06-02T18:10:48.348533Z","shell.execute_reply":"2021-06-02T18:10:48.539978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = 'OverallQual'\ndata = pd.concat([train['SalePrice'], train[var]], axis=1)\nf, ax = plt.subplots(figsize=(8, 6))\nfig = sns.boxplot(x=var, y=\"SalePrice\", data=data)\nfig.axis(ymin=0, ymax=800000);","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:11:51.462638Z","iopub.execute_input":"2021-06-02T18:11:51.463023Z","iopub.status.idle":"2021-06-02T18:11:51.803503Z","shell.execute_reply.started":"2021-06-02T18:11:51.462988Z","shell.execute_reply":"2021-06-02T18:11:51.802363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = 'YearBuilt'\ndata = pd.concat([train['SalePrice'], train[var]], axis=1)\nf, ax = plt.subplots(figsize=(16, 8))\nfig = sns.boxplot(x=var, y=\"SalePrice\", data=data)\nfig.axis(ymin=0, ymax=800000);\nplt.xticks(rotation=90);","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:12:34.278118Z","iopub.execute_input":"2021-06-02T18:12:34.278607Z","iopub.status.idle":"2021-06-02T18:12:37.390978Z","shell.execute_reply.started":"2021-06-02T18:12:34.278577Z","shell.execute_reply":"2021-06-02T18:12:37.389901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr = train.corr()\nf, ax = plt.subplots(figsize=(14, 10))\nsns.heatmap(corr, vmax=.8, square=True);","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:16:08.639409Z","iopub.execute_input":"2021-06-02T18:16:08.63998Z","iopub.status.idle":"2021-06-02T18:16:09.761634Z","shell.execute_reply.started":"2021-06-02T18:16:08.63992Z","shell.execute_reply":"2021-06-02T18:16:09.760651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"k = 10 #number of variables for heatmap\ncols = corr.nlargest(k, 'SalePrice')['SalePrice'].index\nplt.subplots(figsize=(14, 10))\ncm = np.corrcoef(train[cols].values.T)\nsns.set(font_scale=1.25)\nhm = sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 10}, yticklabels=cols.values, xticklabels=cols.values)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:18:36.734724Z","iopub.execute_input":"2021-06-02T18:18:36.735097Z","iopub.status.idle":"2021-06-02T18:18:37.573127Z","shell.execute_reply.started":"2021-06-02T18:18:36.735063Z","shell.execute_reply":"2021-06-02T18:18:37.572021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set()\ncols = ['SalePrice', 'OverallQual', 'GrLivArea', 'GarageCars', 'TotalBsmtSF', 'FullBath', 'YearBuilt']\nsns.pairplot(train[cols], size = 2.5)\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:20:13.386411Z","iopub.execute_input":"2021-06-02T18:20:13.386766Z","iopub.status.idle":"2021-06-02T18:20:23.786354Z","shell.execute_reply.started":"2021-06-02T18:20:13.386736Z","shell.execute_reply":"2021-06-02T18:20:23.785214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total = train.isnull().sum().sort_values(ascending=False)\npercent = (train.isnull().sum()/train.isnull().count()).sort_values(ascending=False)\nmissing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:23:09.964913Z","iopub.execute_input":"2021-06-02T18:23:09.965447Z","iopub.status.idle":"2021-06-02T18:23:10.003913Z","shell.execute_reply.started":"2021-06-02T18:23:09.965413Z","shell.execute_reply":"2021-06-02T18:23:10.003066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.drop((missing_data[missing_data['Total'] > 1]).index,1)\ntrain = train.drop(train.loc[train['Electrical'].isnull()].index)\ntrain.isnull().sum().max() #just checking that there's no missing data missing...","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:26:06.844042Z","iopub.execute_input":"2021-06-02T18:26:06.846179Z","iopub.status.idle":"2021-06-02T18:26:06.864992Z","shell.execute_reply.started":"2021-06-02T18:26:06.84613Z","shell.execute_reply":"2021-06-02T18:26:06.86417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"saleprice_scaled = StandardScaler().fit_transform(train['SalePrice'][:,np.newaxis]);\nlow_range = saleprice_scaled[saleprice_scaled[:,0].argsort()][:10]\nhigh_range= saleprice_scaled[saleprice_scaled[:,0].argsort()][-10:]\nprint('outer range (low) of the distribution:')\nprint(low_range)\nprint('\\nouter range (high) of the distribution:')\nprint(high_range)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:27:30.527379Z","iopub.execute_input":"2021-06-02T18:27:30.527744Z","iopub.status.idle":"2021-06-02T18:27:30.536986Z","shell.execute_reply.started":"2021-06-02T18:27:30.527715Z","shell.execute_reply":"2021-06-02T18:27:30.536106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.sort_values(by = 'GrLivArea', ascending = False)[:2]\ntrain = train.drop(train[train['Id'] == 1299].index)\ntrain = train.drop(train[train['Id'] == 524].index)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:29:23.460438Z","iopub.execute_input":"2021-06-02T18:29:23.460796Z","iopub.status.idle":"2021-06-02T18:29:23.473824Z","shell.execute_reply.started":"2021-06-02T18:29:23.460767Z","shell.execute_reply":"2021-06-02T18:29:23.47267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = 'TotalBsmtSF'\ndata = pd.concat([train['SalePrice'], train[var]], axis=1)\ndata.plot.scatter(x=var, y='SalePrice', ylim=(0,800000));","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:29:49.180906Z","iopub.execute_input":"2021-06-02T18:29:49.181304Z","iopub.status.idle":"2021-06-02T18:29:49.423974Z","shell.execute_reply.started":"2021-06-02T18:29:49.181273Z","shell.execute_reply":"2021-06-02T18:29:49.423185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(train['SalePrice'], fit=norm);\nfig = plt.figure()\nres = stats.probplot(train['SalePrice'], plot=plt)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:33:22.18773Z","iopub.execute_input":"2021-06-02T18:33:22.188128Z","iopub.status.idle":"2021-06-02T18:33:22.736032Z","shell.execute_reply.started":"2021-06-02T18:33:22.188096Z","shell.execute_reply":"2021-06-02T18:33:22.734755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['SalePrice'] = np.log(train['SalePrice'])","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:34:27.93527Z","iopub.execute_input":"2021-06-02T18:34:27.935621Z","iopub.status.idle":"2021-06-02T18:34:27.942147Z","shell.execute_reply.started":"2021-06-02T18:34:27.935593Z","shell.execute_reply":"2021-06-02T18:34:27.940946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(train['SalePrice'], fit=norm);\nfig = plt.figure()\nres = stats.probplot(train['SalePrice'], plot=plt)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:34:45.465977Z","iopub.execute_input":"2021-06-02T18:34:45.466332Z","iopub.status.idle":"2021-06-02T18:34:46.02407Z","shell.execute_reply.started":"2021-06-02T18:34:45.466303Z","shell.execute_reply":"2021-06-02T18:34:46.023061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(train['GrLivArea'], fit=norm);\nfig = plt.figure()\nres = stats.probplot(train['GrLivArea'], plot=plt)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:35:20.282183Z","iopub.execute_input":"2021-06-02T18:35:20.282585Z","iopub.status.idle":"2021-06-02T18:35:20.799542Z","shell.execute_reply.started":"2021-06-02T18:35:20.282544Z","shell.execute_reply":"2021-06-02T18:35:20.79835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['GrLivArea'] = np.log(train['GrLivArea'])","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:35:38.724467Z","iopub.execute_input":"2021-06-02T18:35:38.724849Z","iopub.status.idle":"2021-06-02T18:35:38.73077Z","shell.execute_reply.started":"2021-06-02T18:35:38.724818Z","shell.execute_reply":"2021-06-02T18:35:38.729687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(train['GrLivArea'], fit=norm);\nfig = plt.figure()\nres = stats.probplot(train['GrLivArea'], plot=plt)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:35:59.003984Z","iopub.execute_input":"2021-06-02T18:35:59.004354Z","iopub.status.idle":"2021-06-02T18:35:59.581045Z","shell.execute_reply.started":"2021-06-02T18:35:59.004322Z","shell.execute_reply":"2021-06-02T18:35:59.580123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(train['TotalBsmtSF'], fit=norm);\nfig = plt.figure()\nres = stats.probplot(train['TotalBsmtSF'], plot=plt)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:36:39.194487Z","iopub.execute_input":"2021-06-02T18:36:39.195029Z","iopub.status.idle":"2021-06-02T18:36:39.757852Z","shell.execute_reply.started":"2021-06-02T18:36:39.194995Z","shell.execute_reply":"2021-06-02T18:36:39.75707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['HasBsmt'] = pd.Series(len(train['TotalBsmtSF']), index=train.index)\ntrain['HasBsmt'] = 0 \ntrain.loc[train['TotalBsmtSF']>0,'HasBsmt'] = 1","metadata":{"execution":{"iopub.status.busy":"2021-06-02T18:37:25.950382Z","iopub.execute_input":"2021-06-02T18:37:25.951095Z","iopub.status.idle":"2021-06-02T18:37:25.959456Z","shell.execute_reply.started":"2021-06-02T18:37:25.951044Z","shell.execute_reply":"2021-06-02T18:37:25.958458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}