{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from __future__ import print_function\nfrom ipywidgets import interact, interactive, fixed, interact_manual\nimport ipywidgets as widgets\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings('ignore')\nsns.set_style('white')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-06T07:43:05.464874Z","iopub.execute_input":"2022-07-06T07:43:05.465564Z","iopub.status.idle":"2022-07-06T07:43:06.117576Z","shell.execute_reply.started":"2022-07-06T07:43:05.465514Z","shell.execute_reply":"2022-07-06T07:43:06.116352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(r\"../input/house-prices-advanced-regression-techniques/train.csv\")\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:43:06.119685Z","iopub.execute_input":"2022-07-06T07:43:06.120073Z","iopub.status.idle":"2022-07-06T07:43:06.200395Z","shell.execute_reply.started":"2022-07-06T07:43:06.120041Z","shell.execute_reply":"2022-07-06T07:43:06.199062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## List of categorical columns","metadata":{}},{"cell_type":"code","source":"cat = df.select_dtypes(include=['object']).columns.tolist()\ndf[cat]","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:43:06.201809Z","iopub.execute_input":"2022-07-06T07:43:06.202212Z","iopub.status.idle":"2022-07-06T07:43:06.244562Z","shell.execute_reply.started":"2022-07-06T07:43:06.202180Z","shell.execute_reply":"2022-07-06T07:43:06.243530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in cat:\n    print(df[i].value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:43:06.246959Z","iopub.execute_input":"2022-07-06T07:43:06.247285Z","iopub.status.idle":"2022-07-06T07:43:06.299004Z","shell.execute_reply.started":"2022-07-06T07:43:06.247258Z","shell.execute_reply":"2022-07-06T07:43:06.297824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe().T","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:43:06.300579Z","iopub.execute_input":"2022-07-06T07:43:06.301420Z","iopub.status.idle":"2022-07-06T07:43:06.426136Z","shell.execute_reply.started":"2022-07-06T07:43:06.301374Z","shell.execute_reply":"2022-07-06T07:43:06.424979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = pd.DataFrame(df.isna().sum().sort_values(ascending=False))\ndf1['columns']=df1.index\ndf1['count']=df1.iloc[:,:-1]\ndf1.reset_index(drop=True, inplace=True)\ndf1 = df1.drop(df1.columns[[0]],axis = 1)\ndf1.head(15)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:43:06.428111Z","iopub.execute_input":"2022-07-06T07:43:06.428427Z","iopub.status.idle":"2022-07-06T07:43:06.455896Z","shell.execute_reply.started":"2022-07-06T07:43:06.428399Z","shell.execute_reply":"2022-07-06T07:43:06.454783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 1, sharex=True, figsize=(18,6))\nfig.suptitle('Null Distribution Column-Wise')\nax=sns.barplot(y='columns',x='count',data=df1.head(15))","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:43:06.458197Z","iopub.execute_input":"2022-07-06T07:43:06.458888Z","iopub.status.idle":"2022-07-06T07:43:06.839942Z","shell.execute_reply.started":"2022-07-06T07:43:06.458844Z","shell.execute_reply":"2022-07-06T07:43:06.838569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From these informations we can already see that some features won't be relevant in our exploratory analysis as there are too much missing values \nRemoving features with 30% or less NaN values.","metadata":{}},{"cell_type":"code","source":"# df.count() does not include NaN values\ndf2 = df[[column for column in df if df[column].count() / len(df) >= 0.3]]\ndel df2['Id']\nprint(\"List of dropped columns:\", end=\" \")\nfor c in df.columns:\n    if c not in df2.columns:\n        print(c, end=\", \")\nprint('\\n')\ndf = df2","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:43:06.841209Z","iopub.execute_input":"2022-07-06T07:43:06.841556Z","iopub.status.idle":"2022-07-06T07:43:06.867646Z","shell.execute_reply.started":"2022-07-06T07:43:06.841522Z","shell.execute_reply":"2022-07-06T07:43:06.866368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We split the columns into categorical and numerical columns","metadata":{}},{"cell_type":"code","source":"numerical = df.select_dtypes(exclude=['object']).drop(['MSSubClass'], axis=1).copy()\nl=[]\nfor i in numerical:\n  l.append(i)\nprint(l)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:43:06.869222Z","iopub.execute_input":"2022-07-06T07:43:06.869558Z","iopub.status.idle":"2022-07-06T07:43:06.884679Z","shell.execute_reply.started":"2022-07-06T07:43:06.869528Z","shell.execute_reply":"2022-07-06T07:43:06.883306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ncategorical = df.select_dtypes(include=['object']).copy()\nl=[]\nfor i in categorical:\n  l.append(i)\nprint(l)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:43:06.888213Z","iopub.execute_input":"2022-07-06T07:43:06.889125Z","iopub.status.idle":"2022-07-06T07:43:06.897817Z","shell.execute_reply.started":"2022-07-06T07:43:06.889087Z","shell.execute_reply":"2022-07-06T07:43:06.896729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def f(x):\n    return x","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:43:06.899130Z","iopub.execute_input":"2022-07-06T07:43:06.899644Z","iopub.status.idle":"2022-07-06T07:43:06.906584Z","shell.execute_reply.started":"2022-07-06T07:43:06.899613Z","shell.execute_reply":"2022-07-06T07:43:06.905458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#@\n#interact(f, x=[('one', 10), ('two', 20)])","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:43:06.907979Z","iopub.execute_input":"2022-07-06T07:43:06.908560Z","iopub.status.idle":"2022-07-06T07:43:06.918449Z","shell.execute_reply.started":"2022-07-06T07:43:06.908528Z","shell.execute_reply":"2022-07-06T07:43:06.917571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Checking the count of each type of variables in categorical columns","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x='BldgType',data=categorical.dropna())","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:43:06.920059Z","iopub.execute_input":"2022-07-06T07:43:06.920776Z","iopub.status.idle":"2022-07-06T07:43:07.141087Z","shell.execute_reply.started":"2022-07-06T07:43:06.920732Z","shell.execute_reply":"2022-07-06T07:43:07.139966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(18,20))\nfor index in range(len(categorical.columns)):\n    plt.subplot(9,5,index+1)\n    sns.countplot(x=categorical.iloc[:,index], data=categorical.dropna())\n    plt.xticks(rotation=90)\nfig.tight_layout(pad=1.2)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T08:16:13.150328Z","iopub.execute_input":"2022-07-06T08:16:13.150701Z","iopub.status.idle":"2022-07-06T08:16:20.378697Z","shell.execute_reply.started":"2022-07-06T08:16:13.150671Z","shell.execute_reply":"2022-07-06T08:16:20.377164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Relation between the numerical features and the target variables:","metadata":{}},{"cell_type":"code","source":"sns.scatterplot(x='MSZoning', y='SalePrice',hue='MSZoning',data=df.dropna())","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:56:59.222612Z","iopub.execute_input":"2022-07-06T07:56:59.223030Z","iopub.status.idle":"2022-07-06T07:56:59.560414Z","shell.execute_reply.started":"2022-07-06T07:56:59.222997Z","shell.execute_reply":"2022-07-06T07:56:59.558993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(20,20))\nfor index in range(len(numerical.columns)):\n    plt.subplot(10,5,index+1)\n    sns.scatterplot(x=df.iloc[:,index], y='SalePrice',data=df.dropna())\n    plt.xticks(rotation=90)\nfig.tight_layout(pad=1.0)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T07:56:03.399170Z","iopub.execute_input":"2022-07-06T07:56:03.399658Z","iopub.status.idle":"2022-07-06T07:56:10.074868Z","shell.execute_reply.started":"2022-07-06T07:56:03.399622Z","shell.execute_reply":"2022-07-06T07:56:10.073770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Checking for outliers in numerical columns","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16,8))\nfig=sns.violinplot(y='OverallQual', x ='SalePrice',data=df)\nfig.axis(ymin=0, ymax=800000);","metadata":{"execution":{"iopub.status.busy":"2022-07-06T08:34:37.566138Z","iopub.execute_input":"2022-07-06T08:34:37.566811Z","iopub.status.idle":"2022-07-06T08:34:57.280657Z","shell.execute_reply.started":"2022-07-06T08:34:37.566773Z","shell.execute_reply":"2022-07-06T08:34:57.279010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nval_col1 = df.select_dtypes(exclude=['object']).drop(['MSSubClass'], axis=1).copy()\nfig = plt.figure(figsize=(20,15))\nfor index,col in enumerate(nval_col1):\n    plt.subplot(6,6,index+1)\n    sns.boxplot(nval_col1.loc[:,col].dropna())\nfig.tight_layout(pad=1.0)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T08:17:10.829966Z","iopub.execute_input":"2022-07-06T08:17:10.830374Z","iopub.status.idle":"2022-07-06T08:17:16.422537Z","shell.execute_reply.started":"2022-07-06T08:17:10.830343Z","shell.execute_reply":"2022-07-06T08:17:16.421149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = 'OverallQual'\ndata = pd.concat([df['SalePrice'], df[var]], axis=1)\nplt.figure(figsize=(16,8))\nfig = sns.boxplot(x=var, y=\"SalePrice\", hue='SaleCondition',data=df)\nfig.axis(ymin=0, ymax=800000);","metadata":{"execution":{"iopub.status.busy":"2022-07-06T08:17:16.424477Z","iopub.execute_input":"2022-07-06T08:17:16.424796Z","iopub.status.idle":"2022-07-06T08:17:17.916623Z","shell.execute_reply.started":"2022-07-06T08:17:16.424768Z","shell.execute_reply":"2022-07-06T08:17:17.915310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.hist(figsize=(19, 20), bins=50, xlabelsize=8, ylabelsize=8);","metadata":{"execution":{"iopub.status.busy":"2022-07-06T08:17:17.917791Z","iopub.execute_input":"2022-07-06T08:17:17.918109Z","iopub.status.idle":"2022-07-06T08:17:26.830334Z","shell.execute_reply.started":"2022-07-06T08:17:17.918081Z","shell.execute_reply":"2022-07-06T08:17:26.829068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}