{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-18T08:54:02.783572Z","iopub.execute_input":"2022-07-18T08:54:02.784368Z","iopub.status.idle":"2022-07-18T08:54:02.817218Z","shell.execute_reply.started":"2022-07-18T08:54:02.784251Z","shell.execute_reply":"2022-07-18T08:54:02.816341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')\ntest = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')\nprint('Data Loading is done!')","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:02.819272Z","iopub.execute_input":"2022-07-18T08:54:02.819658Z","iopub.status.idle":"2022-07-18T08:54:02.902759Z","shell.execute_reply.started":"2022-07-18T08:54:02.819623Z","shell.execute_reply":"2022-07-18T08:54:02.901870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Exploring Data Analytics","metadata":{}},{"cell_type":"code","source":"print(\"The shape of Train Data is:\", train.shape)\nprint(\"The shape od Test Data is:\", test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:02.904085Z","iopub.execute_input":"2022-07-18T08:54:02.904827Z","iopub.status.idle":"2022-07-18T08:54:02.910766Z","shell.execute_reply.started":"2022-07-18T08:54:02.904789Z","shell.execute_reply":"2022-07-18T08:54:02.909445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:02.913837Z","iopub.execute_input":"2022-07-18T08:54:02.914764Z","iopub.status.idle":"2022-07-18T08:54:02.959045Z","shell.execute_reply.started":"2022-07-18T08:54:02.914721Z","shell.execute_reply":"2022-07-18T08:54:02.958102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:02.960394Z","iopub.execute_input":"2022-07-18T08:54:02.960962Z","iopub.status.idle":"2022-07-18T08:54:02.995348Z","shell.execute_reply.started":"2022-07-18T08:54:02.960928Z","shell.execute_reply":"2022-07-18T08:54:02.994184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.info())","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:02.996802Z","iopub.execute_input":"2022-07-18T08:54:02.997350Z","iopub.status.idle":"2022-07-18T08:54:03.036910Z","shell.execute_reply.started":"2022-07-18T08:54:02.997316Z","shell.execute_reply":"2022-07-18T08:54:03.035522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test.info())","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:03.038867Z","iopub.execute_input":"2022-07-18T08:54:03.040793Z","iopub.status.idle":"2022-07-18T08:54:03.069947Z","shell.execute_reply.started":"2022-07-18T08:54:03.040748Z","shell.execute_reply":"2022-07-18T08:54:03.068634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:03.071774Z","iopub.execute_input":"2022-07-18T08:54:03.072601Z","iopub.status.idle":"2022-07-18T08:54:04.329636Z","shell.execute_reply.started":"2022-07-18T08:54:03.072563Z","shell.execute_reply":"2022-07-18T08:54:04.328512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\n\nwarnings.filterwarnings(action='ignore')\n\nplt.figure(figsize=(16, 4))\n\nplt.subplot(1, 3, 1)\nplt.title(\"OverallQual\")\nsns.boxplot(train['OverallQual'])\n\nplt.subplot(1, 3, 2)\nplt.title(\"OverallCond\")\nsns.boxplot(train['OverallCond'])\n\nplt.subplot(1, 3, 3)\nplt.title(\"SalePrice\")\nplt.xticks(rotation=45)\nsns.boxplot(train['SalePrice'])","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:04.332944Z","iopub.execute_input":"2022-07-18T08:54:04.333743Z","iopub.status.idle":"2022-07-18T08:54:04.763020Z","shell.execute_reply.started":"2022-07-18T08:54:04.333694Z","shell.execute_reply":"2022-07-18T08:54:04.761521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering\n* Remove Outlier","metadata":{}},{"cell_type":"code","source":"train[(train['OverallQual']<4) & (train['SalePrice']>250000)]","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:04.764828Z","iopub.execute_input":"2022-07-18T08:54:04.765324Z","iopub.status.idle":"2022-07-18T08:54:04.796261Z","shell.execute_reply.started":"2022-07-18T08:54:04.765280Z","shell.execute_reply":"2022-07-18T08:54:04.794908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[(train['OverallCond']<5) & (train['SalePrice']>250000)]","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:04.798190Z","iopub.execute_input":"2022-07-18T08:54:04.799347Z","iopub.status.idle":"2022-07-18T08:54:04.832255Z","shell.execute_reply.started":"2022-07-18T08:54:04.799285Z","shell.execute_reply":"2022-07-18T08:54:04.830735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(train[(train['OverallQual']<4) & (train['SalePrice']>250000)].index, inplace=True)\ntrain.drop(train[(train['OverallCond']<5) & (train['SalePrice']>250000)].index, inplace=True)\ntrain.reset_index(drop=True, inplace=True)\nprint(train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:04.834159Z","iopub.execute_input":"2022-07-18T08:54:04.834717Z","iopub.status.idle":"2022-07-18T08:54:04.859873Z","shell.execute_reply.started":"2022-07-18T08:54:04.834662Z","shell.execute_reply":"2022-07-18T08:54:04.858338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Log transformation of the dependent variable.","metadata":{}},{"cell_type":"code","source":"from scipy.stats import norm\n\nmu, sigma = norm.fit(train['SalePrice'])\nprint(\"The value of mu before log transformation is:\", mu)\nprint(\"The value of sigma before log transformation is:\", sigma)\n\nfig, ax = plt.subplots(figsize=(10, 6))\nsns.histplot(train['SalePrice'], color='skyblue', stat=\"probability\")\nax.xaxis.grid(True)\nax.set(ylabel=\"Frequency\")\nax.set(xlabel=\"SalePrice\")\nax.set(title=\"SalePrice Distribution\")\n\nplt.axvline(mu, color='r', linestyle=\"solid\")\nplt.text(mu+10000, 0.11, 'Mean of SalePrice', rotation=0, color='r')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:04.862100Z","iopub.execute_input":"2022-07-18T08:54:04.862651Z","iopub.status.idle":"2022-07-18T08:54:05.286535Z","shell.execute_reply.started":"2022-07-18T08:54:04.862597Z","shell.execute_reply":"2022-07-18T08:54:05.285151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Log transformation for Standard Deviation\ntrain[\"SalePrice\"] = np.log1p(train[\"SalePrice\"])\n\n(mu, sigma) = norm.fit(train['SalePrice'])\nprint(\"The value of mu after log transformation is:\", mu)\nprint(\"The value of sigma after log transformation is:\", sigma)\n\nfig, ax = plt.subplots(figsize=(10, 6))\nsns.histplot(train['SalePrice'], color='skyblue', stat=\"probability\")\nax.xaxis.grid(True)\nax.set(ylabel=\"Frequency\")\nax.set(xlabel=\"SalePrice\")\nax.set(title=\"SalePrice distribution\")\n\nplt.axvline(mu, color='r', linestyle=\"solid\")\nplt.text(mu+0.05, 0.111, 'Mean of SalePrice', rotation=0, color='r')\nplt.ylim(0, 0.12)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:05.288317Z","iopub.execute_input":"2022-07-18T08:54:05.289768Z","iopub.status.idle":"2022-07-18T08:54:05.650728Z","shell.execute_reply.started":"2022-07-18T08:54:05.289712Z","shell.execute_reply":"2022-07-18T08:54:05.649431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Concat the Data","metadata":{}},{"cell_type":"code","source":"# split the 'Id' column\ntrain_ID = train['Id']\ntest_ID = test['Id']\ntrain.drop(['Id'], axis=1, inplace=True)\ntest.drop(['Id'], axis=1, inplace=True)\ntrain.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:05.652765Z","iopub.execute_input":"2022-07-18T08:54:05.653283Z","iopub.status.idle":"2022-07-18T08:54:05.669445Z","shell.execute_reply.started":"2022-07-18T08:54:05.653233Z","shell.execute_reply":"2022-07-18T08:54:05.667988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# move 'SalePrice' column to variation 'y'\ny = train['SalePrice'].reset_index(drop=True)\ntrain = train.drop(['SalePrice'], axis=1)\ntrain.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:05.673462Z","iopub.execute_input":"2022-07-18T08:54:05.673875Z","iopub.status.idle":"2022-07-18T08:54:05.685141Z","shell.execute_reply.started":"2022-07-18T08:54:05.673838Z","shell.execute_reply":"2022-07-18T08:54:05.683885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# concat the Data\nall_df = pd.concat([train, test]).reset_index(drop=True)\nall_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:05.687627Z","iopub.execute_input":"2022-07-18T08:54:05.688479Z","iopub.status.idle":"2022-07-18T08:54:05.737563Z","shell.execute_reply.started":"2022-07-18T08:54:05.688405Z","shell.execute_reply":"2022-07-18T08:54:05.736133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:05.739859Z","iopub.execute_input":"2022-07-18T08:54:05.740717Z","iopub.status.idle":"2022-07-18T08:54:05.802892Z","shell.execute_reply.started":"2022-07-18T08:54:05.740662Z","shell.execute_reply":"2022-07-18T08:54:05.801194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Processing missing value","metadata":{}},{"cell_type":"code","source":"def check_na(data, head_num=6):\n    isnull_na = (data.isnull().sum()/len(data))*100\n    data_na = isnull_na.drop(isnull_na[isnull_na==0].index).sort_values(ascending=False)\n    missing_data = pd.DataFrame({'Missing Ratio' :data_na,\n                                 'Data Type' : data.dtypes[data_na.index]})\n    print(\"the column and the number of the missing value:\\n\", missing_data.head(head_num))\n    \ncheck_na(all_df, 20)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:05.811176Z","iopub.execute_input":"2022-07-18T08:54:05.815724Z","iopub.status.idle":"2022-07-18T08:54:05.848498Z","shell.execute_reply.started":"2022-07-18T08:54:05.815622Z","shell.execute_reply":"2022-07-18T08:54:05.847046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop columns\nall_df.drop(['PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu', 'LotFrontage'], axis=1, inplace=True)\ncheck_na(all_df, 20)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:05.850517Z","iopub.execute_input":"2022-07-18T08:54:05.851300Z","iopub.status.idle":"2022-07-18T08:54:05.888333Z","shell.execute_reply.started":"2022-07-18T08:54:05.851250Z","shell.execute_reply":"2022-07-18T08:54:05.886911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* processing missing value about dtype object","metadata":{}},{"cell_type":"code","source":"cat_all_vars = train.select_dtypes(exclude=[np.number])\nprint(\"The whole number of all_vars\", len(list(cat_all_vars)))\n\nfinal_cat_vars = []\nfor v in cat_all_vars:\n    if v not in ['PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu', 'LotFrontage']:\n        final_cat_vars.append(v)\nprint(\"The whole number of final_cat_vars\", len(final_cat_vars))\n\nfor i in final_cat_vars:\n    all_df[i] = all_df[i].fillna(all_df[i].mode()[0])\n    \ncheck_na(all_df, 20)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:05.890127Z","iopub.execute_input":"2022-07-18T08:54:05.890805Z","iopub.status.idle":"2022-07-18T08:54:05.975999Z","shell.execute_reply.started":"2022-07-18T08:54:05.890765Z","shell.execute_reply":"2022-07-18T08:54:05.974848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* processing missing value for float64","metadata":{}},{"cell_type":"code","source":"num_all_vars = list(train.select_dtypes(include=[np.number]))\nprint(\"The whole number of all_vars\", len(num_all_vars))\n\nnum_all_vars.remove('LotFrontage')\n\nprint(\"The whole number of num_all_vars\", len(num_all_vars))\nfor i in num_all_vars:\n    all_df[i].fillna(value=all_df[i].median(), inplace=True)\n    \ncheck_na(all_df, 20)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:05.977598Z","iopub.execute_input":"2022-07-18T08:54:05.978513Z","iopub.status.idle":"2022-07-18T08:54:06.032980Z","shell.execute_reply.started":"2022-07-18T08:54:05.978465Z","shell.execute_reply":"2022-07-18T08:54:06.031493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Processing for Skewness values","metadata":{}},{"cell_type":"code","source":"from scipy.stats import skew\n\ndef find_skew(x):\n    return skew(x)\n\nskew_features = all_df[num_all_vars].apply(find_skew).sort_values(ascending=False)\nskew_features","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:06.039506Z","iopub.execute_input":"2022-07-18T08:54:06.040371Z","iopub.status.idle":"2022-07-18T08:54:06.065597Z","shell.execute_reply.started":"2022-07-18T08:54:06.040314Z","shell.execute_reply":"2022-07-18T08:54:06.064216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make skew_features's index without 'LotArea' and make dataframe all_numeric_df by skew_features's index\nskewnewss_index = list(skew_features.index)\nskewnewss_index.remove('LotArea')\nall_numeric_df = all_df.loc[:, skewnewss_index]","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:06.067818Z","iopub.execute_input":"2022-07-18T08:54:06.068714Z","iopub.status.idle":"2022-07-18T08:54:06.077747Z","shell.execute_reply.started":"2022-07-18T08:54:06.068658Z","shell.execute_reply":"2022-07-18T08:54:06.076144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(10, 6))\nax.set_xlim(0, all_numeric_df.max().sort_values(ascending=False)[0])\nax = sns.boxplot(data=all_numeric_df[skewnewss_index], orient=\"h\", palette=\"Set1\")\nax.xaxis.grid(True)\nax.set(ylabel=\"Feature names\")\nax.set(xlabel=\"Numeric values\")\nax.set(title=\"Numeric Distribution of Features Before Box-Cox Transformation\")\nsns.despine(trim=True, left=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T08:54:06.080066Z","iopub.execute_input":"2022-07-18T08:54:06.081033Z","iopub.status.idle":"2022-07-18T08:54:06.868497Z","shell.execute_reply.started":"2022-07-18T08:54:06.080981Z","shell.execute_reply":"2022-07-18T08:54:06.867143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.special import boxcox1p\nfrom scipy.stats import boxcox_normmax\n\nhigh_skew = skew_features[skew_features > 1]\nhigh_skew_index = high_skew.index\nprint(\"The data before Box-Cox Transformation: \\n\", all_df[high_skew_index].head())\n\nfor num_var in high_skew_index:\n    all_df[num_var] = boxcox1p(all_df[num_var], boxcox_normmax(all_df[num_var] + 1))\n    \nprint(\"The data after Box-Cox Transformation: \\n\", all_df[high_skew_index].head())","metadata":{"execution":{"iopub.status.busy":"2022-07-18T09:07:40.892641Z","iopub.execute_input":"2022-07-18T09:07:40.893082Z","iopub.status.idle":"2022-07-18T09:07:41.087684Z","shell.execute_reply.started":"2022-07-18T09:07:40.893050Z","shell.execute_reply":"2022-07-18T09:07:41.086469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Generative derived variables","metadata":{}},{"cell_type":"code","source":"all_df['TotalSF'] = all_df['TotalBsmtSF'] + all_df['1stFlrSF'] + all_df['2ndFlrSF']\nall_df.drop(['TotalBsmtSF', '1stFlrSF', '2ndFlrSF'], axis=1)\n\nall_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-18T09:25:56.247071Z","iopub.execute_input":"2022-07-18T09:25:56.247705Z","iopub.status.idle":"2022-07-18T09:25:56.264634Z","shell.execute_reply.started":"2022-07-18T09:25:56.247654Z","shell.execute_reply":"2022-07-18T09:25:56.263582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_df['Total_Bathrooms'] = (all_df['FullBath'] + (0.5 * all_df['HalfBath']) + all_df['BsmtFullBath'] + (0.5 * all_df['BsmtHalfBath']))\n\nall_df['Total_porch_sf'] = (all_df['OpenPorchSF'] + all_df['3SsnPorch'] + all_df['EnclosedPorch'] + all_df['ScreenPorch'])\n\nall_df = all_df.drop(['FullBath', 'HalfBath', 'BsmtFullBath', 'BsmtHalfBath', 'OpenPorchSF', '3SsnPorch', 'EnclosedPorch','ScreenPorch'], axis=1)\n\nall_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-18T09:39:08.753067Z","iopub.execute_input":"2022-07-18T09:39:08.753808Z","iopub.status.idle":"2022-07-18T09:39:08.778190Z","shell.execute_reply.started":"2022-07-18T09:39:08.753749Z","shell.execute_reply":"2022-07-18T09:39:08.776748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}