{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport pathlib\nimport seaborn as sns\nfrom scipy import stats\nimport matplotlib.pyplot as plt\n\nimport sklearn.dummy\nimport sklearn.model_selection\nimport sklearn.metrics\nimport sklearn.ensemble\nimport sklearn.compose\nimport sklearn.pipeline\nimport sklearn.impute\nimport sklearn.preprocessing","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = pathlib.Path.cwd()\nDATA_DIR = BASE_DIR.parent / 'input' / 'house-prices-advanced-regression-techniques'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_df = pd.read_csv(DATA_DIR / 'train.csv')\ndata_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_df.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Measurement Scales\n\n1. Nonmetric Measurement Scales\n  * Nominal Scales\n  * Ordinal Scales\n2. Metric Measurement Scales\n  * Interval Scales\n  * Ratio Scales","metadata":{}},{"cell_type":"code","source":"train_df, test_df = sklearn.model_selection.train_test_split(data_df, test_size=0.3, random_state=850)\ntrain_df.reset_index(drop=True, inplace=True)\ntest_df.reset_index(drop=True, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Dependent Variable**: Sale Price","metadata":{}},{"cell_type":"code","source":"train_df['SalePrice'].describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loc = train_df['SalePrice'].mean()\nscale = train_df['SalePrice'].std()\n\nplt.figure(figsize=(20, 6))\nax = sns.histplot(train_df['SalePrice'], stat='density')\nmu, std = stats.norm.fit(train_df['SalePrice'])\nxmin, xmax = ax.get_xlim()\nx = np.linspace(xmin, xmax, 1022)\np = stats.norm.pdf(x, mu, std)\nplt.plot(x, p, 'r', linewidth=2)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Skewness:', train_df['SalePrice'].skew())\nprint('Kurtosis:', train_df['SalePrice'].kurt())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"$\\text{Skewness} = \\frac{\\text{Mean} - \\text{Mode}}{\\text{Std. Dev.}}$\n\n1. The data is highly positively skewed.\n2. We have a leptokurtic kurtosis since the the kurtosis of the data is higher than a std. norm. distribution (which is 3).","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nsns.heatmap(train_df.corr(), square=True, vmax=0.8)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nk = 10\ncols = train_df.corr().nlargest(k, 'SalePrice')['SalePrice'].index\ncm = np.corrcoef(train_df[cols].values.T)\nsns.set(font_scale=1.25)\nhm = sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 10}, yticklabels=cols.values, xticklabels=cols.values)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_columns = train_df.columns[(pd.isna(train_df).sum(axis=0) / len(train_df)) > 0.15]\ntrain_df = train_df.drop(columns=drop_columns)\ntest_df = test_df.drop(columns=drop_columns)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_columns = train_df.select_dtypes(include='object').columns\nnumerical_columns = train_df.select_dtypes(exclude='object').columns.drop('SalePrice')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_transformer = sklearn.impute.SimpleImputer(strategy='mean')\n\ncategorical_transformer = sklearn.pipeline.Pipeline(steps=[\n    ('imputer', sklearn.impute.SimpleImputer(strategy='most_frequent')),\n    ('onehot', sklearn.preprocessing.OneHotEncoder(handle_unknown='ignore'))\n])\n\npreprocessor = sklearn.compose.ColumnTransformer(transformers=[\n    ('num', numerical_transformer, numerical_columns),\n    ('cat', categorical_transformer, categorical_columns)\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"baseline = sklearn.dummy.DummyRegressor(strategy='median')\nrf_model = sklearn.ensemble.RandomForestRegressor(random_state=850)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"baseline_data_pipeline = sklearn.pipeline.Pipeline(steps=[\n    ('preprocessor', preprocessor),\n    ('model', baseline)\n])\n\nmodel_data_pipeline = sklearn.pipeline.Pipeline(steps=[\n    ('preprocessor', preprocessor),\n    ('model', rf_model)\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"baseline_data_pipeline.fit(train_df.iloc[:, :-1], train_df.iloc[:, -1])\nbaseline_predictions = baseline_data_pipeline.predict(test_df.iloc[:, :-1])\nprint(sklearn.metrics.mean_absolute_error(test_df.iloc[:, -1], baseline_predictions))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_data_pipeline.fit(train_df.iloc[:, :-1], train_df.iloc[:, -1])\nmodel_predictions = model_data_pipeline.predict(test_df.iloc[:, :-1])\nprint(sklearn.metrics.mean_absolute_error(test_df.iloc[:, -1], model_predictions))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = -1 * sklearn.model_selection.cross_val_score(model_data_pipeline, train_df.iloc[:, :-1], train_df.iloc[:, -1], cv=5, scoring='neg_mean_absolute_error')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores.mean()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}