{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from sklearn.preprocessing import QuantileTransformer, FunctionTransformer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.impute import SimpleImputer\nfrom catboost import CatBoostRegressor\nfrom sklearn.pipeline import Pipeline\nfrom matplotlib import pyplot as plt\nfrom scipy import stats\nimport seaborn as sns\nimport pandas as pd\nimport numpy as np\n\ndef normal(mean, std, color='black', ax=None):\n    x = np.linspace(mean - 4 * std, mean + 4 * std, 200)\n    p = stats.norm.pdf(x, mean, std)\n    if ax is None:\n        plt.plot(x, p, color, linewidth=2)\n    else:\n        ax.plot(x, p, color, linewidth=2)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-12T10:39:30.596968Z","iopub.execute_input":"2022-08-12T10:39:30.597500Z","iopub.status.idle":"2022-08-12T10:39:31.266510Z","shell.execute_reply.started":"2022-08-12T10:39:30.597376Z","shell.execute_reply":"2022-08-12T10:39:31.265262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.concat([pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv'), pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')], axis=0, sort=False)\ntest_id = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')['Id']\ndata.drop(columns='Id', inplace=True)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-12T10:55:21.593669Z","iopub.execute_input":"2022-08-12T10:55:21.594136Z","iopub.status.idle":"2022-08-12T10:55:21.675185Z","shell.execute_reply.started":"2022-08-12T10:55:21.594098Z","shell.execute_reply":"2022-08-12T10:55:21.674080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[:1460].info()","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-12T10:39:31.359241Z","iopub.execute_input":"2022-08-12T10:39:31.359554Z","iopub.status.idle":"2022-08-12T10:39:31.382260Z","shell.execute_reply.started":"2022-08-12T10:39:31.359525Z","shell.execute_reply":"2022-08-12T10:39:31.381448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[:1460].head()","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-12T10:39:31.384239Z","iopub.execute_input":"2022-08-12T10:39:31.384737Z","iopub.status.idle":"2022-08-12T10:39:31.415072Z","shell.execute_reply.started":"2022-08-12T10:39:31.384703Z","shell.execute_reply":"2022-08-12T10:39:31.413628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA step","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"plt.figure(figsize=(30, 25))\nsns.heatmap(data[:1460].corr(), cmap='plasma', annot=True)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-12T10:39:31.416193Z","iopub.execute_input":"2022-08-12T10:39:31.416872Z","iopub.status.idle":"2022-08-12T10:39:37.167867Z","shell.execute_reply.started":"2022-08-12T10:39:31.416825Z","shell.execute_reply":"2022-08-12T10:39:37.166593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[:1460].corr()['SalePrice']","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-12T10:39:37.169376Z","iopub.execute_input":"2022-08-12T10:39:37.169770Z","iopub.status.idle":"2022-08-12T10:39:37.186703Z","shell.execute_reply.started":"2022-08-12T10:39:37.169734Z","shell.execute_reply":"2022-08-12T10:39:37.185622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, ax = plt.subplots(4, 3, figsize=(50, 60))\nfor i, feature in enumerate(['OverallQual', 'FullBath', 'TotRmsAbvGrd', 'GarageCars']):\n    sns.stripplot(data=data[:1460], x=feature, y='SalePrice', ax=ax[i, 0])\n    sns.violinplot(data=data[:1460], x=feature, y='SalePrice', ax=ax[i, 1])\n    sns.boxplot(data=data[:1460], x=feature, y='SalePrice', ax=ax[i, 2])\nplt.show()","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-12T10:39:37.188306Z","iopub.execute_input":"2022-08-12T10:39:37.189406Z","iopub.status.idle":"2022-08-12T10:39:40.520119Z","shell.execute_reply.started":"2022-08-12T10:39:37.189344Z","shell.execute_reply":"2022-08-12T10:39:40.518847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, ax = plt.subplots(3, 2, figsize=(50, 60))\nax = ax.flatten()\nfor i, feature in enumerate(['YearBuilt', 'YearRemodAdd', 'TotalBsmtSF', '1stFlrSF', 'GrLivArea', 'GarageArea']):\n    sns.regplot(data=data[:1460], x=feature, y='SalePrice', scatter_kws={'alpha': 0.3}, ax=ax[i])\n    ax[i].set_title(f'{feature} VS. SalePrice')","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-12T10:39:40.522035Z","iopub.execute_input":"2022-08-12T10:39:40.523000Z","iopub.status.idle":"2022-08-12T10:39:43.439033Z","shell.execute_reply.started":"2022-08-12T10:39:40.522955Z","shell.execute_reply":"2022-08-12T10:39:43.437711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Preprocessing","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"# Convert the non-numeric values present, into numeric values\ndata['MSSubClass'] = data['MSSubClass'].apply(str)\ndata['YrSold'] = data['YrSold'].apply(str)\ndata['MoSold'] = data['MoSold'].apply(str)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-12T10:39:43.440401Z","iopub.execute_input":"2022-08-12T10:39:43.440750Z","iopub.status.idle":"2022-08-12T10:39:43.454058Z","shell.execute_reply.started":"2022-08-12T10:39:43.440716Z","shell.execute_reply":"2022-08-12T10:39:43.452918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Engineering new features","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"data['TotalQual'] = data['OverallQual'] + data['OverallCond']\ndata['TotalBath'] = data['FullBath'] + data['BsmtFullBath'].fillna(0) + (0.5 * data['HalfBath']) + (0.5 * data['BsmtHalfBath'].fillna(0))\ndata['GarageAreaPerCar'] = (data['GarageArea'] / data['GarageCars']).fillna(0)\ndata['TimeTakenToBuildGarage'] = (data['GarageYrBlt'] - data['YearBuilt']).fillna(0)\ndata['AreaPerRoom'] = data['GrLivArea'] / (data['TotRmsAbvGrd'] + data['FullBath'] + data['HalfBath'] + data['KitchenAbvGr'])\ndata['TotalSF'] = data['1stFlrSF'] + data['TotalBsmtSF'].fillna(0) + data['2ndFlrSF'] + data['GrLivArea']\ndata['HighQualSF'] = data['GrLivArea'] + data['1stFlrSF'] + data['2ndFlrSF'] + 0.25 * data['GarageArea'].fillna(0) + 0.5 * data['TotalBsmtSF'].fillna(0) + data['MasVnrArea'].fillna(0)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-12T10:39:43.457641Z","iopub.execute_input":"2022-08-12T10:39:43.458389Z","iopub.status.idle":"2022-08-12T10:39:43.477330Z","shell.execute_reply.started":"2022-08-12T10:39:43.458331Z","shell.execute_reply":"2022-08-12T10:39:43.475976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fixing the skewness of the data","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"for feature in data[:1460].columns.values:\n    if data[:1460][feature].dtype in ['int64', 'float64']:\n        print('Skewness of {}: {:.2f}'.format(feature, data[:1460][feature].skew()))","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-12T10:39:43.478857Z","iopub.execute_input":"2022-08-12T10:39:43.479953Z","iopub.status.idle":"2022-08-12T10:39:43.516675Z","shell.execute_reply.started":"2022-08-12T10:39:43.479915Z","shell.execute_reply":"2022-08-12T10:39:43.515851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modeling","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"# One hot encoding\nfor feature in ['Functional', 'Electrical', 'KitchenQual', 'Exterior1st', 'Exterior2nd', 'SaleType', 'MSZoning', 'Utilities']:\n    data[feature].fillna(data[feature].mode()[0])\n\nfor feature in [column for column in data.columns.values if data[column].dtype == 'object']:\n    dummies = pd.get_dummies(data[feature], prefix=feature)\n    data.drop(columns=[feature], inplace=True)\n    data = pd.concat([data, dummies], axis=1)\n    print(f'Dummy variables created for {feature}')","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-12T10:39:43.518287Z","iopub.execute_input":"2022-08-12T10:39:43.518975Z","iopub.status.idle":"2022-08-12T10:39:43.977990Z","shell.execute_reply.started":"2022-08-12T10:39:43.518926Z","shell.execute_reply":"2022-08-12T10:39:43.975690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = data[:1460]\ntest_data = data[1460:]","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-12T10:39:43.979418Z","iopub.execute_input":"2022-08-12T10:39:43.979730Z","iopub.status.idle":"2022-08-12T10:39:43.988346Z","shell.execute_reply.started":"2022-08-12T10:39:43.979701Z","shell.execute_reply":"2022-08-12T10:39:43.986447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zero_imputer = Pipeline([\n    ('imputing', SimpleImputer(strategy='constant', fill_value=0))\n])\n\nmedian_imputer = Pipeline([\n    ('imputing', SimpleImputer(strategy='median'))\n])\n\nnormal_transform_pipeline = Pipeline([\n    ('transforming', QuantileTransformer(output_distribution='normal')),\n])\n\nuniform_transform_pipeline = Pipeline([\n    ('transforming', QuantileTransformer(output_distribution='uniform')),\n])\n\nlog_transform_pipeline = Pipeline([\n    ('transforming', FunctionTransformer(np.log1p))\n])\n\nnumeric_features = [feature for feature in train_data.columns.values if train_data[feature].dtype in ['int64', 'float64']]\nskewed_columns = [feature for feature in numeric_features if round(abs(train_data[feature].skew())) > 0.5 and feature != 'SalePrice']\n\ndata_preprocessor = ColumnTransformer([\n    ('median_imputer', median_imputer, ['LotFrontage']),\n    ('zero_imputer', zero_imputer, ['GarageCars', 'GarageArea', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath', 'HalfBath', 'GarageCars', 'GarageArea', 'GarageYrBlt']),\n    ('normal_transformer', normal_transform_pipeline, ['LotArea', 'GrLivArea', 'HighQualSF', 'AreaPerRoom', 'TotalSF']),\n    ('uniform_transformer', uniform_transform_pipeline, ['YearBuilt', 'YearRemodAdd', 'BsmtUnfSF', 'TotalBsmtSF']),\n    ('log_transformer', log_transform_pipeline, skewed_columns),\n], remainder='passthrough')","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-12T10:39:43.989999Z","iopub.execute_input":"2022-08-12T10:39:43.990935Z","iopub.status.idle":"2022-08-12T10:39:44.024197Z","shell.execute_reply.started":"2022-08-12T10:39:43.990882Z","shell.execute_reply":"2022-08-12T10:39:44.023089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sale_price_qt = QuantileTransformer(output_distribution='normal').fit(train_data['SalePrice'].to_numpy().reshape(-1, 1))\nsale_price = sale_price_qt.transform(train_data['SalePrice'].to_numpy().reshape(-1, 1)).flatten()\ntrain_data.drop(columns='SalePrice', inplace=True)\ntest_data.drop(columns='SalePrice', inplace=True)\n\ntrain_data = data_preprocessor.fit_transform(train_data)\ntest_data = data_preprocessor.transform(test_data)\nX_train, X_val, y_train, y_val = train_test_split(train_data, sale_price, test_size=0.2, random_state=52)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-12T10:39:44.025869Z","iopub.execute_input":"2022-08-12T10:39:44.026789Z","iopub.status.idle":"2022-08-12T10:39:44.127537Z","shell.execute_reply.started":"2022-08-12T10:39:44.026752Z","shell.execute_reply":"2022-08-12T10:39:44.126438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"model = CatBoostRegressor()\nrandomized_search = model.randomized_search(\n    {\n        'iterations': [1000, 15000, 20000, 25000, 30000],\n        'learning_rate': [0.01, 0.03, 0.05, 0.1, 0.3, 0.5],\n        'l2_leaf_reg': [1, 3, 5, 7, 9, 11, 18],\n        'max_leaves': [8, 16, 32, 64, 128, 256],\n        'early_stopping_rounds': [200],\n        'model_size_reg': [0.1, 0.3, 0.5, 0.7, 0.9],\n    },\n    X=X_train,\n    y=y_train,\n    verbose=False,\n    plot=True\n)","metadata":{"pycharm":{"name":"#%%\n","is_executing":true},"tags":[],"execution":{"iopub.status.busy":"2022-08-12T10:39:44.128824Z","iopub.execute_input":"2022-08-12T10:39:44.129665Z","iopub.status.idle":"2022-08-12T10:44:44.607355Z","shell.execute_reply.started":"2022-08-12T10:39:44.129631Z","shell.execute_reply":"2022-08-12T10:44:44.605791Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CatBoostRegressor(**randomized_search['params']).fit(X_train, y_train, eval_set=(X_val, y_val), plot=True, verbose=False)\npredictions = model.predict(X_val)\nprint(mean_squared_error(y_true=y_val, y_pred=predictions))","metadata":{"pycharm":{"is_executing":true},"execution":{"iopub.status.busy":"2022-08-12T10:44:44.608984Z","iopub.execute_input":"2022-08-12T10:44:44.609332Z","iopub.status.idle":"2022-08-12T10:45:43.298111Z","shell.execute_reply.started":"2022-08-12T10:44:44.609301Z","shell.execute_reply":"2022-08-12T10:45:43.296971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(test_data)\npredictions = sale_price_qt.inverse_transform(predictions.reshape(-1, 1)).flatten()\npd.DataFrame({'Id': test_id, 'SalePrice': predictions}).to_csv('submission.csv', index=False)","metadata":{"pycharm":{"is_executing":true},"execution":{"iopub.status.busy":"2022-08-12T10:52:19.420529Z","iopub.execute_input":"2022-08-12T10:52:19.420999Z","iopub.status.idle":"2022-08-12T10:52:19.460964Z","shell.execute_reply.started":"2022-08-12T10:52:19.420957Z","shell.execute_reply":"2022-08-12T10:52:19.460008Z"},"trusted":true},"execution_count":null,"outputs":[]}]}