{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-21T04:03:57.155202Z","iopub.execute_input":"2022-07-21T04:03:57.155922Z","iopub.status.idle":"2022-07-21T04:03:57.200561Z","shell.execute_reply.started":"2022-07-21T04:03:57.155844Z","shell.execute_reply":"2022-07-21T04:03:57.198823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import OrdinalEncoder, PolynomialFeatures, LabelEncoder\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import mean_squared_error\n# from pycaret.regression import setup, compare_models\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:03:57.207036Z","iopub.execute_input":"2022-07-21T04:03:57.207355Z","iopub.status.idle":"2022-07-21T04:03:58.095574Z","shell.execute_reply.started":"2022-07-21T04:03:57.207316Z","shell.execute_reply":"2022-07-21T04:03:58.094284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import CatBoostRegressor","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:03:58.097113Z","iopub.execute_input":"2022-07-21T04:03:58.097491Z","iopub.status.idle":"2022-07-21T04:03:58.529851Z","shell.execute_reply.started":"2022-07-21T04:03:58.097457Z","shell.execute_reply":"2022-07-21T04:03:58.528056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read train and test sets","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"../input/house-prices-advanced-regression-techniques/train.csv\")\ndf_test = pd.read_csv(\"../input/house-prices-advanced-regression-techniques/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:03:58.532986Z","iopub.execute_input":"2022-07-21T04:03:58.534172Z","iopub.status.idle":"2022-07-21T04:03:58.630758Z","shell.execute_reply.started":"2022-07-21T04:03:58.534127Z","shell.execute_reply":"2022-07-21T04:03:58.628813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Store both dfs lengths","metadata":{}},{"cell_type":"code","source":"m_train = df_train.shape[0]\nm_test = df_test.shape[0]\nm_train, m_test","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:03:58.632402Z","iopub.execute_input":"2022-07-21T04:03:58.633025Z","iopub.status.idle":"2022-07-21T04:03:58.645890Z","shell.execute_reply.started":"2022-07-21T04:03:58.632980Z","shell.execute_reply":"2022-07-21T04:03:58.644526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Concat the dfs","metadata":{}},{"cell_type":"code","source":"df = pd.concat([df_train, df_test])\nassert df.shape[0] == m_train + m_test","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:03:58.648041Z","iopub.execute_input":"2022-07-21T04:03:58.649115Z","iopub.status.idle":"2022-07-21T04:03:58.679354Z","shell.execute_reply.started":"2022-07-21T04:03:58.649074Z","shell.execute_reply":"2022-07-21T04:03:58.678136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:03:58.681082Z","iopub.execute_input":"2022-07-21T04:03:58.681557Z","iopub.status.idle":"2022-07-21T04:03:58.715467Z","shell.execute_reply.started":"2022-07-21T04:03:58.681512Z","shell.execute_reply":"2022-07-21T04:03:58.714213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save target","metadata":{}},{"cell_type":"code","source":"target_col, target = \"SalePrice\", df[\"SalePrice\"]","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:03:58.718026Z","iopub.execute_input":"2022-07-21T04:03:58.718383Z","iopub.status.idle":"2022-07-21T04:03:58.725874Z","shell.execute_reply.started":"2022-07-21T04:03:58.718350Z","shell.execute_reply":"2022-07-21T04:03:58.724907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  Drop Id and target columns","metadata":{}},{"cell_type":"code","source":"cols_to_drop = [\"Id\", target_col]\ndf.drop(cols_to_drop, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:03:58.997833Z","iopub.execute_input":"2022-07-21T04:03:58.998463Z","iopub.status.idle":"2022-07-21T04:03:59.018171Z","shell.execute_reply.started":"2022-07-21T04:03:58.998412Z","shell.execute_reply":"2022-07-21T04:03:59.017084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# View and fix columns data-types","metadata":{}},{"cell_type":"code","source":"df.select_dtypes(object).columns","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:03:59.423861Z","iopub.execute_input":"2022-07-21T04:03:59.424463Z","iopub.status.idle":"2022-07-21T04:03:59.441413Z","shell.execute_reply.started":"2022-07-21T04:03:59.424428Z","shell.execute_reply":"2022-07-21T04:03:59.440417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.select_dtypes(np.number).columns","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:03:59.750975Z","iopub.execute_input":"2022-07-21T04:03:59.752155Z","iopub.status.idle":"2022-07-21T04:03:59.761552Z","shell.execute_reply.started":"2022-07-21T04:03:59.752107Z","shell.execute_reply":"2022-07-21T04:03:59.760523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_to_obj_cols = ['MSSubClass', 'MoSold']\ndf[num_to_obj_cols] = df[num_to_obj_cols].astype(object)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:00.084974Z","iopub.execute_input":"2022-07-21T04:04:00.085693Z","iopub.status.idle":"2022-07-21T04:04:00.094795Z","shell.execute_reply.started":"2022-07-21T04:04:00.085643Z","shell.execute_reply":"2022-07-21T04:04:00.093700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_cat = df.select_dtypes(object).columns.to_list()\ncols_num = df.select_dtypes(np.number).columns.to_list()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:00.317398Z","iopub.execute_input":"2022-07-21T04:04:00.318196Z","iopub.status.idle":"2022-07-21T04:04:00.334901Z","shell.execute_reply.started":"2022-07-21T04:04:00.318144Z","shell.execute_reply":"2022-07-21T04:04:00.333435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Impute categorical columns","metadata":{}},{"cell_type":"code","source":"cols_cat_na = df[cols_cat].isnull().sum()[df[cols_cat].isnull().sum() > 0]\ncols_cat_na","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:00.797517Z","iopub.execute_input":"2022-07-21T04:04:00.798014Z","iopub.status.idle":"2022-07-21T04:04:00.844700Z","shell.execute_reply.started":"2022-07-21T04:04:00.797977Z","shell.execute_reply":"2022-07-21T04:04:00.843528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mode_filled_cols = [\"MSZoning\", \"Utilities\", \"Exterior1st\", \"Exterior2nd\", \"MasVnrType\", \"Electrical\", \"KitchenQual\", \"Functional\", \"SaleType\"]\nfor col in mode_filled_cols:\n    df[col].fillna(df[col].mode()[0], inplace=True)\n\nnone_filled_cols = [\"Alley\", \"BsmtQual\", \"BsmtCond\", \"BsmtExposure\", \"BsmtFinType1\", \"BsmtFinType2\", \"FireplaceQu\", \"GarageType\", \"GarageFinish\", \"GarageQual\", \"GarageCond\", \"PoolQC\", \"Fence\", \"MiscFeature\"]\nfor col in none_filled_cols:\n    df[col].fillna(\"None\", inplace=True)\n    \ndf[cols_cat].isnull().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:00.996383Z","iopub.execute_input":"2022-07-21T04:04:00.996867Z","iopub.status.idle":"2022-07-21T04:04:01.050676Z","shell.execute_reply.started":"2022-07-21T04:04:00.996831Z","shell.execute_reply":"2022-07-21T04:04:01.049434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Change ordinal columns to numeric, and encode accordingly","metadata":{}},{"cell_type":"code","source":"df.select_dtypes(object).columns","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:01.329577Z","iopub.execute_input":"2022-07-21T04:04:01.330346Z","iopub.status.idle":"2022-07-21T04:04:01.344039Z","shell.execute_reply.started":"2022-07-21T04:04:01.330279Z","shell.execute_reply":"2022-07-21T04:04:01.343145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oe = OrdinalEncoder(categories=[['Reg', 'IR1', 'IR2', 'IR3']])\ndf.loc[:, \"LotShape\"] = oe.fit_transform(df[[\"LotShape\"]])","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:01.527304Z","iopub.execute_input":"2022-07-21T04:04:01.528048Z","iopub.status.idle":"2022-07-21T04:04:01.539618Z","shell.execute_reply.started":"2022-07-21T04:04:01.527995Z","shell.execute_reply":"2022-07-21T04:04:01.538239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oe = OrdinalEncoder(categories=[['Gtl', 'Mod', 'Sev']])\ndf.loc[:, \"LandSlope\"] = oe.fit_transform(df[[\"LandSlope\"]])","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:01.705217Z","iopub.execute_input":"2022-07-21T04:04:01.705600Z","iopub.status.idle":"2022-07-21T04:04:01.718775Z","shell.execute_reply.started":"2022-07-21T04:04:01.705569Z","shell.execute_reply":"2022-07-21T04:04:01.717176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"qual_oe = OrdinalEncoder(categories=[['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex']])\nfor col in [\"ExterQual\", \"ExterCond\", 'BsmtQual', 'BsmtCond', 'HeatingQC', 'KitchenQual', 'FireplaceQu', 'GarageQual', 'GarageCond', 'PoolQC']:\n    df.loc[:, col] = qual_oe.fit_transform(df[[col]])","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:01.880800Z","iopub.execute_input":"2022-07-21T04:04:01.881548Z","iopub.status.idle":"2022-07-21T04:04:01.931616Z","shell.execute_reply.started":"2022-07-21T04:04:01.881509Z","shell.execute_reply":"2022-07-21T04:04:01.930670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oe = OrdinalEncoder(categories=[['None', 'No', 'Mn', 'Av', 'Gd']])\ndf.loc[:, \"BsmtExposure\"] = oe.fit_transform(df[[\"BsmtExposure\"]])","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:02.045995Z","iopub.execute_input":"2022-07-21T04:04:02.046599Z","iopub.status.idle":"2022-07-21T04:04:02.058597Z","shell.execute_reply.started":"2022-07-21T04:04:02.046564Z","shell.execute_reply":"2022-07-21T04:04:02.057496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oe = OrdinalEncoder(categories=[['None', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ']])\nfor col in [\"BsmtFinType1\", \"BsmtFinType2\"]:\n    df.loc[:, col] = oe.fit_transform(df[[col]])","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:02.238284Z","iopub.execute_input":"2022-07-21T04:04:02.238914Z","iopub.status.idle":"2022-07-21T04:04:02.254439Z","shell.execute_reply.started":"2022-07-21T04:04:02.238880Z","shell.execute_reply":"2022-07-21T04:04:02.253093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oe = OrdinalEncoder(categories=[['None', 'Unf', 'RFn', 'Fin']])\ndf.loc[:, \"GarageFinish\"] = oe.fit_transform(df[[\"GarageFinish\"]])","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:02.411026Z","iopub.execute_input":"2022-07-21T04:04:02.411870Z","iopub.status.idle":"2022-07-21T04:04:02.424885Z","shell.execute_reply.started":"2022-07-21T04:04:02.411821Z","shell.execute_reply":"2022-07-21T04:04:02.422982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oe = OrdinalEncoder(categories=[['N', 'P', 'Y']])\ndf.loc[:, \"PavedDrive\"] = oe.fit_transform(df[[\"PavedDrive\"]])","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:02.603111Z","iopub.execute_input":"2022-07-21T04:04:02.603512Z","iopub.status.idle":"2022-07-21T04:04:02.617498Z","shell.execute_reply.started":"2022-07-21T04:04:02.603480Z","shell.execute_reply":"2022-07-21T04:04:02.616187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oe = OrdinalEncoder(categories=[['None', 'MnWw', 'GdWo', 'MnPrv', 'GdPrv']])\ndf.loc[:, \"Fence\"] = oe.fit_transform(df[[\"Fence\"]])","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:02.837518Z","iopub.execute_input":"2022-07-21T04:04:02.838567Z","iopub.status.idle":"2022-07-21T04:04:02.850535Z","shell.execute_reply.started":"2022-07-21T04:04:02.838526Z","shell.execute_reply":"2022-07-21T04:04:02.849306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_cat = df.select_dtypes(object).columns.to_list()\ncols_num = df.select_dtypes(np.number).columns.to_list()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:02.986704Z","iopub.execute_input":"2022-07-21T04:04:02.987841Z","iopub.status.idle":"2022-07-21T04:04:03.001144Z","shell.execute_reply.started":"2022-07-21T04:04:02.987785Z","shell.execute_reply":"2022-07-21T04:04:03.000019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Impute numerical columns","metadata":{}},{"cell_type":"code","source":"cols_num_na = df[cols_num].isnull().sum()[df[cols_num].isnull().sum() > 0]\ncols_num_na","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:06.435820Z","iopub.execute_input":"2022-07-21T04:04:06.436437Z","iopub.status.idle":"2022-07-21T04:04:06.452650Z","shell.execute_reply.started":"2022-07-21T04:04:06.436268Z","shell.execute_reply":"2022-07-21T04:04:06.451433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zero_filled_cols = [\"BsmtFinSF1\", \"BsmtFinSF2\", \"BsmtUnfSF\", \"TotalBsmtSF\", \"BsmtFullBath\", \"BsmtHalfBath\", \"GarageCars\", \"GarageArea\"]\nfor col in zero_filled_cols:\n    df[col].fillna(0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:06.627217Z","iopub.execute_input":"2022-07-21T04:04:06.627869Z","iopub.status.idle":"2022-07-21T04:04:06.636446Z","shell.execute_reply.started":"2022-07-21T04:04:06.627830Z","shell.execute_reply":"2022-07-21T04:04:06.635037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Thanks to: https://www.kaggle.com/code/t3kuwabara/houseprices-test2\ndf[\"LotFrontage\"] = df.groupby(\"Neighborhood\")[\"LotFrontage\"].transform(lambda x: x.fillna(x.median()))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_num_na = df[cols_num].isnull().sum()[df[cols_num].isnull().sum() > 0]\ncols_num_na","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:06.825018Z","iopub.execute_input":"2022-07-21T04:04:06.825438Z","iopub.status.idle":"2022-07-21T04:04:06.840343Z","shell.execute_reply.started":"2022-07-21T04:04:06.825404Z","shell.execute_reply":"2022-07-21T04:04:06.839382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def catboost_imputer(df, cols_num_na):\n    \n    # save columns to impute and drop them from df\n    cols_to_impute = df[cols_num_na]\n    df.drop(cols_num_na, axis=1, inplace=True)\n    \n    for col in cols_num_na:\n        # define X and y\n        X, y = df.copy(), cols_to_impute[col]\n\n        # get train and test sets\n        train_indexes, test_indexes = ~y.isnull(), y.isnull()\n        X_train, X_test = X.loc[train_indexes, :], X.loc[test_indexes, :]\n        y_train, y_test = y.loc[train_indexes], y.loc[test_indexes]\n        \n        model = CatBoostRegressor(max_depth=8, random_seed=10,\n                                  subsample=0.65, n_estimators=1000,\n                                  cat_features=cols_cat, verbose=0)\n        model.fit(X_train, y_train)\n        cols_to_impute.loc[test_indexes, col] = model.predict(X_test)\n        df.loc[:, col] = cols_to_impute[col]\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:17.229114Z","iopub.execute_input":"2022-07-21T04:04:17.229671Z","iopub.status.idle":"2022-07-21T04:04:17.242916Z","shell.execute_reply.started":"2022-07-21T04:04:17.229602Z","shell.execute_reply":"2022-07-21T04:04:17.241427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = catboost_imputer(df, cols_num_na.index)\ndf.isnull().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:18.105386Z","iopub.execute_input":"2022-07-21T04:04:18.106489Z","iopub.status.idle":"2022-07-21T04:06:43.862349Z","shell.execute_reply.started":"2022-07-21T04:04:18.106443Z","shell.execute_reply":"2022-07-21T04:06:43.861465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature selection","metadata":{}},{"cell_type":"code","source":"def get_del_cols(df):\n    # return the columns where no more than one value in the categorical features\n    # exists in the test data - those features can be ignored\n    cols_to_drop = []\n    for col in df.select_dtypes(object).columns:\n        col_vals = df[col].unique()\n        n_vals = len(col_vals)\n        n_irrelevant = 0\n        for val in col_vals:\n            if val not in df[col][m_train:].values:\n                n_irrelevant += 1\n        if n_irrelevant >= n_vals - 1:\n            cols_to_drop.append(col)\n    return cols_to_drop","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:06:43.864544Z","iopub.execute_input":"2022-07-21T04:06:43.865506Z","iopub.status.idle":"2022-07-21T04:06:43.874693Z","shell.execute_reply.started":"2022-07-21T04:06:43.865458Z","shell.execute_reply":"2022-07-21T04:06:43.873288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_to_drop = get_del_cols(df)\ndf.drop(cols_to_drop, axis=1, inplace=True)\nfor col_to_drop in cols_to_drop:\n    cols_cat.remove(col_to_drop)\n    print(\"Dropped \", col_to_drop)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:06:43.876276Z","iopub.execute_input":"2022-07-21T04:06:43.876685Z","iopub.status.idle":"2022-07-21T04:06:43.934908Z","shell.execute_reply.started":"2022-07-21T04:06:43.876624Z","shell.execute_reply":"2022-07-21T04:06:43.933605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"## Features from the Internet","metadata":{}},{"cell_type":"code","source":"df[\"HighQualSF\"] = df[\"GrLivArea\"]+df[\"1stFlrSF\"] + df[\"2ndFlrSF\"]+0.5*df[\"GarageArea\"]+0.5*df[\"TotalBsmtSF\"]+1*df[\"MasVnrArea\"]\n\ndf[\"SqFtPerRoom\"] = df[\"GrLivArea\"] / (df[\"TotRmsAbvGrd\"] +\n                                       df[\"FullBath\"] +\n                                       df[\"HalfBath\"] +\n                                       df[\"KitchenAbvGr\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:06:43.939174Z","iopub.execute_input":"2022-07-21T04:06:43.939594Z","iopub.status.idle":"2022-07-21T04:06:43.951019Z","shell.execute_reply.started":"2022-07-21T04:06:43.939561Z","shell.execute_reply":"2022-07-21T04:06:43.949657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## My features","metadata":{}},{"cell_type":"code","source":"n_stories_dict = {\n    \"1Story\": 1.0,\n    \"1.5Fin\": 1.5,\n    \"1.5Unf\": 1.5,\n    \"2Story\": 2.0,\n    \"2.5Fin\": 2.5,\n    \"2.5Unf\": 2.5,\n    \"SFoyer\": 2.0,\n    \"SLvl\": 2.0,\n}\n\ndf[\"n_stories\"] = df[\"HouseStyle\"].replace(n_stories_dict)\ndf[\"n_stories\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:06:43.952871Z","iopub.execute_input":"2022-07-21T04:06:43.953238Z","iopub.status.idle":"2022-07-21T04:06:43.983008Z","shell.execute_reply.started":"2022-07-21T04:06:43.953208Z","shell.execute_reply":"2022-07-21T04:06:43.981209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"age_sold\"] = df[\"YrSold\"] - df[\"YearBuilt\"]\ndf[\"age_sold_Remod\"] = df[\"YrSold\"] - df[\"YearRemodAdd\"]\ndf[\"GarageYrSold\"] = df[\"YrSold\"] - df[\"GarageYrBlt\"]","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:06:43.984994Z","iopub.execute_input":"2022-07-21T04:06:43.985537Z","iopub.status.idle":"2022-07-21T04:06:43.998617Z","shell.execute_reply.started":"2022-07-21T04:06:43.985496Z","shell.execute_reply":"2022-07-21T04:06:43.996447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"CentralAir\"] = df[\"CentralAir\"].replace({\"Y\": 1, \"N\": 0})\ndf[\"CentralAir\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:06:44.001330Z","iopub.execute_input":"2022-07-21T04:06:44.001872Z","iopub.status.idle":"2022-07-21T04:06:44.019877Z","shell.execute_reply.started":"2022-07-21T04:06:44.001821Z","shell.execute_reply":"2022-07-21T04:06:44.018107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['total_area_hq'] = df[\"HighQualSF\"] - df['LowQualFinSF']","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:06:44.021908Z","iopub.execute_input":"2022-07-21T04:06:44.022361Z","iopub.status.idle":"2022-07-21T04:06:44.031544Z","shell.execute_reply.started":"2022-07-21T04:06:44.022322Z","shell.execute_reply":"2022-07-21T04:06:44.029441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Apply polynomial features","metadata":{}},{"cell_type":"code","source":"cols_to_poly = ['LotArea', 'OverallQual', 'OverallCond', 'YearBuilt', 'ExterQual', 'ExterCond','BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinSF1','BsmtFinType2', 'BsmtFinSF2', 'TotalBsmtSF', 'total_area_hq', 'GrLivArea', 'FullBath', 'HalfBath', 'SqFtPerRoom', 'KitchenQual', 'KitchenAbvGr', 'Fireplaces', 'FireplaceQu', 'GarageFinish', 'GarageCars', 'GarageArea', 'GarageQual', 'GarageCond', 'PoolArea', 'YrSold', 'LotFrontage', 'HighQualSF',]","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:06:44.033408Z","iopub.execute_input":"2022-07-21T04:06:44.033847Z","iopub.status.idle":"2022-07-21T04:06:44.041601Z","shell.execute_reply.started":"2022-07-21T04:06:44.033809Z","shell.execute_reply":"2022-07-21T04:06:44.040327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"poly = PolynomialFeatures(degree=2, include_bias=False)\ncols_df = df[cols_to_poly]\ndf.drop(cols_to_poly, axis=1, inplace=True)\ncols_array = poly.fit_transform(cols_df)\ncols_df = pd.DataFrame(cols_array, columns=poly.get_feature_names_out())\ndf = pd.concat((df.reset_index(drop=True), cols_df.reset_index(drop=True)), axis=1)\n\ndf.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:06:44.045752Z","iopub.execute_input":"2022-07-21T04:06:44.046196Z","iopub.status.idle":"2022-07-21T04:06:44.157359Z","shell.execute_reply.started":"2022-07-21T04:06:44.046161Z","shell.execute_reply":"2022-07-21T04:06:44.156185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split test and train","metadata":{}},{"cell_type":"code","source":"X_train, X_test = df[:m_train], df[m_train:]\ny_train, _ = target[:m_train], target[m_train:]\nn = X_train.shape[1]\nX_train.shape, y_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:06:44.159089Z","iopub.execute_input":"2022-07-21T04:06:44.159689Z","iopub.status.idle":"2022-07-21T04:06:44.169664Z","shell.execute_reply.started":"2022-07-21T04:06:44.159619Z","shell.execute_reply":"2022-07-21T04:06:44.168243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_ohe = pd.get_dummies(df)\n# df_ohe[target_col] = target.reset_index(drop=True)\n# _ = setup(data=df_ohe[:m_train], target='SalePrice')\n# compare_models()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:06:44.171378Z","iopub.execute_input":"2022-07-21T04:06:44.171888Z","iopub.status.idle":"2022-07-21T04:06:44.178494Z","shell.execute_reply.started":"2022-07-21T04:06:44.171840Z","shell.execute_reply":"2022-07-21T04:06:44.177026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CatBoost Grid Search ","metadata":{}},{"cell_type":"code","source":"# cat_features = df.select_dtypes(object).columns.to_list()\n# model = CatBoostRegressor(random_state=10, cat_features=cat_features)\n# kwargs = {\n#     \"n_estimators\": [17000, 20000, 25000],\n#     \"max_depth\": [8],\n#     \"subsample\": [.65],\n#     \"reg_lambda\": [0.1], \n# }\n# clf = GridSearchCV(model, kwargs, verbose=1, n_jobs=2)\n# clf.fit(X_train, y_train)\n# print(clf.best_score_)\n# print(clf.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:07:01.977839Z","iopub.execute_input":"2022-07-21T04:07:01.978257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CatBoost Params:\n    {'max_depth': 8, 'n_estimators': 20_000, 'reg_lambda': 0.1, 'subsample': 0.65}","metadata":{}},{"cell_type":"markdown","source":"# Training - CatBoost with Best Parameters","metadata":{}},{"cell_type":"code","source":"model = CatBoostRegressor(max_depth=8, random_seed=10,\n                          subsample=0.65, n_estimators=20_000,\n                          cat_features=cols_cat, verbose=0)\nmodel.fit(X_train, y_train, plot=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize Features' Importance","metadata":{}},{"cell_type":"code","source":"print(model.best_score_)\nfe = model.get_feature_importance(prettified=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 200))\nsns.barplot(y=\"Feature Id\", x=\"Importances\", data=fe)\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make a submission","metadata":{}},{"cell_type":"code","source":"sub_name = \"submission.csv\"\npd.DataFrame(model.predict(X_test), \n            index=range(1461, len(df)+1), \n            columns=['SalePrice']).reset_index().\\\n            rename(columns={'index': 'id'}).to_csv(sub_name, index=False)","metadata":{},"execution_count":null,"outputs":[]}]}