{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport optuna\nimport matplotlib.pyplot as plt\nimport eli5","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-15T08:49:45.762805Z","iopub.execute_input":"2022-07-15T08:49:45.763278Z","iopub.status.idle":"2022-07-15T08:49:45.768743Z","shell.execute_reply.started":"2022-07-15T08:49:45.763243Z","shell.execute_reply":"2022-07-15T08:49:45.767952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from eli5.sklearn import PermutationImportance\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import mean_squared_error, mean_squared_log_error, make_scorer\nfrom sklearn.ensemble import RandomForestRegressor\n\nfrom optuna.visualization import plot_intermediate_values\nfrom optuna.visualization import plot_optimization_history\nfrom optuna.visualization import plot_parallel_coordinate\nfrom optuna.visualization import plot_param_importances\nfrom optuna.visualization import plot_slice","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:45.796941Z","iopub.execute_input":"2022-07-15T08:49:45.797823Z","iopub.status.idle":"2022-07-15T08:49:45.803449Z","shell.execute_reply.started":"2022-07-15T08:49:45.797785Z","shell.execute_reply":"2022-07-15T08:49:45.802433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.options.mode.chained_assignment = None\npd.set_option('display.max_columns', None)\n\nSEED = 42\nnp.random.seed(SEED)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:45.824731Z","iopub.execute_input":"2022-07-15T08:49:45.825768Z","iopub.status.idle":"2022-07-15T08:49:45.830482Z","shell.execute_reply.started":"2022-07-15T08:49:45.825728Z","shell.execute_reply":"2022-07-15T08:49:45.829767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_raw = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')\ntest_raw = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')\nprint(train_raw.shape)\nprint(test_raw.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:45.854728Z","iopub.execute_input":"2022-07-15T08:49:45.855144Z","iopub.status.idle":"2022-07-15T08:49:45.908087Z","shell.execute_reply.started":"2022-07-15T08:49:45.855101Z","shell.execute_reply":"2022-07-15T08:49:45.907069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Check the graphic again\nfig, ax = plt.subplots()\n\nsns.scatterplot(data=train_raw, x=\"GrLivArea\", y=\"SalePrice\", ax=ax)\nplt.ylabel('SalePrice', fontsize=13)\nplt.xlabel('GrLivArea', fontsize=13)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:45.910074Z","iopub.execute_input":"2022-07-15T08:49:45.910714Z","iopub.status.idle":"2022-07-15T08:49:46.125699Z","shell.execute_reply.started":"2022-07-15T08:49:45.910657Z","shell.execute_reply":"2022-07-15T08:49:46.124886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_to_drop = train_raw[(train_raw['GrLivArea']>4000) & (train_raw['SalePrice']>600000)][\"Id\"]\nid_to_drop = list(id_to_drop)\nprint(train_raw.shape)\ntrain_raw.drop(id_to_drop,inplace=True)\nprint(train_raw.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:46.127180Z","iopub.execute_input":"2022-07-15T08:49:46.127498Z","iopub.status.idle":"2022-07-15T08:49:46.139287Z","shell.execute_reply.started":"2022-07-15T08:49:46.127471Z","shell.execute_reply":"2022-07-15T08:49:46.138170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all = pd.concat([train_raw, test_raw], axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:46.140488Z","iopub.execute_input":"2022-07-15T08:49:46.140815Z","iopub.status.idle":"2022-07-15T08:49:46.161050Z","shell.execute_reply.started":"2022-07-15T08:49:46.140788Z","shell.execute_reply":"2022-07-15T08:49:46.160094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:46.162795Z","iopub.execute_input":"2022-07-15T08:49:46.163245Z","iopub.status.idle":"2022-07-15T08:49:46.222748Z","shell.execute_reply.started":"2022-07-15T08:49:46.163205Z","shell.execute_reply":"2022-07-15T08:49:46.222024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_ratio = df_all.isna().sum().div(len(df_all)).mul(100).sort_values(ascending=False)\nmissing_ratio = missing_ratio[missing_ratio>0]\n\nfig, ax = plt.subplots(1,1, figsize=(20,5))\nax.bar(missing_ratio.index, missing_ratio.values)\nax.tick_params(axis='x', rotation=90)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:46.223748Z","iopub.execute_input":"2022-07-15T08:49:46.224644Z","iopub.status.idle":"2022-07-15T08:49:46.633398Z","shell.execute_reply.started":"2022-07-15T08:49:46.224604Z","shell.execute_reply":"2022-07-15T08:49:46.632460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Filling missing values","metadata":{}},{"cell_type":"code","source":"df_all[\"PoolQC\"] = df_all[\"PoolQC\"].fillna(\"None\")\ndf_all[\"MiscFeature\"] = df_all[\"MiscFeature\"].fillna(\"None\")\ndf_all[\"Alley\"] = df_all[\"Alley\"].fillna(\"None\")\ndf_all[\"Fence\"] = df_all[\"Fence\"].fillna(\"None\")\ndf_all[\"FireplaceQu\"] = df_all[\"FireplaceQu\"].fillna(\"None\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:46.634999Z","iopub.execute_input":"2022-07-15T08:49:46.635618Z","iopub.status.idle":"2022-07-15T08:49:46.647215Z","shell.execute_reply.started":"2022-07-15T08:49:46.635572Z","shell.execute_reply":"2022-07-15T08:49:46.645937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Group by neighborhood and fill in missing value by the median LotFrontage of all the neighborhood\ndf_all[\"LotFrontage\"] = df_all.groupby(\"Neighborhood\")[\"LotFrontage\"].transform(lambda group: group.fillna(group.median()))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:46.648935Z","iopub.execute_input":"2022-07-15T08:49:46.649273Z","iopub.status.idle":"2022-07-15T08:49:46.674099Z","shell.execute_reply.started":"2022-07-15T08:49:46.649245Z","shell.execute_reply":"2022-07-15T08:49:46.673241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in ('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):\n    df_all[col] = df_all[col].fillna('None')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:46.675526Z","iopub.execute_input":"2022-07-15T08:49:46.675890Z","iopub.status.idle":"2022-07-15T08:49:46.684114Z","shell.execute_reply.started":"2022-07-15T08:49:46.675835Z","shell.execute_reply":"2022-07-15T08:49:46.683110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in ('GarageYrBlt', 'GarageArea', 'GarageCars'):\n    df_all[col] = df_all[col].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:46.685335Z","iopub.execute_input":"2022-07-15T08:49:46.685648Z","iopub.status.idle":"2022-07-15T08:49:46.696937Z","shell.execute_reply.started":"2022-07-15T08:49:46.685622Z","shell.execute_reply":"2022-07-15T08:49:46.696025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in ('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):\n    df_all[col] = df_all[col].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:46.699913Z","iopub.execute_input":"2022-07-15T08:49:46.700273Z","iopub.status.idle":"2022-07-15T08:49:46.710182Z","shell.execute_reply.started":"2022-07-15T08:49:46.700243Z","shell.execute_reply":"2022-07-15T08:49:46.709153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in ('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):\n    df_all[col] = df_all[col].fillna('None')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:46.711371Z","iopub.execute_input":"2022-07-15T08:49:46.711819Z","iopub.status.idle":"2022-07-15T08:49:46.723427Z","shell.execute_reply.started":"2022-07-15T08:49:46.711789Z","shell.execute_reply":"2022-07-15T08:49:46.722356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['MSZoning'] = df_all['MSZoning'].fillna(df_all['MSZoning'].mode()[0])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:46.724937Z","iopub.execute_input":"2022-07-15T08:49:46.725942Z","iopub.status.idle":"2022-07-15T08:49:46.734054Z","shell.execute_reply.started":"2022-07-15T08:49:46.725898Z","shell.execute_reply":"2022-07-15T08:49:46.733280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all[\"MasVnrType\"] = df_all[\"MasVnrType\"].fillna(\"None\")\ndf_all[\"MasVnrArea\"] = df_all[\"MasVnrArea\"].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:46.735101Z","iopub.execute_input":"2022-07-15T08:49:46.735778Z","iopub.status.idle":"2022-07-15T08:49:46.746187Z","shell.execute_reply.started":"2022-07-15T08:49:46.735747Z","shell.execute_reply":"2022-07-15T08:49:46.745244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all = df_all.drop(['Utilities'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:46.747760Z","iopub.execute_input":"2022-07-15T08:49:46.748296Z","iopub.status.idle":"2022-07-15T08:49:46.761061Z","shell.execute_reply.started":"2022-07-15T08:49:46.748264Z","shell.execute_reply":"2022-07-15T08:49:46.759831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all[\"Functional\"] = df_all[\"Functional\"].fillna(\"Typ\")\ndf_all['Electrical'] = df_all['Electrical'].fillna(df_all['Electrical'].mode()[0])\ndf_all['KitchenQual'] = df_all['KitchenQual'].fillna(df_all['KitchenQual'].mode()[0])\ndf_all['Exterior1st'] = df_all['Exterior1st'].fillna(df_all['Exterior1st'].mode()[0])\ndf_all['Exterior2nd'] = df_all['Exterior2nd'].fillna(df_all['Exterior2nd'].mode()[0])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:46.762762Z","iopub.execute_input":"2022-07-15T08:49:46.763411Z","iopub.status.idle":"2022-07-15T08:49:46.779306Z","shell.execute_reply.started":"2022-07-15T08:49:46.763376Z","shell.execute_reply":"2022-07-15T08:49:46.778291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['SaleType'] = df_all['SaleType'].fillna(df_all['SaleType'].mode()[0])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:46.780444Z","iopub.execute_input":"2022-07-15T08:49:46.780916Z","iopub.status.idle":"2022-07-15T08:49:46.787711Z","shell.execute_reply.started":"2022-07-15T08:49:46.780860Z","shell.execute_reply":"2022-07-15T08:49:46.786711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_columns = df_all.select_dtypes(exclude=\"number\").columns","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:46.788955Z","iopub.execute_input":"2022-07-15T08:49:46.789399Z","iopub.status.idle":"2022-07-15T08:49:46.800089Z","shell.execute_reply.started":"2022-07-15T08:49:46.789369Z","shell.execute_reply":"2022-07-15T08:49:46.799155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(11,4, figsize=(25,60))\nfor i, ax in enumerate(axes.reshape(-1)):\n    if i<len(cat_columns):\n        sns.boxplot(x=cat_columns[i], y=\"SalePrice\", data=df_all, ax=ax)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:46.801525Z","iopub.execute_input":"2022-07-15T08:49:46.802288Z","iopub.status.idle":"2022-07-15T08:49:55.988636Z","shell.execute_reply.started":"2022-07-15T08:49:46.802243Z","shell.execute_reply":"2022-07-15T08:49:55.987559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['SalePrice'] = df_all.pop('SalePrice')\n\ncorrelations = np.abs(df_all.select_dtypes('number').corr())\nmask = np.zeros_like(correlations)\nmask[np.triu_indices_from(mask)] = True\n\nfig, ax = plt.subplots(1,1, figsize=(16,11))\nsns.heatmap(correlations, mask=mask, linewidths=.5, cmap=\"Blues\",vmax=1, ax=ax)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:55.990153Z","iopub.execute_input":"2022-07-15T08:49:55.990792Z","iopub.status.idle":"2022-07-15T08:49:56.898007Z","shell.execute_reply.started":"2022-07-15T08:49:55.990748Z","shell.execute_reply":"2022-07-15T08:49:56.896980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['total_house_area'] = df_all[['TotalBsmtSF','TotalBsmtSF','1stFlrSF','2ndFlrSF','GrLivArea']].sum(axis=1)\ndf_all['total_bathrooms'] = df_all[['FullBath','HalfBath','BsmtFullBath','BsmtHalfBath']].sum(axis=1)\ndf_all['remodelling'] = np.where(df_all['YearRemodAdd']==df_all['YearBuilt'], 1,0)\ndf_all['new_garage'] = np.where(df_all['GarageYrBlt']==df_all['YearBuilt'], 1,0)\n\ndf_all['LotFrontage'].fillna(df_all['LotFrontage'].mean(), inplace=True)\ndf_all['MasVnrArea'].fillna(df_all['MasVnrArea'].mean(), inplace=True)\ndf_all['GarageYrBlt'].fillna(0, inplace=True)\ndf_all['MSSubClass'] = df_all['MSSubClass'].astype(\"category\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:56.899241Z","iopub.execute_input":"2022-07-15T08:49:56.899571Z","iopub.status.idle":"2022-07-15T08:49:56.917075Z","shell.execute_reply.started":"2022-07-15T08:49:56.899541Z","shell.execute_reply":"2022-07-15T08:49:56.916230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_to_drop=['MoSold', 'MiscVal', 'PoolArea', 'ScreenPorch', '3SsnPorch','EnclosedPorch', 'GarageCars','YrSold']\n# uniportant_eli5 = ['Condition2_RRNn', 'Condition2_nan', 'GarageType_nan', 'SaleCondition_nan', 'PavedDrive_nan',\n#                    'HouseStyle_nan', 'LotConfig_nan', 'Alley_nan', 'BsmtFinType1_nan','BsmtExposure_nan', 'MSZoning_nan',\n#                    'MiscFeature_nan', 'CentralAir_nan', 'LotShape_nan', 'Heating_nan', 'PoolQC_nan', 'Street_nan', 'MasVnrType_nan', 'Exterior2nd_Other',\n#                    'Foundation_Stone', 'Condition2_RRAn', 'Condition2_RRAe', 'land_shapecont_IR3HLS', 'MiscFeature_TenC', 'land_shapecont_IR2HLS']\n\ndf_all.drop(cols_to_drop, axis=1, inplace=True)\ndf_all['SalePrice'] = df_all.pop('SalePrice')\n\ncorrelations = np.abs(df_all.select_dtypes('number').corr())\nmask = np.zeros_like(correlations)\nmask[np.triu_indices_from(mask)] = True\n\nfig, ax = plt.subplots(1,1, figsize=(16,11))\nsns.heatmap(correlations, mask=mask, linewidths=.5, cmap=\"Blues\",vmax=1, ax=ax)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:56.918262Z","iopub.execute_input":"2022-07-15T08:49:56.919254Z","iopub.status.idle":"2022-07-15T08:49:57.768093Z","shell.execute_reply.started":"2022-07-15T08:49:56.919204Z","shell.execute_reply":"2022-07-15T08:49:57.767055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ordered Manual Encoding","metadata":{}},{"cell_type":"code","source":"mapping1 = {'None':0,'Ex':1,'Gd':2,'TA':3,'Fa':4,'Po':5,}\ncols_map = ['FireplaceQu', 'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'HeatingQC', 'KitchenQual', 'FireplaceQu', 'GarageQual', 'GarageCond']\nfor c in cols_map:\n    df_all[c] = df_all[c].replace(mapping1)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:57.769685Z","iopub.execute_input":"2022-07-15T08:49:57.770654Z","iopub.status.idle":"2022-07-15T08:49:57.818004Z","shell.execute_reply.started":"2022-07-15T08:49:57.770608Z","shell.execute_reply":"2022-07-15T08:49:57.817211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mapping2 = {'Typ':0,'Min1':1,'Min2':2,'Mod':3,'Maj1':4,'Maj2':5,'Sev':6,'Sal':7}\ndf_all['Functional'] = df_all['Functional'].replace(mapping2)\ncols_map.append('Functional')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:57.819246Z","iopub.execute_input":"2022-07-15T08:49:57.819876Z","iopub.status.idle":"2022-07-15T08:49:57.830303Z","shell.execute_reply.started":"2022-07-15T08:49:57.819824Z","shell.execute_reply":"2022-07-15T08:49:57.829310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mapping3 = {'Y':0,'P':1,'N':2}\ndf_all['PavedDrive'] = df_all['PavedDrive'].replace(mapping3)\ncols_map.append('PavedDrive')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:57.832271Z","iopub.execute_input":"2022-07-15T08:49:57.832714Z","iopub.status.idle":"2022-07-15T08:49:57.844098Z","shell.execute_reply.started":"2022-07-15T08:49:57.832671Z","shell.execute_reply":"2022-07-15T08:49:57.842928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Label encoding","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\ncols_le = ['MSSubClass', 'OverallQual', 'OverallCond']\n# process columns, apply LabelEncoder to categorical features\nfor c in cols_le:\n    lbl = LabelEncoder() \n    lbl.fit(list(df_all[c].values)) \n    df_all[c] = lbl.transform(list(df_all[c].values))\n\n# shape        \nprint('Shape all_data: {}'.format(df_all.shape))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:57.845406Z","iopub.execute_input":"2022-07-15T08:49:57.846134Z","iopub.status.idle":"2022-07-15T08:49:57.859966Z","shell.execute_reply.started":"2022-07-15T08:49:57.846097Z","shell.execute_reply":"2022-07-15T08:49:57.859165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"One-Hot-Encoding","metadata":{}},{"cell_type":"code","source":"all_categorical = set(df_all.select_dtypes(['object']).columns)\nleft_for_ohe = all_categorical.difference(set(cols_le+cols_map))\nleft_for_ohe","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:57.861328Z","iopub.execute_input":"2022-07-15T08:49:57.861811Z","iopub.status.idle":"2022-07-15T08:49:57.872824Z","shell.execute_reply.started":"2022-07-15T08:49:57.861780Z","shell.execute_reply":"2022-07-15T08:49:57.871825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all_onehot = pd.get_dummies(df_all, dummy_na=True, columns=left_for_ohe)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:57.873985Z","iopub.execute_input":"2022-07-15T08:49:57.874664Z","iopub.status.idle":"2022-07-15T08:49:57.924650Z","shell.execute_reply.started":"2022-07-15T08:49:57.874598Z","shell.execute_reply":"2022-07-15T08:49:57.923643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unimportant_features = []\nif len(unimportant_features)>0:\n    df_all_onehot.drop(unimportant_features, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:57.929283Z","iopub.execute_input":"2022-07-15T08:49:57.929632Z","iopub.status.idle":"2022-07-15T08:49:57.934631Z","shell.execute_reply.started":"2022-07-15T08:49:57.929602Z","shell.execute_reply":"2022-07-15T08:49:57.933561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all_onehot.tail()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:57.936562Z","iopub.execute_input":"2022-07-15T08:49:57.937061Z","iopub.status.idle":"2022-07-15T08:49:58.066094Z","shell.execute_reply.started":"2022-07-15T08:49:57.937027Z","shell.execute_reply":"2022-07-15T08:49:58.065121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = df_all_onehot[~df_all_onehot['SalePrice'].isna()].drop(['SalePrice','Id'], axis=1)\nX_test = df_all_onehot[df_all_onehot['SalePrice'].isna()].drop(['SalePrice'], axis=1).set_index('Id')\ny = df_all_onehot[~df_all_onehot['SalePrice'].isna()].pop('SalePrice')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:58.067240Z","iopub.execute_input":"2022-07-15T08:49:58.067579Z","iopub.status.idle":"2022-07-15T08:49:58.090726Z","shell.execute_reply.started":"2022-07-15T08:49:58.067548Z","shell.execute_reply":"2022-07-15T08:49:58.089788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:58.092257Z","iopub.execute_input":"2022-07-15T08:49:58.092584Z","iopub.status.idle":"2022-07-15T08:49:58.216929Z","shell.execute_reply.started":"2022-07-15T08:49:58.092556Z","shell.execute_reply":"2022-07-15T08:49:58.216168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:58.217914Z","iopub.execute_input":"2022-07-15T08:49:58.218565Z","iopub.status.idle":"2022-07-15T08:49:58.343539Z","shell.execute_reply.started":"2022-07-15T08:49:58.218534Z","shell.execute_reply":"2022-07-15T08:49:58.342729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#X_train.fillna(-1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:58.344711Z","iopub.execute_input":"2022-07-15T08:49:58.345184Z","iopub.status.idle":"2022-07-15T08:49:58.348644Z","shell.execute_reply.started":"2022-07-15T08:49:58.345151Z","shell.execute_reply":"2022-07-15T08:49:58.347936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model optimisation","metadata":{}},{"cell_type":"code","source":"def log_rmse(y_true, y_pred):\n    return mean_squared_log_error(y_true, y_pred)\n\nmsle = make_scorer(log_rmse)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:49:58.349613Z","iopub.execute_input":"2022-07-15T08:49:58.350336Z","iopub.status.idle":"2022-07-15T08:49:58.362429Z","shell.execute_reply.started":"2022-07-15T08:49:58.350297Z","shell.execute_reply":"2022-07-15T08:49:58.361720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Random Forest","metadata":{}},{"cell_type":"code","source":"optuna.logging.set_verbosity(optuna.logging.WARNING)\n\ndef objective(trial):\n    n_est = trial.suggest_int('n_estimators', 200, 450, log=False)\n    max_d = trial.suggest_int('max_depth', 2, 15, log=False)\n    min_sf = trial.suggest_int('min_samples_leaf', 2, 4, log=False)\n    \n    reg_f = RandomForestRegressor(n_estimators=n_est,\n                                  max_depth=max_d,\n                                  min_samples_leaf=min_sf,\n                                  random_state=22) \n    \n    score = cross_val_score(reg_f,\n                            X_train, y,\n                            scoring=msle,\n                            n_jobs=-1,\n                            cv=3)\n    msle_mean = score.mean()\n    return msle_mean\n\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=180, show_progress_bar=True)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-15T08:49:58.363853Z","iopub.execute_input":"2022-07-15T08:49:58.364955Z","iopub.status.idle":"2022-07-15T09:13:39.259515Z","shell.execute_reply.started":"2022-07-15T08:49:58.364907Z","shell.execute_reply":"2022-07-15T09:13:39.258564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(study.best_trial.value)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:13:39.260901Z","iopub.execute_input":"2022-07-15T09:13:39.261773Z","iopub.status.idle":"2022-07-15T09:13:39.267694Z","shell.execute_reply.started":"2022-07-15T09:13:39.261725Z","shell.execute_reply":"2022-07-15T09:13:39.266822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_optimization_history(study)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:13:39.269337Z","iopub.execute_input":"2022-07-15T09:13:39.270006Z","iopub.status.idle":"2022-07-15T09:13:39.321390Z","shell.execute_reply.started":"2022-07-15T09:13:39.269970Z","shell.execute_reply":"2022-07-15T09:13:39.320627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_param_importances(study)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:13:39.322618Z","iopub.execute_input":"2022-07-15T09:13:39.323179Z","iopub.status.idle":"2022-07-15T09:13:44.534078Z","shell.execute_reply.started":"2022-07-15T09:13:39.323135Z","shell.execute_reply":"2022-07-15T09:13:44.533035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The final RegF model","metadata":{}},{"cell_type":"code","source":"print(study.best_value)\nprint(study.best_params)\nn_estimators = study.best_params['n_estimators']\nmax_depth = study.best_params['max_depth']\nmin_sf = study.best_params['min_samples_leaf']","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:13:44.535552Z","iopub.execute_input":"2022-07-15T09:13:44.535984Z","iopub.status.idle":"2022-07-15T09:13:44.543634Z","shell.execute_reply.started":"2022-07-15T09:13:44.535946Z","shell.execute_reply":"2022-07-15T09:13:44.542201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train_eli5, X_val, y_train_eli5, y_val = train_test_split(X_train, y, test_size=0.25, random_state=22)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:13:44.545202Z","iopub.execute_input":"2022-07-15T09:13:44.546352Z","iopub.status.idle":"2022-07-15T09:13:44.568331Z","shell.execute_reply.started":"2022-07-15T09:13:44.546304Z","shell.execute_reply":"2022-07-15T09:13:44.566578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model = RandomForestRegressor(**study.best_params)\nbest_model.fit(X_train_eli5, y_train_eli5)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:13:44.570062Z","iopub.execute_input":"2022-07-15T09:13:44.571296Z","iopub.status.idle":"2022-07-15T09:13:51.885038Z","shell.execute_reply.started":"2022-07-15T09:13:44.571239Z","shell.execute_reply":"2022-07-15T09:13:51.884302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import tree\nplt.figure(figsize=(13,8))  # set plot size (denoted in inches)\ntree.plot_tree(best_model.estimators_[1], max_depth=2, fontsize=10, impurity=True, precision=0, rounded=True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:37:14.518983Z","iopub.execute_input":"2022-07-15T09:37:14.519487Z","iopub.status.idle":"2022-07-15T09:37:15.300500Z","shell.execute_reply.started":"2022-07-15T09:37:14.519451Z","shell.execute_reply":"2022-07-15T09:37:15.299779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# RegF importances","metadata":{}},{"cell_type":"code","source":"feat_importances = pd.Series(best_model.feature_importances_, index=X_train.columns).sort_values(ascending=True)\nfeat_importances[-10:].plot(kind='barh')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:13:51.886408Z","iopub.execute_input":"2022-07-15T09:13:51.886991Z","iopub.status.idle":"2022-07-15T09:13:52.139256Z","shell.execute_reply.started":"2022-07-15T09:13:51.886954Z","shell.execute_reply":"2022-07-15T09:13:52.138491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_importances","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:13:52.140831Z","iopub.execute_input":"2022-07-15T09:13:52.141537Z","iopub.status.idle":"2022-07-15T09:13:52.151334Z","shell.execute_reply.started":"2022-07-15T09:13:52.141487Z","shell.execute_reply":"2022-07-15T09:13:52.150413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"perm = PermutationImportance(best_model, n_iter=35).fit(X_val, y_val)\neli5_importances = perm.feature_importances_\neli5_importances[-25:]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:13:52.152510Z","iopub.execute_input":"2022-07-15T09:13:52.153162Z","iopub.status.idle":"2022-07-15T09:25:08.728296Z","shell.execute_reply.started":"2022-07-15T09:13:52.153121Z","shell.execute_reply":"2022-07-15T09:25:08.727448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eli5.show_weights(perm, top=30, feature_names=list(X_val.columns))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:25:08.729407Z","iopub.execute_input":"2022-07-15T09:25:08.729952Z","iopub.status.idle":"2022-07-15T09:25:08.741636Z","shell.execute_reply.started":"2022-07-15T09:25:08.729917Z","shell.execute_reply":"2022-07-15T09:25:08.740640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"permutation_eli5 = pd.DataFrame(data = perm.feature_importances_,index=X_val.columns.tolist(), columns=[\"Importance\"]).sort_values(by='Importance', ascending=False)\nprint(list(permutation_eli5[-30:].index))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:25:08.743040Z","iopub.execute_input":"2022-07-15T09:25:08.743377Z","iopub.status.idle":"2022-07-15T09:25:08.753625Z","shell.execute_reply.started":"2022-07-15T09:25:08.743350Z","shell.execute_reply":"2022-07-15T09:25:08.752632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pdpbox import pdp, get_dataset, info_plots\n\npdp_goals = pdp.pdp_isolate(model=best_model,\n                            dataset=X_val,\n                            model_features=X_val.columns,\n                            feature='total_house_area',\n                            num_grid_points=50)\npdp.pdp_plot(pdp_goals, 'total_house_area', figsize=(12,6))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T10:09:00.950176Z","iopub.execute_input":"2022-07-15T10:09:00.950636Z","iopub.status.idle":"2022-07-15T10:09:04.157669Z","shell.execute_reply.started":"2022-07-15T10:09:00.950601Z","shell.execute_reply":"2022-07-15T10:09:04.156703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pdp_goals = pdp.pdp_isolate(model=best_model,\n                            dataset=X_val,\n                            model_features=X_val.columns,\n                            feature='YearBuilt',\n                            num_grid_points=30)\npdp.pdp_plot(pdp_goals, 'YearBuilt', figsize=(12,6))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T10:11:03.779037Z","iopub.execute_input":"2022-07-15T10:11:03.779460Z","iopub.status.idle":"2022-07-15T10:11:06.001388Z","shell.execute_reply.started":"2022-07-15T10:11:03.779428Z","shell.execute_reply":"2022-07-15T10:11:06.000416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pdp_goals = pdp.pdp_isolate(model=best_model,\n                            dataset=X_val,\n                            model_features=X_val.columns,\n                            feature='OverallQual',\n                            num_grid_points=30)\npdp.pdp_plot(pdp_goals, 'OverallQual', figsize=(12,6))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T10:11:34.932813Z","iopub.execute_input":"2022-07-15T10:11:34.933263Z","iopub.status.idle":"2022-07-15T10:11:35.753288Z","shell.execute_reply.started":"2022-07-15T10:11:34.933226Z","shell.execute_reply":"2022-07-15T10:11:35.752103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_to_plot = ['total_house_area', 'OverallQual']\ninter1  =  pdp.pdp_interact(model=best_model,\n                            dataset=X_val,\n                            model_features=X_val.columns,\n                            features=features_to_plot)\n\npdp.pdp_interact_plot(pdp_interact_out=inter1, feature_names=features_to_plot, plot_type='contour')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T10:13:38.415793Z","iopub.execute_input":"2022-07-15T10:13:38.416222Z","iopub.status.idle":"2022-07-15T10:13:43.227728Z","shell.execute_reply.started":"2022-07-15T10:13:38.416189Z","shell.execute_reply":"2022-07-15T10:13:43.226656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install rfpimp --quiet","metadata":{"execution":{"iopub.status.busy":"2022-07-15T10:43:58.828169Z","iopub.execute_input":"2022-07-15T10:43:58.828610Z","iopub.status.idle":"2022-07-15T10:44:13.832647Z","shell.execute_reply.started":"2022-07-15T10:43:58.828577Z","shell.execute_reply":"2022-07-15T10:44:13.831531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Yes, this is bad syntax, but it's how their docs recommend usage\nfrom rfpimp import *\n\ndrop_imp = dropcol_importances(best_model, X_train_eli5, y_train_eli5, X_val, y_val)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T10:44:18.949970Z","iopub.execute_input":"2022-07-15T10:44:18.950436Z","iopub.status.idle":"2022-07-15T11:18:03.307255Z","shell.execute_reply.started":"2022-07-15T10:44:18.950391Z","shell.execute_reply":"2022-07-15T11:18:03.305526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_imp[:10].sort_values(by=['Importance'], ascending=True).plot(kind='barh').set(xlabel=\"Drop Column Importance Score\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T11:21:58.860115Z","iopub.execute_input":"2022-07-15T11:21:58.860663Z","iopub.status.idle":"2022-07-15T11:21:59.103916Z","shell.execute_reply.started":"2022-07-15T11:21:58.860624Z","shell.execute_reply":"2022-07-15T11:21:59.103028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"best_model.fit(X_train, y)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:25:08.754795Z","iopub.execute_input":"2022-07-15T09:25:08.755273Z","iopub.status.idle":"2022-07-15T09:25:18.520241Z","shell.execute_reply.started":"2022-07-15T09:25:08.755233Z","shell.execute_reply":"2022-07-15T09:25:18.519109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = best_model.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:25:18.521763Z","iopub.execute_input":"2022-07-15T09:25:18.522172Z","iopub.status.idle":"2022-07-15T09:25:18.672082Z","shell.execute_reply.started":"2022-07-15T09:25:18.522138Z","shell.execute_reply":"2022-07-15T09:25:18.670917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:25:18.673553Z","iopub.execute_input":"2022-07-15T09:25:18.673984Z","iopub.status.idle":"2022-07-15T09:25:18.681252Z","shell.execute_reply.started":"2022-07-15T09:25:18.673945Z","shell.execute_reply":"2022-07-15T09:25:18.680154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:25:18.683138Z","iopub.execute_input":"2022-07-15T09:25:18.683680Z","iopub.status.idle":"2022-07-15T09:25:18.695099Z","shell.execute_reply.started":"2022-07-15T09:25:18.683617Z","shell.execute_reply":"2022-07-15T09:25:18.694009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(data=predictions, columns=[\"SalePrice\"], index=X_test.index).reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:25:18.696627Z","iopub.execute_input":"2022-07-15T09:25:18.697602Z","iopub.status.idle":"2022-07-15T09:25:18.706679Z","shell.execute_reply.started":"2022-07-15T09:25:18.697420Z","shell.execute_reply":"2022-07-15T09:25:18.705936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:25:18.708211Z","iopub.execute_input":"2022-07-15T09:25:18.708810Z","iopub.status.idle":"2022-07-15T09:25:18.722090Z","shell.execute_reply.started":"2022-07-15T09:25:18.708775Z","shell.execute_reply":"2022-07-15T09:25:18.721049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:25:18.723837Z","iopub.execute_input":"2022-07-15T09:25:18.724621Z","iopub.status.idle":"2022-07-15T09:25:18.741136Z","shell.execute_reply.started":"2022-07-15T09:25:18.724584Z","shell.execute_reply":"2022-07-15T09:25:18.740147Z"},"trusted":true},"execution_count":null,"outputs":[]}]}