{"cells":[{"metadata":{"_execution_state":"idle","_uuid":"944d942e241129b15c592e26330d6e84d2ded322","_cell_guid":"b570dbce-d7d9-4208-8e07-81e9fec540fd"},"cell_type":"markdown","source":"# Zillow, Stacking Ensemble of Regressors\n\n### Overview\nBelow, a Stacking Ensemble of Regressors built for the Zillow competition with code annotations for help. The model achieves 0.065...  accuracy in log error. \n\nThe training steps are in markdown becauee of the time limitation that Kaggle impose on Kernels. \n\nIdeas, thoughts and suggestions on how to improve this model welcome, Twitter DM @jamesdhope."},{"metadata":{"_uuid":"0b6b58ed12c897041b14a6c7fc3161312aadadea","_cell_guid":"ca410937-4eff-417d-b225-1c8acfbb2ab9","collapsed":true},"cell_type":"markdown","source":"# Phase 1: Import the Data "},{"metadata":{"_uuid":"c9eac1bfdcec34e070db2493fa4667e80dcbc4b7","_cell_guid":"576e9229-8756-4308-9b0e-fa0c60f97cbd","collapsed":true,"trusted":false},"cell_type":"code","source":"# Load in our libraries\nimport pandas as pd\nimport numpy as np\nimport re\nimport sklearn\nimport xgboost as xgb\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.tools as tls\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Going to use these 5 base models for the stacking\nfrom sklearn.ensemble import RandomForestRegressor, AdaBoostRegressor, ExtraTreesRegressor, GradientBoostingRegressor\nfrom sklearn.svm import LinearSVR\nfrom sklearn.cross_validation import KFold;","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1463b4683573e6f2f640bebff2e427860435bde0","_cell_guid":"6be3ffe8-5291-401c-bc41-561326bdd055","collapsed":true,"trusted":false},"cell_type":"code","source":"# Load in the train and test datasets\ntrain = pd.read_csv('../input/properties_2016.csv')\ntrain_label = pd.read_csv('../input/train_2016_v2.csv')\n\n# Store our passenger ID for easy access\nParcelID = train['parcelid']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c15f1d88ccc7ed6d388ec75237c9abc0c756cd48","_cell_guid":"b3b43bb9-d760-41c1-b330-e55c865b4765","collapsed":true,"trusted":false},"cell_type":"code","source":"train.head(3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3957969fdedb92e005fd0b53332c6baf99b6c935","_cell_guid":"a998329a-df0e-4335-9b42-e8a8a4822303"},"cell_type":"markdown","source":"# Phase 2: Data Preparation\n\n### Feature Engineering"},{"metadata":{"_uuid":"0fcb14ff2c76f34300a3264da6c78c2b6b1aa358","_cell_guid":"5c7bb0a2-59aa-4085-8ba2-823c70d12cb8","collapsed":true,"trusted":false},"cell_type":"code","source":"# OneHotEncoding\ntrain['has_basement'] = train[\"basementsqft\"].apply(lambda x: 0 if np.isnan(x) else 1).astype(float)\ntrain['hashottuborspa'] = train[\"hashottuborspa\"].apply(lambda x: 0 if np.isnan(x) else 1).astype(float)\ntrain['has_pool'] = train[\"poolcnt\"].apply(lambda x: 0 if np.isnan(x) else 1).astype(float)\ntrain['has_airconditioning'] = train[\"airconditioningtypeid\"].apply(lambda x: 0 if np.isnan(x) else 1).astype(float)\n\n# Columns to be consolidated\ntrain['yardbuildingsqft17'] = train['yardbuildingsqft17'].apply(lambda x: 0 if np.isnan(x) else x).astype(float)\ntrain['yardbuildingsqft26'] = train['yardbuildingsqft26'].apply(lambda x: 0 if np.isnan(x) else x).astype(float)\ntrain['yard_building_square_feet'] = train['yardbuildingsqft17'].astype(int) + train['yardbuildingsqft26'].astype(float)\n\n# Assume some more friendly feature names\ntrain.rename(columns={'fireplacecnt':'fireplace_count'}, inplace=True)\ntrain.rename(columns={'bedroomcnt':'bedroom_count'}, inplace=True)\ntrain.rename(columns={'bathroomcnt':'bathroom_count'}, inplace=True)\ntrain.rename(columns={'calculatedfinishedsquarefeet':'square_feet'}, inplace=True)\ntrain.rename(columns={'garagecarcnt':'garage_car_count'}, inplace=True)\ntrain.rename(columns={'garagetotalsqft':'garage_square_feet'}, inplace=True)\ntrain.rename(columns={'hashottuborspa':'has_hottub_or_spa'}, inplace=True)\n\ntrain.rename(columns={'landtaxvaluedollarcnt':'land_tax'}, inplace=True)\ntrain.rename(columns={'lotsizesquarefeet':'lot_size_square_feet'}, inplace=True)\ntrain.rename(columns={'taxvaluedollarcnt':'tax_value'}, inplace=True)\ntrain.rename(columns={'taxamount':'tax_amount'}, inplace=True)\ntrain.rename(columns={'structuretaxvaluedollarcnt':'structure_tax_value'}, inplace=True)\ntrain.rename(columns={'yearbuilt':'year_built'}, inplace=True)\n\ntrain.rename(columns={'roomcnt':'room_count'}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aec7e8ba71feb8ff001b8082a17f8d9dfd12ffb7","_cell_guid":"2a1b523e-0a8a-42a2-a8bb-a552ab53421d"},"cell_type":"markdown","source":"### Impute values "},{"metadata":{"_uuid":"84e8e7604bd659fe37ca6083e5edc383a8673b35","_cell_guid":"35abae7b-af68-4423-a58d-be63aa6c5fb1","collapsed":true,"trusted":false},"cell_type":"code","source":"# Impute zero for NaN for these features\ntrain['fireplace_count'] = train['fireplace_count'].apply(lambda x: 0 if np.isnan(x) else x).astype(float)\n\n# Impute median value for NaN for these features\ntrain['bathroom_count'] = train['bathroom_count'].fillna(train['bathroom_count'].median()).astype(float)\ntrain['bedroom_count'] = train['bedroom_count'].fillna(train['bedroom_count'].median()).astype(float)\ntrain['room_count'] = train['room_count'].fillna(train['room_count'].median()).astype(float)\n\ntrain['tax_amount'] = train['tax_amount'].fillna(train['tax_amount'].median()).astype(float)\ntrain['land_tax'] = train['land_tax'].fillna(train['land_tax'].median()).astype(float)\ntrain['tax_value'] = train['tax_value'].fillna(train['tax_value'].median()).astype(float)\ntrain['structure_tax_value'] = train['structure_tax_value'].fillna(train['structure_tax_value'].median()).astype(float)\ntrain['garage_square_feet'] = train['garage_square_feet'].fillna(train['garage_square_feet'].median()).astype(float)\ntrain['garage_car_count'] = train['garage_car_count'].fillna(train['garage_car_count'].median()).astype(float)\ntrain['fireplace_count'] = train['fireplace_count'].fillna(train['fireplace_count'].median()).astype(float)\ntrain['square_feet'] = train['square_feet'].fillna(train['square_feet'].median()).astype(float)\ntrain['year_built'] = train['year_built'].fillna(train['year_built'].median()).astype(float)\ntrain['lot_size_square_feet'] = train['lot_size_square_feet'].fillna(train['lot_size_square_feet'].median()).astype(float)\n\ntrain['longitude'] = train['longitude'].fillna(train['longitude'].median()).astype(float)\ntrain['latitude'] = train['latitude'].fillna(train['latitude'].median()).astype(float)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cef84263a7d19642b22cda48bdb4f84d82a47f79","_cell_guid":"3a350aac-f0bd-4f3a-815a-0255312324fa"},"cell_type":"markdown","source":"### Feature selection"},{"metadata":{"_uuid":"f33f8c810fa99dbbcfbd8b23e54efd684dae5f0c","_cell_guid":"1336f2be-0c08-4b66-b857-778ca3c61e9c","collapsed":true,"trusted":false},"cell_type":"code","source":"# Drop indistinct features\ndrop_elements = ['assessmentyear']\n\n# Drop any columns insufficiently described\ndrop_elements = drop_elements + ['airconditioningtypeid', 'basementsqft', 'architecturalstyletypeid', 'buildingclasstypeid', 'buildingqualitytypeid', 'calculatedbathnbr', 'decktypeid', 'finishedfloor1squarefeet',\n                 'fips', 'heatingorsystemtypeid', 'rawcensustractandblock',\n                 'numberofstories', 'storytypeid', 'threequarterbathnbr', 'typeconstructiontypeid', 'unitcnt', 'censustractandblock', 'fireplaceflag', 'taxdelinquencyflag', 'taxdelinquencyyear',\n                ]\n\n# Drop any duplicated columns\ndrop_elements = drop_elements + ['fullbathcnt', 'finishedsquarefeet6', 'finishedsquarefeet12', 'finishedsquarefeet13', 'finishedsquarefeet15', 'finishedsquarefeet50', 'yardbuildingsqft17', 'yardbuildingsqft26']\n\n# Land use data\ndrop_elements = drop_elements + ['propertycountylandusecode', 'propertylandusetypeid', 'propertyzoningdesc']\n\n# We'll make do with a binary feature here\ndrop_elements = drop_elements + ['pooltypeid10', 'pooltypeid2', 'pooltypeid7', 'poolsizesum', 'poolcnt']\n\n# We'll use the longitude and latitutde as features \ndrop_elements = drop_elements + ['regionidzip', 'regionidneighborhood', 'regionidcity', 'regionidcounty']\n\nprint(\"dropping features\", drop_elements)\n\ntrain = train.drop(drop_elements, axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"158a3749367a8667e04ca406f591995732e7f238","_cell_guid":"0a9bb043-e6e1-42b4-9621-a1b0ff9cd4a6","collapsed":true,"trusted":false},"cell_type":"code","source":"train.head(5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e6a08ca9809277bc6f06618863675965b70259b5","_cell_guid":"3d01262f-40b9-4dbb-975e-82c76b673f40"},"cell_type":"markdown","source":"### Obtain the training set\nFor this project, we will need to obtain the training set by merging the house features with the labelled data on the parcelid."},{"metadata":{"_uuid":"6b68d65ab5e75b817a87047d535a8ca44c7b218a","_cell_guid":"afc70e79-ea80-443d-84a5-7103832be4f3","collapsed":true,"trusted":false},"cell_type":"code","source":"# Create Numpy arrays of train, test and target dataframes to feed into our models\nimport pandas as pd\nimport random\n\n# Obtain the training by merging both train and train_label on the parcelid\ncommon = train.merge(train_label,on=['parcelid'])\n#common.to_csv('common.csv')\n\n# Split the merged set into the training set and labels\ny_train = common['logerror']\nx_train = common.drop(['logerror', 'parcelid'], axis=1)\n\n# OneHotEncode the date information and drop the original\nx_train = common\nx_train['year'], x_train['month'], x_train['day'] = x_train['transactiondate'].str.split('-').str\nx_train = common.drop(['transactiondate'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c9c9dd7ec1a5f54d1c73c6b0fc90fbe0b5d87cc9","_cell_guid":"d5c8ecba-e645-4a68-8e1f-dcc23b8cbf5c","collapsed":true,"trusted":false},"cell_type":"code","source":"#config\ncolormap = plt.cm.viridis\nplt.figure(figsize=(15,15))\nplt.title('Pearson Correlation of Features', y=1.05, size=15)\n\n#create the correlation map\ncorr = x_train.astype(float).corr()\n\n#create a mask for the null values\nmask = corr.isnull()\n\n#plot the heatmap\nsns.heatmap(corr, mask=mask, linewidths=0.1,vmax=1.0, square=True, cmap=colormap, linecolor='white', annot=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"55517f5a1ad2b3db4e46a8a63e53d6a7982c8f07","_cell_guid":"02a319c3-2b2a-4f78-b7a2-dbb67b334854","collapsed":true,"trusted":false},"cell_type":"code","source":"x_train = x_train.drop(['logerror', 'parcelid'], axis=1)\n\n# Convert training set and labels to Numpy array\nx_train = x_train.values\ny_train = y_train.values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"843a0b8df09739c7d010c05710ea54e81ca61308","_cell_guid":"e079ea65-a7f5-4c99-9809-41d6cf3e8e33"},"cell_type":"markdown","source":"### Prepare the test set, with the date information required for prediction\nNow let's populate the test data with the additional date information. We'll need to do this, else we won't be able to feed the best data into the models. We'll assign random dates to properties to avoid bias."},{"metadata":{"_uuid":"e4fd2e46e2f47d0c299544a213d2fa6d2a869887","_cell_guid":"90da4df1-b279-4054-80cc-0dae1497b578","collapsed":true,"trusted":false},"cell_type":"code","source":"# Obtain the full test set, with the date for each time period as an additional feature\ntrain = train.drop(['parcelid'], axis=1)\n#train.to_csv('train.csv')\n\ndef get_random_day():\n    return pd.DataFrame(np.random.randint(1,30,len(train)))\n                   \n# Create a Test Set with date column for each time period we are being asked to predict for\nx_test_201610 = train\nx_test_201610['year'], x_test_201610['month'], x_test_201610['day'] = [2016,10,get_random_day()]\n#x_test_201610.to_csv(\"x_train_201610.csv\")\nx_test_201610 = x_test_201610.values\n\nx_test_201611 = train\nx_test_201611['year'], x_test_201611['month'], x_test_201611['day'] = [2016,11,get_random_day()]\nx_test_201611 = x_test_201611.values\n\nx_test_201612 = train\nx_test_201612['year'], x_test_201612['month'], x_test_201612['day'] = [2016,12,get_random_day()]\nx_test_201612 = x_test_201612.values\n\nx_test_201710 = train\nx_test_201710['year'], x_test_201710['month'], x_test_201710['day'] = [2017,10,get_random_day()]\nx_test_201710 = x_test_201710.values\n\nx_test_201711 = train\nx_test_201711['year'], x_test_201711['month'], x_test_201711['day'] = [2017,11,get_random_day()]\nx_test_201711 = x_test_201711.values\n\nx_test_201712 = train\nx_test_201712['year'], x_test_201712['month'], x_test_201712['day'] = [2017,12,get_random_day()]\n#x_test_201712.to_csv(\"x_train_201712.csv\")\nx_test_201712 = x_test_201712.values\n\nprint(\"test sets populated with random transaction dates\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7d79b8b58b3ece01987724ef6a2294288efda9cd","_cell_guid":"7c7b0c2f-bc0d-48ad-b109-e80b335c14fa"},"cell_type":"markdown","source":"### Scale the data"},{"metadata":{"_uuid":"9b86b4e39cedf72fb8de435a918aa79579daa01d","_cell_guid":"8d860285-d2f4-45c7-93e6-0c86e889ed5e","collapsed":true,"trusted":false},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\nx_train = scaler.fit_transform(x_train)\nx_test_201610 = scaler.transform(x_test_201610)\nx_test_201611 = scaler.transform(x_test_201611)\nx_test_201612 = scaler.transform(x_test_201612)\nx_test_201710 = scaler.transform(x_test_201710)\nx_test_201711 = scaler.transform(x_test_201711)\nx_test_201712 = scaler.transform(x_test_201712)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c4bd7e02518e09afd50fd4e6fb5224ce25d863a0","_cell_guid":"aed2b79a-69ee-46c3-958f-e3e79e916530"},"cell_type":"markdown","source":"# 3. Construct the Models\n\n### Extend the Sklearn classifier methods\n\nSklearnHelper will extend the inbuilt methods (such as train, predict and fit) common to all the Sklearn classifiers. Therefore this cuts out redundancy as won't need to write the same methods five times if we wanted to invoke five different classifiers."},{"metadata":{"_uuid":"e63d7f122617eb97ca1f21155515620963271513","_cell_guid":"2f574de0-34e6-4dcf-ada3-619d4f206895","collapsed":true,"trusted":false},"cell_type":"code","source":"# Class to extend the Sklearn classifier\nclass SklearnHelper(object):\n    def __init__(self, clf, seed=0, params=None):\n        params['random_state'] = seed\n        self.clf = clf(**params)\n\n    def train(self, x_train, y_train):\n        self.clf.fit(x_train, y_train)\n\n    def predict(self, x):\n        return self.clf.predict(x)\n    \n    def fit(self,x,y):\n        return self.clf.fit(x,y)\n    \n    def feature_importances(self,x,y):\n        return(self.clf.fit(x,y).feature_importances_)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"33b648d19f2568f418f4b8ecb0652204e88c43f6","_cell_guid":"36589d07-1e4d-466f-af6d-42a4c9d4dbce"},"cell_type":"markdown","source":"### XValidation\nLet's define a function to perform cross validation"},{"metadata":{"_uuid":"2b76fc3193db86b0c409c43c67a8065af9b165cb","_cell_guid":"221ab0c6-4f39-471a-a982-188b8a5895d8","collapsed":true,"trusted":false},"cell_type":"code","source":"def get_oof(clf, x_train, y_train, x_test_201610, x_test_201611, x_test_201612, x_test_201710, x_test_201711, x_test_201712):\n    oof_train = np.zeros((ntrain,))\n    \n    oof_test_201610 = np.zeros((ntest,))\n    oof_test_201611 = np.zeros((ntest,))\n    oof_test_201612 = np.zeros((ntest,))\n    oof_test_201710 = np.zeros((ntest,))    \n    oof_test_201711 = np.zeros((ntest,))\n    oof_test_201712 = np.zeros((ntest,))\n    \n    oof_test_skf_201610 = np.empty((NFOLDS, ntest))\n    oof_test_skf_201611 = np.empty((NFOLDS, ntest))\n    oof_test_skf_201612 = np.empty((NFOLDS, ntest))\n    oof_test_skf_201710 = np.empty((NFOLDS, ntest))\n    oof_test_skf_201711 = np.empty((NFOLDS, ntest))\n    oof_test_skf_201712 = np.empty((NFOLDS, ntest))\n    \n    #train_index: indicies of training set\n    #test_index: indicies of testing set\n     \n    for i, (train_index, test_index) in enumerate(kf):\n        #break the dataset down into two sets, train and test\n        x_tr = x_train[train_index]\n        y_tr = y_train[train_index]\n        x_te = x_train[test_index]\n        \n        clf.train(x_tr, y_tr)\n        \n        #make a predition on the test data subset\n        oof_train[test_index] = clf.predict(x_te)\n        \n        #use the model trained on the first fold to make a prediction on the entire test data \n        oof_test_skf_201610[i, :] = clf.predict(x_test_201610)\n        oof_test_skf_201611[i, :] = clf.predict(x_test_201611)\n        oof_test_skf_201612[i, :] = clf.predict(x_test_201612)\n        oof_test_skf_201710[i, :] = clf.predict(x_test_201710)\n        oof_test_skf_201711[i, :] = clf.predict(x_test_201711)\n        oof_test_skf_201712[i, :] = clf.predict(x_test_201712)\n    \n    #take an average of all of the folds\n    oof_test_201610[:] = oof_test_skf_201610.mean(axis=0)\n    oof_test_201611[:] = oof_test_skf_201611.mean(axis=0)\n    oof_test_201612[:] = oof_test_skf_201612.mean(axis=0)\n    oof_test_201710[:] = oof_test_skf_201710.mean(axis=0)\n    oof_test_201711[:] = oof_test_skf_201711.mean(axis=0)\n    oof_test_201712[:] = oof_test_skf_201712.mean(axis=0)\n    \n    return oof_train.reshape(-1, 1), oof_test_201610.reshape(-1, 1), oof_test_201611.reshape(-1, 1), oof_test_201612.reshape(-1, 1), oof_test_201710.reshape(-1, 1), oof_test_201711.reshape(-1, 1), oof_test_201712.reshape(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cdc271ece4d57f274fc0f6978f9c4d42b1bafed0","_cell_guid":"db19cc6b-6e5b-4f50-9679-63165ceb6bb6"},"cell_type":"markdown","source":"### Create a Dict data type to hold the parameters for our First Level Models"},{"metadata":{"_uuid":"07c66106a0b7662c77d05cde3a95ffccb08d380b","_cell_guid":"0cbe4452-fe89-48d1-9cef-41603baac358","collapsed":true,"trusted":false},"cell_type":"code","source":"SEED = 0 # for reproducibility\nNFOLDS = 5 # set folds for out-of-fold prediction\n\n# Put in our parameters for said classifiers\n# Random Forest parameters\nrf_params = {\n    'n_jobs': -1,\n    'n_estimators': 500,\n     'warm_start': True, \n     #'max_features': 0.2,\n    'max_depth': 6,\n    'min_samples_leaf': 2,\n    'max_features' : 'sqrt',\n    'verbose': 0\n}\n\n# Extra Trees Parameters\net_params = {\n    'n_jobs': -1,\n    'n_estimators':500,\n    #'max_features': 0.5,\n    'max_depth': 8,\n    'min_samples_leaf': 2,\n    'verbose': 0\n}\n\n# AdaBoost parameters\nada_params = {\n    'n_estimators': 400,\n    'learning_rate' : 0.75\n}\n\n# Support Vector Classifier parameters \nsvm_params = {\n    'C' : 0.025,\n    'epsilon':0.1\n    }\n\n# Gradient Boosting parameters\ngb_regressor_params = {\n    'n_estimators':500, \n    'learning_rate':0.1,\n    'max_depth':1, \n    'random_state':0, \n    'loss':'ls'\n}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b8e6f9330ff52a1b81ee1a6394689350a7704f6e","_cell_guid":"0af06cce-9e89-4db0-a730-7f021950b927","collapsed":true,"trusted":false},"cell_type":"code","source":"### Create 5 objects that represent our 4 models","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3f5a23faf9628ae089a4f1581037814a52a5a55a","_cell_guid":"20fdc863-10c5-41b1-963c-6f7a5018b52b","collapsed":true,"trusted":false},"cell_type":"code","source":"rf = SklearnHelper(clf=RandomForestRegressor, seed=SEED, params=rf_params)\net = SklearnHelper(clf=ExtraTreesRegressor, seed=SEED, params=et_params)\nada = SklearnHelper(clf=AdaBoostRegressor, seed=SEED, params=ada_params)\ngb_regressor = SklearnHelper(clf=GradientBoostingRegressor, seed=SEED, params=gb_regressor_params)\nsvm = SklearnHelper(clf=LinearSVR, seed=SEED, params=svm_params)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5d759b8bfdd5d2d6d0baa8e59ff1f0502a736f09","_cell_guid":"4becdfed-bf4c-47c0-b840-a15109fab3e4"},"cell_type":"markdown","source":"### Train our First Level Models\n\n#### Model Training marked down here. Allow 2-4 hours on single CPU. "},{"metadata":{"_uuid":"c184170fcf6b720b9255078da59d99dfcafb5ced","_cell_guid":"b7112c06-469b-4fc3-95e6-3feb340b8dc1","collapsed":true,"trusted":false},"cell_type":"code","source":"ntrain = x_train.shape[0]\nprint(ntrain)\nntest = x_test_201610.shape[0] #need the size of a test set\nprint(ntest)\n\nkf = KFold(ntrain, n_folds= NFOLDS, random_state=SEED)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4923e1a4755f4ed745706e7924bbcfc410cbb5e7","_cell_guid":"700993a8-4f3d-4fa6-a1a2-94b7666beed3","collapsed":true,"trusted":false},"cell_type":"code","source":"#svm_oof_train, svm_oof_test_201610, svm_oof_test_201611, svm_oof_test_201612, svm_oof_test_201710, svm_oof_test_201711, svm_oof_test_201712 = get_oof(svm,x_train, y_train, x_test_201610, x_test_201611, x_test_201612, x_test_201710, x_test_201711, x_test_201712) # Support Vector Classifier\nprint(\"SVM Training is complete\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c8415c6609a7b3bdd0565535c109cf4c98104f25","_cell_guid":"d1f32875-8511-4891-aaef-8a8b88d4b801","collapsed":true,"trusted":false},"cell_type":"code","source":"#et_oof_train, et_oof_test_201610, et_oof_test_201611, et_oof_test_201612, et_oof_test_201710, et_oof_test_201711, et_oof_test_201712 = get_oof(et, x_train, y_train, x_test_201610, x_test_201611, x_test_201612, x_test_201710, x_test_201711, x_test_201712) # Extra Trees\nprint(\"Extra Trees Regressor Training is complete\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"32f6d5cc2e094f70026a9c30473d4fae861891bf","_cell_guid":"c25b416c-91ff-4ed9-96a0-12595d88314c","collapsed":true,"trusted":false},"cell_type":"code","source":"#rf_oof_train, rf_oof_test_201610, rf_oof_test_201611, rf_oof_test_201612, rf_oof_test_201710, rf_oof_test_201711, rf_oof_test_201712 = get_oof(rf,x_train, y_train, x_test_201610, x_test_201611, x_test_201612, x_test_201710, x_test_201711, x_test_201712) # Random Forest\nprint(\"Random Forest Regressor Training is complete\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d9cbd00163ac96f71067bb4155a9d92f82e630c2","_cell_guid":"68f29415-5d2c-43cf-8398-0a01ed8ac580","collapsed":true,"trusted":false},"cell_type":"code","source":"#ada_oof_train, ada_oof_test_201610, ada_oof_test_201611, ada_oof_test_201612, ada_oof_test_201710, ada_oof_test_201711, ada_oof_test_201712 = get_oof(ada, x_train, y_train, x_test_201610, x_test_201611, x_test_201612, x_test_201710, x_test_201711, x_test_201712) # AdaBoost \nprint(\"Ada Boost Regressor Training is complete\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a13761794752ec9ad920ce38fb468dd5251664ee","_cell_guid":"6c11d0ea-f071-4172-aca4-158298108a2f","collapsed":true,"trusted":false},"cell_type":"code","source":"#gb_regressor_oof_train, gb_regressor_oof_test_201610, gb_regressor_oof_test_201611, gb_regressor_oof_test_201612, gb_regressor_oof_test_201710, gb_regressor_oof_test_201711, gb_regressor_oof_test_201712 = get_oof(gb_regressor,x_train,y_train,x_test_201610, x_test_201611, x_test_201612, x_test_201710, x_test_201711, x_test_201712)\nprint(\"Gradient Boost Regressor Training is complete\")\n\nprint(\"Training is complete\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ac2e4c53c7cafa565c11eb42c36a9d9adbc8c91c","_cell_guid":"2debcfba-7b09-4c10-8b4f-a6e4e9206143"},"cell_type":"markdown","source":"### Extract feature importances for our Second Level "},{"metadata":{"_uuid":"e69f59ae2cf82a323bd865741873397ff72330a6","_cell_guid":"c7a9663e-1e75-4514-8f6d-4e4683e9ceee","collapsed":true,"trusted":false},"cell_type":"code","source":"rf_feature = rf.feature_importances(x_train,y_train)\nprint(\"rf_feature\", rf_feature)\net_feature = et.feature_importances(x_train, y_train)\nprint(\"et_feature\", et_feature)\nada_feature = ada.feature_importances(x_train, y_train)\nprint(\"ada_feature\", ada_feature)\ngb_regressor_feature = gb_regressor.feature_importances(x_train,y_train)\nprint(\"gb_regressor_feature\", gb_regressor_feature)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2292e76d6c9495e7ed8c42f220c4848426a66609","_cell_guid":"8b6c62f7-59c6-4e2f-9a41-72f797666138","collapsed":true,"trusted":false},"cell_type":"code","source":"rf_features = [0.03177995, 0.02151717,  0.15271272,  0.00370392,  0.00672652,  0.01860676,\n  0.00366968,  0.08548836,  0.06296792,  0.06384852,  0.01510663,  0.05273821,\n  0.10866998,  0.09620006,  0.06764444,  0.11359643,  0.00080572,  0.01050976,\n  0.00891371,  0.0032457,   0,          0.02464635,  0.0469015]\net_features = [0.06465583,  0.0572915,   0.12032578,  0.00991171,  0.01124228,  0.00960876,\n  0.01485536,  0.06740794,  0.05175181,  0.05436677,  0.01772004,  0.05594463,\n  0.09801529,  0.0533328,   0.0450343,   0.08253611,  0.00201233,  0.02357432,\n  0.03032919,  0.00296583,  0,          0.061361,    0.06575642] \nada_features = [8.36785346e-03,   3.79894667e-03,   7.05391914e-02,   7.82563418e-05,\n   3.22502690e-07,   1.36595920e-02,   0.00000000e+00,   6.15640675e-02,\n   5.32715094e-02,   3.73193212e-02,   1.70693107e-02,   1.18344505e-01,\n   1.72005440e-01,   3.48224492e-02,   4.48032666e-02,   3.87885107e-02,\n   0.00000000e+00,   7.54103834e-03,   0.00000000e+00,   1.06703836e-02,\n   0.00000000e+00,   1.25617605e-01,   1.81738430e-01]\ngb_regressor_features = [0.02,   0.012,  0.246,  0,     0.016,  0.002,  0.004,  0.114,  0.068,  0.02,\n  0.004,  0.012,  0.158,  0.056,  0.06,   0.16,   0,     0.022,  0,     0,     0,\n  0.026,  0 ]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"578f4c329cfab523e375a2f2c7f8b29edd386112","_cell_guid":"a606e988-ec10-409a-a8c1-b603ebf68ba1","collapsed":true,"trusted":false},"cell_type":"code","source":"cols = train.columns.values\n# Create a dataframe with features\nfeature_dataframe = pd.DataFrame( { \n    'features': cols,\n    'Random Forest feature importances': rf_features,\n    'Extra Trees  feature importances': et_features,\n    'AdaBoost feature importances': ada_features,\n    'Gradient Regressor feature importances': gb_regressor_features\n    })\n\nfeature_dataframe.head(3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"008bb1c4ca4c39ece9a305f07b8398aab4eb3a3d","_cell_guid":"aa74e562-8ed2-4ab4-b4fe-9ac503e947c6"},"cell_type":"markdown","source":"Let's have a look at our feature importances"},{"metadata":{"_uuid":"a280c6d6f106525c60b923abdc3ff7ec37529eaa","_cell_guid":"06317179-465d-4028-9902-582c941df55e","collapsed":true,"trusted":false},"cell_type":"code","source":"# Scatter plot \ntrace = go.Scatter(\n    y = feature_dataframe['Random Forest feature importances'].values,\n    x = feature_dataframe['features'].values,\n    mode='markers',\n    marker=dict(\n        sizemode = 'diameter',\n        sizeref = 1,\n        size = 25,\n        color = feature_dataframe['Random Forest feature importances'].values,\n        colorscale='Portland',\n        showscale=True\n    ),\n    text = feature_dataframe['features'].values\n)\ndata = [trace]\n\nlayout= go.Layout(\n    autosize= True,\n    title= 'Random Forest feature importances',\n    hovermode= 'closest',\n    yaxis=dict(\n        title= 'Feature Importance',\n        ticklen= 5,\n        gridwidth= 2\n    ),\n    showlegend= False\n)\nfig = go.Figure(data=data, layout=layout)\npy.iplot(fig)\n\n\n# Scatter plot \ntrace = go.Scatter(\n    y = feature_dataframe['Extra Trees  feature importances'].values,\n    x = feature_dataframe['features'].values,\n    mode='markers',\n    marker=dict(\n        sizemode = 'diameter',\n        sizeref = 1,\n        size = 25,\n        color = feature_dataframe['Extra Trees  feature importances'].values,\n        colorscale='Portland',\n        showscale=True\n    ),\n    text = feature_dataframe['features'].values\n)\ndata = [trace]\n\nlayout= go.Layout(\n    autosize= True,\n    title= 'Extra Trees  feature importances',\n    hovermode= 'closest',\n    yaxis=dict(\n        title= 'Feature Importance',\n        ticklen= 5,\n        gridwidth= 2\n    ),\n    showlegend= False\n)\nfig = go.Figure(data=data, layout=layout)\npy.iplot(fig)\n\n# Scatter plot \ntrace = go.Scatter(\n    y = feature_dataframe['AdaBoost feature importances'].values,\n    x = feature_dataframe['features'].values,\n    mode='markers',\n    marker=dict(\n        sizemode = 'diameter',\n        sizeref = 1,\n        size = 25,\n        color = feature_dataframe['AdaBoost feature importances'].values,\n        colorscale='Portland',\n        showscale=True\n    ),\n    text = feature_dataframe['features'].values\n)\ndata = [trace]\n\nlayout= go.Layout(\n    autosize= True,\n    title= 'AdaBoost feature importances',\n    hovermode= 'closest',\n    yaxis=dict(\n        title= 'Feature Importance',\n        ticklen= 5,\n        gridwidth= 2\n    ),\n    showlegend= False\n)\nfig = go.Figure(data=data, layout=layout)\npy.iplot(fig)\n\n\n\n# Scatter plot \ntrace = go.Scatter(\n    y = feature_dataframe['Gradient Regressor feature importances'].values,\n    x = feature_dataframe['features'].values,\n    mode='markers',\n    marker=dict(\n        sizemode = 'diameter',\n        sizeref = 1,\n        size = 25,\n        color = feature_dataframe['Gradient Regressor feature importances'].values,\n        colorscale='Portland',\n        showscale=True\n    ),\n    text = feature_dataframe['features'].values\n)\ndata = [trace]\n\nlayout= go.Layout(\n    autosize= True,\n    title= 'Gradient Regressor Feature Importance',\n    hovermode= 'closest',\n    yaxis=dict(\n        title= 'Feature Importance',\n        ticklen= 5,\n        gridwidth= 2\n    ),\n    showlegend= False\n)\nfig = go.Figure(data=data, layout=layout)\npy.iplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f2b382b706141c821004ad4a48030b4ad25cbfea","_cell_guid":"99b09b6e-8ff5-463a-8b64-e4afb1d4e360"},"cell_type":"markdown","source":"Let's have a look at the feature importances across all of the emsemble models"},{"metadata":{"_uuid":"8f96c800abe9703e95f8a209ffa89cb23bae7fb0","_cell_guid":"0fa94d5c-26d9-4369-83d3-2e6e1e43166c","collapsed":true,"trusted":false},"cell_type":"code","source":"feature_dataframe['mean'] = feature_dataframe.mean(axis= 1) # axis = 1 computes the mean row-wise\nfeature_dataframe.head(50)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"507eac2505a372712a33155f8ad2aca540155a03","_cell_guid":"84c7e626-e35e-4427-a21f-33273c1a69c5","collapsed":true,"trusted":false},"cell_type":"code","source":"y = feature_dataframe['mean'].values\nx = feature_dataframe['features'].values\ndata = [go.Bar(\n            x= x,\n             y= y,\n            width = 0.5,\n            marker=dict(\n               color = feature_dataframe['mean'].values,\n            colorscale='Portland',\n            showscale=True,\n            reversescale = False\n            ),\n            opacity=0.6\n        )]\n\nlayout= go.Layout(\n    autosize= True,\n    title= 'Barplots of Mean Feature Importance',\n    hovermode= 'closest',\n    yaxis=dict(\n        title= 'Feature Importance',\n        ticklen= 5,\n        gridwidth= 2\n    ),\n    showlegend= False\n)\nfig = go.Figure(data=data, layout=layout)\npy.iplot(fig, filename='bar-direct-labels')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5e78d236db416c10df670fd1abf48d9de65e3363","_cell_guid":"70467185-2771-4b97-b5d7-c3a8ec6e874e"},"cell_type":"markdown","source":"### Create a DataFrame with our First Level features"},{"metadata":{"_uuid":"a56faf1f2372c35d9ed4d1ec295850493e787493","_cell_guid":"1f2c7250-7dae-4441-b725-ec9cba7d12d1","collapsed":true,"trusted":false},"cell_type":"code","source":"#predictions from first layer become data input for second layer\nbase_predictions_train = pd.DataFrame( \n    {\n    'RandomForest': rf_oof_train.ravel(),\n    'ExtraTrees': et_oof_train.ravel(),\n    'AdaBoost': ada_oof_train.ravel(),\n    'GradientRegressor': gb_regressor_oof_train.ravel()\n    })\n#predictions for all instances in the training set\nbase_predictions_train.head(3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0d3bca2d9e1a063ec6b474d73ec509fdf77aebc3","_cell_guid":"71971a27-ea3c-437b-8b0a-9eccc3ef3edd"},"cell_type":"markdown","source":"### Let's look at the correlation between the regressors"},{"metadata":{"_uuid":"125c08bb6aab1ca8094e5a259fa741eff657e5eb","_cell_guid":"cb5171e4-ac39-4eb0-a5db-0634ee5d2ef2","collapsed":true,"trusted":false},"cell_type":"code","source":"data = [\n    go.Heatmap(\n        z= base_predictions_train.astype(float).corr().values ,\n        x=base_predictions_train.columns.values,\n        y= base_predictions_train.columns.values,\n          colorscale='Portland',\n            showscale=True,\n            reversescale = True\n    )\n]\npy.iplot(data, filename='labelled-heatmap')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7adac01295bea18155617f0e8c13aae8e9d1926e","_cell_guid":"9480ee79-2702-4c13-ac6e-64e6f0e8efca","collapsed":true,"trusted":false},"cell_type":"code","source":"x_train = np.concatenate(( et_oof_train, rf_oof_train, gb_regressor_oof_train, svm_oof_train), axis=1)\n\nx_test_201610 = np.concatenate(( et_oof_test_201610, rf_oof_test_201610, gb_regressor_oof_test_201610, svm_oof_test_201610), axis=1)\nx_test_201611 = np.concatenate(( et_oof_test_201611, rf_oof_test_201611, gb_regressor_oof_test_201611, svm_oof_test_201611), axis=1)\nx_test_201612 = np.concatenate(( et_oof_test_201612, rf_oof_test_201612, gb_regressor_oof_test_201612, svm_oof_test_201612), axis=1)\nx_test_201710 = np.concatenate(( et_oof_test_201710, rf_oof_test_201710, gb_regressor_oof_test_201710, svm_oof_test_201710), axis=1)\nx_test_201711 = np.concatenate(( et_oof_test_201711, rf_oof_test_201711, gb_regressor_oof_test_201711, svm_oof_test_201711), axis=1)\nx_test_201712 = np.concatenate(( et_oof_test_201712, rf_oof_test_201712, gb_regressor_oof_test_201712, svm_oof_test_201712), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6d47cfc4a5b0d5bc2d3f10c626cea6e017adc46c","_cell_guid":"6439ddd1-5171-462a-8820-5bd4f053aa7e"},"cell_type":"markdown","source":"### Use XGBRegressor as Second Level Regressor on First Level Features"},{"metadata":{"_uuid":"f5bfdbcf7074154c9137d8fd7dd16ed4b49e199e","_cell_guid":"5ef7ed01-4d55-4db7-b0af-53ae302b2a90","collapsed":true,"trusted":false},"cell_type":"code","source":"#gbm = xgb.XGBRegressor(\n    #learning_rate = 0.02,\n# n_estimators= 2000,\n# max_depth= 4,\n# min_child_weight= 2,\n# #gamma=1,\n# gamma=0.9,                        \n# subsample=0.8,\n# colsample_bytree=0.8,\n# objective= 'reg:linear',\n# nthread= -1,\n# scale_pos_weight=1\n# ).fit(x_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5dd7130761e73bb4f1db1fe4588790f5f91534fd","_cell_guid":"554c0dbc-fac8-4c2f-ae6b-f5084b13e62e","collapsed":true},"cell_type":"markdown","source":"# Phase 4: Make predictions using the model\n\n### Use the Emsemble Model to make predictions"},{"metadata":{"_uuid":"ebe9dcb9930663f84e6f38eaf73ddbbd0f86ace6","_cell_guid":"0697dbba-4d23-4d06-8630-d73098a7f846","collapsed":true,"trusted":false},"cell_type":"code","source":"#generate the predictions\n#predictions_201610 = gbm.predict(x_test_201610).round(4)\n#predictions_201611 = gbm.predict(x_test_201611).round(4)\n#predictions_201612 = gbm.predict(x_test_201612).round(4)\n\n#predictions_201710 = gbm.predict(x_test_201710).round(4)\n#predictions_201711 = gbm.predict(x_test_201711).round(4)\n#predictions_201712 = gbm.predict(x_test_201712).round(4)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9ece9b46c367d3a4844778d1cdc022596300e945","_cell_guid":"15e2e51e-fba7-4572-af10-4586bd40a7d4","collapsed":true,"trusted":false},"cell_type":"code","source":" #StackingSubmission = pd.DataFrame({ '201610': predictions_201610, \n #                                         '201611': predictions_201611,\n #                                         '201612': predictions_201612,\n #                                         '201710': predictions_201710,\n #                                         '201711': predictions_201711,\n #                                         '201712': predictions_201712,\n #                                         'ParcelId': ParcelID,\n #                            })\n#print(StackingSubmission)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}