{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Intro","metadata":{}},{"cell_type":"markdown","source":"**House Price Prediction**\n* Problem type = regression\n\n**Nice Links**\n* https://www.kaggle.com/code/angqx95/data-science-workflow-top-2-with-tuning\n* https://www.kaggle.com/code/alexisbcook/pipelines","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns # cool graph\nimport matplotlib.pyplot as plt # graph\n\nfilepath = '../input/home-data-for-ml-course/train.csv'\ndata = pd.read_csv(filepath)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-05T16:24:22.678548Z","iopub.execute_input":"2022-07-05T16:24:22.678998Z","iopub.status.idle":"2022-07-05T16:24:22.71148Z","shell.execute_reply.started":"2022-07-05T16:24:22.678964Z","shell.execute_reply":"2022-07-05T16:24:22.709992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.dtypes.unique()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:22.749324Z","iopub.execute_input":"2022-07-05T16:24:22.750377Z","iopub.status.idle":"2022-07-05T16:24:22.758229Z","shell.execute_reply.started":"2022-07-05T16:24:22.750336Z","shell.execute_reply":"2022-07-05T16:24:22.757376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = list(data.select_dtypes('object')\n                .nunique().sort_values()\n                .index)\n\nnum_cols = list(data.select_dtypes(include=['int64', 'float64'])\n                .nunique().sort_values()\n                .index)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:22.838923Z","iopub.execute_input":"2022-07-05T16:24:22.839624Z","iopub.status.idle":"2022-07-05T16:24:22.861831Z","shell.execute_reply.started":"2022-07-05T16:24:22.839587Z","shell.execute_reply":"2022-07-05T16:24:22.861026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Skimming","metadata":{}},{"cell_type":"code","source":"# This code here will save column names with 0 null value -> .isna() will return true & false, and if we sum it, true = 1, false = 0.\nzero_na = list(data.isna().sum()[data.isna().sum() == 0].index)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:22.908349Z","iopub.execute_input":"2022-07-05T16:24:22.909051Z","iopub.status.idle":"2022-07-05T16:24:22.924341Z","shell.execute_reply.started":"2022-07-05T16:24:22.909012Z","shell.execute_reply":"2022-07-05T16:24:22.922882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('column with na =',data.isna().sum()[data.isna().sum() > 0].count(),'\\n')\npd.DataFrame(data.isna().sum()[data.isna().sum() > 0].sort_values()/(data.count().max()), columns = ['na_percentage'])","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:22.988068Z","iopub.execute_input":"2022-07-05T16:24:22.988639Z","iopub.status.idle":"2022-07-05T16:24:23.027333Z","shell.execute_reply.started":"2022-07-05T16:24:22.988607Z","shell.execute_reply":"2022-07-05T16:24:23.026015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"High NA count:\n* LotFrontage     0.177397\n* FireplaceQu     0.472603\n* Fence           0.807534\n* Alley           0.937671\n* MiscFeature     0.963014\n* PoolQC          0.995205","metadata":{}},{"cell_type":"markdown","source":"**High NA count column description**\n\n```LotFrontage```: Linear feet of *street connected to property* (ind: teras)\n\nData with valid NA:\n\n* ```FireplaceQu```: Fireplace quality\n\n       Ex\tExcellent - Exceptional Masonry Fireplace\n       Gd\tGood - Masonry Fireplace in main level\n       TA\tAverage - Prefabricated Fireplace in main living area or Masonry Fireplace in basement\n       Fa\tFair - Prefabricated Fireplace in basement\n       Po\tPoor - Ben Franklin Stove\n       NA\tNo Fireplace\n    \n    \n* ```Fence```: Fence quality\n\t\t\n       GdPrv\tGood Privacy\n       MnPrv\tMinimum Privacy\n       GdWo\tGood Wood\n       MnWw\tMinimum Wood/Wire\n       NA\tNo Fence\n\n\n* ```Alley```: Type of alley access to property\n\n       Grvl\tGravel\n       Pave\tPaved\n       NA \tNo alley access\n\n\n* ```MiscFeature```: Miscellaneous feature not covered in other categories\n\t\t\n       Elev\tElevator\n       Gar2\t2nd Garage (if not described in garage section)\n       Othr\tOther\n       Shed\tShed (over 100 SF)\n       TenC\tTennis Court\n       NA\tNone\n\n\n* ```PoolQC```: Pool quality\n\t\t\n       Ex\tExcellent\n       Gd\tGood\n       TA\tAverage/Typical\n       Fa\tFair\n       NA\tNo Pool\n       \n**Other categorical data with NA as a valid value**\n* Basement Data\n    * ```BsmtQual```: Evaluates the height of the basement\n    * ```BsmtCond```: Evaluates the general condition of the basement\n    * ```BsmtExposure```: Refers to walkout or garden level walls\n    * ```BsmtFinType1```: Rating of basement finished area\n    * ```BsmtFinType2```: Rating of basement finished area (if multiple types)\n\n*NA = no basement*\n\n* Garage Data\n    * ```GarageType```: Garage location\n    * ```GarageFinish```: Interior finish of the garage\n    * ```GarageQual```: Garage quality\n    * ```GarageCond```: Garage condition\n\n*NA = no garage*","metadata":{"execution":{"iopub.status.busy":"2022-07-03T14:20:52.110712Z","iopub.execute_input":"2022-07-03T14:20:52.111202Z","iopub.status.idle":"2022-07-03T14:20:52.120357Z","shell.execute_reply.started":"2022-07-03T14:20:52.11116Z","shell.execute_reply":"2022-07-03T14:20:52.118494Z"}}},{"cell_type":"markdown","source":"Other than the data with valid NA, only leaves (*data with not valid NA values*):\n* Electrical      0.000685\n* MasVnrType      0.005479\n* MasVnrArea      0.005479\n* LotFrontage     0.177397\n\nI like to explore & process this kind of data separately.","metadata":{}},{"cell_type":"code","source":"# save column names with valid NA\nvalid_na = ['BsmtQual', 'BsmtCond', 'BsmtFinType1', 'BsmtExposure', 'BsmtFinType2', 'GarageCond', 'GarageQual', \n            'GarageFinish', 'GarageType', 'GarageYrBlt', 'FireplaceQu', 'Fence', 'Alley', 'MiscFeature', 'PoolQC']","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:23.056461Z","iopub.execute_input":"2022-07-05T16:24:23.057659Z","iopub.status.idle":"2022-07-05T16:24:23.06413Z","shell.execute_reply.started":"2022-07-05T16:24:23.057612Z","shell.execute_reply":"2022-07-05T16:24:23.062713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save column names with non valid NA\nnonvalid_na = ['Electrical', 'MasVnrType', 'MasVnrArea', 'LotFrontage']\n\n# MasVnrType, Electrical = Categorical","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:23.123442Z","iopub.execute_input":"2022-07-05T16:24:23.124656Z","iopub.status.idle":"2022-07-05T16:24:23.128992Z","shell.execute_reply.started":"2022-07-05T16:24:23.124601Z","shell.execute_reply":"2022-07-05T16:24:23.128175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Cleaning (+Feature Selection Part 1)","metadata":{}},{"cell_type":"markdown","source":"## Column category (*temporary*)","metadata":{}},{"cell_type":"markdown","source":"**Data by dtypes**","metadata":{}},{"cell_type":"code","source":"# numerical columns\nprint(num_cols)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:23.192161Z","iopub.execute_input":"2022-07-05T16:24:23.192753Z","iopub.status.idle":"2022-07-05T16:24:23.198291Z","shell.execute_reply.started":"2022-07-05T16:24:23.192711Z","shell.execute_reply":"2022-07-05T16:24:23.196994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# categorical columns\nprint(cat_cols)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:23.254333Z","iopub.execute_input":"2022-07-05T16:24:23.254706Z","iopub.status.idle":"2022-07-05T16:24:23.2597Z","shell.execute_reply.started":"2022-07-05T16:24:23.254675Z","shell.execute_reply":"2022-07-05T16:24:23.258963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data by \"NA\" category**","metadata":{}},{"cell_type":"code","source":"# don't have any na at all. 0 null values.\nprint(zero_na)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:23.316977Z","iopub.execute_input":"2022-07-05T16:24:23.317741Z","iopub.status.idle":"2022-07-05T16:24:23.323397Z","shell.execute_reply.started":"2022-07-05T16:24:23.317701Z","shell.execute_reply":"2022-07-05T16:24:23.322179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data with valid NA. NA = some value, usually NA means \"don't have xx\"\nprint(valid_na)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:23.379517Z","iopub.execute_input":"2022-07-05T16:24:23.379954Z","iopub.status.idle":"2022-07-05T16:24:23.386327Z","shell.execute_reply.started":"2022-07-05T16:24:23.379918Z","shell.execute_reply":"2022-07-05T16:24:23.385456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data with non-valid NA. NA = missing data.\nprint(nonvalid_na)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:23.434673Z","iopub.execute_input":"2022-07-05T16:24:23.435205Z","iopub.status.idle":"2022-07-05T16:24:23.442821Z","shell.execute_reply.started":"2022-07-05T16:24:23.435162Z","shell.execute_reply":"2022-07-05T16:24:23.441779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cleaning Valid NA vs Nonvalid NA","metadata":{}},{"cell_type":"markdown","source":"Plan: \n1. Data with valid NA -> Change NA to some value\n2. Data with nonvalid NA -> Impute","metadata":{}},{"cell_type":"markdown","source":"### Valid NA","metadata":{}},{"cell_type":"code","source":"data[valid_na].head()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:23.502613Z","iopub.execute_input":"2022-07-05T16:24:23.503031Z","iopub.status.idle":"2022-07-05T16:24:23.529824Z","shell.execute_reply.started":"2022-07-05T16:24:23.502997Z","shell.execute_reply":"2022-07-05T16:24:23.528496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some ideas (obvious ordinal data types):\n* `BsmtQual`: na = 0, poor = 1, etc\n* `BsmtCond`: na = 0, poor = 1, etc\n* `BsmtFinType1`: na = 0, unf = 1, etc\n* `BsmtFinType2`: na = 0, unf = 1, etc\n* [BsmtExposure](https://denveregresswindow.com/walk-out-basements/): na = 0, No = 1, etc\n* `GarageCond`: na = 0, poor = 1, etc\n* `GarageQual`: na = 0, poor = 1, etc\n* `GarageFinish`: na = 0, unf = 1, etc\n* `FireplaceQu`: na = 0, poor = 1, etc\n* `Fence`: na = 0, minimum wood = 1, minimum privacy = 2, good wood = 3, good privacy = 4\n* `Alley`: na = 0, gravel = 1, paved = 2\n* `PoolQC`: na = 0, fair = 1, etc\n\nNot quite sure (will be explored):\n* `MiscFeature`: na = 0 -> transformed into count_MiscFeature (?), or check with swarmplot\n* `GarageType` -> Check with swarmplot\n\nNo ideas (will be dropped):\n* `GarageYrBlt`, na = no garage...","metadata":{}},{"cell_type":"markdown","source":"**'Not quite sure' exploration**","metadata":{}},{"cell_type":"code","source":"# Check the data, how the column is inputted.\ndata[['MiscFeature', 'GarageType', 'SalePrice']].dropna().head()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:23.597216Z","iopub.execute_input":"2022-07-05T16:24:23.598629Z","iopub.status.idle":"2022-07-05T16:24:23.615798Z","shell.execute_reply.started":"2022-07-05T16:24:23.598556Z","shell.execute_reply":"2022-07-05T16:24:23.614113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['GarageType'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:23.666554Z","iopub.execute_input":"2022-07-05T16:24:23.667566Z","iopub.status.idle":"2022-07-05T16:24:23.677589Z","shell.execute_reply.started":"2022-07-05T16:24:23.667519Z","shell.execute_reply":"2022-07-05T16:24:23.676094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Making sure there's no input like \"Shed, TenC\", because it's possible for a house have multiple MiscFeature\ndata['MiscFeature'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:23.73481Z","iopub.execute_input":"2022-07-05T16:24:23.735488Z","iopub.status.idle":"2022-07-05T16:24:23.744054Z","shell.execute_reply.started":"2022-07-05T16:24:23.735448Z","shell.execute_reply":"2022-07-05T16:24:23.743108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the boxplot -> very useful to skim if a variable variety influence the target variable.\nsns.boxplot(data=data,x='GarageType',y='SalePrice')","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:23.80197Z","iopub.execute_input":"2022-07-05T16:24:23.802624Z","iopub.status.idle":"2022-07-05T16:24:24.057807Z","shell.execute_reply.started":"2022-07-05T16:24:23.802583Z","shell.execute_reply":"2022-07-05T16:24:24.056378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the pivot table with SalePrice min, max, mean, median. Sorted by median.\ntemp = data[['MiscFeature', 'GarageType', 'SalePrice']].groupby('GarageType').agg({'SalePrice': ['min', 'max', 'mean', 'median']})\ntemp.columns = ['price_min', 'price_max', 'price_mean', 'price_median']\ntemp.sort_values('price_median')","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:24.060096Z","iopub.execute_input":"2022-07-05T16:24:24.060388Z","iopub.status.idle":"2022-07-05T16:24:24.085534Z","shell.execute_reply.started":"2022-07-05T16:24:24.060361Z","shell.execute_reply":"2022-07-05T16:24:24.083978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Conclusion:** Na = 0, CarPort = 1, Detchd = 2, 2Types = 3, Basment = 4, Attchd = 5, BuiltIn = 6","metadata":{}},{"cell_type":"code","source":"# Check the boxplot\nsns.boxplot(data=data,x='MiscFeature',y='SalePrice')","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:24.087605Z","iopub.execute_input":"2022-07-05T16:24:24.088172Z","iopub.status.idle":"2022-07-05T16:24:24.303699Z","shell.execute_reply.started":"2022-07-05T16:24:24.088123Z","shell.execute_reply":"2022-07-05T16:24:24.302188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pivot table\ntemp = data[['MiscFeature', 'GarageType', 'SalePrice']].groupby('MiscFeature').agg({'SalePrice': ['min', 'max', 'mean', 'median']})\ntemp.columns = ['price_min', 'price_max', 'price_mean', 'price_median']\ntemp.sort_values('price_median')","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:24.306305Z","iopub.execute_input":"2022-07-05T16:24:24.306682Z","iopub.status.idle":"2022-07-05T16:24:24.329531Z","shell.execute_reply.started":"2022-07-05T16:24:24.306652Z","shell.execute_reply":"2022-07-05T16:24:24.328293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Conclusion:** Na = 0, Othr = 1, Shed = 2, Gar2 = 3, TenC = 4, Elev = 5","metadata":{}},{"cell_type":"markdown","source":"Conclusion:\n* `BsmtQual`: na = 0, poor = 1, etc\n* `BsmtCond`: na = 0, poor = 1, etc\n* `BsmtFinType1`: na = 0, unf = 1, etc\n* `BsmtFinType2`: na = 0, unf = 1, etc\n* [BsmtExposure](https://denveregresswindow.com/walk-out-basements/): na = 0, No = 1, etc\n* `GarageCond`: na = 0, poor = 1, etc\n* `GarageQual`: na = 0, poor = 1, etc\n* `GarageFinish`: na = 0, unf = 1, etc\n* `FireplaceQu`: na = 0, poor = 1, etc\n* `Fence`: na = 0, minimum wood = 1, minimum privacy = 2, good wood = 3, good privacy = 4\n* `Alley`: na = 0, gravel = 1, paved = 2\n* `PoolQC`: na = 0, fair = 1, etc\n* `MiscFeature`: Na = 0, Othr = 1, Shed = 2, Gar2 = 3, TenC = 4, Elev = 5\n* `GarageType`: Na = 0, CarPort = 1, Detchd = 2, 2Types = 3, Basment = 4, Attchd = 5, BuiltIn = 6\n\n`GarageYrBlt` = Drop","metadata":{}},{"cell_type":"code","source":"# Dropping GarageYrBlt (manually lol)\nvalid_na = ['BsmtQual', 'BsmtCond', 'BsmtFinType1', 'BsmtFinType2', 'BsmtExposure', 'GarageCond', 'GarageQual', \n            'GarageFinish', 'GarageType', 'FireplaceQu', 'Fence', 'Alley', 'MiscFeature', 'PoolQC']","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:24.332031Z","iopub.execute_input":"2022-07-05T16:24:24.332506Z","iopub.status.idle":"2022-07-05T16:24:24.338119Z","shell.execute_reply.started":"2022-07-05T16:24:24.332462Z","shell.execute_reply":"2022-07-05T16:24:24.336956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set categories for the ordinal encoder. I don't know the fast way to do this.\nBsmtQual = ['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex']\nBsmtCond = ['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex']\nGarageCond = ['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex']\nGarageQual = ['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex']\nFireplaceQu = ['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex']\nPoolQC = ['NA', 'Fa', 'TA', 'Gd', 'Ex']\nBsmtFinType1 = ['NA', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ']\nBsmtFinType2 = ['NA', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ']\nBsmtExposure = ['NA', 'No', 'Mn', 'Av', 'Gd']\nGarageFinish = ['NA', 'Unf', 'RFn', 'Fin']\nFence = ['NA', 'MnWw', 'GdWo', 'MnPrv', 'GdPrv']\nAlley = ['NA', 'Grvl', 'Pave']\nMiscFeature = ['NA', 'Othr', 'Shed', 'Gar2', 'TenC', 'Elev']\nGarageType = ['NA', 'CarPort', 'Detchd', '2Types', 'Basment', 'Attchd', 'BuiltIn']\n\nvalidna_ordinal = [BsmtQual, BsmtCond, BsmtFinType1, BsmtFinType2, BsmtExposure, GarageCond, GarageQual, GarageFinish, \n                   GarageType, FireplaceQu, Fence, Alley, MiscFeature, PoolQC]","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:24.33955Z","iopub.execute_input":"2022-07-05T16:24:24.340001Z","iopub.status.idle":"2022-07-05T16:24:24.355257Z","shell.execute_reply.started":"2022-07-05T16:24:24.339969Z","shell.execute_reply":"2022-07-05T16:24:24.353831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Tidying up using pipeline & column transformer\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\n\nfillvalidna = SimpleImputer(strategy='constant',\n                           fill_value='NA')\n\nvalidna_enc = OrdinalEncoder(categories = validna_ordinal,\n                             handle_unknown = 'use_encoded_value',\n                             unknown_value = -1)\n\nvalidna_pipe = Pipeline([('fillvna', fillvalidna), \n                         ('validenc', validna_enc)])","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:24.356878Z","iopub.execute_input":"2022-07-05T16:24:24.357324Z","iopub.status.idle":"2022-07-05T16:24:24.367311Z","shell.execute_reply.started":"2022-07-05T16:24:24.357279Z","shell.execute_reply":"2022-07-05T16:24:24.366125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(validna_pipe.fit_transform(X = data[valid_na]), columns = valid_na).head()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:24.369008Z","iopub.execute_input":"2022-07-05T16:24:24.369486Z","iopub.status.idle":"2022-07-05T16:24:24.406609Z","shell.execute_reply.started":"2022-07-05T16:24:24.369444Z","shell.execute_reply":"2022-07-05T16:24:24.405478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**column + processor**\n* Cols = valid_na\n* Preprocessor = fillvalidna (imputer), validna_enc (OrdinalEncoder)","metadata":{}},{"cell_type":"markdown","source":"### Nonvalid NA","metadata":{}},{"cell_type":"markdown","source":"Data with not valid NA values:\n* Electrical      0.0685%\n* MasVnrType      0.5479%\n* MasVnrArea      0.5479%\n* LotFrontage     17.7397%\n\nPlan = Impute (*categorical* = frequent, *numerical* = median)","metadata":{}},{"cell_type":"code","source":"data[nonvalid_na].head()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:24.408871Z","iopub.execute_input":"2022-07-05T16:24:24.40925Z","iopub.status.idle":"2022-07-05T16:24:24.428298Z","shell.execute_reply.started":"2022-07-05T16:24:24.409219Z","shell.execute_reply":"2022-07-05T16:24:24.427128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Separate categorical & numerical manually lol\ncat_nonvalid_na = ['Electrical', 'MasVnrType']\nnum_nonvalid_na = ['MasVnrArea', 'LotFrontage']","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:24.430829Z","iopub.execute_input":"2022-07-05T16:24:24.431345Z","iopub.status.idle":"2022-07-05T16:24:24.436556Z","shell.execute_reply.started":"2022-07-05T16:24:24.431295Z","shell.execute_reply":"2022-07-05T16:24:24.435411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**[Categorical data with nonvalid NA] explore**","metadata":{}},{"cell_type":"code","source":"# Check the pivot table with saleprice\ntemp = data.groupby('Electrical').agg({'SalePrice': ['mean', 'median', 'min', 'max']})\ntemp.columns = ['price_mean', 'price_median', 'price_min', 'price_max']\ntemp.sort_values('price_median')","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:24.520173Z","iopub.execute_input":"2022-07-05T16:24:24.520897Z","iopub.status.idle":"2022-07-05T16:24:24.542593Z","shell.execute_reply.started":"2022-07-05T16:24:24.520845Z","shell.execute_reply":"2022-07-05T16:24:24.539642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the pivot table with saleprice\ntemp = data.groupby('MasVnrType').agg({'SalePrice': ['mean', 'median', 'min', 'max']})\ntemp.columns = ['price_mean', 'price_median', 'price_min', 'price_max']\ntemp.sort_values('price_median')","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:24.59131Z","iopub.execute_input":"2022-07-05T16:24:24.59174Z","iopub.status.idle":"2022-07-05T16:24:24.614208Z","shell.execute_reply.started":"2022-07-05T16:24:24.591708Z","shell.execute_reply":"2022-07-05T16:24:24.612942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**[Numerical data with nonvalid NA] explore**","metadata":{}},{"cell_type":"code","source":"# Check correlation with SalePrice\ndata[num_nonvalid_na + ['SalePrice']].corr()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:24.66229Z","iopub.execute_input":"2022-07-05T16:24:24.662754Z","iopub.status.idle":"2022-07-05T16:24:24.677488Z","shell.execute_reply.started":"2022-07-05T16:24:24.662717Z","shell.execute_reply":"2022-07-05T16:24:24.676284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the scatterplot & linear regression\nfig = plt.figure(figsize=(14,4))\nfor index,col in enumerate(num_nonvalid_na):\n    plt.subplot(1,2,index+1)\n    sns.regplot(x=col, y='SalePrice', data=data)\nfig.tight_layout(pad=1.0)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:24.72938Z","iopub.execute_input":"2022-07-05T16:24:24.730069Z","iopub.status.idle":"2022-07-05T16:24:25.439279Z","shell.execute_reply.started":"2022-07-05T16:24:24.730033Z","shell.execute_reply":"2022-07-05T16:24:25.438021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Nonvalid NA data cleaning (categorical + numerical)**","metadata":{}},{"cell_type":"code","source":"# median imputer for numerical data\nmedian_imputer = SimpleImputer(strategy = 'median')\n\n# most frequent imputer for categorical data\nfreq_imputer = SimpleImputer(strategy = 'most_frequent')","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:25.441812Z","iopub.execute_input":"2022-07-05T16:24:25.442228Z","iopub.status.idle":"2022-07-05T16:24:25.447991Z","shell.execute_reply.started":"2022-07-05T16:24:25.442193Z","shell.execute_reply":"2022-07-05T16:24:25.446766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check if our median imputer works\npd.DataFrame(median_imputer.fit_transform(data[num_nonvalid_na]),\n            columns = num_nonvalid_na).head()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:25.450108Z","iopub.execute_input":"2022-07-05T16:24:25.451114Z","iopub.status.idle":"2022-07-05T16:24:25.473601Z","shell.execute_reply.started":"2022-07-05T16:24:25.451066Z","shell.execute_reply":"2022-07-05T16:24:25.472326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check if our frequent imputer works\npd.DataFrame(freq_imputer.fit_transform(data[cat_nonvalid_na]),\n            columns = cat_nonvalid_na).head()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:25.476236Z","iopub.execute_input":"2022-07-05T16:24:25.476605Z","iopub.status.idle":"2022-07-05T16:24:25.496705Z","shell.execute_reply.started":"2022-07-05T16:24:25.476575Z","shell.execute_reply":"2022-07-05T16:24:25.495595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Column name + processor:**\n* num_nonvalid_na -> median_imputer\n* cat_nonvalid_na -> freq_imputer","metadata":{}},{"cell_type":"markdown","source":"**Nonvalid NA data preprocessing (scaling + encoding)**","metadata":{}},{"cell_type":"code","source":"# Scaler for numerical scaling\nfrom sklearn.preprocessing import RobustScaler\n\n# prepare categories for ordinal scaler\nms_categories = [] #median sorted categories. Better than random / auto categories...\nfor col in cat_nonvalid_na:\n    temp = data.groupby(col).agg({'SalePrice': ['median']})\n    temp.columns = ['median']\n    ms_categories.append(np.array(temp.sort_values('median').index))\n\nnvna_scaler = RobustScaler()\nnvna_encoder = OrdinalEncoder(categories = ms_categories,\n                              handle_unknown = 'use_encoded_value',\n                              unknown_value = -1)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:25.498392Z","iopub.execute_input":"2022-07-05T16:24:25.498725Z","iopub.status.idle":"2022-07-05T16:24:25.514452Z","shell.execute_reply.started":"2022-07-05T16:24:25.498695Z","shell.execute_reply":"2022-07-05T16:24:25.513466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Column transformer + Pipeline to combine them both\nfrom sklearn.compose import ColumnTransformer\n\n# I don't have the best naming sense. nvna = nonvalid na\n# numnvna_cleaner = impute numerical -> scale the numerical\nnumnvna_cleaner = Pipeline(steps = [('imputer', median_imputer),\n                                    ('scaler', nvna_scaler),\n                                   ])\n\n#catnvna_cleaner = impute categorical -> encode the categorical\ncatnvna_cleaner = Pipeline(steps = [('imputer', freq_imputer),\n                                    ('encoder', nvna_encoder),\n                                   ])\n\n# Combining them both\nnonvalidna_pipe = ColumnTransformer(\n    transformers=[\n        ('num', numnvna_cleaner, num_nonvalid_na),\n        ('cat', catnvna_cleaner, cat_nonvalid_na),\n    ])","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:25.51594Z","iopub.execute_input":"2022-07-05T16:24:25.517363Z","iopub.status.idle":"2022-07-05T16:24:25.527143Z","shell.execute_reply.started":"2022-07-05T16:24:25.517279Z","shell.execute_reply":"2022-07-05T16:24:25.525876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Final cleaner","metadata":{}},{"cell_type":"code","source":"# Final cleaner = cleaner for validna + cleaner for nonvalid na.\nna_cleaner = ColumnTransformer(\n    transformers=[\n        ('vna', validna_pipe, valid_na),\n        ('nvna', nonvalidna_pipe, nonvalid_na),\n    ])","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:25.528529Z","iopub.execute_input":"2022-07-05T16:24:25.528858Z","iopub.status.idle":"2022-07-05T16:24:25.539785Z","shell.execute_reply.started":"2022-07-05T16:24:25.528828Z","shell.execute_reply":"2022-07-05T16:24:25.538568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save the column with na\ndata_with_na = valid_na + nonvalid_na\n\n# to remind you, we also have zero_na (no null value)\nprint(zero_na)\nprint()\nprint(data_with_na)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:25.541259Z","iopub.execute_input":"2022-07-05T16:24:25.541603Z","iopub.status.idle":"2022-07-05T16:24:25.550958Z","shell.execute_reply.started":"2022-07-05T16:24:25.541571Z","shell.execute_reply":"2022-07-05T16:24:25.549842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data cleaning test with final cleaner\ncleaned_na = pd.DataFrame(na_cleaner.fit_transform(data),\n                          columns = data_with_na)\n\n# concat cleaned data to orginal data + rearrange the columns\ndata_clean = (pd.concat([data[zero_na], cleaned_na], axis = 1)\n              [list(data.drop(['GarageYrBlt'], axis = 1).columns)]) # rearrange columns\n\n# check if our cleaner works correctly\ndata_clean.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:25.552018Z","iopub.execute_input":"2022-07-05T16:24:25.552347Z","iopub.status.idle":"2022-07-05T16:24:25.621364Z","shell.execute_reply.started":"2022-07-05T16:24:25.552315Z","shell.execute_reply":"2022-07-05T16:24:25.6203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# .isna().sum() sorted descending -> the top should be column with biggest null value, if it's 0, our cleaner works perfectly\ndata_clean.isna().sum().sort_values(ascending=False).head() ","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:25.624212Z","iopub.execute_input":"2022-07-05T16:24:25.625574Z","iopub.status.idle":"2022-07-05T16:24:25.639775Z","shell.execute_reply.started":"2022-07-05T16:24:25.625533Z","shell.execute_reply":"2022-07-05T16:24:25.638155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**column + preprocessor:**\n* data_with_na -> na_cleaner","metadata":{}},{"cell_type":"markdown","source":"# Data Exploration (+ Feature Selection part 2)","metadata":{}},{"cell_type":"markdown","source":"Some of the data (data with na), already explored in the previous section. In this section we're gonna explore the rest of the data, or data without na values. The colums of those data already contained in \"zero_na\" variable.\n\nResources:\n- https://www.kaggle.com/code/angqx95/data-science-workflow-top-2-with-tuning","metadata":{}},{"cell_type":"code","source":"print(zero_na) # zero na = zna for easy naming :v","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:25.642088Z","iopub.execute_input":"2022-07-05T16:24:25.642559Z","iopub.status.idle":"2022-07-05T16:24:25.648827Z","shell.execute_reply.started":"2022-07-05T16:24:25.642516Z","shell.execute_reply":"2022-07-05T16:24:25.647757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check how many unique dtypes, dtypes can be bool, float, int, object, etc\nprint(f'\\nUnique data values: {data[zero_na].dtypes.unique()}')","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:25.650197Z","iopub.execute_input":"2022-07-05T16:24:25.650869Z","iopub.status.idle":"2022-07-05T16:24:25.661801Z","shell.execute_reply.started":"2022-07-05T16:24:25.650822Z","shell.execute_reply":"2022-07-05T16:24:25.660632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# separate num & categorical column\nnum_cols_zna = [item for item in data[zero_na].select_dtypes(include=['int64']).columns if item not in ['Id', 'MSSubClass']]\ncat_cols_zna = list(data[zero_na].select_dtypes(include=['O']).columns) + ['MSSubClass']\n# MSSubClass is a nominal data","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:25.663527Z","iopub.execute_input":"2022-07-05T16:24:25.664149Z","iopub.status.idle":"2022-07-05T16:24:25.679946Z","shell.execute_reply.started":"2022-07-05T16:24:25.664115Z","shell.execute_reply":"2022-07-05T16:24:25.678132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create variable to contain the separated column. Remember, zna = zero na :v\nzna_num = data[num_cols_zna]\nzna_cat = data[cat_cols_zna]","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:25.683187Z","iopub.execute_input":"2022-07-05T16:24:25.684781Z","iopub.status.idle":"2022-07-05T16:24:25.69447Z","shell.execute_reply.started":"2022-07-05T16:24:25.684727Z","shell.execute_reply":"2022-07-05T16:24:25.693006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zna_num.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:25.719128Z","iopub.execute_input":"2022-07-05T16:24:25.719561Z","iopub.status.idle":"2022-07-05T16:24:25.738254Z","shell.execute_reply.started":"2022-07-05T16:24:25.719526Z","shell.execute_reply":"2022-07-05T16:24:25.737055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zna_cat.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:25.789519Z","iopub.execute_input":"2022-07-05T16:24:25.789953Z","iopub.status.idle":"2022-07-05T16:24:25.818484Z","shell.execute_reply.started":"2022-07-05T16:24:25.789894Z","shell.execute_reply":"2022-07-05T16:24:25.817171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Univariate Analysis","metadata":{}},{"cell_type":"markdown","source":"**Numerical data**","metadata":{}},{"cell_type":"code","source":"# Numerical data univariate (one variable) analysis = histogram\nfig = plt.figure(figsize=(18,16))\nfor index,col in enumerate(zna_num):\n    plt.subplot(7,5,index+1)\n    sns.histplot(data.loc[:,col], kde=False)\nfig.tight_layout(pad=1.0)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:25.896311Z","iopub.execute_input":"2022-07-05T16:24:25.89738Z","iopub.status.idle":"2022-07-05T16:24:31.762354Z","shell.execute_reply.started":"2022-07-05T16:24:25.897331Z","shell.execute_reply":"2022-07-05T16:24:31.760757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Categorical data**","metadata":{}},{"cell_type":"code","source":"# Categorical data univariate (one variable) analysis = barplot (countplot)\nfig = plt.figure(figsize=(18,20))\nfor index,col in enumerate(zna_cat):\n    plt.subplot(8,6,index+1)\n    sns.countplot(x=col, data=zna_cat)\n    plt.xticks(rotation=90)\nfig.tight_layout(pad=1.0)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:31.765251Z","iopub.execute_input":"2022-07-05T16:24:31.765736Z","iopub.status.idle":"2022-07-05T16:24:35.346869Z","shell.execute_reply.started":"2022-07-05T16:24:31.765688Z","shell.execute_reply":"2022-07-05T16:24:35.345646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Bivariate Analysis","metadata":{}},{"cell_type":"markdown","source":"For the numerical bivariate analysis, I used 2 type of plot. \n1. One with all of the data \n2. the other only uses the data with the X value greater than 0. \n\nIn this analysis, we want to check the influence of this one particular variable. If the value of this variable is 0, it really doesn't tell us anything, it just means that the variable also influenced by other variables. Eg: y = ax + b, where b is influence from other variables. If the x value is 0, then y = b, in other words, the y variable is only influenced by other variable, and well, it doesn't really tell us anything about the variable we wanted to explore","metadata":{}},{"cell_type":"markdown","source":"**Correlation matrix**","metadata":{}},{"cell_type":"code","source":"# All data\nplt.figure(figsize=(14,12))\ncorrelation = zna_num.corr()\nsns.heatmap(correlation, mask = ((correlation < 0.2) & (correlation > -0.2)), linewidth=0.5)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:35.348544Z","iopub.execute_input":"2022-07-05T16:24:35.350325Z","iopub.status.idle":"2022-07-05T16:24:36.170968Z","shell.execute_reply.started":"2022-07-05T16:24:35.350269Z","shell.execute_reply":"2022-07-05T16:24:36.169748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X != 0\nplt.figure(figsize=(14,12))\ncorrelation = zna_num[zna_num != 0].corr()\nsns.heatmap(correlation, mask = ((correlation < 0.2) & (correlation > -0.2)), linewidth=0.5, cmap='mako', center = 0)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:36.173997Z","iopub.execute_input":"2022-07-05T16:24:36.174655Z","iopub.status.idle":"2022-07-05T16:24:38.547682Z","shell.execute_reply.started":"2022-07-05T16:24:36.174619Z","shell.execute_reply":"2022-07-05T16:24:38.546016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Scatter plot**","metadata":{}},{"cell_type":"code","source":"# All data\nfig = plt.figure(figsize=(20,20))\nfor index,col in enumerate(zna_num.drop('SalePrice', axis = 1)):\n    plt.subplot(8,4,index+1)\n    sns.regplot(x=col, y='SalePrice', data=zna_num,\n               scatter_kws={\"color\": \"red\"}, line_kws={\"color\": \"#F6BE00\"})\nfig.tight_layout(pad=1.0)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:38.549978Z","iopub.execute_input":"2022-07-05T16:24:38.550507Z","iopub.status.idle":"2022-07-05T16:24:48.082388Z","shell.execute_reply.started":"2022-07-05T16:24:38.550458Z","shell.execute_reply":"2022-07-05T16:24:48.08102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X != 0\nfig = plt.figure(figsize=(20,20))\nfor index,col in enumerate(zna_num.drop('SalePrice', axis = 1)):\n    plt.subplot(8,4,index+1)\n    sns.regplot(x=col, y='SalePrice', data=zna_num[zna_num != 0],\n               scatter_kws={\"color\": \"blue\"}, line_kws={\"color\": \"green\"}) \nfig.tight_layout(pad=1.0)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:48.084824Z","iopub.execute_input":"2022-07-05T16:24:48.085722Z","iopub.status.idle":"2022-07-05T16:24:56.776563Z","shell.execute_reply.started":"2022-07-05T16:24:48.085665Z","shell.execute_reply":"2022-07-05T16:24:56.77523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# selecting num feature, remove MoSold, YrSold because of the curve.\nnum_cols_zna = ['LotArea', 'OverallQual', 'OverallCond', 'YearBuilt', 'YearRemodAdd', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', \n                '1stFlrSF', '2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'BedroomAbvGr', \n                'KitchenAbvGr', 'TotRmsAbvGrd', 'Fireplaces', 'GarageCars', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', \n                'ScreenPorch', 'PoolArea', 'MiscVal']","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:56.778234Z","iopub.execute_input":"2022-07-05T16:24:56.779311Z","iopub.status.idle":"2022-07-05T16:24:56.78609Z","shell.execute_reply.started":"2022-07-05T16:24:56.779269Z","shell.execute_reply":"2022-07-05T16:24:56.784319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Categorical data**","metadata":{}},{"cell_type":"code","source":"# alternatively, you can use violinplot, boxplot, swarmplot, etc\nfig = plt.figure(figsize=(18,20))\nfor index,col in enumerate(zna_cat):\n    plt.subplot(7,4,index+1)\n    sns.boxplot(x=col, y='SalePrice', data=data)\n    plt.xticks(rotation=90)\nfig.tight_layout(pad=1.0)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:24:56.787718Z","iopub.execute_input":"2022-07-05T16:24:56.78811Z","iopub.status.idle":"2022-07-05T16:25:03.870357Z","shell.execute_reply.started":"2022-07-05T16:24:56.788079Z","shell.execute_reply":"2022-07-05T16:25:03.869132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# selecting cat cols, only remove utlities though, because it seems that 'Utilities' only contain 2 variable, and one of the variable only contain 1 value.\ncat_cols_zna = ['MSZoning', 'Street', 'LotShape', 'LandContour', 'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2', \n                'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'ExterQual', 'ExterCond', 'Foundation', 'Heating', \n                'HeatingQC', 'CentralAir', 'KitchenQual', 'Functional', 'PavedDrive', 'SaleType', 'SaleCondition', 'MSSubClass']","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:25:03.872302Z","iopub.execute_input":"2022-07-05T16:25:03.87304Z","iopub.status.idle":"2022-07-05T16:25:03.880998Z","shell.execute_reply.started":"2022-07-05T16:25:03.872994Z","shell.execute_reply":"2022-07-05T16:25:03.879192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# combining columns\nzero_na = num_cols_zna + cat_cols_zna","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:25:03.885236Z","iopub.execute_input":"2022-07-05T16:25:03.885582Z","iopub.status.idle":"2022-07-05T16:25:03.896241Z","shell.execute_reply.started":"2022-07-05T16:25:03.885553Z","shell.execute_reply":"2022-07-05T16:25:03.894994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preprocessing","metadata":{}},{"cell_type":"markdown","source":"## Categorical Columns Processing","metadata":{}},{"cell_type":"markdown","source":"Categorical data dtypes should be \"O\", or \"object\". MSSubClass is still in int64, so we wanted to change it.","metadata":{}},{"cell_type":"code","source":"# Functional transformer to fit a function into pipeline\nfrom sklearn.preprocessing import FunctionTransformer\n\n# create the function to convert dtypes\ndef to_object(x):\n    return pd.DataFrame(x).astype(object)\n\n# create the functional transformer\nstring_transform = FunctionTransformer(to_object)\n\n# check if it works\nstring_transform.fit_transform(data[cat_cols_zna]).head()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:25:03.897978Z","iopub.execute_input":"2022-07-05T16:25:03.898686Z","iopub.status.idle":"2022-07-05T16:25:03.930178Z","shell.execute_reply.started":"2022-07-05T16:25:03.898638Z","shell.execute_reply":"2022-07-05T16:25:03.928656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preparing categories for the ordinal encoder, so it's not chosen randomly.\nms_categories = [] #median sorted categories. Better than random / auto categories...\nfor col in cat_cols_zna:\n    temp = data.groupby(col).agg({'SalePrice': ['median']})\n    temp.columns = ['median']\n    ms_categories.append(np.array(temp.sort_values('median').index))\n    \n# defining our encoder\ncat_encoder = OrdinalEncoder(categories = ms_categories, \n                             handle_unknown='use_encoded_value',\n                             unknown_value = -1)\n\n# pipelining our categorical preprocessor: change all dtypes into object -> encode the values.\ncat_processor = Pipeline([\n        ('tostring', string_transform),\n        ('allcategorical', cat_encoder),\n    ])","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:25:03.932606Z","iopub.execute_input":"2022-07-05T16:25:03.933127Z","iopub.status.idle":"2022-07-05T16:25:03.999553Z","shell.execute_reply.started":"2022-07-05T16:25:03.933062Z","shell.execute_reply":"2022-07-05T16:25:03.998289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's check our cat_processor if it works\npd.DataFrame(cat_processor.fit_transform(data[cat_cols_zna]), columns = cat_cols_zna).head()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:25:04.001149Z","iopub.execute_input":"2022-07-05T16:25:04.001512Z","iopub.status.idle":"2022-07-05T16:25:04.051512Z","shell.execute_reply.started":"2022-07-05T16:25:04.001477Z","shell.execute_reply":"2022-07-05T16:25:04.050344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Column + preprocessor:**\n* cat_cols_zna -> cat_processor","metadata":{}},{"cell_type":"markdown","source":"## Numerical Columns Processing","metadata":{}},{"cell_type":"code","source":"# strategy = scale.\nprint(num_cols_zna)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:25:04.053095Z","iopub.execute_input":"2022-07-05T16:25:04.053491Z","iopub.status.idle":"2022-07-05T16:25:04.05964Z","shell.execute_reply.started":"2022-07-05T16:25:04.05346Z","shell.execute_reply":"2022-07-05T16:25:04.058269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_scaler = RobustScaler()\n\n# check if num_scaler works\npd.DataFrame(num_scaler.fit_transform(data[num_cols_zna]), columns=num_cols_zna).head()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:25:04.061345Z","iopub.execute_input":"2022-07-05T16:25:04.061633Z","iopub.status.idle":"2022-07-05T16:25:04.109391Z","shell.execute_reply.started":"2022-07-05T16:25:04.061607Z","shell.execute_reply":"2022-07-05T16:25:04.107832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Column + preprocessor**\n* num_cols_zna -> num_scaler","metadata":{}},{"cell_type":"markdown","source":"## Final processor","metadata":{}},{"cell_type":"markdown","source":"**Column + preprocessor**\n* num_cols_zna -> num_scaler\n* cat_cols_zna -> cat_processor\n\nTo remind you, zna = zero_na\n* zero_na = num_cols_zna + cat_cols_zna","metadata":{}},{"cell_type":"code","source":"# combine num_scaler and cat_processor\nzerona_cleaner = ColumnTransformer(\n    transformers = [('num', num_scaler, num_cols_zna),\n                    ('cat', cat_processor, cat_cols_zna)\n                   ]\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:25:04.111429Z","iopub.execute_input":"2022-07-05T16:25:04.112062Z","iopub.status.idle":"2022-07-05T16:25:04.116939Z","shell.execute_reply.started":"2022-07-05T16:25:04.112027Z","shell.execute_reply":"2022-07-05T16:25:04.115973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check if it works\npd.DataFrame(zerona_cleaner.fit_transform(data[zero_na]), columns = zero_na).head()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:25:04.11834Z","iopub.execute_input":"2022-07-05T16:25:04.118923Z","iopub.status.idle":"2022-07-05T16:25:04.18859Z","shell.execute_reply.started":"2022-07-05T16:25:04.118866Z","shell.execute_reply":"2022-07-05T16:25:04.187564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# combine data with null cleaner + data without null processor.\nfinal_processor = ColumnTransformer(\n    transformers = [('na_data', na_cleaner, data_with_na),\n                    ('nona_data', zerona_cleaner, zero_na),\n                   ]\n)\n\n# combine the selected final feature\nfinal_feature = data_with_na + zero_na\n\n# check if the final_processor works\npd.DataFrame(final_processor.fit_transform(data), columns = final_feature).head()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:25:04.190023Z","iopub.execute_input":"2022-07-05T16:25:04.190565Z","iopub.status.idle":"2022-07-05T16:25:04.283986Z","shell.execute_reply.started":"2022-07-05T16:25:04.190532Z","shell.execute_reply":"2022-07-05T16:25:04.282755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modelling","metadata":{}},{"cell_type":"code","source":"# Separate the training data\nX_train = data[final_feature]\ny_train = data[['SalePrice']]","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:25:04.285438Z","iopub.execute_input":"2022-07-05T16:25:04.285782Z","iopub.status.idle":"2022-07-05T16:25:04.294996Z","shell.execute_reply.started":"2022-07-05T16:25:04.285751Z","shell.execute_reply":"2022-07-05T16:25:04.293343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.svm import SVR\nfrom xgboost import XGBRegressor\n\n# evaluation\nfrom sklearn.model_selection import cross_val_score","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:25:04.297093Z","iopub.execute_input":"2022-07-05T16:25:04.297858Z","iopub.status.idle":"2022-07-05T16:25:04.308525Z","shell.execute_reply.started":"2022-07-05T16:25:04.297807Z","shell.execute_reply":"2022-07-05T16:25:04.306851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Decision tree\ndt = DecisionTreeRegressor(random_state = 14)\nmodel_dt = Pipeline(steps=[('preprocessor', final_processor),\n                           ('model', dt),\n                          ])\n\nscores = -1 * cross_val_score(model_dt, X_train, y_train,\n                              cv=5,\n                              scoring='neg_mean_absolute_error')\n\nprint(\"Average MAE score (across experiments):\")\nprint(scores.mean())","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:25:04.309852Z","iopub.execute_input":"2022-07-05T16:25:04.31043Z","iopub.status.idle":"2022-07-05T16:25:04.871029Z","shell.execute_reply.started":"2022-07-05T16:25:04.310395Z","shell.execute_reply":"2022-07-05T16:25:04.86975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# KNN\nknn = KNeighborsRegressor()\nmodel_knn = Pipeline(steps=[('preprocessor', final_processor),\n                            ('model', knn),\n                          ])\n\nscores = -1 * cross_val_score(model_knn, X_train, y_train,\n                              cv=5,\n                              scoring='neg_mean_absolute_error')\n\nprint(\"Average MAE score (across experiments):\")\nprint(scores.mean())","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:25:04.872699Z","iopub.execute_input":"2022-07-05T16:25:04.873545Z","iopub.status.idle":"2022-07-05T16:25:05.600941Z","shell.execute_reply.started":"2022-07-05T16:25:04.873494Z","shell.execute_reply":"2022-07-05T16:25:05.599636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Random forest\nrf = RandomForestRegressor(random_state = 14)\nmodel_rf = Pipeline(steps=[('preprocessor', final_processor),\n                            ('model', rf),\n                          ])\n\nscores = -1 * cross_val_score(model_rf, X_train, y_train,\n                              cv=5,\n                              scoring='neg_mean_absolute_error')\n\nprint(\"Average MAE score (across experiments):\")\nprint(scores.mean())","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:25:05.602607Z","iopub.execute_input":"2022-07-05T16:25:05.60376Z","iopub.status.idle":"2022-07-05T16:25:14.257976Z","shell.execute_reply.started":"2022-07-05T16:25:05.603705Z","shell.execute_reply":"2022-07-05T16:25:14.256736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SVM\nsvr = SVR()\nmodel_svr = Pipeline(steps=[('preprocessor', final_processor),\n                            ('model', svr),\n                          ])\n\nscores = -1 * cross_val_score(model_svr, X_train, y_train,\n                              cv=5,\n                              scoring='neg_mean_absolute_error')\n\nprint(\"Average MAE score (across experiments):\")\nprint(scores.mean())","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:25:14.259875Z","iopub.execute_input":"2022-07-05T16:25:14.260675Z","iopub.status.idle":"2022-07-05T16:25:15.527247Z","shell.execute_reply.started":"2022-07-05T16:25:14.260624Z","shell.execute_reply":"2022-07-05T16:25:15.525663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# XGB\nxgb = XGBRegressor(n_estimators=1000, learning_rate=0.05, random_state = 14)\nmodel_xgb = Pipeline(steps=[('preprocessor', final_processor),\n                            ('model', xgb),\n                          ])\n\nscores = -1 * cross_val_score(model_xgb, X_train, y_train,\n                              cv=5,\n                              scoring='neg_mean_absolute_error')\n\nprint(\"Average MAE score (across experiments):\")\nprint(scores.mean())","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:25:15.528357Z","iopub.execute_input":"2022-07-05T16:25:15.528748Z","iopub.status.idle":"2022-07-05T16:25:53.782924Z","shell.execute_reply.started":"2022-07-05T16:25:15.528713Z","shell.execute_reply":"2022-07-05T16:25:53.781705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = pd.read_csv('../input/home-data-for-ml-course/test.csv')\n\n# Generate test predictions\nmodel_xgb.fit(X_train, y_train)\npreds_test = model_xgb.predict(X_test)\n\n# Save predictions in format used for competition scoring\noutput = pd.DataFrame({'Id': X_test.Id,\n                       'SalePrice': preds_test})\noutput.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T16:25:53.784327Z","iopub.execute_input":"2022-07-05T16:25:53.785258Z","iopub.status.idle":"2022-07-05T16:26:02.804888Z","shell.execute_reply.started":"2022-07-05T16:25:53.785217Z","shell.execute_reply":"2022-07-05T16:26:02.803741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Don't feel like tuning it (I still don't know why it takes me a super long time to tune a model...). \n\nReached my target already haha,\n\n![image.png](attachment:6eea3457-9d7d-4bdd-8703-49960d7e3119.png)","metadata":{},"attachments":{"6eea3457-9d7d-4bdd-8703-49960d7e3119.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAGoAAAB1CAYAAACmoXTZAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAABRESURBVHhe7Z0HWBTX2sf/W2CXtgiCFEUELDRRxCAYNfYClg9JvNfkMX62L89NYvxiL9drYqJXE2+MMcmnGONNseVJLDFXjREVu9GIFEUUlbZEXeoCu+xS5jvn7CxNFEwk7pD56XmYc2bmTPnP+573zJnZkXAEWBg74j7BkaM/Ifl6GqqrOejLK6A3GCCzlmPws8Ph42UPB1slzv18BQUlZZBaOUBlZwsFVwk/P3cE9g6DU5cwjBkaydcofGRvEfhpi0GptEFGegqc7GxgJQPaOTnAwUEFpZyDR3sX1FTp4evdCX5dOsPNxRnWCls4OzmB48pJeTdIZEr0CY9EeycVX6PweSyLWrduHQ4fPgydTseXtB6GigqS9KjhaiCXyVFVXc3KpVIZSRLIZFJUGitZGSQSlug/ikKphLW1NZu2dGxtbTF69GjMnz+fL2maFgtFRdqzZw+f+2PQkwvCaDRASkQw76RUKme6gAjIdp1liES8WFZWVoIRqT4TJ058pFgtFmr48OF/iCU1ptJohIG0T3Q3qRjUoqhIDJMBMaRSKRHJGjK5nC8RFtSyjh49yuceRMr/bZanIRLFiliHvYMDOxBraysiCNGHuD6aqDhyIoxCoYTSxlawIlGaO78tFuppQ0Ww5gWxsbVjiU7TMiEL1FIEI9SfHVEogSAKJRBEoQSCKJRAEIUSCKJQAqHFdyb69+/PTz1ZoqKiEBYWxucaotFosGnTJj73ZHjppZfg6+vL50zcvn0b27dvh42NDWbPno0ePXrgwoULiIuLY/Np+bRp01BQUIDdu3ezstbg7Nmz/NSDPHWLcnd3R7du3Viigg0dOhT+/v4s7+3tzS/15Bg0aBAiIyNrt0lTp06d2LxXXnmFXZAXL17EqFGjMHnyZFY+depUDB48GFlZWSz/NHjqFlWfJUuWICIiAitXrsQvv/zCl5qgIjo5OSEpKYlZWmOam29mx44duHXrFpYvX86X1EG37+HhgTfeeIMtl5KSgjNnzmDOnDk4cuQINm/ezC/ZOli0RTUHFW7nzp344IMPsHTpUuZ65s6dy+bR4YH9+/ezE0jv7tP5X3/9NaZPn87mN4aKaWdnh/LycmYhjS8+evPX1dWVuWN6B/7+/fvMmrKzs1tdpOaweKFefvllduf8tddeY27x+PHjGDFiBMaOHcvm0xNvbkOio6Nx6dIlTJgwocl2z9HRkQkwcuRIrFixAmvXrmUXAb0YKHQYJycnBzNnzkRiYiJzy3QdKv7TxqKFGjBgADp27MhcQmpqKivbu3cvu9McFBTE8lVVVTh9+jQyMzOh1+sRHx/PhkNCQkLY/PpQV0YDhNWrV2PIkCHMAqnI9GKg0DoWLlzIxoaoCw0PD2cWS/kjXP+jsGih7O3tmQVQl2SGCkbzHTp0YHk6TV2TmZKSElRWVjIBGkPbLiq0edzn1KlTOHnyJDw9PRtYYJcuXfDCCy8gOTmZBRrvvPMOFi1ahC1btjRZ7x+BRQtVVlYGo9EIhULBlwDBwcEsT9sPCp3u3Lkzm6a0b9+ejfJS62rMwIEDWftG63gUU6ZMYX+3bt2KwMBAJiwVibZftI6ngUULRV2aWq1mbsd8cmNiYqBUKmujQjpwSE8enU+vdtpO0TaNWkNj6HzavsXGxrJpuh4N1/Py8mrro/WHhoZi3759zBVaCvJ7mkJ+8tHU1PDD360I3Ub9RKHRFrWCjRs3orq6mrk1GvnRcJlGfdT10TZr/fr1bMSX5r/66ivWF2oMfTCHdmapmIcOHWJtWUZGBtsG3R51eePHj2eifffdd2wd6mrpYwg0paenIyEhoXbfnjSP0kJSYTC2qB81+LlB/NTTISAgAG5ubuyOgdmtUaFoH4dGa9T6Gs9/FDTSu3v3boushgpIoz8aYLQmJxJO8lMPYvHhuZm0tDScOHHioSI0N78x58+fb7Fro8u1tkjNIRihmkKr1bL7dPn5+XxJ20Uwru/PQJtwfX92RKEEgiiUQBCFEgiiUAKhxULRZ79FWo/mzm+LhRo5chQ/JdIaNHd+W9yPony4fj2OHPnxqb3Z0RahlkRF+t833+RLmuaxhLJE6I3Sf61bhxs30vmSR9O9ew/Mmz+f3ZwVEoIXygwV68CB7/lc04wbN56JJETajFCUHw4cwLp17/O5hsyfvwBjx43jc8KjTQlFaewKherqGtPmhDJDxaII1dU1ps0K1dYQ70wIBFEogSAKJRBEoQSCKJRAEIUSCKJQAkEUSiCIQgkEUSiB8GSEqjJAW6SFoYrP16Pi3m1c+eU2Csr5gsaQdTW3ryDptqbJ9RkGLXJSzyMtu+ltUIov7cQX50yv4jTGoNVCq617x6o+FVo1rp5Ng/oh8+vzQD38cTeVGu8n3c61R52H5qD3+n5PKkrZxc0dE8T5+E3ndufUlesLk7mNL4ZxARHjuKhx47i+QX24179I40rqr3spjpsUGsSFR5FloiK5gLCXuC2XtbXzaUrft5gbQ9YdTuoYM6wPFxz9Lnc0r26+KZVyBxf34NacMjQqN3K6rH3clMCuXOC0HZy6/rzyO9zeheM5/z4j2f4N7RPETVj8H+52eb1l6qWm6ik/s5rr4tP1gdTgXJSruYMr6rYTFhTE/eXjKw3OQ0vS7xIqfcfLXC9ycv/vm83cJJ+GQl3cMIQbvjKB0/AHrr9/lFscGMt9mcHnS05xiwNGcGvO1Qnz66EVXM/YbdxtPq+/f4h7jdRrXoem5M2TuL4rEhocqL78Irci8A3uYH5dmSnlc3tfDeOmTp/xgFDp2yZxvWfuqBOmPJ3bOCGIe/tEwwvFlB5eT+NUSMQbNGIDl8TXm7VjZoPt6O+f4lY8+xz38eUHL6pHpd/l+owOMdh7chum9nNE3TuBFC1y09Xw6+4HB/43DyWOAQgIS0aehr9Zb3TF6E83YkYfpSlPcAwMQOgVNQr4nyCtUWcgSeaLgLoXCuEX3Bu6bDXK+Dyl5sZ5JPSLQC8HvoBHl7ARq43L8M4MP77EjAG2Ia8jbtk4eJp/k1Hug9CwKuTd1fIFdTy8nsZocGDTDoTPfhE9WL138OP2e5g7//na7Ugc++HVze8gxKbUVNBCfpdQ3UZG1x1oA1ToM2AQLqVeQynvq2tyTyH+6hiE+JpUkKh8EdHfF/XPbc6Zo0iPDkUXXkupbwSGuSXiyk1Tnp7gxPgE+PUNRHu+hJKbchEug0IblHFlF7Dh7RzMXza2QbkJBVyDIxDiWXd5ceVJ+OmEN0ICXPkSE4+upyG6S19g691ZmDLMkeVrNGk4nT0IAV75uPzdNmz46CP8e28iqnwiEO77eD/13WpRn9uENXhPtQUxw2Ix8fmxiJx8BAM/exdD2/EL1JKELbGxiImKwJyLQ7H5rRG14kmUvbBo0xRcmB2Bsc+TZZ6LwCrDPGz670B+CYoW185nY0if+j+bY8DZDctxY8YCTPDkix7CvUP/IPsXi0Hj10K19HPM8K/3i8KPUQ+Qh2/f217PmojImhxk+mix5/Xn8ekdFXr37Ii737+K6DnfIu9hgdNDaDWhsn5YjvlHfDBzzfuI++QjrP9bB+yatQCHHgjMfDBk3gIs+vtyDL/3LyxZ/VPtQdCr/MOFK6GNWo73P9iGjz5ZhYHXF2P2lkRyCvllKq7g54ThCO9Wd4J1iXH457kYLP5Lc66KuNuQiVg4dwFWzuiKowvn4LOrFfycx6tHd4bsn+5/aq2JoStFdsoeVMbsx2fzY/HckFgs3vIt3jSswKfxJfxCLaNVhOIqLmAbcRevbngXf430hYubL/q/uBLrZ93Fmm8S+aXMqNC1fwRxg9F4bcs2DDu7EN8km3xf9v73sSt4DbbMjibtlApewVF4M+49eH+4GccK2CKoSf0ZR2LC0V1mynNV1/H5km8xbFEsPEr5cJmE1JyBdAOaCJuVHXuTbUdgCNm/z5d1wKcbfwCt+vHqIda04VuMrmdNFEknXwyUTMSYIfXEk3dExIhIxKfe5gtaRusIVayGuiIUvvWCAIqjyhW6rHzQZrQ4cQ827kpk02Yk8vZw9TMiX2Nq0DU5Sege6NUgUJHYO0JVk1EblFy79BP6BgfWLsPlX0PSnfvYPGswwiNJW0BS5KvboT+3CjHPLsRh9j5zHo4TK4/PYavUonL3QuUtDej7iy2rx0ST1kSQuPjBV5EItZov4NFkXoGfx2N+jqKpUPBxky5rFzelQXieycVN6MpFr6oLz3VZJDwfGsTN/15jyt/4ghvvM4J768dcfh0SypLwfFgwCbPvmfKFx1Zw/sHTuV3XS03LlOdzFz+byQXXhr/pXFz0OO7L66blH5Zof6dhWF3KxS/ryfWasY27ojGV6TVJXNy0YG7cJ1dq12ucHqyHJnqsdcfVOF3cMLrBdn49tpob6lPXTWlpaqU2yhNTtuzE6IwlGECudn//QASRdqbghe1YPsp01Um9JyNu5wu49Y9RbD5NUR/rMWv3qtqAw/bZZdj/thd2Tg41LUMitVcOemBt3N+Yi2FRlXY4evmYlm85Cjy79Hss6XAQU8JN2w4In4bTvT5pFKg0jy6haWsy03PmVnwQdKx2O88uuIqYLz/CJC9+gRbS+k8h0dssxL+pnBr2tOrDlWuJC1RBZccXNAG9fWNUqOBQrxpdwiqMvhSFk/NC+ZLfANs/AxQOKiia7Go8Ieh2dOQ8qB5+Hh6FoB8Xyz62GameUxHlX9dpbquIz/UJhFbrR4k8WUShBIIolEAQhRIIolACQRRKIIhCCQRRKIEgCiUQRKEEgiiUQBCFEgiiUAJBFEogiEIJBEGMR5k+5k+SRMqmyX8G/UQZ/bxDDUe/PMCx6baKRQtFRZHLpOxTDi2BfoqhqrqmTQpmsULJiDhyOf+wHuHUiePIzbqBiBA35N7TIDenFKWVCpQrbdA3pCcG9n2GXxKoqqpGdSt9P+NpYZFCyYgVyWUmkZIuX8QXn32NouJ7cHZSYNZfB6OwohK5aj3S6TtVNk7IVOfCQWGNpfNeh5en6fGeqmoiFrGutoLFBRPMkniRDu3bg3PHDyNPrUZpqREa+mBmjRUklRzKiosgkVpBZd8OympAKZMgOfEsrqeaPuhI66B1tRUs6khYm8S7u2X/WIlDRxKgNQJGIo5WV457d4vx6/0i5GUXo7BAi2JtMTIyb0FJlFIq7aGUluPOjUT89J+9rA5aF62zLWBRQtHAgXL46DFs+3IHSisqoK+UwMbeHtVSa1ST9iirmLgzW3vI7VW4V1QGuY0eBilprwwlyMzOg6ZAh1MnjyMjzfSMu7lOoWMxR2EKwU27cyzhLOyIEI52tsRqSlBgyEelzBo27q5IVlcgt9iI8kopsbZyqPM05K8SsGmPwiI7FJQpwCmckZ58mdVF62wLVmUxQtF+EuV+iQ5Ozq4YENkfPj6+KC4phqG6ClKFBFb045QKO2JQnqioksLR2QleHbvATuUIJzs7lJSVoaaag7O7O4kMtUi/fpXVaa5byFiOUKQzS8kvqkCnTl1gawVkZ+Yg89Yd+Hj6oH9EPwT3DILKmkPmzTTY2qng2t4L7m6+COzRG95unvBwdYWhIA+3E3/GjcwsxP98mtVprlvIWMwRmN2TtrgC1ta26OjRAT7eHeDiYIN2cjm4SsC/W0+42ijgbK2HLXGHyjINJGVF6OzuR6K+auSrb6JMmw+XDi64ry3C5RSTRf1W16e/cRBb6eucB2/VvjjX0rInjQUJZfpbpi1AWeE96LVl+DX3DkqL80mEd5d9K1dGunzaggJoiwqQlJwEHQk2KsBBXmOErKoMbq528PLuDFsPTxJgKFBUZnp78LfoxJXEY+WkeXj/001YM/d5rD1a0uKy1sDifEJ5oRqVhXdgJTHi+s3bJJjQI+laGlIunUZe6gUo5JVo56yCn583evcNh1/3PsjPvo5iTR6UNkroK/RwbdcJg4dEY9ig8Xytwsdi7kxYW5GOLLnyD3yzG9eSLkGpsMaPCaeRkpYORwcn/Nf4MaQNcoGttRRW1qSHK7cmFgMiYhZ6B/tAQYRVSGUo0VdAbudOQng33CssxFvzprGbt8ZK4jsfE+rSdhzOgKxrNCZH+bG3Glta9qSxGKGsSDtEo7OEIwdx8thRErERKykpQ3J6OiqMNZBZydHNrxuxkv5wdpChkIhw8vQFhIT0RugzYTDoSlFB3KWE9JfzSyvh5B0Kdc4dvPfuEnZnvbLqMV9DtzAsxvXRoQpKR6/OcGynQn5xGfQGAxztHaC0koGrqULKzVv4JTUXxWU6yGQy9OtHv+9eg5PHT+NK8lVo8vMhI2bp370LPNxdEBNjcn3muoWM5QhFrnpK14BgtHdyhXsHZ+LaiiG1kjI3aEU6ro6kX+tsL0NNlRG2JPq7SdqwpKtpyCSd3l/vlpB8FukA55K6qmFvb4fIsCBWp7luISP7+/Llb/HTTx0psQYaSiuUStxKTyGdWBsQY0I7Jwc4OKiglHPwaO9ChNLD17sT/Lp0hpuLM+kE28LZyYm0ReWkvBtxf0r0fiYCLs6ObIyqLQx5WFTURwf9KD2CQzB46Gg42NmSCM4RA/r2QY8unvD394eDsyu8OntDaWuN1LTryC8ogr2dHLakh+zZ2R/FOiO6hvRFDz/zcIfwRaJY3HhU/QHDnf/egrMn4tl32o1GIzgSULi6dSeRXxWqyouh0xtJ8KeAhHSQra1tiIu0Qc8+vdF/0AC2flsaQLT4gcMLZ06RSPBHGCoMUBBX6OLSCVX6ItJmkZkkSpQRUTniLlX2KvQk7o62cZS2NnBokUJRGg/FZ1xLgTr7DkoKClBRbvq9F4WtHdq5uKKjt2+tQBRxKP4PhgYW4sMtJixaKDNUMPFxMQEIJWJh4bnIwxGFEgiiUAJBFEogiEIJBFEogSAKJRBEoQSCKJRAEIUSCKJQggD4f4Ch5Ao/Pik3AAAAAElFTkSuQmCC"}}}]}