{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# House Prices:Data cleaning, visualization and modeling","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2022-08-06T16:20:32.419401Z","iopub.execute_input":"2022-08-06T16:20:32.419771Z","iopub.status.idle":"2022-08-06T16:20:32.424587Z","shell.execute_reply.started":"2022-08-06T16:20:32.419739Z","shell.execute_reply":"2022-08-06T16:20:32.423277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv(\"../input/house-prices-advanced-regression-techniques/sample_submission.csv\")\ntest = pd.read_csv(\"../input/house-prices-advanced-regression-techniques/test.csv\")\ntrain = pd.read_csv(\"../input/house-prices-advanced-regression-techniques/train.csv\")\n#Creating a copy of the train and test datasets\nc_test  = test.copy()\nc_train  = train.copy()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:32.596614Z","iopub.execute_input":"2022-08-06T16:20:32.597007Z","iopub.status.idle":"2022-08-06T16:20:32.681059Z","shell.execute_reply.started":"2022-08-06T16:20:32.596973Z","shell.execute_reply":"2022-08-06T16:20:32.680109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:32.762564Z","iopub.execute_input":"2022-08-06T16:20:32.763147Z","iopub.status.idle":"2022-08-06T16:20:32.809663Z","shell.execute_reply.started":"2022-08-06T16:20:32.763093Z","shell.execute_reply":"2022-08-06T16:20:32.808710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:32.935328Z","iopub.execute_input":"2022-08-06T16:20:32.935729Z","iopub.status.idle":"2022-08-06T16:20:32.965050Z","shell.execute_reply.started":"2022-08-06T16:20:32.935668Z","shell.execute_reply":"2022-08-06T16:20:32.963996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c_train['train'] = 1\nc_test['train'] = 0\ndf = pd.concat([c_train, c_test], axis=0, sort=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:33.078418Z","iopub.execute_input":"2022-08-06T16:20:33.079176Z","iopub.status.idle":"2022-08-06T16:20:33.107510Z","shell.execute_reply.started":"2022-08-06T16:20:33.079134Z","shell.execute_reply":"2022-08-06T16:20:33.106585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:33.210348Z","iopub.execute_input":"2022-08-06T16:20:33.210746Z","iopub.status.idle":"2022-08-06T16:20:33.242291Z","shell.execute_reply.started":"2022-08-06T16:20:33.210702Z","shell.execute_reply":"2022-08-06T16:20:33.241428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# [c for c in df]:list of column names\nNAN = [(c, df[c].isna().mean() * 100) for c in df]\nNAN = pd.DataFrame(NAN, columns=[\"column_name\", \"percentage\"])","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:33.330521Z","iopub.execute_input":"2022-08-06T16:20:33.330927Z","iopub.status.idle":"2022-08-06T16:20:33.368088Z","shell.execute_reply.started":"2022-08-06T16:20:33.330891Z","shell.execute_reply":"2022-08-06T16:20:33.366810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(NAN)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:33.465294Z","iopub.execute_input":"2022-08-06T16:20:33.465830Z","iopub.status.idle":"2022-08-06T16:20:33.475058Z","shell.execute_reply.started":"2022-08-06T16:20:33.465794Z","shell.execute_reply":"2022-08-06T16:20:33.474090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get features with more than 50% of missing values\nNAN = NAN[NAN.percentage > 50]\nNAN.sort_values(\"percentage\", ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:33.617445Z","iopub.execute_input":"2022-08-06T16:20:33.617992Z","iopub.status.idle":"2022-08-06T16:20:33.648732Z","shell.execute_reply.started":"2022-08-06T16:20:33.617958Z","shell.execute_reply":"2022-08-06T16:20:33.647629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We can drop PoolQC, MiscFeature, Alley and Fence features because they have more than 80% of missing values.\ndf = df.drop(['Alley','PoolQC','Fence','MiscFeature'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:33.781368Z","iopub.execute_input":"2022-08-06T16:20:33.781724Z","iopub.status.idle":"2022-08-06T16:20:33.798332Z","shell.execute_reply.started":"2022-08-06T16:20:33.781679Z","shell.execute_reply":"2022-08-06T16:20:33.797217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# select numerical and categorical features\n# include是把'object'选出来，exclude是除了'object'其他类型的选出来\nobject_columns_df = df.select_dtypes(include=['object'])\nnumerical_columns_df = df.select_dtypes(exclude=['object'])","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:33.938072Z","iopub.execute_input":"2022-08-06T16:20:33.938724Z","iopub.status.idle":"2022-08-06T16:20:33.949074Z","shell.execute_reply.started":"2022-08-06T16:20:33.938669Z","shell.execute_reply":"2022-08-06T16:20:33.947824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"object_columns_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:34.091242Z","iopub.execute_input":"2022-08-06T16:20:34.091814Z","iopub.status.idle":"2022-08-06T16:20:34.101213Z","shell.execute_reply.started":"2022-08-06T16:20:34.091772Z","shell.execute_reply":"2022-08-06T16:20:34.100153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_columns_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:34.349105Z","iopub.execute_input":"2022-08-06T16:20:34.349801Z","iopub.status.idle":"2022-08-06T16:20:34.361136Z","shell.execute_reply.started":"2022-08-06T16:20:34.349759Z","shell.execute_reply":"2022-08-06T16:20:34.359905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Deeling with categorical feature","metadata":{}},{"cell_type":"code","source":"# Number of null values in each feature\nnull_counts = object_columns_df.isnull().sum()\nprint(\"Number of null values in each colum:\\n{}\".format(null_counts))","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:34.401196Z","iopub.execute_input":"2022-08-06T16:20:34.401926Z","iopub.status.idle":"2022-08-06T16:20:34.418093Z","shell.execute_reply.started":"2022-08-06T16:20:34.401879Z","shell.execute_reply":"2022-08-06T16:20:34.416838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* We will fill -- BsmtQual, BsmtCond, BsmtExposure, BsmtFinType1, BsmtFinType2, GarageType, GarageFinish, GarageQual, FireplaceQu, GarageCond -- with \"None\" (Take a look in the data description). \n* We will fill the rest of features with th most frequent value (using its own most frequent value).","metadata":{}},{"cell_type":"code","source":"columns_None = ['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','GarageType','GarageFinish','GarageQual','FireplaceQu','GarageCond']\nobject_columns_df[columns_None]= object_columns_df[columns_None].fillna('None')","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:34.548318Z","iopub.execute_input":"2022-08-06T16:20:34.549044Z","iopub.status.idle":"2022-08-06T16:20:34.569865Z","shell.execute_reply.started":"2022-08-06T16:20:34.548997Z","shell.execute_reply":"2022-08-06T16:20:34.568701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_with_lowNA = ['MSZoning','Utilities','Exterior1st','Exterior2nd','MasVnrType','Electrical','KitchenQual','Functional','SaleType']\n#fill missing values for each column (using its own most frequent value)\nobject_columns_df[columns_with_lowNA] = object_columns_df[columns_with_lowNA].fillna(object_columns_df.mode().iloc[0])","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:34.702357Z","iopub.execute_input":"2022-08-06T16:20:34.702908Z","iopub.status.idle":"2022-08-06T16:20:34.750663Z","shell.execute_reply.started":"2022-08-06T16:20:34.702862Z","shell.execute_reply":"2022-08-06T16:20:34.749725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Deeling with categorical feature","metadata":{}},{"cell_type":"code","source":"# Number of null values in each feature\nnull_counts = numerical_columns_df.isnull().sum()\nprint(\"Number of null values in each column:\\n{}\".format(null_counts))","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:34.959126Z","iopub.execute_input":"2022-08-06T16:20:34.959496Z","iopub.status.idle":"2022-08-06T16:20:34.969513Z","shell.execute_reply.started":"2022-08-06T16:20:34.959458Z","shell.execute_reply":"2022-08-06T16:20:34.968260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1. Fill GarageYrBlt and LotFrontage\n2. Fill the rest of columns with 0","metadata":{}},{"cell_type":"code","source":"print((numerical_columns_df['YrSold']-numerical_columns_df['YearBuilt']).median())\nprint(numerical_columns_df[\"LotFrontage\"].median())","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:35.181801Z","iopub.execute_input":"2022-08-06T16:20:35.182156Z","iopub.status.idle":"2022-08-06T16:20:35.189493Z","shell.execute_reply.started":"2022-08-06T16:20:35.182125Z","shell.execute_reply":"2022-08-06T16:20:35.188278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(numerical_columns_df['GarageYrBlt'])","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:35.293872Z","iopub.execute_input":"2022-08-06T16:20:35.294437Z","iopub.status.idle":"2022-08-06T16:20:35.301204Z","shell.execute_reply.started":"2022-08-06T16:20:35.294401Z","shell.execute_reply":"2022-08-06T16:20:35.300197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* So we will fill the year with 1979 and the Lot frontage with 68","metadata":{}},{"cell_type":"code","source":"print(numerical_columns_df['YrSold'])","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:35.350246Z","iopub.execute_input":"2022-08-06T16:20:35.350861Z","iopub.status.idle":"2022-08-06T16:20:35.358295Z","shell.execute_reply.started":"2022-08-06T16:20:35.350823Z","shell.execute_reply":"2022-08-06T16:20:35.357059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_columns_df['GarageYrBlt'] = numerical_columns_df['GarageYrBlt'].fillna(numerical_columns_df['YrSold']-35)\nnumerical_columns_df['LotFrontage'] = numerical_columns_df['LotFrontage'].fillna(68)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:35.454582Z","iopub.execute_input":"2022-08-06T16:20:35.455301Z","iopub.status.idle":"2022-08-06T16:20:35.463424Z","shell.execute_reply.started":"2022-08-06T16:20:35.455261Z","shell.execute_reply":"2022-08-06T16:20:35.462490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Fill the rest of columns with 0","metadata":{}},{"cell_type":"code","source":"numerical_columns_df= numerical_columns_df.fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:35.625328Z","iopub.execute_input":"2022-08-06T16:20:35.625939Z","iopub.status.idle":"2022-08-06T16:20:35.630907Z","shell.execute_reply.started":"2022-08-06T16:20:35.625899Z","shell.execute_reply":"2022-08-06T16:20:35.630044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* We finally end up with a clean dataset\n* After making some plots we found that we have some colums with low variance so we decide to delete them\n\n","metadata":{}},{"cell_type":"code","source":"object_columns_df['Utilities'].value_counts().plot(kind='bar',figsize=[10,3])\nobject_columns_df['Utilities'].value_counts() ","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:35.856527Z","iopub.execute_input":"2022-08-06T16:20:35.857156Z","iopub.status.idle":"2022-08-06T16:20:36.119251Z","shell.execute_reply.started":"2022-08-06T16:20:35.857096Z","shell.execute_reply":"2022-08-06T16:20:36.117986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"object_columns_df['Street'].value_counts().plot(kind='bar',figsize=[10,3])\nobject_columns_df['Street'].value_counts() ","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:36.121761Z","iopub.execute_input":"2022-08-06T16:20:36.122223Z","iopub.status.idle":"2022-08-06T16:20:36.253575Z","shell.execute_reply.started":"2022-08-06T16:20:36.122177Z","shell.execute_reply":"2022-08-06T16:20:36.252665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"object_columns_df['Condition2'].value_counts().plot(kind='bar',figsize=[10,3])\nobject_columns_df['Condition2'].value_counts() ","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:36.256100Z","iopub.execute_input":"2022-08-06T16:20:36.256416Z","iopub.status.idle":"2022-08-06T16:20:36.405920Z","shell.execute_reply.started":"2022-08-06T16:20:36.256386Z","shell.execute_reply":"2022-08-06T16:20:36.405056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"object_columns_df['RoofMatl'].value_counts().plot(kind='bar',figsize=[10,3])\nobject_columns_df['RoofMatl'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:36.407217Z","iopub.execute_input":"2022-08-06T16:20:36.407616Z","iopub.status.idle":"2022-08-06T16:20:36.573939Z","shell.execute_reply.started":"2022-08-06T16:20:36.407587Z","shell.execute_reply":"2022-08-06T16:20:36.572775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"object_columns_df['Heating'].value_counts().plot(kind='bar',figsize=[10,3])\nobject_columns_df['Heating'].value_counts() #======> Drop feature one Type","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:36.576493Z","iopub.execute_input":"2022-08-06T16:20:36.576943Z","iopub.status.idle":"2022-08-06T16:20:36.720047Z","shell.execute_reply.started":"2022-08-06T16:20:36.576898Z","shell.execute_reply":"2022-08-06T16:20:36.719120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"object_columns_df = object_columns_df.drop(['Heating','RoofMatl','Condition2','Street','Utilities'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:36.722341Z","iopub.execute_input":"2022-08-06T16:20:36.722646Z","iopub.status.idle":"2022-08-06T16:20:36.730351Z","shell.execute_reply.started":"2022-08-06T16:20:36.722615Z","shell.execute_reply":"2022-08-06T16:20:36.729241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Now we will create some new features","metadata":{}},{"cell_type":"code","source":"numerical_columns_df['Age_House']= (numerical_columns_df['YrSold']-numerical_columns_df['YearBuilt'])\nnumerical_columns_df['Age_House'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:36.735674Z","iopub.execute_input":"2022-08-06T16:20:36.736033Z","iopub.status.idle":"2022-08-06T16:20:36.750573Z","shell.execute_reply.started":"2022-08-06T16:20:36.736002Z","shell.execute_reply":"2022-08-06T16:20:36.749391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Negatif = numerical_columns_df[numerical_columns_df['Age_House'] < 0]\nNegatif","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:36.887451Z","iopub.execute_input":"2022-08-06T16:20:36.887961Z","iopub.status.idle":"2022-08-06T16:20:36.912131Z","shell.execute_reply.started":"2022-08-06T16:20:36.887927Z","shell.execute_reply":"2022-08-06T16:20:36.911269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Like we see here tha the minimun is -1 ???\n* It is strange to find that the house was sold in 2007 before the YearRemodAdd 2009.\n\nSo we decide to change the year of sold to 2009\n","metadata":{}},{"cell_type":"code","source":"numerical_columns_df.loc[numerical_columns_df['YrSold'] < numerical_columns_df['YearBuilt'],'YrSold' ] = 2009\nnumerical_columns_df['Age_House']= (numerical_columns_df['YrSold']-numerical_columns_df['YearBuilt'])\nnumerical_columns_df['Age_House'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:37.033879Z","iopub.execute_input":"2022-08-06T16:20:37.034396Z","iopub.status.idle":"2022-08-06T16:20:37.048001Z","shell.execute_reply.started":"2022-08-06T16:20:37.034354Z","shell.execute_reply":"2022-08-06T16:20:37.046832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* TotalBsmtBath : Sum of : BsmtFullBath and 1/2 BsmtHalfBath\n* TotalBath : Sum of : FullBath and 1/2 HalfBath\n* TotalSA : Sum of : 1stFlrSF and 2ndFlrSF and basement area\n\n","metadata":{}},{"cell_type":"code","source":"numerical_columns_df['TotalBsmtBath'] = numerical_columns_df['BsmtFullBath'] + numerical_columns_df['BsmtHalfBath']*0.5\nnumerical_columns_df['TotalBath'] = numerical_columns_df['FullBath'] + numerical_columns_df['HalfBath']*0.5 \nnumerical_columns_df['TotalSA']=numerical_columns_df['TotalBsmtSF'] + numerical_columns_df['1stFlrSF'] + numerical_columns_df['2ndFlrSF']","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:37.176488Z","iopub.execute_input":"2022-08-06T16:20:37.176874Z","iopub.status.idle":"2022-08-06T16:20:37.189291Z","shell.execute_reply.started":"2022-08-06T16:20:37.176839Z","shell.execute_reply":"2022-08-06T16:20:37.188504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_columns_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:37.327825Z","iopub.execute_input":"2022-08-06T16:20:37.328372Z","iopub.status.idle":"2022-08-06T16:20:37.355563Z","shell.execute_reply.started":"2022-08-06T16:20:37.328337Z","shell.execute_reply":"2022-08-06T16:20:37.354498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Now the next step is to encode categorical features\n* Ordinal categories features - Mapping from 0 to N\n\n","metadata":{}},{"cell_type":"code","source":"print(object_columns_df['ExterQual'])","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:37.417218Z","iopub.execute_input":"2022-08-06T16:20:37.417572Z","iopub.status.idle":"2022-08-06T16:20:37.424702Z","shell.execute_reply.started":"2022-08-06T16:20:37.417540Z","shell.execute_reply":"2022-08-06T16:20:37.423650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bin_map  = {'TA':2,'Gd':3, 'Fa':1,'Ex':4,'Po':1,'None':0,'Y':1,'N':0,'Reg':3,'IR1':2,'IR2':1,'IR3':0,\"None\" : 0,\n            \"No\" : 2, \"Mn\" : 2, \"Av\": 3,\"Gd\" : 4,\"Unf\" : 1, \"LwQ\": 2, \"Rec\" : 3,\"BLQ\" : 4, \"ALQ\" : 5, \"GLQ\" : 6\n            }\nobject_columns_df['ExterQual'] = object_columns_df['ExterQual'].map(bin_map)\nobject_columns_df['ExterCond'] = object_columns_df['ExterCond'].map(bin_map)\nobject_columns_df['BsmtCond'] = object_columns_df['BsmtCond'].map(bin_map)\nobject_columns_df['BsmtQual'] = object_columns_df['BsmtQual'].map(bin_map)\nobject_columns_df['HeatingQC'] = object_columns_df['HeatingQC'].map(bin_map)\nobject_columns_df['KitchenQual'] = object_columns_df['KitchenQual'].map(bin_map)\nobject_columns_df['FireplaceQu'] = object_columns_df['FireplaceQu'].map(bin_map)\nobject_columns_df['GarageQual'] = object_columns_df['GarageQual'].map(bin_map)\nobject_columns_df['GarageCond'] = object_columns_df['GarageCond'].map(bin_map)\nobject_columns_df['CentralAir'] = object_columns_df['CentralAir'].map(bin_map)\nobject_columns_df['LotShape'] = object_columns_df['LotShape'].map(bin_map)\nobject_columns_df['BsmtExposure'] = object_columns_df['BsmtExposure'].map(bin_map)\nobject_columns_df['BsmtFinType1'] = object_columns_df['BsmtFinType1'].map(bin_map)\nobject_columns_df['BsmtFinType2'] = object_columns_df['BsmtFinType2'].map(bin_map)\n\nPavedDrive =   {\"N\" : 0, \"P\" : 1, \"Y\" : 2}\nobject_columns_df['PavedDrive'] = object_columns_df['PavedDrive'].map(PavedDrive)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:37.501973Z","iopub.execute_input":"2022-08-06T16:20:37.502573Z","iopub.status.idle":"2022-08-06T16:20:37.564504Z","shell.execute_reply.started":"2022-08-06T16:20:37.502524Z","shell.execute_reply":"2022-08-06T16:20:37.562841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Will we use One hot encoder to encode the rest of categorical features ","metadata":{}},{"cell_type":"code","source":"#Select categorical features\nrest_object_columns = object_columns_df.select_dtypes(include=['object'])\n#Using One hot encoder\nobject_columns_df = pd.get_dummies(object_columns_df, columns=rest_object_columns.columns)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:37.642150Z","iopub.execute_input":"2022-08-06T16:20:37.642490Z","iopub.status.idle":"2022-08-06T16:20:37.676526Z","shell.execute_reply.started":"2022-08-06T16:20:37.642461Z","shell.execute_reply":"2022-08-06T16:20:37.675712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" object_columns_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:37.791718Z","iopub.execute_input":"2022-08-06T16:20:37.792224Z","iopub.status.idle":"2022-08-06T16:20:37.812138Z","shell.execute_reply.started":"2022-08-06T16:20:37.792191Z","shell.execute_reply":"2022-08-06T16:20:37.811313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(numerical_columns_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:37.935275Z","iopub.execute_input":"2022-08-06T16:20:37.935842Z","iopub.status.idle":"2022-08-06T16:20:37.957642Z","shell.execute_reply.started":"2022-08-06T16:20:37.935804Z","shell.execute_reply":"2022-08-06T16:20:37.956274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# standardize\nstd_numerical_columns_df = (numerical_columns_df - numerical_columns_df.mean()) / numerical_columns_df.std()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:38.075944Z","iopub.execute_input":"2022-08-06T16:20:38.076553Z","iopub.status.idle":"2022-08-06T16:20:38.106345Z","shell.execute_reply.started":"2022-08-06T16:20:38.076488Z","shell.execute_reply":"2022-08-06T16:20:38.105480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(numerical_columns_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:38.210511Z","iopub.execute_input":"2022-08-06T16:20:38.211122Z","iopub.status.idle":"2022-08-06T16:20:38.232898Z","shell.execute_reply.started":"2022-08-06T16:20:38.211066Z","shell.execute_reply":"2022-08-06T16:20:38.231824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(std_numerical_columns_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:38.356786Z","iopub.execute_input":"2022-08-06T16:20:38.357169Z","iopub.status.idle":"2022-08-06T16:20:38.378335Z","shell.execute_reply.started":"2022-08-06T16:20:38.357133Z","shell.execute_reply":"2022-08-06T16:20:38.377439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Concat Categorical (after encoding) and numerical features","metadata":{}},{"cell_type":"code","source":"print(std_numerical_columns_df['train'])","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:38.490844Z","iopub.execute_input":"2022-08-06T16:20:38.491213Z","iopub.status.idle":"2022-08-06T16:20:38.498216Z","shell.execute_reply.started":"2022-08-06T16:20:38.491178Z","shell.execute_reply":"2022-08-06T16:20:38.496822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(object_columns_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:38.635661Z","iopub.execute_input":"2022-08-06T16:20:38.636276Z","iopub.status.idle":"2022-08-06T16:20:38.650170Z","shell.execute_reply.started":"2022-08-06T16:20:38.636224Z","shell.execute_reply":"2022-08-06T16:20:38.649372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final = pd.concat([object_columns_df, std_numerical_columns_df], axis=1,sort=False)\ndf_final.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:38.777574Z","iopub.execute_input":"2022-08-06T16:20:38.778115Z","iopub.status.idle":"2022-08-06T16:20:38.804972Z","shell.execute_reply.started":"2022-08-06T16:20:38.778081Z","shell.execute_reply":"2022-08-06T16:20:38.804022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final = df_final.drop(['Id',],axis=1)\n\ndf_train = df_final[df_final['train'] > 0]\ndf_train = df_train.drop(['train',],axis=1)\n\n\ndf_test = df_final[df_final['train'] < 0]\ndf_test = df_test.drop(['SalePrice'],axis=1)\ndf_test = df_test.drop(['train',],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:38.917860Z","iopub.execute_input":"2022-08-06T16:20:38.918222Z","iopub.status.idle":"2022-08-06T16:20:38.937769Z","shell.execute_reply.started":"2022-08-06T16:20:38.918191Z","shell.execute_reply":"2022-08-06T16:20:38.936938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Separate Train and Targets","metadata":{}},{"cell_type":"code","source":"target= df_train['SalePrice']\ndf_train = df_train.drop(['SalePrice'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:39.059392Z","iopub.execute_input":"2022-08-06T16:20:39.060019Z","iopub.status.idle":"2022-08-06T16:20:39.066542Z","shell.execute_reply.started":"2022-08-06T16:20:39.059969Z","shell.execute_reply":"2022-08-06T16:20:39.065709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"from sklearn import preprocessing\nfrom sklearn.model_selection import train_test_split\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nimport sklearn.metrics as metrics\nimport math","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:39.191592Z","iopub.execute_input":"2022-08-06T16:20:39.192238Z","iopub.status.idle":"2022-08-06T16:20:41.219121Z","shell.execute_reply.started":"2022-08-06T16:20:39.192177Z","shell.execute_reply":"2022-08-06T16:20:41.218221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train,x_test,y_train,y_test = train_test_split(df_train,target,test_size=0.33,random_state=0)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:41.221668Z","iopub.execute_input":"2022-08-06T16:20:41.222136Z","iopub.status.idle":"2022-08-06T16:20:41.235041Z","shell.execute_reply.started":"2022-08-06T16:20:41.222091Z","shell.execute_reply":"2022-08-06T16:20:41.233640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\n\nreg = LinearRegression()\nreg = reg.fit(df_train, target)\npre_y = reg.predict(df_test)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:41.236744Z","iopub.execute_input":"2022-08-06T16:20:41.237079Z","iopub.status.idle":"2022-08-06T16:20:41.414767Z","shell.execute_reply.started":"2022-08-06T16:20:41.237047Z","shell.execute_reply":"2022-08-06T16:20:41.413818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\n        \"Id\": test[\"Id\"],\n        \"SalePrice\": pre_y\n    })\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:41.416298Z","iopub.execute_input":"2022-08-06T16:20:41.416721Z","iopub.status.idle":"2022-08-06T16:20:41.707935Z","shell.execute_reply.started":"2022-08-06T16:20:41.416663Z","shell.execute_reply":"2022-08-06T16:20:41.706970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}