{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-11T07:54:33.620678Z","iopub.status.idle":"2022-07-11T07:54:33.621752Z","shell.execute_reply.started":"2022-07-11T07:54:33.621487Z","shell.execute_reply":"2022-07-11T07:54:33.621510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ntrain_data = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')\ntest_data = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')\n\nprint(train_data.shape, test_data.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:14:32.949305Z","iopub.execute_input":"2022-07-13T04:14:32.949702Z","iopub.status.idle":"2022-07-13T04:14:33.051512Z","shell.execute_reply.started":"2022-07-13T04:14:32.949608Z","shell.execute_reply":"2022-07-13T04:14:33.050503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:14:35.749172Z","iopub.execute_input":"2022-07-13T04:14:35.749578Z","iopub.status.idle":"2022-07-13T04:14:35.791502Z","shell.execute_reply.started":"2022-07-13T04:14:35.749535Z","shell.execute_reply":"2022-07-13T04:14:35.790198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dealing with Null Columns","metadata":{}},{"cell_type":"code","source":"nullCols = [column for column, value in train_data.isnull().sum().items() if value > 0]\nnullCols_test = [column for column, value in test_data.isnull().sum().items() if value > 0]\nprint('Null Columns in train_data:', ', '.join(nullCols))\nprint()\nprint('Null Columns in test_data:', ', '.join(nullCols_test))\nprint()\nprint('Missing Null Column in test_data but present in train_data:', set(nullCols).difference(set(nullCols_test)))","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:14:39.864262Z","iopub.execute_input":"2022-07-13T04:14:39.864614Z","iopub.status.idle":"2022-07-13T04:14:39.890355Z","shell.execute_reply.started":"2022-07-13T04:14:39.864576Z","shell.execute_reply":"2022-07-13T04:14:39.889263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Details about Null Columns\n\n*  Null Column => `LotFrontage`\n\n    Linear feet of street connected to property. Datatype-float64.\n\n    Imputing using mean\n\n*  Null Column => `Alley`\n    \n    Type of alley access to property. Datatype- Object\n    \n    Replace NaN with 'No_Alley_Access'\n    \n*  Null Column => `MasVnrType`\n\n    Masonry veneer type. Datatype- Object\n    \n    Replace NaN with mode\n    \n*  Null Column => `MasVnrArea`\n\n    Masonry veneer area in square feet. Datatype- float64\n    \n    Replace NaN with mean\n    \n*  Null Column => `BsmtQual`\n\n    Evaluates the height of the basement. Datatype- Object\n    \n    Replace NaN with 'No_Basement'\n    \n*  Null Column => `BsmtCond`\n\n    Evaluates the general condition of the basement. Datatype-Object\n    \n    Replace NaN with 'No_Basement'\n    \n*  Null Column => `BsmtExposure`\n\n    Refers to walkout or garden level walls. Datatype-Object\n    \n    Replace NaN with 'No_Basement'\n    \n*  Null Column => `BsmtFinType1`\n\n    Rating of basement finished area. Datatype-Object\n    \n    Replace NaN with 'No_Basement'\n    \n*  Null Column => `BsmtFinType2`\n\n    Rating of basement finished area (if multiple types). Datatype-Object\n    \n    Replace NaN with 'No_Basement'\n    \n*  Null Column => `Electrical`\n\n    Electrical system. Datatype-Object\n    \n    Replace NaN with mode\n    \n*  Null Column => `FireplaceQu`\n\n    Number of fireplaces. Datatype-Object, convert to int\n    \n    Replace NaN with mode\n    \n*  Null Column => `GarageType`\n\n     Garage location. Datatype-Object\n    \n    Replace NaN with 'No_Garage'\n    \n*  Null Column => `GarageYrBlt`\n\n     Year garage was built. Datatype-float\n    \n    Replace NaN with 'No_Garage'\n    \n    \n*  Null Column => `GarageFinish`\n\n    Interior Finish. Datatype-Object\n    \n    Replace NaN with 'No_Garage'\n    \n*  Null Column => `GarageQual`\n\n    Quality. Datatype-Object\n    \n    Replace NaN with 'No_Garage'\n    \n*  Null Column => `GarageCond`\n\n    Garage Condition. Datatype-Object\n    \n    Replace NaN with 'No_Garage'\n    \n*  Null Column => `PoolQc`\n\n    Pool Quality. Datatype-Object\n    \n    Replace NaN with 'No_Pool'\n    \n*  Null Column => `Fence`\n\n    Fence Quality. Datatype-Object\n    \n    Replace NaN with 'No_Fence'\n    \n*  Null Column => `MiscFeature`\n\n    Misc Features. Datatype-Object\n    \n    Replace NaN with 'No_Features'","metadata":{"jupyter":{"source_hidden":true}}},{"cell_type":"code","source":"from scipy import stats as st\nimport numpy as np\n\ndef train_test_nan(nullCols, nullCols_test, df=train_data, df2=test_data):\n    \n    basement = ['BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2']\n    garage = ['GarageType','GarageFinish', 'GarageQual', 'GarageCond']\n    for column in nullCols:\n        \n        if column == 'LotFrontage':\n            \n            nullCols_test.remove(column)\n            lot_mean = np.mean(df[column])\n            df[column].fillna(lot_mean, inplace=True)\n            df2[column].fillna(lot_mean, inplace=True)\n            \n        elif column == 'Alley':\n            \n            nullCols_test.remove(column)\n            df[column].fillna('No_Access', inplace=True)\n            df2[column].fillna('No_Access', inplace=True)\n\n            \n        elif column == 'MasVnrType':\n            \n            nullCols_test.remove(column)\n            type_mode = st.mode(train_data.MasVnrType)[0][0]\n            df[column].fillna(type_mode, inplace=True)\n            df2[column].fillna(type_mode, inplace=True)\n\n            \n        elif column == 'MasVnrArea':\n            \n            nullCols_test.remove(column)\n            area_mean = np.mean(df[column])\n            df[column].fillna(area_mean, inplace=True)\n            df2[column].fillna(area_mean, inplace=True)\n\n            \n        elif column in basement:\n            \n            nullCols_test.remove(column)\n            df[column].fillna('No_Basement', inplace=True)\n            df2[column].fillna('No_Basement', inplace=True)\n            \n        elif column == 'Electrical':\n\n            electrical_mode = st.mode(df[column])[0][0]\n            df[column].fillna(type_mode, inplace=True)\n            df2[column].fillna(type_mode, inplace=True)\n            \n        elif column == 'FireplaceQu':\n\n            nullCols_test.remove(column)\n            mode = st.mode(df[column])[0][0]\n            df[column].fillna(mode, inplace=True)\n            df2[column].fillna(mode, inplace=True)\n\n        elif column == 'GarageYrBlt':\n\n            nullCols_test.remove(column)\n            blt_mean = (np.mean(df[column]))\n            df[column].fillna(blt_mean, inplace=True)\n            df2[column].fillna(blt_mean, inplace=True)\n\n        elif column in garage:\n\n            nullCols_test.remove(column)\n            df[column].fillna('No_Garage',inplace=True)\n            df2[column].fillna('No_Garage',inplace=True)\n            \n        elif column == 'PoolQc':\n\n            nullCols_test.remove(column)\n            df[column].fillna('No_Pool', inplace=True)\n            df2[column].fillna('No_Pool', inplace=True)\n            \n        elif column == 'Fence':\n\n            nullCols_test.remove(column)\n            df[column].fillna('No_Fence', inplace=True)\n            df2[column].fillna('No_Fence', inplace=True)\n\n            \n        else :\n\n            nullCols_test.remove(column)\n            df[column].fillna('No_feature', inplace=True)\n            df2[column].fillna('No_feature', inplace=True)\n    \n    print('Imputation Done Successfully on Train Data')\n    \n    for col in nullCols_test:\n        \n        if (df2[col].dtype == int or df2[col].dtype == float):\n            df2[col].fillna(df[col].mean(), inplace=True)\n        else:\n            df2[col].fillna(st.mode(df[col])[0][0], inplace=True)\n            \n    print('Imputation Done Successfully on Test Data')\n    \n\ntrain_test_nan(nullCols, nullCols_test)\n            ","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:16:19.409206Z","iopub.execute_input":"2022-07-13T04:16:19.409553Z","iopub.status.idle":"2022-07-13T04:16:19.844518Z","shell.execute_reply.started":"2022-07-13T04:16:19.409521Z","shell.execute_reply":"2022-07-13T04:16:19.843551Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\nplt.figure(figsize=(30,20))\ncor = train_data.corr()\nsns.heatmap(cor, annot=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:17:17.502988Z","iopub.execute_input":"2022-07-13T04:17:17.503896Z","iopub.status.idle":"2022-07-13T04:17:23.001780Z","shell.execute_reply.started":"2022-07-13T04:17:17.503858Z","shell.execute_reply":"2022-07-13T04:17:23.000899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Numerical Features","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nnum_cols = train_data.select_dtypes(exclude='object').columns.to_list()\nnum_cols.remove('SalePrice')\nnum_cols.remove('Id')\n\nfig, axes = plt.subplots(6, 6, figsize=(20,15), sharey=True,\n                        )\n\nplt.subplots_adjust(left=0.1,\n                    bottom=0.1, \n                    right=0.9, \n                    top=0.9, \n                    wspace=0.4, \n                    hspace=0.4)\n\nj, k, l=0, 0, 0\nfor i in range(36):\n    if (j>5):\n        j=0\n        l+=1\n    sns.histplot(train_data[num_cols[k]], ax=axes[l, j], color='green')\n    j+=1\n    k+=1","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:17:23.003141Z","iopub.execute_input":"2022-07-13T04:17:23.004133Z","iopub.status.idle":"2022-07-13T04:17:29.030611Z","shell.execute_reply.started":"2022-07-13T04:17:23.004096Z","shell.execute_reply":"2022-07-13T04:17:29.029689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.compose import make_column_transformer\n\nnum_cols = train_data.select_dtypes(exclude='object').columns.to_list()\nnum_cols.remove('SalePrice')\nnum_cols.remove('Id')\n\nmct = make_column_transformer(\n    (MinMaxScaler(), num_cols)\n)\n\ntrain_X = pd.DataFrame(mct.fit_transform(train_data[num_cols]))\ntrain_X.columns = num_cols\ntrain_y = train_data['SalePrice']\n\ntest_data_n = pd.DataFrame(mct.transform(test_data[num_cols]))\ntest_data_n.columns = num_cols\n\nplt.figure(figsize=(20,8))\nplt.xticks(rotation=45)\nsns.boxplot(data = train_X)\n\n# train_data[num_cols].plot(kind='box', figsize=(15,10))","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:18:13.119542Z","iopub.execute_input":"2022-07-13T04:18:13.119904Z","iopub.status.idle":"2022-07-13T04:18:13.803916Z","shell.execute_reply.started":"2022-07-13T04:18:13.119870Z","shell.execute_reply":"2022-07-13T04:18:13.802683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Selecting Numerical Features using Lasso Regression","metadata":{}},{"cell_type":"code","source":"sel_features = abs(cor['SalePrice'])\nsel_features = sel_features[sel_features > .3]\nsel_features = sel_features.index.to_list()\n\nplt.figure(figsize=(15,7))\nsns.heatmap(train_data[sel_features[:-1]].corr(), annot=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:19:41.692454Z","iopub.execute_input":"2022-07-13T04:19:41.692832Z","iopub.status.idle":"2022-07-13T04:19:43.040687Z","shell.execute_reply.started":"2022-07-13T04:19:41.692795Z","shell.execute_reply":"2022-07-13T04:19:43.039622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import Lasso\nfrom sklearn.compose import make_column_transformer\nfrom sklearn.preprocessing import MinMaxScaler\n\nX = train_data[sel_features[:-1]]\ny = train_data['SalePrice']\n\nmct = make_column_transformer(\n(MinMaxScaler(), sel_features[:-1])\n)\n\nX = pd.DataFrame(mct.fit_transform(X))\nX.columns = sel_features[:-1]\n\nlasso = Lasso()\nlasso.fit(X, y)\n\nplt.figure(figsize=(20, 10))\n\nsns.barplot(x=sel_features[:-1], y=lasso.coef_)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:20:57.458405Z","iopub.execute_input":"2022-07-13T04:20:57.458779Z","iopub.status.idle":"2022-07-13T04:20:57.941795Z","shell.execute_reply.started":"2022-07-13T04:20:57.458724Z","shell.execute_reply":"2022-07-13T04:20:57.940765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop = ['GarageArea', 'GarageYrBlt', 'OpenPorchSF', 'FullBath']\n\nfor col in drop:\n\n    sel_features.remove(col)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:21:03.081134Z","iopub.execute_input":"2022-07-13T04:21:03.081481Z","iopub.status.idle":"2022-07-13T04:21:03.086539Z","shell.execute_reply.started":"2022-07-13T04:21:03.081450Z","shell.execute_reply":"2022-07-13T04:21:03.085286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nnum_cols = sel_features[:-1]\n\nfig, axes = plt.subplots(5,4, figsize=(15, 10), sharey = True)\nplt.suptitle('ScatterPlots', fontsize=16)\nplt.subplots_adjust(left=0.1,\n                    bottom=0.1, \n                    right=0.9, \n                    top=0.9, \n                    wspace=0.4, \n                    hspace=0.4)\n\ni, j = 0, 0\nfor col in num_cols:\n    if(j>3):\n        i+=1\n        j=0\n    sns.scatterplot(x = col, y = train_data['SalePrice'], data=train_X , ax=axes[i,j], color='teal')\n    j+=1\n    \nfor i in range(4):\n    fig.delaxes(axes[4,i])\nfig.delaxes(axes[3,3])\nfig.delaxes(axes[3,2])\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:21:12.109718Z","iopub.execute_input":"2022-07-13T04:21:12.110513Z","iopub.status.idle":"2022-07-13T04:21:13.814561Z","shell.execute_reply.started":"2022-07-13T04:21:12.110473Z","shell.execute_reply":"2022-07-13T04:21:13.813668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Label Encoding and One Hot Encoding Categorical Variables","metadata":{}},{"cell_type":"code","source":"def ordinal_encoder():\n    train_cp = train_data.copy()\n    test_cp = test_data.copy()\n    \n    mapping = {\n        ('Gtl', 'Mod', 'Sev'): {'Gtl':1,\n                                      'Mod':2,\n                                      'Sev':3},\n        \n        ('Gd', 'TA', 'Ex', 'Fa'): {'Ex':5,\n                                  'Gd':4,\n                                  'TA':3,\n                                  'Fa':2,\n                                  'Po':1},\n        \n        ('TA', 'Gd', 'Fa', 'Po', 'Ex'): {'Ex':5,\n                                  'Gd':4,\n                                  'TA':3,\n                                  'Fa':2,\n                                  'Po':1},\n        \n        ('Gd', 'TA', 'Ex', 'No_Basement', 'Fa'): {'Ex':5,\n                                  'Gd':4,\n                                  'TA':3,\n                                  'Fa':2,\n                                  'Po':1,\n                                  'No_Basement':0},\n        \n        ('TA', 'Gd', 'No_Basement', 'Fa', 'Po'): {'Ex':5,\n                                  'Gd':4,\n                                  'TA':3,\n                                  'Fa':2,\n                                  'Po':1,\n                                  'No_Basement':0},\n        \n        ('No', 'Gd', 'Mn', 'Av', 'No_Basement'): {\n                                  'Gd':4,\n                                  'Av':3,\n                                  'Mn':2,\n                                  'No':1,\n                                  'No_Basement':0},\n        \n        ('GLQ', 'ALQ', 'Unf', 'Rec', 'BLQ', 'No_Basement', 'LwQ'): {\n            'GLQ':6,\n            'ALQ':5,\n            'BLQ':4,\n            'Rec':3,\n            'LwQ':2,\n            'Unf':1,\n            'No_Basement':0\n        },\n        \n        ('Unf', 'BLQ', 'No_Basement', 'ALQ', 'Rec', 'LwQ', 'GLQ'): {\n            'GLQ':6,\n            'ALQ':5,\n            'BLQ':4,\n            'Rec':3,\n            'LwQ':2,\n            'Unf':1,\n            'No_Basement':0\n        },\n        \n        ('Ex', 'Gd', 'TA', 'Fa', 'Po'): {'Ex':5,\n                                  'Gd':4,\n                                  'TA':3,\n                                  'Fa':2,\n                                  'Po':1},\n        \n        ('Y', 'N'): {'Y':1,\n                    'N':0},\n        \n        \n        ('Gd', 'TA', 'Ex', 'Fa'): {'Ex':5,\n                                  'Gd':4,\n                                  'TA':3,\n                                  'Fa':2,\n                                  'Po':1},\n        \n        ('Gd', 'TA', 'Fa', 'Ex', 'Po'): {'Ex':5,\n                                  'Gd':4,\n                                  'TA':3,\n                                  'Fa':2,\n                                  'Po':1,\n                                  'No_Fireplace': 0},\n        \n        ('TA', 'Fa', 'Gd', 'No_Garage', 'Ex', 'Po'): {'Ex':5,\n                                  'Gd':4,\n                                  'TA':3,\n                                  'Fa':2,\n                                  'Po':1,\n                                  'No_Garage': 0},\n        \n        ('TA', 'Fa', 'No_Garage', 'Gd', 'Po', 'Ex'): {'Ex':5,\n                                  'Gd':4,\n                                  'TA':3,\n                                  'Fa':2,\n                                  'Po':1,\n                                  'No_Garage': 0},\n        \n        ('No_feature', 'Ex', 'Fa', 'Gd'): {'Ex':5,\n                                  'Gd':4,\n                                  'TA':3,\n                                  'Fa':2,\n                                  'No_feature': 0}\n              }\n    \n    label_en_cols = ['LandSlope', 'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond',\n                    'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',\n                    'HeatingQC', 'CentralAir', 'KitchenQual', 'FireplaceQu',\n                    'GarageQual', 'GarageCond', 'PoolQC']\n    \n    for i, col in enumerate(label_en_cols):\n        \n        key = tuple(train_cp[col].unique())\n        train_cp[col] = train_cp[col].map(mapping[key])\n        test_cp[col] = test_cp[col].map(mapping[key])\n        \n    print('Encoding Completed')\n    \n    return train_cp, test_cp","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:21:19.871398Z","iopub.execute_input":"2022-07-13T04:21:19.871830Z","iopub.status.idle":"2022-07-13T04:21:19.892611Z","shell.execute_reply.started":"2022-07-13T04:21:19.871793Z","shell.execute_reply":"2022-07-13T04:21:19.891562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\n\n\ndef encoders():\n    \n    num_cols = train_data.select_dtypes(exclude='object').columns.to_list()\n\n    label_en_cols = ['LandSlope', 'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond',\n                    'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',\n                    'HeatingQC', 'CentralAir', 'KitchenQual', 'FireplaceQu',\n                    'GarageQual', 'GarageCond', 'PoolQC']\n\n    one_hot_cols = ['MSZoning', 'Street', 'Alley', 'LotShape', 'LandContour',\n                   'Utilities', 'LotConfig', 'Neighborhood', 'Condition1',\n                   'Condition2', 'BldgType', 'HouseStyle', 'RoofStyle',\n                   'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType',\n                   'Foundation', 'Heating', 'Electrical', 'Functional',\n                   'GarageType', 'GarageFinish', 'PavedDrive', 'Fence',\n                   'MiscFeature', 'SaleType', 'SaleCondition']\n## Label Encoding\n    train_copy, test_copy = ordinal_encoder()\n\n## One Hot Encoding\n    ## Getting a column name for ohe cols\n    encoded_cols = []\n    for col in one_hot_cols:\n        encoded_cols += ([f\"{col}_{cat}\" for cat in list(train_data[col].unique())])\n\n    ohe = OneHotEncoder(sparse=False)\n    temp = pd.DataFrame(ohe.fit_transform(train_data[one_hot_cols]))\n    temp_test = pd.DataFrame(ohe.transform(test_data[one_hot_cols]))\n    temp.columns, temp_test.columns = encoded_cols, encoded_cols\n    \n    train_copy.drop(num_cols, axis=1, inplace=True)\n    train_copy.drop(one_hot_cols, axis=1, inplace=True)\n    train_copy = pd.concat([train_copy, temp, train_data[['SalePrice']]], axis=1)\n    \n    test_copy.drop(num_cols[:-1], axis=1, inplace=True)\n    test_copy.drop(one_hot_cols, axis=1, inplace=True)\n    test_copy = pd.concat([test_copy, temp_test], axis=1)\n\n    return (train_copy, test_copy)\n\n\ntrain_cat, test_cat = encoders()\nprint(train_cat.shape, test_cat.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:21:50.366409Z","iopub.execute_input":"2022-07-13T04:21:50.366816Z","iopub.status.idle":"2022-07-13T04:21:50.468954Z","shell.execute_reply.started":"2022-07-13T04:21:50.366777Z","shell.execute_reply":"2022-07-13T04:21:50.467868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.feature_selection import SelectFromModel\n\nsel = SelectFromModel(RandomForestClassifier(n_estimators=500))\nsel.fit(train_cat[train_cat.columns[:-1]], train_cat['SalePrice'])\n\nsupport = sel.get_support()\nfeatures = test_cat.columns[support]\nprint(features)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:21:55.252217Z","iopub.execute_input":"2022-07-13T04:21:55.252568Z","iopub.status.idle":"2022-07-13T04:22:03.728650Z","shell.execute_reply.started":"2022-07-13T04:21:55.252536Z","shell.execute_reply":"2022-07-13T04:22:03.727695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sel_features","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:22:14.946092Z","iopub.execute_input":"2022-07-13T04:22:14.946445Z","iopub.status.idle":"2022-07-13T04:22:14.953234Z","shell.execute_reply.started":"2022-07-13T04:22:14.946413Z","shell.execute_reply":"2022-07-13T04:22:14.952200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_features = ['LandSlope', 'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond',\n       'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'HeatingQC',\n       'KitchenQual', 'FireplaceQu', 'GarageQual', 'GarageCond', 'MSZoning_FV',\n       'MSZoning_RH', 'LotShape_Reg', 'LotShape_IR3', 'LandContour_HLS',\n       'LotConfig_Inside', 'LotConfig_FR2', 'LotConfig_Corner',\n       'LotConfig_FR3', 'Neighborhood_Somerst', 'Neighborhood_OldTown',\n       'Neighborhood_BrkSide', 'Neighborhood_SawyerW', 'Neighborhood_MeadowV',\n       'Neighborhood_ClearCr', 'Neighborhood_NPkVill', 'Condition1_Feedr',\n       'Condition1_PosN', 'BldgType_1Fam', 'BldgType_Twnhs',\n       'HouseStyle_2Story', 'HouseStyle_1.5Fin', 'HouseStyle_SLvl',\n       'RoofStyle_Hip', 'RoofStyle_Mansard', 'Exterior1st_CemntBd',\n       'Exterior1st_AsbShng', 'Exterior1st_Stucco', 'Exterior1st_Stone',\n       'Exterior1st_ImStucc', 'Exterior2nd_CmentBd', 'Exterior2nd_Stucco',\n       'Exterior2nd_Brk Cmn', 'Exterior2nd_Stone', 'Exterior2nd_Other',\n       'MasVnrType_None', 'MasVnrType_Stone', 'MasVnrType_BrkCmn',\n       'Foundation_PConc', 'Foundation_CBlock', 'Foundation_BrkTil',\n       'Electrical_SBrkr', 'Electrical_None', 'Functional_Sev',\n       'GarageType_Detchd', 'GarageType_CarPort', 'GarageType_Basment',\n       'GarageFinish_RFn', 'GarageFinish_Fin', 'GarageFinish_No_Garage',\n       'Fence_No_Fence', 'Fence_GdWo', 'Fence_MnWw', 'SaleType_Oth',\n       'SaleCondition_Normal', 'SaleCondition_Alloca',\n       'LotFrontage', 'OverallQual', 'YearBuilt', 'YearRemodAdd',\n       'MasVnrArea', 'BsmtFinSF1', 'TotalBsmtSF', '1stFlrSF',\n       '2ndFlrSF', 'GrLivArea', 'TotRmsAbvGrd', 'Fireplaces','GarageCars',\n       'WoodDeckSF']\n\nX_train = pd.concat([train_X[sel_features[:-1]], train_cat], axis=1)\nX_train = X_train[selected_features]\ny_train = train_data['SalePrice']\n\nX_predict = pd.concat([test_data_n[sel_features[:-1]], test_cat], axis=1)\nX_predict = X_predict[selected_features]","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:23:21.277275Z","iopub.execute_input":"2022-07-13T04:23:21.277916Z","iopub.status.idle":"2022-07-13T04:23:21.297830Z","shell.execute_reply.started":"2022-07-13T04:23:21.277861Z","shell.execute_reply":"2022-07-13T04:23:21.296865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Development\n\n### `Linear Regression`","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import cross_val_score as cvs\nfrom sklearn.metrics import mean_squared_error as mse\nfrom sklearn.metrics import r2_score as r2\nimport numpy as np\n\nfolds = KFold(n_splits=10, random_state=42, shuffle=True)\nscore_rmse, score_r2 = [], []\n\nfor train_index, test_index in folds.split(X_train):\n    train_x = X_train.iloc[train_index]\n    train_y = y_train.iloc[train_index]\n    test_x = X_train.iloc[test_index]\n    test_y = y_train.iloc[test_index]\n    \n    lr = LinearRegression()\n    lr.fit(train_x, train_y)\n    y_preds = lr.predict(test_x)\n    score_rmse.append(mse((test_y),(y_preds), squared=False))\n    score_r2.append(r2(test_y, y_preds))\n    \nprint('Rmse - {}'.format(np.mean(score_rmse)))\nprint('R2 Score- {}'.format(np.mean(score_r2)))\n   \n","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:23:30.167643Z","iopub.execute_input":"2022-07-13T04:23:30.168377Z","iopub.status.idle":"2022-07-13T04:23:30.364067Z","shell.execute_reply.started":"2022-07-13T04:23:30.168335Z","shell.execute_reply":"2022-07-13T04:23:30.363046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = lr.predict(X_predict)\n\nsub = pd.DataFrame({'Id':test_data['Id'], \n                   'SalePrice': results})\n\nsub.head()\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T06:23:48.718565Z","iopub.execute_input":"2022-07-12T06:23:48.719149Z","iopub.status.idle":"2022-07-12T06:23:48.747728Z","shell.execute_reply.started":"2022-07-12T06:23:48.719106Z","shell.execute_reply":"2022-07-12T06:23:48.746139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### `Random Forest Regressor`","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\n\nfolds = KFold(n_splits=10, random_state=42, shuffle=True)\nscore_rmse, score_r2 = [], []\n\nfor train_index, test_index in folds.split(X_train):\n    train_x = X_train.iloc[train_index]\n    train_y = y_train.iloc[train_index]\n    test_x = X_train.iloc[test_index]\n    test_y = y_train.iloc[test_index]\n    \n    rfr = RandomForestRegressor()\n    rfr.fit(train_x, train_y)\n    y_preds = rfr.predict(test_x)\n    score_rmse.append(mse((test_y),(y_preds) , squared=False))\n    score_r2.append(r2(test_y, y_preds))\n    \nprint('Rmse - {}'.format(np.mean(score_rmse)))\nprint('R2 Score- {}'.format(np.mean(score_r2)))","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:23:40.556351Z","iopub.execute_input":"2022-07-13T04:23:40.557255Z","iopub.status.idle":"2022-07-13T04:23:54.024173Z","shell.execute_reply.started":"2022-07-13T04:23:40.557216Z","shell.execute_reply":"2022-07-13T04:23:54.022952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = rfr.predict(X_predict)\nsub['SalePrice'] = results\nsub.head()\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T07:54:33.664134Z","iopub.status.idle":"2022-07-11T07:54:33.665221Z","shell.execute_reply.started":"2022-07-11T07:54:33.664956Z","shell.execute_reply":"2022-07-11T07:54:33.664979Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### `XGBoost Regressor`","metadata":{}},{"cell_type":"code","source":"from xgboost import XGBRegressor\n\n'''xgb = XGBRegressor(colsample_bytree = 0.5, n_estimators = 6000,\n            max_depth = 4, learning_rate = 0.01, gamma = 0.45,\n            subsample = 0.5, random_state = 11, reg_alpha = 0.00006,\n            reg_lambda = None, nthread = -1)'''\n\n\nfolds = KFold(n_splits=10, random_state=42, shuffle=True)\nscore_rmse, score_r2 = [], []\n\nfor train_index, test_index in folds.split(X_train):\n    train_x = X_train.iloc[train_index]\n    train_y = y_train.iloc[train_index]\n    test_x = X_train.iloc[test_index]\n    test_y = y_train.iloc[test_index]\n    \n    xgb = XGBRegressor(colsample_bytree = 0.5, n_estimators = 6000,\n            max_depth = 4, learning_rate = 0.01, gamma = 0.45,\n            subsample = 0.5, random_state = 11, reg_alpha = 0.00006,\n            reg_lambda = None, nthread = -1)\n    xgb.fit(train_x, train_y)\n    y_preds = xgb.predict(test_x)\n    score_rmse.append(mse((test_y),(y_preds) , squared=False))\n    score_r2.append(r2(test_y, y_preds))\n    \nprint('Rmse - {}'.format(np.mean(score_rmse)))\nprint('R2 Score- {}'.format(np.mean(score_r2)))","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:24:01.191122Z","iopub.execute_input":"2022-07-13T04:24:01.192053Z","iopub.status.idle":"2022-07-13T04:26:24.393613Z","shell.execute_reply.started":"2022-07-13T04:24:01.192015Z","shell.execute_reply":"2022-07-13T04:26:24.392505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = xgb.predict(X_predict)\n\nsub = pd.DataFrame({'Id':test_data['Id'], \n                   'SalePrice': results})\n\nsub.head()\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T06:29:07.114854Z","iopub.execute_input":"2022-07-12T06:29:07.115738Z","iopub.status.idle":"2022-07-12T06:29:07.216704Z","shell.execute_reply.started":"2022-07-12T06:29:07.115679Z","shell.execute_reply":"2022-07-12T06:29:07.215096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### `Light GBM`","metadata":{}},{"cell_type":"code","source":"# from lightgbm import LGBMRegressor\n# import optuna\n\n# def objective(trial):\n    \n#     params = {\n#         \"device_type\": trial.suggest_categorical(\"device_type\", ['gpu']),\n#         \"n_estimators\": trial.suggest_categorical(\"n_estimators\", [10000]),\n#         \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.01, 0.3),\n#         \"num_leaves\": trial.suggest_int(\"num_leaves\", 20, 3000, step=20),\n#         \"max_depth\": trial.suggest_int(\"max_depth\", 3, 12),\n#     }\n        \n#     lgbr = LGBMRegressor(**params)\n    \n#     folds = KFold(n_splits=10, random_state=42, shuffle=True)\n#     score_rmse, score_r2 = [], []\n\n#     for train_index, test_index in folds.split(X_train):\n#         train_x = X_train.iloc[train_index]\n#         train_y = y_train.iloc[train_index]\n#         test_x = X_train.iloc[test_index]\n#         test_y = y_train.iloc[test_index]\n        \n#         lgbr.fit(train_x, train_y)\n#         y_preds = lgbr.predict(test_x)\n#         score_rmse.append(mse((test_y),(y_preds) , squared=False))\n#         score_r2.append(r2(test_y, y_preds))\n    \n#     return np.mean(score_r2) \n    \n    \n# study = optuna.create_study(direction='maximize')\n# study.optimize(objective, n_trials=25)\n# print('Number of finished trials:', len(study.trials))\n# print('Best trial:', study.best_trial.params)\n# print('Accuracy:{}'.format(study.best_trial.value))\n\n'''Best Params for LGBMRegressor'''\n'''Best trial: {'device_type': 'gpu', 'n_estimators': 10000, 'learning_rate': 0.1955558655688105, 'num_leaves': 2620, 'max_depth': 5}'''","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:51:59.087838Z","iopub.execute_input":"2022-07-11T08:51:59.088185Z","iopub.status.idle":"2022-07-11T09:51:01.469272Z","shell.execute_reply.started":"2022-07-11T08:51:59.088155Z","shell.execute_reply":"2022-07-11T09:51:01.468319Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightgbm import LGBMRegressor\n\n'''{'device_type': 'gpu', 'n_estimators': 10000, 'learning_rate': 0.10774643320342608, 'num_leaves': 2280, 'max_depth': 4}'''\n\n\nfolds = KFold(n_splits=10, random_state=42, shuffle=True)\nscore_rmse, score_r2 = [], []\n\nfor train_index, test_index in folds.split(X_train):\n    train_x = X_train.iloc[train_index]\n    train_y = y_train.iloc[train_index]\n    test_x = X_train.iloc[test_index]\n    test_y = y_train.iloc[test_index]\n    \n    lgbr = LGBMRegressor(**{'device_type': 'gpu', 'n_estimators': 10000, \n                            'learning_rate': 0.1955558655688105, 'num_leaves': 2620, \n                            'max_depth': 5})\n    lgbr.fit(train_x, train_y)\n    y_preds = lgbr.predict(test_x)\n    score_rmse.append(mse((test_y),(y_preds) , squared=False))\n    score_r2.append(r2(test_y, y_preds))\n    \nprint('Rmse - {}'.format(np.mean(score_rmse)))\nprint('R2 Score- {}'.format(np.mean(score_r2)))","metadata":{"execution":{"iopub.status.busy":"2022-07-13T04:27:00.629474Z","iopub.execute_input":"2022-07-13T04:27:00.630529Z","iopub.status.idle":"2022-07-13T04:29:01.628615Z","shell.execute_reply.started":"2022-07-13T04:27:00.630482Z","shell.execute_reply":"2022-07-13T04:29:01.627578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\nfolds = KFold(n_splits=10, random_state=42, shuffle=True)\nscore_rmse, score_r2, i = [], [], 0\n\nmodel = tf.keras.Sequential([\n    tf.keras.layers.Dense(units=128, activation='relu'),\n    tf.keras.layers.Dense(units=64, activation='relu'),\n    tf.keras.layers.Dense(units=32, activation='relu'),\n    tf.keras.layers.Dense(units=1, activation='linear'),\n])\n\nmodel.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(.003),\n             metrics=['mse'])\n\n\nfor train_index, test_index in folds.split(X_train):\n    i+=1\n    local_rmse = []\n    train_x = X_train.iloc[train_index]\n    train_y = y_train.iloc[train_index]\n    test_x = X_train.iloc[test_index]\n    test_y = y_train.iloc[test_index]\n    \n    model.fit(train_x, train_y, epochs=200, verbose=0)\n    y_preds = model.predict(test_x)\n    local_rmse.append(mse((test_y),(y_preds) , squared=False))\n    score_rmse.append(mse((test_y),(y_preds) , squared=False))\n    score_r2.append(r2(test_y, y_preds))\n    \n    print('Rmse {} iteration - {}'.format(i, np.mean(local_rmse)))\n    \nprint('Rmse - {}'.format(np.mean(score_rmse)))\nprint('R2 Score- {}'.format(np.mean(score_r2)))","metadata":{"execution":{"iopub.status.busy":"2022-07-13T05:11:37.634214Z","iopub.execute_input":"2022-07-13T05:11:37.634793Z","iopub.status.idle":"2022-07-13T05:14:35.795092Z","shell.execute_reply.started":"2022-07-13T05:11:37.634748Z","shell.execute_reply":"2022-07-13T05:14:35.794098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = model.predict(X_predict).ravel()\n\nsub = pd.DataFrame({'Id':test_data['Id'], \n                   'SalePrice': results})\n\nsub.head()\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T05:15:06.920057Z","iopub.execute_input":"2022-07-13T05:15:06.921033Z","iopub.status.idle":"2022-07-13T05:15:07.039952Z","shell.execute_reply.started":"2022-07-13T05:15:06.920980Z","shell.execute_reply":"2022-07-13T05:15:07.039052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}