{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nimport numpy as np\nfrom scipy.stats import norm\nfrom sklearn.preprocessing import StandardScaler\nfrom scipy import stats\nfrom scipy.stats import norm, skew","metadata":{"papermill":{"duration":1.262286,"end_time":"2022-07-11T07:31:48.938977","exception":false,"start_time":"2022-07-11T07:31:47.676691","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-18T16:39:03.078118Z","iopub.execute_input":"2022-07-18T16:39:03.078644Z","iopub.status.idle":"2022-07-18T16:39:04.445754Z","shell.execute_reply.started":"2022-07-18T16:39:03.078488Z","shell.execute_reply":"2022-07-18T16:39:04.444620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train and test dataframe\ndf_train = pd.read_csv('../input/home-data-for-ml-course/train.csv')\ndf_test = pd.read_csv('../input/home-data-for-ml-course/test.csv')\n\n\n#Numerical columns\nnum_col = [col for col in df_train.select_dtypes([\"int\", \"float\"])]\nnum_col.remove(\"SalePrice\")\nnum_col.remove(\"Id\")\n\n\n#Categorical columns\ncat_col = [col for col in df_train.columns if df_train[col].dtype == \"object\"]\n\n#Tereshold for onehot encoding\ntr_one = 6\n#Categorical for onehot encoding\ncat_col_one = [col for col in cat_col if df_train[col].nunique() <= tr_one ]\n#All remaining colomns\ncat_col_lef =  [col for col in cat_col if df_train[col].nunique() > tr_one ] ","metadata":{"papermill":{"duration":0.08037,"end_time":"2022-07-11T07:31:49.023825","exception":false,"start_time":"2022-07-11T07:31:48.943455","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-18T16:39:40.211408Z","iopub.execute_input":"2022-07-18T16:39:40.212290Z","iopub.status.idle":"2022-07-18T16:39:40.328742Z","shell.execute_reply.started":"2022-07-18T16:39:40.212246Z","shell.execute_reply":"2022-07-18T16:39:40.327562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Lets check cardinality in the train and test sets for columns which are going to be encoded as onehot ","metadata":{}},{"cell_type":"code","source":"for i in range(len(cat_col_one)):\n    if df_train[cat_col_one[i]].nunique() != df_test[cat_col_one[i]].nunique() :\n        print(cat_col_one[i])\n        print(\"features in train: \",df_train[cat_col_one[i]].unique())\n        print(\"features in test: \",df_test[cat_col_one[i]].unique())","metadata":{"execution":{"iopub.status.busy":"2022-07-18T16:39:48.719072Z","iopub.execute_input":"2022-07-18T16:39:48.719431Z","iopub.status.idle":"2022-07-18T16:39:48.748396Z","shell.execute_reply.started":"2022-07-18T16:39:48.719401Z","shell.execute_reply":"2022-07-18T16:39:48.747444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Important point:\n### Number of items in these features are not the same in the train and test set so we should use reindexing in pandas! ","metadata":{}},{"cell_type":"code","source":"missing_values = df_train.isna().sum(axis=0)/df_train.shape[0]\nmissing_values = missing_values.loc[missing_values > 0]\nmissing_values.sort_values(ascending=True)\nmissing_values.plot( kind='bar',\n                     title = 'Missing Columns',\n                     ylabel = '% of missing',\n                     ylim = (0,1.2),\n                     grid = True,\n                     figsize=(16,8))","metadata":{"papermill":{"duration":0.357217,"end_time":"2022-07-11T07:31:49.385304","exception":false,"start_time":"2022-07-11T07:31:49.028087","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-18T16:39:54.945357Z","iopub.execute_input":"2022-07-18T16:39:54.945777Z","iopub.status.idle":"2022-07-18T16:39:55.345358Z","shell.execute_reply.started":"2022-07-18T16:39:54.945740Z","shell.execute_reply":"2022-07-18T16:39:55.343842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# This is very importnat to know what does NA mean in each features.\n\n### Alley: Type of alley access to property\n\n       Grvl\tGravel\n       Pave\tPaved\n       NA \tNo alley access\n       \n### FireplaceQu: Fireplace quality\n\n       Ex\tExcellent - Exceptional Masonry Fireplace\n       Gd\tGood - Masonry Fireplace in main level\n       TA\tAverage - Prefabricated Fireplace in main living area or Masonry Fireplace in basement\n       Fa\tFair - Prefabricated Fireplace in basement\n       Po\tPoor - Ben Franklin Stove\n       NA\tNo Fireplace\n       \n### PoolQC: Pool quality\n\t\t\n       Ex\tExcellent\n       Gd\tGood\n       TA\tAverage/Typical\n       Fa\tFair\n       NA\tNo Pool\n       \n ### Fence: Fence quality\n\t\t\n       GdPrv\tGood Privacy\n       MnPrv\tMinimum Privacy\n       GdWo\tGood Wood\n       MnWw\tMinimum Wood/Wire\n       NA\tNo Fence\n       \n### MiscFeature: Miscellaneous feature not covered in other categories\n\t\t\n       Elev\tElevator\n       Gar2\t2nd Garage (if not described in garage section)\n       Othr\tOther\n       Shed\tShed (over 100 SF)\n       TenC\tTennis Court\n       NA\tNone\n       \n       \n### As you can see in all of these features NA does not mean no value. It makes sense to imput them by \"no_features\" not drop the column! Notice that it is also the case for features strating with Garage and Base\n","metadata":{"papermill":{"duration":0.004663,"end_time":"2022-07-11T07:31:49.394850","exception":false,"start_time":"2022-07-11T07:31:49.390187","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def fill_missing(dataframe):\n#All columns with at least one missing value\n    col_mis = [col for col in dataframe if dataframe[col].isnull().sum()>0]\n    print(\"Columns with at least one missing value:\\n\",col_mis)\n\n    \n    gar_col = [col for col in dataframe if col.startswith(\"Garag\") and dataframe[col].isnull().sum()>0]\n    bas_col = [col for col in dataframe if col.startswith(\"Bsm\") and dataframe[col].isnull().sum()>0]\n\n#Filling categorical features starting with Bsm and Gar with a no_base and no_garg\n#Filling numerical features starting with Bsm and Gar with -1  \n    for item in bas_col:\n\n        if(dataframe[item].dtype == \"object\"):\n            dataframe[item].fillna(\"no_basement\",inplace=True)\n        else:\n            dataframe[item].fillna(-1,inplace=True)\n\n        col_mis.remove(item)     \n    \n    for item in gar_col:\n\n        if(dataframe[item].dtype == \"object\"):\n            dataframe[item].fillna(\"no_garage\",inplace=True) \n        else:\n            dataframe[item].fillna(-1,inplace=True)\n\n        col_mis.remove(item)      \n    \n    \n    #Renaming other colomns which nan means  \n    dataframe[\"Alley\"].fillna(\"no_alley\",inplace=True)\n    col_mis.remove(\"Alley\")\n    dataframe[\"FireplaceQu\"].fillna(\"no_firepla\",inplace=True)\n    col_mis.remove(\"FireplaceQu\")\n    dataframe[\"PoolQC\"].fillna(\"no_pool\",inplace=True)\n    col_mis.remove(\"PoolQC\")\n    dataframe[\"Fence\"].fillna(\"no_fence\",inplace=True)\n    col_mis.remove(\"Fence\")\n    dataframe[\"MiscFeature\"].fillna(\"no_misc\",inplace=True)\n    col_mis.remove(\"MiscFeature\")\n\n    if len(col_mis)>0:\n        #Filling remaining item\n        #For categorical features, fill with mode\n        #For numerical features, fill with median\n        col_rem_cat = [col for col in col_mis if dataframe[col].dtype == \"object\"]\n        col_rem_num = [col for col in col_mis if dataframe[col].dtype != \"object\"]\n\n        for item in col_rem_cat:\n            dataframe[item].fillna(dataframe[item].mode()[0],inplace=True)\n\n        for item in col_rem_num:\n                dataframe[item].fillna(dataframe[item].median(),inplace=True)      \n\n    else:\n        return dataframe\n\n    return dataframe","metadata":{"execution":{"iopub.execute_input":"2022-07-11T07:31:49.406173Z","iopub.status.busy":"2022-07-11T07:31:49.405397Z","iopub.status.idle":"2022-07-11T07:31:49.428700Z","shell.execute_reply":"2022-07-11T07:31:49.427918Z"},"papermill":{"duration":0.031752,"end_time":"2022-07-11T07:31:49.431159","exception":false,"start_time":"2022-07-11T07:31:49.399407","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Check missing values in the train set\nX_no_mis = fill_missing(df_train)\nprint(X_no_mis.shape)\nprint(X_no_mis.isnull().sum().sort_values(ascending=False))","metadata":{"execution":{"iopub.execute_input":"2022-07-11T07:31:49.443232Z","iopub.status.busy":"2022-07-11T07:31:49.442337Z","iopub.status.idle":"2022-07-11T07:31:49.468139Z","shell.execute_reply":"2022-07-11T07:31:49.466569Z"},"papermill":{"duration":0.034133,"end_time":"2022-07-11T07:31:49.470458","exception":false,"start_time":"2022-07-11T07:31:49.436325","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def encode_cat(data):\n\n#Numerical part of the data\n    df_num = data[num_col]\n    #print(df_num.shape)\n\n#Part of data with onehot encoding\n    df_cat_one = data[cat_col_one]\n    df_cat_one = pd.get_dummies(df_cat_one)\n    #print(df_cat_one.shape)\n\n#part of data with numerical encoding\n    data[cat_col_lef] = data[cat_col_lef].astype('category')\n    df_cat_enc = pd.DataFrame()\n    for col in cat_col_lef:\n         df_cat_enc[col] = data[col].cat.codes\n\n    #print(df_cat_enc.shape)\n#Finall  data\n    X = pd.concat([df_num,df_cat_one,df_cat_enc],axis=1)  \n\n    return X   ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Notice that sale_price is droped \nX_full = encode_cat(X_no_mis)\nprint(X_full.shape)\nprint(X_full.info())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_cor = 20 #number of features selected by correlation matrix\n#saleprice correlation matrix\ncorrmat = df_train[num_col+['SalePrice']].corr()\ncols = corrmat.nlargest(n_cor, 'SalePrice')['SalePrice'].index\ncm = np.corrcoef(df_train[cols].values.T)\nsns.set(font_scale=1.25)\nplt.subplots(figsize=(20,12))\nhm = sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 10}, yticklabels=cols.values, xticklabels=cols.values)\nplt.show()\n\n#cols.values","metadata":{"execution":{"iopub.execute_input":"2022-07-11T07:31:49.700699Z","iopub.status.busy":"2022-07-11T07:31:49.700376Z","iopub.status.idle":"2022-07-11T07:31:50.629330Z","shell.execute_reply":"2022-07-11T07:31:50.628468Z"},"papermill":{"duration":0.937256,"end_time":"2022-07-11T07:31:50.631621","exception":false,"start_time":"2022-07-11T07:31:49.694365","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_selection import mutual_info_regression as MI\n\nmi = MI(X_full,y=df_train[\"SalePrice\"])\n\n#Dict for saving mi values\nmi_dict = {}\nkeys = [col for col in X_full.columns]\nvalues = mi\n\nfor i in range(len(keys)):\n    mi_dict[keys[i]] = values[i]\n\n\nn_mi = 20 #number of features selected by MI method \nsel_fea_mi = []\nval_sort = sorted(mi_dict.values(),reverse=True)\n\nfor st in val_sort[:n_mi]:\n    for k,v in mi_dict.items():\n        if v==st:\n            sel_fea_mi.append(k)    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Now time to define a model","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_absolute_error\nfrom xgboost import XGBRegressor\n\nModel = XGBRegressor(n_estimators=1200, max_depth=9, eta=0.1, subsample=0.7, colsample_bytree=0.8)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Select features\ny = df_train[\"SalePrice\"]\n\n# for features selected by correlation matrix\nsel_fea_cor = [col for col in cols.values]\nsel_fea_cor.remove(\"SalePrice\")\n\n# Comments one of the methods\nX = X_full[sel_fea_cor]\n#X = X_full[sel_fea_mi]\n\n# Splitting\nX_train, X_vald, y_train, y_vald = train_test_split(X,y,test_size = 0.2, random_state = 123)\n\nModel.fit(X_train,y_train)\npreds = Model.predict(X_vald)\nscore = mean_absolute_error(y_vald, preds)\nprint(score)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check missing values in the test set\nX_test_no_mis = fill_missing(df_test)\nprint(X_test_no_mis.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = encode_cat(df_test)  \nX_test = X_test.reindex(columns = X.columns, fill_value=0)\nprint(X_test.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = Model.predict(X_test)\noutput = pd.DataFrame({'Id': df_test.Id,\n                       'SalePrice': test_preds})\noutput.to_csv('submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}