{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nimport numpy as np\nfrom scipy.stats import norm\nfrom sklearn.preprocessing import StandardScaler\nfrom scipy import stats\nfrom scipy.stats import norm, skew\n\nmpl.rcParams['figure.figsize'] = 18,8","metadata":{"execution":{"iopub.execute_input":"2022-07-11T07:31:47.684159Z","iopub.status.busy":"2022-07-11T07:31:47.683659Z","iopub.status.idle":"2022-07-11T07:31:48.935968Z","shell.execute_reply":"2022-07-11T07:31:48.934838Z"},"papermill":{"duration":1.262286,"end_time":"2022-07-11T07:31:48.938977","exception":false,"start_time":"2022-07-11T07:31:47.676691","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## In this notebook we are going to check two important points:\n### 1- How outliers affect the final results?\n### 2- Do features with zero mutual information change the final results?    ","metadata":{}},{"cell_type":"code","source":"#train and test dataframe\ndf_train = pd.read_csv('../input/home-data-for-ml-course/train.csv')\ndf_test = pd.read_csv('../input/home-data-for-ml-course/test.csv')\n\n\n#Numerical columns\nnum_col = [col for col in df_train.select_dtypes([\"int\", \"float\"])]\nnum_col.remove(\"SalePrice\")\nnum_col.remove(\"Id\")\n\n\n#Categorical columns\ncat_col = [col for col in df_train.columns if df_train[col].dtype == \"object\"]\n\n#Tereshold for onehot encoding\ntr_one = 4\n#Categorical for onehot encoding\ncat_col_one = [col for col in cat_col if df_train[col].nunique() <= tr_one ]\n#All remaining colomns\ncat_col_lef =  [col for col in cat_col if df_train[col].nunique() > tr_one ] ","metadata":{"execution":{"iopub.execute_input":"2022-07-11T07:31:48.949464Z","iopub.status.busy":"2022-07-11T07:31:48.948714Z","iopub.status.idle":"2022-07-11T07:31:49.021462Z","shell.execute_reply":"2022-07-11T07:31:49.020588Z"},"papermill":{"duration":0.08037,"end_time":"2022-07-11T07:31:49.023825","exception":false,"start_time":"2022-07-11T07:31:48.943455","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fill_missing(dataframe):\n#All columns with at least one missing value\n    col_mis = [col for col in dataframe if dataframe[col].isnull().sum()>0]\n    print(\"Columns with at least one missing value:\\n\",col_mis)\n\n    \n    gar_col = [col for col in dataframe if col.startswith(\"Garag\") and dataframe[col].isnull().sum()>0]\n    bas_col = [col for col in dataframe if col.startswith(\"Bsm\") and dataframe[col].isnull().sum()>0]\n\n#Filling categorical features starting with Bsm and Gar with a no_base and no_garg\n#Filling numerical features starting with Bsm and Gar with -1  \n    for item in bas_col:\n\n        if(dataframe[item].dtype == \"object\"):\n            dataframe[item].fillna(\"no_basement\",inplace=True)\n        else:\n            dataframe[item].fillna(-1,inplace=True)\n\n        col_mis.remove(item)     \n    \n    for item in gar_col:\n\n        if(dataframe[item].dtype == \"object\"):\n            dataframe[item].fillna(\"no_garage\",inplace=True) \n        else:\n            #dataframe[item].fillna(-1,inplace=True)\n            dataframe[item].fillna(dataframe[item].median(),inplace=True)\n\n        col_mis.remove(item)      \n    \n    \n    #Renaming other colomns which nan means  \n    dataframe[\"Alley\"].fillna(\"no_alley\",inplace=True)\n    col_mis.remove(\"Alley\")\n    dataframe[\"FireplaceQu\"].fillna(\"no_firepla\",inplace=True)\n    col_mis.remove(\"FireplaceQu\")\n    dataframe[\"PoolQC\"].fillna(\"no_pool\",inplace=True)\n    col_mis.remove(\"PoolQC\")\n    dataframe[\"Fence\"].fillna(\"no_fence\",inplace=True)\n    col_mis.remove(\"Fence\")\n    dataframe[\"MiscFeature\"].fillna(\"no_misc\",inplace=True)\n    col_mis.remove(\"MiscFeature\")\n\n    if len(col_mis)>0:\n        #Filling remaining item\n        #For categorical features, fill with mode\n        #For numerical features, fill with median\n        col_rem_cat = [col for col in col_mis if dataframe[col].dtype == \"object\"]\n        col_rem_num = [col for col in col_mis if dataframe[col].dtype != \"object\"]\n\n        for item in col_rem_cat:\n            dataframe[item].fillna(dataframe[item].mode()[0],inplace=True)\n\n        for item in col_rem_num:\n                dataframe[item].fillna(dataframe[item].median(),inplace=True)      \n\n    else:\n        return dataframe\n\n    return dataframe","metadata":{"execution":{"iopub.execute_input":"2022-07-11T07:31:49.406173Z","iopub.status.busy":"2022-07-11T07:31:49.405397Z","iopub.status.idle":"2022-07-11T07:31:49.428700Z","shell.execute_reply":"2022-07-11T07:31:49.427918Z"},"papermill":{"duration":0.031752,"end_time":"2022-07-11T07:31:49.431159","exception":false,"start_time":"2022-07-11T07:31:49.399407","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## For more details on above imputing see https://www.kaggle.com/code/ahmadmehraby/add-new-features","metadata":{}},{"cell_type":"code","source":"#Check missing values in the train set\nX_no_mis = fill_missing(df_train)\nprint(X_no_mis.shape)\nprint(X_no_mis.isnull().sum().sort_values(ascending=False))","metadata":{"execution":{"iopub.execute_input":"2022-07-11T07:31:49.443232Z","iopub.status.busy":"2022-07-11T07:31:49.442337Z","iopub.status.idle":"2022-07-11T07:31:49.468139Z","shell.execute_reply":"2022-07-11T07:31:49.466569Z"},"papermill":{"duration":0.034133,"end_time":"2022-07-11T07:31:49.470458","exception":false,"start_time":"2022-07-11T07:31:49.436325","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def encode_cat(data):\n\n#Numerical part of the data\n    df_num = data[num_col]\n    #print(df_num.shape)\n\n#Part of data with onehot encoding\n    df_cat_one = data[cat_col_one]\n    df_cat_one = pd.get_dummies(df_cat_one)\n    #print(df_cat_one.shape)\n\n#part of data with numerical encoding\n    data[cat_col_lef] = data[cat_col_lef].astype('category')\n    df_cat_enc = pd.DataFrame()\n    for col in cat_col_lef:\n         df_cat_enc[col] = data[col].cat.codes\n\n    #print(df_cat_enc.shape)\n#Finall  data\n    X = pd.concat([df_num,df_cat_one,df_cat_enc],axis=1)  \n\n    return X   ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Notice that sale_price is droped \nX_full = encode_cat(X_no_mis)\nprint(X_full.shape)\nprint(X_full.info())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_selection import mutual_info_regression as MI\n\nmi = MI(X_full,y=df_train[\"SalePrice\"])\n\n#Dict for saving mi values\nmi_dict = {}\nkeys = [col for col in X_full.columns]\nvalues = mi\n\nfor i in range(len(keys)):\n    mi_dict[keys[i]] = values[i]\n  ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Finding features with muatual information above a treshold\ndef fea_from_mi(mi_cr): #mi_cr is a treshold value to select all features above it\n      sel_fea = []\n\n      for k,v in mi_dict.items():\n            if v>mi_cr:\n                  sel_fea.append(k)           \n      return sel_fea                    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Now time to define a model","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_absolute_error\nfrom xgboost import XGBRegressor\n\nModel = XGBRegressor(n_estimators=1500, max_depth=10, eta=0.1, subsample=0.7, colsample_bytree=0.8)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Target\ny = df_train[\"SalePrice\"]\n\n# Finding results for full features\nX = X_full\n\n# Splitting\nX_train, X_vald, y_train, y_vald = train_test_split(X,y,test_size = 0.2, random_state = 123)\n\nModel.fit(X_train,y_train)\npreds = Model.predict(X_vald)\nscore_full = mean_absolute_error(y_vald, preds)\n\nres_dict = {} #An empty dict to save results\n\nres_dict[\"Full\"] = score_full","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a list value for cr_mi to check dependency of the results\ncr = [0.00,0.01,0.05,0.10,0.15,0.20,0.25,0.30,0.35,0.40,0.45,0.50]\nkey_res = [f\"cr_mi={cr[i]}\" for i in range(len(cr))]\nval_res = []\n\n\nfor i in range(len(cr)):\n      #Select features\n\n      features = fea_from_mi(cr[i])\n      X = X_full[features]\n\n      # Splitting\n      X_train, X_vald, y_train, y_vald = train_test_split(X,y,test_size = 0.2, random_state = 123)\n\n      Model.fit(X_train,y_train)\n      preds = Model.predict(X_vald)\n      score = mean_absolute_error(y_vald, preds)\n      val_res.append(score)\n      print(\"for cr =\",cr[i],\"score is\",score)\n\nfor i in range(len(key_res)):\n    res_dict[key_res[i]] = val_res[i]      ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"key_res.insert(0,\"Full\")\nplt.bar(range(len(res_dict)), res_dict.values(),tick_label=key_res)\nplt.xlabel(\"Critical value for MI\")\nplt.ylabel(\"Score\")\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Now let's see how outliers might affect the results ","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\n\n#Target\ny = df_train[\"SalePrice\"]\n\nsns.boxplot(y)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Clearly there are some outliers","metadata":{}},{"cell_type":"code","source":"def data_after_removing_out(fr_iqr):\n#fr_iqr, fraction of IQR to define a reliable range \n      y = df_train[\"SalePrice\"]\n      Q1,Q3 = y.quantile([0.25,0.75])\n      IQR = Q3 - Q1\n      df = pd.concat([X_full,y],axis=1)\n      upper = Q3 + fr_iqr*IQR\n      lower = Q1 - fr_iqr*IQR\n\n      l1 = np.where(y<lower)[0]\n      l2 = np.where(y>upper)[0]\n\n      print(\"Number of outliers below range\",len(l1))\n      print(\"Number of outliers above range\",len(l2))\n\n      ind_to_be_remove = [*l1 , *l2]\n\n      df.drop(ind_to_be_remove,inplace=True)\n      yf = df[\"SalePrice\"]\n      Xf = df.drop([\"SalePrice\"],axis=1)\n\n      return Xf,yf\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X,y = data_after_removing_out(1.4)\n\n#New boxplot\nsns.boxplot(y)\nprint(X.shape)\n\n# Splitting\nX_train, X_vald, y_train, y_vald = train_test_split(X,y,test_size = 0.2, random_state = 123)\n\nModel.fit(X_train,y_train)\npreds = Model.predict(X_vald)\nscore = mean_absolute_error(y_vald, preds)\nprint(score)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Let's see score for different value of fr_iqr","metadata":{}},{"cell_type":"code","source":"list_fr = [1.2,1.3,1.4,1.5,1.6,1.8,2]\nfor item in list_fr:\n      X,y = data_after_removing_out(item)\n      X_train, X_vald, y_train, y_vald = train_test_split(X,y,test_size = 0.2, random_state = 123)\n\n      Model.fit(X_train,y_train)\n      preds = Model.predict(X_vald)\n      score = mean_absolute_error(y_vald, preds)\n      print(f\"For fr_iqr = {item} score is \",score)\n      ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Let's see outlier in other numerical columns\nfr_iqr = 1.6\nX_help,y_help = data_after_removing_out(fr_iqr)\nfor item in num_col:\n      Q1,Q3 = X_help[item].quantile([0.25,0.75])\n      IQR = Q3 - Q1\n       \n      upper = Q3 + fr_iqr*IQR\n      lower = Q1 - fr_iqr*IQR\n\n      l1 = np.where(X_help[item]<lower)[0]\n      l2 = np.where(X_help[item]>upper)[0]\n      lt = [*l1 , *l2]\n      if(len(lt)>0):\n            print(f\"Number of outliar in column {item} is\",len(lt))\n      ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### As you can see there are other outliers in numerical columns after removing outliers in the sale_price","metadata":{}},{"cell_type":"code","source":"X,y = data_after_removing_out(1.8)\nX_train, X_vald, y_train, y_vald = train_test_split(X,y,test_size = 0.2, random_state = 12)\n\nModel.fit(X_train,y_train)\npreds = Model.predict(X_vald)\nscore = mean_absolute_error(y_vald, preds)\n\nprint(score)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check missing values in the test set\nX_test_no_mis = fill_missing(df_test)\nprint(X_test_no_mis.shape)\n\nX_test = encode_cat(df_test)  \nX_test = X_test.reindex(columns = X.columns, fill_value=0)\nprint(X_test.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = Model.predict(X_test)\noutput = pd.DataFrame({'Id': df_test.Id,\n                       'SalePrice': test_preds})\noutput.to_csv('submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}