{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"76b3b76b8b211e3709d20664204026a9a50f3522"},"cell_type":"markdown","source":"## Train data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_path = '../input/train.csv'\ntrain_data = pd.read_csv(train_path)\ntrain_data.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"978f0fe342e674e11cc315e516b203e31e923afb"},"cell_type":"markdown","source":"## Test Data"},{"metadata":{"trusted":true,"_uuid":"f7be9757008aeadb24349bd854ccdc1082c0e2f0"},"cell_type":"code","source":"test_path = '../input/test.csv'\ntest_data = pd.read_csv(test_path)\ntest_data.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7b763e43c704e0fc2d0b169521724e64fb1558a0"},"cell_type":"markdown","source":"## Count of missing values according to columns"},{"metadata":{"trusted":true,"_uuid":"7562f03bd965eff6ad9905e1f14ecc03bcbce1ed","scrolled":false},"cell_type":"code","source":"miss_col = (train_data.isnull().sum())\nprint(miss_col[miss_col>0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"44f713a627cad112b4e666a6df0cd545e6a2c002"},"cell_type":"markdown","source":"## 1. Dropping missing values"},{"metadata":{"trusted":true,"_uuid":"fee2ccef7ea0e7ff50160891c723ee621bbd6c52"},"cell_type":"code","source":"cols_with_missing = [col for col in train_data.columns if train_data[col].isnull().any()]\ndcols_train = train_data.drop(cols_with_missing,axis=1)\ndcols_test = test_data.drop(cols_with_missing,axis=1)\n\ndcols_train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"97111d7beb072adebf399f3e49af0c9b144f8b3e"},"cell_type":"markdown","source":"## Imputing missing values"},{"metadata":{"_kg_hide-output":false,"_kg_hide-input":false,"trusted":true,"_uuid":"b3f3e8f7040c257609c317ec5621adcf9f1feaf2"},"cell_type":"code","source":"from sklearn.impute import SimpleImputer\n\n# imputes numeric data and leaves rest as it is\ndef impute_numeric(data):\n    my_imputer = SimpleImputer()\n\n    int_data = data.select_dtypes('int64')\n    float_data =data.select_dtypes('float64')\n    numeric_data=pd.concat([int_data,float_data],axis=1)\n\n    num_imputed = my_imputer.fit_transform(numeric_data)\n    num_imputed = pd.DataFrame(np.column_stack(list(zip(*num_imputed))),columns=numeric_data.columns)\n\n    data_imputed = data.copy()\n    data_imputed.update(num_imputed)\n    return data_imputed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e358705e375e01440071f85655a963de218da93"},"cell_type":"code","source":"# impute numeric in train data\ndata_imputed = impute_numeric(train_data)\n\n# find missing values left from non numeric\nimp_miss = [col for col in data_imputed.columns if data_imputed[col].isnull().any()]\n# drop from train, test imputed frames\ndata_imputed = data_imputed.drop(imp_miss,axis=1)\ntest_imputed = impute_numeric(test_data.drop(imp_miss,axis=1))\ndata_imputed.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"02bb18043ac416272bed6c582cdd1c65e596409d"},"cell_type":"markdown","source":"## Extension to imputation"},{"metadata":{"trusted":true,"_uuid":"1756f4579657ad89a21bc669afad2121fa300ab9","scrolled":false},"cell_type":"code","source":"data_ext = train_data.copy()\ntest_ext = test_data.copy()\n\n# add col_missing to train and test data\ncols_miss = [col for col in data_ext.columns if data_ext[col].isnull().any()]\nfor col in cols_miss:\n    data_ext[col+\"_missing\"]=data_ext[col].isnull()\n    test_ext[col+\"_missing\"]=test_ext[col].isnull()\n\ndata_ext = impute_numeric(data_ext)\n\n# drop non numeric missing data\ncols_miss = [col for col in data_ext.columns if data_ext[col].isnull().any()]\ndata_ext = data_ext.drop(cols_miss,axis=1)\ntest_ext = impute_numeric(test_data.drop(cols_miss,axis=1))\ndata_ext.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d9caf966e2eddc2f383606a246bfbdecc2e66039"},"cell_type":"markdown","source":"# Compare methods"},{"metadata":{"trusted":true,"_uuid":"e467bdb4766b7cb5e585d87ddb189761dcfc24a9"},"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_absolute_error\n\ndef get_error(data,features):\n    model = RandomForestRegressor(random_state=1)\n\n    X=data[features]\n    y=data['SalePrice']\n    train_X, val_X, train_y, val_y = train_test_split(X,y,random_state=1) \n\n    model.fit(train_X,train_y)\n    val_predictions = model.predict(val_X)\n    return mean_absolute_error(val_predictions, val_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c98f55f5138fcd83b35ec015dfd1e94ab1ef174a"},"cell_type":"code","source":"def get_numeric_cols(data):\n    int_cols = data.select_dtypes(\"float64\")\n    float_cols = data.select_dtypes(\"int64\")\n    num_cols = pd.concat([int_cols,float_cols],axis=1)\n    return num_cols.columns.values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cd618f55daeefb2dc1e9e117c68cfb34e8f9946b"},"cell_type":"markdown","source":"## 1. For dropping columns"},{"metadata":{"trusted":true,"_uuid":"7324444784fcb7a8df0f83f70c97bd2cd6674031"},"cell_type":"code","source":"features = get_numeric_cols(dcols_train.drop([\"SalePrice\"],axis=1))\nget_error(dcols_train,features)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7caa1edb6518dd173b48552df8112e563f53776c"},"cell_type":"markdown","source":"## 2. For imputing"},{"metadata":{"trusted":true,"_uuid":"2a5fc313c52b9f341c3bfcb2820b9f6a43f6509a"},"cell_type":"code","source":"features=get_numeric_cols(data_imputed.drop(['SalePrice'],axis=1))\n\nget_error(data_imputed,features)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4a6eed8a2c9f52758cc33b5e50840f58bcc559e5"},"cell_type":"markdown","source":"## 3. For imputation with extension"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"eed057e0d58189463eef9431b12d9189ed1ec11e"},"cell_type":"code","source":"features=get_numeric_cols(data_ext.drop(['SalePrice'],axis=1))\nbool_features=data_ext.select_dtypes(\"bool\").columns.values\nfeatures=np.concatenate((features,bool_features))\nget_error(data_ext,features)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"91035854c6390d91947e8052ff5afda65c306792"},"cell_type":"markdown","source":"## Conclusion"},{"metadata":{"_uuid":"dc87105d0bcfb2a76fd608ebc00e6c989970e381"},"cell_type":"markdown","source":"Imputation had better results than dropping the columns but imputation with extension performed worse,\nAs stated in the tutorial imputation with extension works sometimes. In this case it has more error than dropping the columns with missing values."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}