{ "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0, "cells": [ { "cell_type": "markdown", "metadata": { "_cell_guid": "dc8b0bdb-d065-f634-1a0c-aef468799974", "_active": false }, "source": null, "outputs": [], "execution_count": null, "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "68664a09-ee88-7140-925c-af8f778bee28", "_active": false }, "outputs": [], "source": "# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output.", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ab4a1813-80b6-c5af-8eaf-4e788b80fff8", "_active": false }, "outputs": [], "source": "# Adding needed libraries and reading data\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn import ensemble, tree, linear_model\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.metrics import r2_score, mean_squared_error\nfrom sklearn.utils import shuffle\n\n%matplotlib inline\nimport warnings\nwarnings.filterwarnings('ignore')\n\ntrain = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d44c314f-8392-7c52-5925-1fca00251fe4", "_active": false }, "outputs": [], "source": "train.head()\ntrain = train.drop(train[train['Id'] == 1299].index)\ntrain = train.drop(train[train['Id'] == 524].index)\ntrain_labels=train['SalePrice']", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f6d80db5-2212-9890-3d06-81b7f7f9c24e", "_active": false }, "outputs": [], "source": "#Checking for missing data\nNAs = pd.concat([train.isnull().sum(), test.isnull().sum()], axis=1, keys=['Train', 'Test'])\nNaNs = NAs[NAs.sum(axis=1)>0]", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "673675b8-d7ff-916b-9988-8e172f6d3a28", "_active": false }, "outputs": [], "source": "#train_labels = train.pop('SalePrice')\n\nfeatures = pd.concat([train, test],keys=['train','test'])\nrate=pd.concat([NaNs['Train']*1.0/len(train),NaNs['Test']*1.0/len(test)],axis=1,keys=['train_rate','test_rate'])", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ee204375-260f-3195-9f74-ca98238148b8", "_active": false }, "outputs": [], "source": "rate.sort_values(by=['train_rate'],ascending=False)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "317847af-d49c-0fba-834e-e0acd64e72dd", "_active": false }, "outputs": [], "source": "#删掉超过一半以上缺失值的特征\nfeatures=features.drop(['PoolQC','MiscFeature','Alley','Fence','FireplaceQu'],1)\n#plt.scatter(features[:1460]['GrLivArea'],train_labels)\n#features[:1460].sort_values(by='GrLivArea',ascending=False)[:2]['Id']\n\n\nplt.scatter(features.loc['train']['GrLivArea'],train_labels)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ca33e3cc-fb36-8694-ad5b-e0ae9a04ed77", "_active": false }, "outputs": [], "source": "\ncorrmat = features.corr()\ncorrmat['SalePrice'].sort_values(ascending=False)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3046e77b-c965-801b-ab11-723796c0fc4f", "_active": false }, "outputs": [], "source": "##填充缺失值\n#features['MSSubClass']=features['MSSubClass'].astype(str)\n\n# GarageYrBlt \nfeatures['GarageYrBlt'] = features['GarageYrBlt'].fillna(features['GarageYrBlt'].mode()[0])\n\n# Exterior2nd\nfeatures['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode()[0])\n\n# Exterior1st\nfeatures['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode()[0])\n\n# Utilities\nfeatures['Utilities'] = features['Utilities'].fillna(features['Utilities'].mode()[0])\n\n# Functional\nfeatures['Functional'] = features['Functional'].fillna(features['Functional'].mode()[0])\n\n# MasVnrArea\nfeatures['MasVnrArea'] = features['MasVnrArea'].fillna(features['MasVnrArea'].mode()[0])\n\n# GarageArea\nfeatures['GarageArea'] = features['GarageArea'].fillna(features['GarageArea'].mean())\n\n# MSSubClass as str\nfeatures['MSSubClass'] = features['MSSubClass'].astype(str)\n\n# MSZoning NA in pred. filling with most popular values\nfeatures['MSZoning'] = features['MSZoning'].fillna(features['MSZoning'].mode()[0])\n\n# LotFrontage NA in all. I suppose NA means 0\nfeatures['LotFrontage'] = features['LotFrontage'].fillna(features['LotFrontage'].mean())\n\n# Converting OverallCond to str\nfeatures.OverallCond = features.OverallCond.astype(str)\n\n# MasVnrType NA in all. filling with most popular values\nfeatures['MasVnrType'] = features['MasVnrType'].fillna(features['MasVnrType'].mode()[0])\n\n# BsmtQual, BsmtCond, BsmtExposure, BsmtFinType1, BsmtFinType2\n# NA in all. NA means No basement\nfor col in ('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2','BsmtUnfSF','BsmtHalfBath','BsmtFullBath','BsmtFinSF2','BsmtFinSF1'):\n features[col] = features[col].fillna('NoBSMT')\n\n# TotalBsmtSF NA in pred. I suppose NA means 0\nfeatures['TotalBsmtSF'] = features['TotalBsmtSF'].fillna(0)\n\n# Electrical NA in pred. filling with most popular values\nfeatures['Electrical'] = features['Electrical'].fillna(features['Electrical'].mode()[0])\n\n# KitchenAbvGr to categorical\nfeatures['KitchenAbvGr'] = features['KitchenAbvGr'].astype(str)\n\n# KitchenQual NA in pred. filling with most popular values\nfeatures['KitchenQual'] = features['KitchenQual'].fillna(features['KitchenQual'].mode()[0])\n\n\n# GarageType, GarageFinish, GarageQual NA in all. NA means No Garage\nfor col in ('GarageType','GarageCond', 'GarageFinish', 'GarageQual'):\n features[col] = features[col].fillna('NoGRG')\n\n# GarageCars NA in pred. I suppose NA means 0\nfeatures['GarageCars'] = features['GarageCars'].fillna(0.0)\n\n# SaleType NA in pred. filling with most popular values\nfeatures['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode()[0])\n\n# Year and Month to categorical\nfeatures['YrSold'] = features['YrSold'].astype(str)\nfeatures['MoSold'] = features['MoSold'].astype(str)\n\nfeatures.drop(['Utilities', 'RoofMatl', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'Heating', 'LowQualFinSF',\n 'BsmtFullBath', 'BsmtHalfBath', 'Functional', 'GarageCond', 'WoodDeckSF',\n 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal'],\n axis=1, inplace=True)\n# Adding total sqfootage feature and removing Basement, 1st and 2nd floor features\nfeatures['TotalSF'] = features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']\nfeatures.drop(['TotalBsmtSF', '1stFlrSF', '2ndFlrSF'], axis=1, inplace=True)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6ac0c8f8-6137-d25d-1e05-6b2e8346637c", "_active": false }, "outputs": [], "source": "corrmat = features.corr()\ncorrmat['SalePrice'].sort_values(ascending=False)", "execution_state": "idle" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "95459f30-b698-9fcf-0062-c10e2ccc97be", "_active": false }, "source": "log transformation", "outputs": [], "execution_count": null, "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "988c3e55-7286-1eb7-b012-79c8b341bbf1", "_active": false }, "outputs": [], "source": "#train_labels=train['SalePrice']\nfeatures=features.drop('SalePrice',axis=1)\nax=sns.distplot(train_labels)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "dc676ec4-c8cc-632b-6f4c-cadd9f5b4217", "_active": false }, "outputs": [], "source": "train_labels = np.log(train_labels)\nax=sns.distplot(train_labels)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c9ed9af9-ac4d-3e07-a4a9-fcabdd702766", "_active": false }, "outputs": [], "source": "features_numeric=features._get_numeric_data()\nfeatures_numeric.columns\nfeatures_numeric_standardized=(features_numeric-features_numeric.mean())/features_numeric.std()\nfeatures[features.index.duplicated(keep=False)]", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0cba76dd-ce9c-34d0-f753-094ac21193a5", "_active": false }, "outputs": [], "source": "features.columns", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "73daba69-2ec9-bce5-6465-3d7062abaf7c", "_active": false, "collapsed": false }, "outputs": [], "source": "# Getting Dummies from Condition1 and Condition2\nconditions = set([x for x in features['Condition1']] + [x for x in features['Condition2']])\ndummies = pd.DataFrame(data=np.zeros((len(features.index), len(conditions))),\n index=features.index, columns=conditions)\nfor i, cond in enumerate(zip(features['Condition1'], features['Condition2'])):\n dummies.ix[i, cond] = 1\nfeatures = pd.concat([features, dummies.add_prefix('Condition_')], axis=1)\nfeatures.drop(['Condition1', 'Condition2'], axis=1, inplace=True)\n\n# Getting Dummies from Exterior1st and Exterior2nd\nexteriors = set([x for x in features['Exterior1st']] + [x for x in features['Exterior2nd']])\ndummies = pd.DataFrame(data=np.zeros((len(features.index), len(exteriors))),\n index=features.index, columns=exteriors)\nfor i, ext in enumerate(zip(features['Exterior1st'], features['Exterior2nd'])):\n dummies.ix[i, ext] = 1\nfeatures = pd.concat([features, dummies.add_prefix('Exterior_')], axis=1)\nfeatures.drop(['Exterior1st', 'Exterior2nd'], axis=1, inplace=True)\n# Getting Dummies from all other categorical vars\nfor col in features.dtypes[features.dtypes == 'object'].index:\n for_dummy = features.pop(col)\n features = pd.concat([features, pd.get_dummies(for_dummy, prefix=col)], axis=1)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "844b968f-0501-4f62-4bd3-08626b772e0c", "_active": false }, "outputs": [], "source": "### Copying features\nfeatures_standardized = features.copy()\n\n### Replacing numeric features by standardized values\nfeatures_standardized.update(features_numeric_standardized)\nfeatures_standardized._get_numeric_data()", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ff8da28d-a871-9312-c52a-440145b187e3", "_active": false }, "outputs": [], "source": "### Splitting features\ntrain_features = features.loc['train'].drop('Id', axis=1).select_dtypes(include=[np.number]).values\ntest_features = features.loc['test'].drop('Id', axis=1).select_dtypes(include=[np.number]).values\n\n### Splitting standardized features\ntrain_features_st = features_standardized.loc['train'].drop('Id', axis=1).select_dtypes(include=[np.number]).values\ntest_features_st = features_standardized.loc['test'].drop('Id', axis=1).select_dtypes(include=[np.number]).values", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0a4194b3-de4f-e923-880a-636dafa6fbfd", "_active": false }, "outputs": [], "source": "len(train_labels)\n#len(train_features)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "29d78206-26aa-18d6-3de2-ea3bc89565e7", "_active": false }, "outputs": [], "source": "### Shuffling train sets\ntrain_features_st, train_features, train_labels = shuffle(train_features_st, train_features, train_labels, random_state = 5)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3d923f05-2503-e75d-84e3-e294d39a03d0", "_active": false }, "outputs": [], "source": "### Splitting\nx_train, x_test, y_train, y_test = train_test_split(train_features, train_labels, test_size=0.1, random_state=200)\nx_train_st, x_test_st, y_train_st, y_test_st = train_test_split(train_features_st, train_labels, test_size=0.1, random_state=200)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9a0267e6-1319-ca4f-b843-3e0c0532d8f6", "_active": false }, "outputs": [], "source": "### Splitting\nx_train, x_test, y_train, y_test = train_test_split(train_features, train_labels, test_size=0.1, random_state=200)\nx_train_st, x_test_st, y_train_st, y_test_st = train_test_split(train_features_st, train_labels, test_size=0.1, random_state=200)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0554a6cd-61e2-1b9c-9abc-d361c5cc324d", "_active": false }, "outputs": [], "source": "# Prints R2 and RMSE scores\ndef get_score(prediction, lables): \n print('R2: {}'.format(r2_score(prediction, lables)))\n print('RMSE: {}'.format(np.sqrt(mean_squared_error(prediction, lables))))\n\n# Shows scores for train and validation sets \ndef train_test(estimator, x_trn, x_tst, y_trn, y_tst):\n prediction_train = estimator.predict(x_trn)\n # Printing estimator\n print(estimator)\n # Printing train scores\n get_score(prediction_train, y_trn)\n prediction_test = estimator.predict(x_tst)\n # Printing test scores\n print(\"Test\")\n get_score(prediction_test, y_tst)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f05afcc2-d5e3-2317-3094-bb9387262c34", "_active": false }, "outputs": [], "source": "ENSTest = linear_model.ElasticNetCV(alphas=[0.0001, 0.0005, 0.001, 0.01, 0.1, 1, 10], l1_ratio=[.01, .1, .5, .9, .99], max_iter=5000).fit(x_train_st, y_train_st)\ntrain_test(ENSTest, x_train_st, x_test_st, y_train_st, y_test_st)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "66147f32-84b6-a394-72aa-b967fe79fcee", "_active": false }, "outputs": [], "source": "# Average R2 score and standart deviation of 5-fold cross-validation\nscores = cross_val_score(ENSTest, train_features_st, train_labels, cv=5)\nprint(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3310f004-c433-9f0b-f8ae-1acc20e77cd4", "_active": false }, "outputs": [], "source": "GBest = ensemble.GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=3, max_features='sqrt',\n min_samples_leaf=15, min_samples_split=10, loss='huber').fit(x_train, y_train)\ntrain_test(GBest, x_train, x_test, y_train, y_test)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "708ddf16-0de0-50a8-3b2b-1fa7684d0c75", "_active": false }, "outputs": [], "source": "# Average R2 score and standart deviation of 5-fold cross-validation\nscores = cross_val_score(GBest, train_features_st, train_labels, cv=5)\nprint(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "26fc11d6-43a1-2ce8-cec9-e2437584f031", "_active": false }, "outputs": [], "source": "# Retraining models\nGB_model = GBest.fit(train_features, train_labels)\nENST_model = ENSTest.fit(train_features_st, train_labels)", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1f686134-87e2-bc4d-1ac1-b216384c01f0", "_active": false }, "outputs": [], "source": "## Getting our SalePrice estimation\nFinal_labels = (np.exp(GB_model.predict(test_features)) + np.exp(ENST_model.predict(test_features_st))) / 2", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bbcfa637-87ac-7163-96f8-4e51e90be7fd", "_active": false }, "outputs": [], "source": "## Saving to CSV\npd.DataFrame({'Id': test.Id, 'SalePrice': Final_labels}).to_csv('2017-03-15.csv', index =False) ", "execution_state": "idle" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5f6e1991-e0c9-e1ee-3bac-3ccdcc7389c0", "_active": true, "collapsed": false }, "outputs": [], "source": "from sklearn.model_selection import KFold\nfrom sklearn import linear_model\nfrom sklearn.metrics import make_scorer\nfrom sklearn.ensemble import BaggingRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn import svm\nfrom sklearn.metrics import r2_score\nfrom sklearn.ensemble import AdaBoostRegressor\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.model_selection import GridSearchCV\ndef lets_try(train,labels):\n results={}\n def test_model(clf):\n \n cv = KFold(n_splits=5,shuffle=True,random_state=45)\n r2 = make_scorer(r2_score)\n r2_val_score = cross_val_score(clf, train, labels, cv=cv,scoring=r2)\n scores=[r2_val_score.mean()]\n return scores\n\n clf = linear_model.LinearRegression()\n results[\"Linear\"]=test_model(clf)\n \n clf = linear_model.Ridge()\n results[\"Ridge\"]=test_model(clf)\n \n clf = linear_model.BayesianRidge()\n results[\"Bayesian Ridge\"]=test_model(clf)\n \n clf = linear_model.HuberRegressor()\n results[\"Hubber\"]=test_model(clf)\n \n clf = linear_model.Lasso(alpha=1e-4)\n results[\"Lasso\"]=test_model(clf)\n \n clf = BaggingRegressor()\n results[\"Bagging\"]=test_model(clf)\n \n clf = RandomForestRegressor()\n results[\"RandomForest\"]=test_model(clf)\n \n clf = AdaBoostRegressor()\n results[\"AdaBoost\"]=test_model(clf)\n \n clf = svm.SVR()\n results[\"SVM RBF\"]=test_model(clf)\n \n clf = svm.SVR(kernel=\"linear\")\n results[\"SVM Linear\"]=test_model(clf)\n \n results = pd.DataFrame.from_dict(results,orient='index')\n results.columns=[\"R Square Score\"] \n results=results.sort(columns=[\"R Square Score\"],ascending=False)\n results.plot(kind=\"bar\",title=\"Model Scores\")\n axes = plt.gca()\n axes.set_ylim([0.5,1])\n return results\n\nlets_try(train_features, train_labels)", "execution_state": "idle" }, { "metadata": { "_cell_guid": "09cf4551-b237-ca5c-b653-dac631bf0f27", "_active": false, "collapsed": false }, "source": null, "execution_count": null, "cell_type": "code", "outputs": [], "execution_state": "idle" } ] }