{ "cells": [ { "cell_type": "markdown", "metadata": { "_cell_guid": "61426f5f-7e20-6216-9de9-002f5ea09325" }, "source": [ "Trying out basic commands according to https://www.kaggle.com/apapiu/house-prices-advanced-regression-techniques/regularized-linear-models/comments, on top of which, model selection with LASSO\n", "http://scikit-learn.org/stable/auto_examples/feature_selection/plot_select_from_model_boston.html#sphx-glr-auto-examples-feature-selection-plot-select-from-model-boston-py" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1c7d0032-f34c-cb3d-1610-d988a6339669" }, "outputs": [], "source": [ "# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "09c253ad-aaeb-3961-abab-dd2ed5bcd009" }, "outputs": [], "source": [ "train = pd.read_csv(\"../input/train.csv\")\n", "test = pd.read_csv(\"../input/test.csv\")\n", "train.shape, test.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0d6f54f3-5f3d-34fb-8cd3-744217d8ae5e" }, "outputs": [], "source": [ "#train.describe()\n", "#train.info()" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "eb464830-8271-2c60-4de5-1e2031c57eee" }, "source": [ "**n = 1460 samples**\n", "\n", "\n", "**d = 79 features** 80 columns: 1st ID 80th SalePrice (Y)\n", "\n", "\n", "Some contain lots of nulls such as \n", "PoolQC 7 non-null object\n", "\n", "Non-numerical features about half\n", "\n", "**Goal: predict SalePrice** \n", "Predict a quantity with <100K samples " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1bb4c34b-da9c-7dab-7587-e2a753238fe0" }, "outputs": [], "source": [ "#train[\"SalePrice\"].hist();" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "42827178-14d5-f921-e3fe-b718585fd452" }, "outputs": [], "source": [ "#prices = np.log1p(train[\"SalePrice\"])\n", "#prices.hist()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f0a37ce9-e351-5f44-758d-3fe69dd893c4" }, "outputs": [], "source": [ "train[\"SalePrice\"] = np.log1p(train[\"SalePrice\"])\n", "y_train = train[\"SalePrice\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ec1c039c-badd-d028-eaf8-a59e6455db36" }, "outputs": [], "source": [ "all_data = pd.concat((train.loc[:,'MSSubClass':'SaleCondition'],\n", " test.loc[:,'MSSubClass':'SaleCondition']))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "76699ad6-8e38-cb2f-2195-428714e47cf6" }, "outputs": [], "source": [ "from scipy.stats import skew\n", "from scipy.stats.stats import pearsonr\n", "#log transform skewed numeric features:\n", "numeric_feats = train.dtypes[train.dtypes != \"object\"].index\n", "\n", "skewed_feats = train[numeric_feats].apply(lambda x: skew(x.dropna())) #compute skewness\n", "skewed_feats = skewed_feats[skewed_feats > 0.75]\n", "skewed_feats = skewed_feats.index" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fd61e005-f6a6-20eb-6683-6240aa95853a" }, "outputs": [], "source": [ "'SalePrice' in skewed_feats" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "84df0bab-9a69-69a4-db05-e17e8667cb1f" }, "outputs": [], "source": [ "all_data[skewed_feats] = np.log1p(all_data[skewed_feats])\n", "#Convert categorical variable into dummy/indicator variables\n", "all_data = pd.get_dummies(all_data)\n", "all_data = all_data.fillna(all_data.mean())\n", "all_data.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1722c268-3782-56d0-da7a-43c2b9b98436" }, "outputs": [], "source": [ "X_train = all_data[:train.shape[0]]\n", "X_test = all_data[train.shape[0]:]\n", "X_train.shape, X_test.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "00245d63-6150-f37d-2bc9-d4a852571044" }, "outputs": [], "source": [ "#X_train.info" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "57876bbd-0673-adfa-e0a4-86cabc749356" }, "source": [ "1. Random split into training and test sets.\n", "\n", " [1]: http://scikit-learn.org/stable/modules/cross_validation.html\n", " [2]: http://scikit-learn.org/stable/modules/linear_model.html#ridge-regression" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2b8f2df7-7eca-ebcb-177f-fd557bd833df" }, "outputs": [], "source": [ "#from sklearn.model_selection import train_test_split\n", "#X_train, X_test, y_train, y_test = train_test_split(train_data, train_target, test_size=0.4, random_state=0)\n", "#X_train.shape, X_test.shape, y_train.shape, y_test.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ae6c0d26-5d4e-1b58-1fe3-a1d6028103e8" }, "outputs": [], "source": [ "from sklearn.model_selection import cross_val_score\n", "from sklearn import linear_model\n", "from sklearn.metrics import mean_squared_error\n", "from sklearn.feature_selection import SelectFromModel\n", "from sklearn.linear_model import LassoCV\n", "\n", "#reg = linear_model.Ridge(alpha = .5)\n", "#reg.fit(X_train, y_train) \n", "#y_pred = reg.predict(X_test)\n", "#mean_squared_error(y_test, y_pred) " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2f53c134-850c-d0b1-44f9-65f059b26d7e" }, "outputs": [], "source": [ "#mean_squared_error(y_test, y_pred, multioutput='raw_values')" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "32bd794e-e060-7cf8-8d71-5c7340b1fd3c" }, "source": [ "-2. Cross validation" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "794ed225-5d0e-a4c8-07dc-4361051482a3" }, "source": [ "> RidgeCV implements ridge regression with built-in cross-validation of\n", "> the alpha parameter. The object works in the same way as GridSearchCV\n", "> except that it defaults to Generalized Cross-Validation (GCV), an\n", "> efficient form of leave-one-out cross-validation." ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "ae328cba-89c3-34c0-54c2-afe4eb1b806a" }, "source": [ "RidgeCV(alphas=(0.1, 1.0, 10.0), fit_intercept=True, normalize=False, scoring=None, cv=None, gcv_mode=None, store_cv_values=False)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "43e94938-6b24-9f07-ebd8-54cdd491b24f" }, "source": [ "cv:\n", " None, to use the efficient Leave-One-Out cross-validation\n", " \n", " integer, to specify the number of folds." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ada68074-094e-e616-ac56-1dc26e0b620c" }, "outputs": [], "source": [ "#reg = linear_model.RidgeCV(alphas = [0.01, 0.1, 1.0, 10])\n", "#reg.fit(X_train, y_train)\n", "#reg.alpha_ " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5746fa4a-112e-480a-9801-9441c7dd9b2e" }, "outputs": [], "source": [ "#reg = linear_model.RidgeCV(alphas = [2.0, 5., 10, 20., 50.])\n", "#reg.fit(X_train, y_train)\n", "#reg.alpha_ " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "20eda3cf-f6d0-2389-b4b6-98654407cc91" }, "outputs": [], "source": [ "#reg = linear_model.RidgeCV(alphas = [6., 8., 10., 12., 14., 17.], fit_intercept=True, \\\n", " # normalize=False, scoring=None, cv=8, gcv_mode=None, store_cv_values=False)\n", "#reg.fit(X_train, y_train)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "cf38d5bd-e667-489a-951c-27e0d27fdf9c" }, "outputs": [], "source": [ "#print(reg.alpha_)\n", "#y_pred = reg.predict(X_test)\n", "#y_pred = np.expm1(y_pred)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9b414f6d-b0a8-f679-5170-7331a7478eed" }, "outputs": [], "source": [ "#output = pd.DataFrame(test['Id'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "cc4d452a-1943-b33e-7ae5-d371c4766126" }, "outputs": [], "source": [ "#output['SalePrice'] = y_pred" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7cbb4012-28dc-1c27-c4d7-1a8aeec94001" }, "outputs": [], "source": [ "#output.to_csv('RidgeRegression_firsttry_12232016.csv', index=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "311ddf8a-9a76-8ac8-da23-c586c7aa2c28" }, "outputs": [], "source": [ "# We use the base estimator LassoCV since the L1 norm promotes sparsity of features.\n", "clf = LassoCV()\n", "\n", "# Set a minimum threshold, using default threshold 1e-100 ~ 1e-4 first\n", "sfm = SelectFromModel(clf, threshold=1e-100)\n", "sfm.fit(X_train, y_train)\n", "X_transform = sfm.transform(X_train)\n", "n_features = X_transform.shape[1]\n", "print(n_features)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2d2f739c-5d31-a523-38ff-6ea86f67f244" }, "outputs": [], "source": [ "print(n_features)\n", "print(X_train.shape)\n", "X_index = np.linspace(0, 287, 288)\n", "xv, yv = np.meshgrid(X_index, range(1))\n", "print(xv.shape)\n", "selected_features = sfm.transform(xv)\n", "print(selected_features)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1b05c921-8fdc-f743-b0d1-c0cd7079f7f3" }, "outputs": [], "source": [ "X_selected_tr = X_train.as_matrix()\n", "print(X_selected_tr.shape)\n", "X_selected_tr = X_selected_tr[:, [3, 5, 6, 8, 22, 24, 26, 27, 28]]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "85c1635a-72cf-fb6a-49f2-fb51081c9080" }, "outputs": [], "source": [ "X_transform - X_selected_tr #Same even though features return type is not integer" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "984b1a91-f809-1013-5d84-d932ed6fdc15" }, "outputs": [], "source": [ "\n", "\n", "# Reset the threshold till the number of features equals two.\n", "# Note that the attribute can be set directly instead of repeatedly\n", "# fitting the metatransformer.\n", "#while n_features > 2:\n", "# sfm.threshold += 0.1\n", "# X_transform = sfm.transform(X)\n", "# n_features = X_transform.shape[1]\n", "\n", "# Plot the selected two features from X.\n", "import matplotlib.pyplot as plt\n", "plt.title(\n", " \"Feature selected from LASSO \"\n", " \"threshold %0.3f.\" % sfm.threshold)\n", "feature1 = X_transform[:, 3]\n", "#feature2 = X_transform[:, 1]\n", "plt.plot(feature1, 'r.')\n", "plt.xlabel(\"Feature number 1\")\n", "plt.ylabel(\"Feature value\")\n", "#plt.ylim([np.min(feature2), np.max(feature2)])\n", "#plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "068ac5d3-623d-4ff1-0066-25e4268b1d3e" }, "outputs": [], "source": [ "from sklearn.linear_model import LassoCV\n", "reg = linear_model.LassoCV(alphas = [1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2])\n", "reg.fit(X_transform, y_train)\n", "print(reg.alpha_)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4caf0aef-7e6a-efdc-a8e8-b58a0a52fd07" }, "outputs": [], "source": [ "X_test_selected = X_test.as_matrix()[:, [3, 5, 6, 8, 22, 24, 26, 27, 28]]\n", "#y_pred = reg.predict(X_test_selected)\n", "#y_pred = np.expm1(y_pred)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "15897a51-893a-0a81-6b00-d901fac581cc" }, "outputs": [], "source": [ "#output = pd.DataFrame(test['Id'])\n", "#output['SalePrice'] = y_pred\n", "#output.to_csv('Lasso_selected9_03132017.csv', index=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7f99e508-1b56-c4af-affc-4353e91690e8" }, "outputs": [], "source": [ "import xgboost as xgb\n", "\n", "\n", "regr = xgb.XGBRegressor(\n", " colsample_bytree=1,\n", " gamma=0.0,\n", " learning_rate=0.7,\n", " max_depth=7,\n", " min_child_weight=1.5,\n", " #n_estimators=7429, \n", " reg_alpha=0.6,\n", " reg_lambda=0.6,\n", " subsample=0.5,\n", " #seed=42,\n", " silent=1)\n", "\n", "regr.fit(X_transform, y_train)\n", "\n", "y_pred = regr.predict(X_test_selected)\n", "y_pred = np.expm1(y_pred)\n", "\n", "output = pd.DataFrame(test['Id'])\n", "output['SalePrice'] = y_pred\n", "output.to_csv('xgboost_selested9features.csv', index=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "41ab9e6c-e7cf-6d5b-1d04-eb885f801085" }, "outputs": [], "source": [ "xgboost parameters from \n", "https://www.kaggle.com/zenstat/house-prices-advanced-regression-techniques/xgboost-lasso-copied/code" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }