{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "61426f5f-7e20-6216-9de9-002f5ea09325"
      },
      "source": [
        "Trying out basic commands according to https://www.kaggle.com/apapiu/house-prices-advanced-regression-techniques/regularized-linear-models/comments"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1c7d0032-f34c-cb3d-1610-d988a6339669"
      },
      "outputs": [],
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load in \n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "# Input data files are available in the \"../input/\" directory.\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
        "\n",
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
        "\n",
        "# Any results you write to the current directory are saved as output."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "09c253ad-aaeb-3961-abab-dd2ed5bcd009"
      },
      "outputs": [],
      "source": [
        "train = pd.read_csv(\"../input/train.csv\")\n",
        "test = pd.read_csv(\"../input/test.csv\")\n",
        "train.shape, test.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0d6f54f3-5f3d-34fb-8cd3-744217d8ae5e"
      },
      "outputs": [],
      "source": [
        "#train.describe()\n",
        "#train.info()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "eb464830-8271-2c60-4de5-1e2031c57eee"
      },
      "source": [
        "**n = 1460 samples**\n",
        "\n",
        "\n",
        "**d = 79 features** 80 columns: 1st ID 80th SalePrice (Y)\n",
        "\n",
        "\n",
        "Some contain lots of nulls such as \n",
        "PoolQC           7 non-null object\n",
        "\n",
        "Non-numerical features about half\n",
        "\n",
        "**Goal: predict SalePrice** \n",
        "Predict a quantity with <100K samples "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1bb4c34b-da9c-7dab-7587-e2a753238fe0"
      },
      "outputs": [],
      "source": [
        "#train[\"SalePrice\"].hist();"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "42827178-14d5-f921-e3fe-b718585fd452"
      },
      "outputs": [],
      "source": [
        "#prices = np.log1p(train[\"SalePrice\"])\n",
        "#prices.hist()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f0a37ce9-e351-5f44-758d-3fe69dd893c4"
      },
      "outputs": [],
      "source": [
        "train[\"SalePrice\"] = np.log1p(train[\"SalePrice\"])\n",
        "y_train = train[\"SalePrice\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ec1c039c-badd-d028-eaf8-a59e6455db36"
      },
      "outputs": [],
      "source": [
        "all_data = pd.concat((train.loc[:,'MSSubClass':'SaleCondition'],\n",
        "                      test.loc[:,'MSSubClass':'SaleCondition']))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "76699ad6-8e38-cb2f-2195-428714e47cf6"
      },
      "outputs": [],
      "source": [
        "from scipy.stats import skew\n",
        "from scipy.stats.stats import pearsonr\n",
        "#log transform skewed numeric features:\n",
        "numeric_feats = train.dtypes[train.dtypes != \"object\"].index\n",
        "\n",
        "skewed_feats = train[numeric_feats].apply(lambda x: skew(x.dropna())) #compute skewness\n",
        "skewed_feats = skewed_feats[skewed_feats > 0.75]\n",
        "skewed_feats = skewed_feats.index"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "fd61e005-f6a6-20eb-6683-6240aa95853a"
      },
      "outputs": [],
      "source": [
        "'SalePrice' in skewed_feats"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "84df0bab-9a69-69a4-db05-e17e8667cb1f"
      },
      "outputs": [],
      "source": [
        "all_data[skewed_feats] = np.log1p(all_data[skewed_feats])\n",
        "#Convert categorical variable into dummy/indicator variables\n",
        "all_data = pd.get_dummies(all_data)\n",
        "all_data = all_data.fillna(all_data.mean())\n",
        "all_data.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1722c268-3782-56d0-da7a-43c2b9b98436"
      },
      "outputs": [],
      "source": [
        "X_train = all_data[:train.shape[0]]\n",
        "X_test = all_data[train.shape[0]:]\n",
        "X_train.shape, X_test.shape"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "57876bbd-0673-adfa-e0a4-86cabc749356"
      },
      "source": [
        "1. Random split into training and test sets.\n",
        "\n",
        "  [1]: http://scikit-learn.org/stable/modules/cross_validation.html\n",
        "  [2]: http://scikit-learn.org/stable/modules/linear_model.html#ridge-regression"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2b8f2df7-7eca-ebcb-177f-fd557bd833df"
      },
      "outputs": [],
      "source": [
        "#from sklearn.model_selection import train_test_split\n",
        "#X_train, X_test, y_train, y_test = train_test_split(train_data, train_target, test_size=0.4, random_state=0)\n",
        "#X_train.shape, X_test.shape, y_train.shape, y_test.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ae6c0d26-5d4e-1b58-1fe3-a1d6028103e8"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn import linear_model\n",
        "from sklearn.metrics import mean_squared_error\n",
        "\n",
        "#reg = linear_model.Ridge(alpha = .5)\n",
        "#reg.fit(X_train, y_train) \n",
        "#y_pred = reg.predict(X_test)\n",
        "#mean_squared_error(y_test, y_pred)  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2f53c134-850c-d0b1-44f9-65f059b26d7e"
      },
      "outputs": [],
      "source": [
        "#mean_squared_error(y_test, y_pred, multioutput='raw_values')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "32bd794e-e060-7cf8-8d71-5c7340b1fd3c"
      },
      "source": [
        "2. Cross validation\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "794ed225-5d0e-a4c8-07dc-4361051482a3"
      },
      "source": [
        "> RidgeCV implements ridge regression with built-in cross-validation of\n",
        "> the alpha parameter. The object works in the same way as GridSearchCV\n",
        "> except that it defaults to Generalized Cross-Validation (GCV), an\n",
        "> efficient form of leave-one-out cross-validation."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "ae328cba-89c3-34c0-54c2-afe4eb1b806a"
      },
      "source": [
        "RidgeCV(alphas=(0.1, 1.0, 10.0), fit_intercept=True, normalize=False, scoring=None, cv=None, gcv_mode=None, store_cv_values=False)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "43e94938-6b24-9f07-ebd8-54cdd491b24f"
      },
      "source": [
        "cv:\n",
        "    None, to use the efficient Leave-One-Out cross-validation\n",
        "    \n",
        "    integer, to specify the number of folds."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ada68074-094e-e616-ac56-1dc26e0b620c"
      },
      "outputs": [],
      "source": [
        "reg = linear_model.LassoCV(alphas = [0.0001, 0.0005, 0.001, 0.005])\n",
        "reg.fit(X_train, y_train)\n",
        "reg.alpha_   "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5746fa4a-112e-480a-9801-9441c7dd9b2e"
      },
      "outputs": [],
      "source": [
        "#reg = linear_model.RidgeCV(alphas = [2.0, 5., 10, 20., 50.])\n",
        "#reg.fit(X_train, y_train)\n",
        "#reg.alpha_   "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "20eda3cf-f6d0-2389-b4b6-98654407cc91"
      },
      "outputs": [],
      "source": [
        "#reg = linear_model.RidgeCV(alphas = [6., 8., 10., 12., 14., 17.], fit_intercept=True, \\\n",
        "                           #normalize=False, scoring=None, cv=8, gcv_mode=None, store_cv_values=False)\n",
        "#reg.fit(X_train, y_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cf38d5bd-e667-489a-951c-27e0d27fdf9c"
      },
      "outputs": [],
      "source": [
        "print(reg.alpha_)\n",
        "y_pred = reg.predict(X_test)\n",
        "y_pred = np.expm1(y_pred)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9b414f6d-b0a8-f679-5170-7331a7478eed"
      },
      "outputs": [],
      "source": [
        "output = pd.DataFrame(test['Id'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cc4d452a-1943-b33e-7ae5-d371c4766126"
      },
      "outputs": [],
      "source": [
        "output['SalePrice'] = y_pred"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "09f54864-66a4-2b1c-6cdc-d448fa294e51"
      },
      "outputs": [],
      "source": [
        "output"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7cbb4012-28dc-1c27-c4d7-1a8aeec94001"
      },
      "outputs": [],
      "source": [
        "output.to_csv('LASSO0319.csv', index=False)"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}