{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "69ef641d-894f-909d-2d1b-41af2946cc01"
      },
      "source": [
        "## A proxy for $/sqft and the interest on 1/2-baths"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9d0649e3-584f-ea7c-19e2-c1a208a0e8fe"
      },
      "outputs": [],
      "source": [
        "%matplotlib inline\n",
        "import matplotlib.pylab as plt\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import statsmodels.api as sm\n",
        "import statsmodels.formula.api as smf\n",
        "from scipy import stats\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.preprocessing import LabelBinarizer\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "import itertools as itertools\n",
        "from sklearn.metrics import log_loss\n",
        "\n",
        "def get_skf_indexes(df, target, kfold=4):\n",
        "    X = df.values\n",
        "    y = df[target].values\n",
        "    skf = StratifiedKFold(n_splits=4);\n",
        "    skf.get_n_splits(X, y);\n",
        "    indexes = [[],[]]\n",
        "    for train_index, test_index in skf.split(X, y):\n",
        "        indexes[0].append(train_index) # Training indexes\n",
        "        indexes[1].append(test_index) # test indexes\n",
        "    return indexes\n",
        "\n",
        "\n",
        "def get_lr_perf(df_train, df_test, feature='__to_check', target='response', n_quantile=20):\n",
        "    results = {}\n",
        "    # Inputs\n",
        "    xtrain = df_train[feature].values.reshape(-1,1)\n",
        "    ytrain = df_train[target].values\n",
        "    xtest = df_test[feature].values.reshape(-1,1)\n",
        "    ytest = df_test[target].values\n",
        "    # Evaluation as a single feature\n",
        "    lr = LogisticRegression()\n",
        "    lr.fit(xtrain, ytrain);\n",
        "    yptrain = lr.predict_proba(xtrain)\n",
        "    yptest = lr.predict_proba(xtest)\n",
        "    results['train.num'] = np.round(log_loss(ytrain, yptrain), 6)\n",
        "    results['test.num'] = np.round(log_loss(ytest, yptest), 6)\n",
        "    # Evaluation as a categorical feature using quantile buckets\n",
        "    bins = np.unique(np.percentile(xtrain, np.arange(n_quantile, 100, n_quantile)))\n",
        "    xtrainq = np.digitize(xtrain, bins)\n",
        "    xtestq = np.digitize(xtest, bins)\n",
        "    lb = LabelBinarizer()\n",
        "    x1 = lb.fit_transform(xtrainq)\n",
        "    x2 = lb.transform(xtestq)\n",
        "    lr.fit(x1, ytrain);\n",
        "    yptrain = lr.predict_proba(x1)\n",
        "    yptest = lr.predict_proba(x2)\n",
        "    results['train.cat'] = np.round(log_loss(ytrain, yptrain), 6)\n",
        "    results['test.cat'] = np.round(log_loss(ytest, yptest), 6)\n",
        "    return results"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "56aef204-2103-a022-3dad-09bfd5135454"
      },
      "source": [
        "### 1) Price/sqft proxy using price, and number of bedrooms/bathrooms:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "713d1747-bc9e-a5cc-4fc0-da5247cdb5da"
      },
      "outputs": [],
      "source": [
        "df = pd.read_json('../input/train.json')\n",
        "df['response'] = \"0\"\n",
        "df.loc[df.interest_level=='medium', 'response'] = \"1\"\n",
        "df.loc[df.interest_level=='high', 'response'] = \"2\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "46703a71-1b57-4034-b40a-c75f41b1bcd3"
      },
      "source": [
        "In this first approach the aim is to parametrize the set of parameters of the following equation that miniminize log-loss from an \"univariant\" point of view:\n",
        "\n",
        "$$\\frac{Price}{A + N_{bedrooms}\\vert _{C_1}^{C_2} + B\u00b7N_{bathrooms}\\vert _{D_1}^{D_2}}$$\n",
        "\n",
        "- A baseline value (A) is set to avoid 0 divisions and to set the relative weight of the number of rooms regarding to the price.\n",
        "- The relative weights of the number of bedrooms and bathrooms are adjusted using parameter B.\n",
        "- Bedrooms and bathrooms are clipped using C and D respectively.\n",
        "\n",
        "For each set of parameters I'll output two performance measures using a stratified 4-fold CV approach:\n",
        "\n",
        "- log-loss of the logistic classifier using as input the computed feature ('train.num' and 'test.num')\n",
        "- Since most of the classifiers will use a \"tree-based\" method, I'll perform bucketization (5% percentiles) of the computed feature and dummification. The resulting set of 20 features will be used as the input for a logistic classifier."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "8c73329e-2095-fa1e-d664-f347fba388fe"
      },
      "outputs": [],
      "source": [
        "# Parameters to check\n",
        "AA = (0.1, 0.5, 1, 2)\n",
        "CC = ((0, 4), (0, 3), (1, 4), (1, 3), (0, 2))\n",
        "DD = ((0, 3), (0, 2), (1, 3), (1, 2))\n",
        "BB = (0, 0.25, 0.5, 1, 2)\n",
        "# Reduced set of parameters to run here\n",
        "AA = (0.5, 1, 2)\n",
        "CC = ((0, 4), (0, 3), (1, 4), (1, 3))\n",
        "DD = ((0, 3), (0, 2))\n",
        "BB = (0.25, 0.5, 1)\n",
        "# Stratified kfold\n",
        "idx_train, idx_test = get_skf_indexes(df, 'response', kfold=2) # kfold=4, set to 2 to quickly run here\n",
        "# Get results\n",
        "Y = pd.DataFrame()\n",
        "for iper, (i_train, i_test) in enumerate(zip(idx_train, idx_test)):\n",
        "    print(iper)\n",
        "    df_train = df.iloc[i_train, :].copy()\n",
        "    df_test = df.iloc[i_test, :].copy()\n",
        "    # For each parameter combination\n",
        "    for A, C, D, B in itertools.product(AA, CC, DD, BB):\n",
        "        df_train['__to_check'] = (df_train.price / (A + df_train.bedrooms.clip(C[0], C[1]) + B*df_train.bathrooms.clip(D[0], D[1]))).values\n",
        "        df_test['__to_check'] = (df_test.price / (A + df_test.bedrooms.clip(C[0], C[1]) + B*df_test.bathrooms.clip(D[0], D[1]))).values\n",
        "        results = get_lr_perf(df_train, df_test, feature='__to_check', target='response', n_quantile=20)\n",
        "        results.update({'fold': iper, 'params': {'A':A, 'B': B, 'C': C, 'D':D}})\n",
        "        Y =  Y.append(pd.DataFrame(pd.Series(results)).transpose())\n",
        "for i in ['train.cat', 'train.num', 'test.cat', 'test.num']:\n",
        "    Y[i] = Y[i].astype(float)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ae331c22-197c-77ab-6dfe-72dd7b0cd1a1"
      },
      "outputs": [],
      "source": [
        "Y.sort_values('test.cat')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "3ff8c1b1-31ed-886c-64e3-160c4f799ffa"
      },
      "source": [
        "From these results we can conclude than the best proxy for price/sqft using price, bedrooms and bathrooms is:\n",
        "$$\\frac{Price}{1 + N_{bedrooms}\\vert _{1}^{4} + 0.5\u00b7N_{bathrooms}\\vert _{0}^{2}}$$"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "39fc574e-bb95-61ea-3636-e5e9f70f93ac"
      },
      "source": [
        "### 2) Uninteresting half bathrooms\n",
        "\n",
        "You'll have noticed that half bathrooms show mean interests far above the mean interest level.\n",
        "\n",
        "Let's compute a boolean feature for half-bathrooms and clip the number of bathrooms to [0, 4]:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9f8d925a-bdc0-dece-82c0-28e8dd5f4d1d"
      },
      "outputs": [],
      "source": [
        "df['half_bathrooms'] = ((np.round(df.bathrooms) - df.bathrooms)!=0).astype(float) # Half bathrooms? 1.5, 2.5, 3.5...\n",
        "df['bathrooms'] = df.bathrooms.clip(0,4) # Reduce outlier effects"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "4bb2bf61-af6f-4d16-845d-519a9240275e"
      },
      "source": [
        "Let's demonstrate the half bathrooms unininterest from a statistical point of view. We'll fit two models with and without using the half bathrooms boolean variable. I'll use a likelihood ratio test to demonstrate that the model including the feature has better fit:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "bb96cb15-e241-de5a-ccaf-9c17695537ec"
      },
      "outputs": [],
      "source": [
        "# Build two models with and without 'half_bathrooms' feature\n",
        "formula1 = 'response ~ bathrooms'\n",
        "formula2 = 'response ~ bathrooms + half_bathrooms'\n",
        "model1 = smf.glm(formula=formula1, data=df, family=sm.families.Binomial())\n",
        "model2 = smf.glm(formula=formula2, data=df, family=sm.families.Binomial())\n",
        "result1 = model1.fit()\n",
        "result2 = model2.fit()\n",
        "# Likelihood ratio test\n",
        "llf_1 = result1.llf\n",
        "llf_2 = result2.llf\n",
        "df_1 = result1.df_resid \n",
        "df_2 = result2.df_resid \n",
        "lrdf = (df_1 - df_2)\n",
        "lrstat = -2*(llf_1 - llf_2)\n",
        "lr_pvalue = stats.chi2.sf(lrstat, df=lrdf)\n",
        "# Print results\n",
        "print(formula1)\n",
        "print(result1.summary())\n",
        "print(formula2)\n",
        "print(result2.summary())\n",
        "print('Likelihood ratio test', lr_pvalue)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "b0c02533-093c-0215-49f6-a8b0724de39c"
      },
      "source": [
        "These results can also be noticed by using a barplot showing the interest frequencies depending on the number of bathrooms:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ae396e83-7f8a-985e-1218-4ae6407b86f6"
      },
      "outputs": [],
      "source": [
        "x = pd.crosstab(df.bathrooms, df.interest_level)[['low', 'medium', 'high']]\n",
        "x.div(x.sum(1), 0).plot(kind='bar', color=['red', 'yellow', 'green'], stacked=True);"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "87833377-f5c1-ccfd-41d4-3f38e9dbbebe",
        "collapsed": true
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}