{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "f3638001-2222-f436-9dc7-2c7c5ddb0bb8"
      },
      "source": [
        "EDA - TFIDF and other features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "16f04b35-71e9-662c-d675-526b50450744"
      },
      "outputs": [],
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load in \n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "# Input data files are available in the \"../input/\" directory.\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
        "\n",
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
        "\n",
        "# Any results you write to the current directory are saved as output."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ae4b0f07-2c9b-04d8-4726-de748cebf5ef"
      },
      "outputs": [],
      "source": [
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "% matplotlib inline\n",
        "import seaborn as sns\n",
        "color = sns.color_palette()\n",
        "sns.set(style=\"whitegrid\", color_codes=True)\n",
        "sns.set(font_scale=1)\n",
        "\n",
        "\n",
        "import plotly.plotly as py\n",
        "import plotly.graph_objs as go\n",
        "from plotly import tools\n",
        "\n",
        "from plotly.offline import download_plotlyjs, init_notebook_mode, iplot\n",
        "init_notebook_mode(connected=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7f148f88-7d20-4a66-b23d-d14ebdbd41c1"
      },
      "outputs": [],
      "source": [
        "# Reading the json train and test files\n",
        "\n",
        "train = pd.read_json(\"../input/train.json\")\n",
        "test = pd.read_json(\"../input/test.json\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5a0ad1ac-5e86-3b63-89b0-8e5eaac8d4fc"
      },
      "outputs": [],
      "source": [
        "train.describe() # for numerical features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ea880082-2f62-862b-8dee-28f693f78b33"
      },
      "outputs": [],
      "source": [
        "int_level = train['interest_level'].value_counts()\n",
        "\n",
        "plt.figure(figsize=(8,4))\n",
        "sns.barplot(int_level.index, int_level.values, alpha=0.8, color=color[1])\n",
        "plt.ylabel('Number of Occurrences', fontsize=12)\n",
        "plt.xlabel('Interest level', fontsize=12)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "53cf0073-e5dd-a7ed-ecdc-65a96db68f67"
      },
      "outputs": [],
      "source": [
        "# Average price of property by interest level\n",
        "\n",
        "\n",
        "int_level_price = train['price'].groupby(train['interest_level']).mean()\n",
        "int_level_bath = train['bathrooms'].groupby(train['interest_level']).mean()\n",
        "int_level_bed = train['bedrooms'].groupby(train['interest_level']).mean()\n",
        "\n",
        "\n",
        "\n",
        "plt.figure(figsize=(8,4))\n",
        "sns.barplot(int_level_price.index, int_level_price.values, alpha=0.8, color=color[2])\n",
        "plt.ylabel(' Mean Price', fontsize=12)\n",
        "plt.xlabel('Interest level', fontsize=12)\n",
        "plt.show()\n",
        "\n",
        "\n",
        "\n",
        "plt.figure(figsize=(8,4))\n",
        "sns.barplot(int_level_bath.index, int_level_bath.values, alpha=0.8, color=color[4])\n",
        "plt.ylabel(' Avg No of Bathrooms', fontsize=12)\n",
        "plt.xlabel('Interest level', fontsize=12)\n",
        "plt.show()\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "plt.figure(figsize=(8,4))\n",
        "sns.barplot(int_level_bath.index, int_level_bath.values, alpha=0.8, color=color[5])\n",
        "plt.ylabel('Avg No of Bedrooms', fontsize=12)\n",
        "plt.xlabel('Interest level', fontsize=12)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "a3530627-dbde-79eb-838b-830b8b495eac"
      },
      "source": [
        "Creating manager skill feature. Thanks to this script https://www.kaggle.com/den3b81/two-sigma-connect-rental-listing-inquiries/improve-perfomances-using-manager-features.\n",
        "\n",
        "This feature would have to be built into the cross validation code so that 'cheating' doesn't happen during CV and also mapped to test set before modeling."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ea636908-ac69-f3a4-a106-29af8466a09e"
      },
      "outputs": [],
      "source": [
        "from sklearn import preprocessing\n",
        "\n",
        "lbl = preprocessing.LabelEncoder()\n",
        "lbl.fit(list(train['manager_id'].values))\n",
        "train['manager_id'] = lbl.transform(list(train['manager_id'].values))\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4ea0bf65-ec74-101e-c8c2-82161368a967"
      },
      "outputs": [],
      "source": [
        "temp = pd.get_dummies(train.interest_level)\n",
        "temp = pd.concat([train.manager_id, temp], axis=1).groupby(train['manager_id']).mean()\n",
        "temp.columns = ['manager_id','high_frac','low_frac','medium_frac']\n",
        "temp['manager_skill'] = temp['high_frac']*2 + temp['medium_frac'] + temp['low_frac']*0.2\n",
        "temp.index = temp.manager_id\n",
        "del temp['manager_id']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "70d54dad-7d6c-f38b-189c-65e6081390bc"
      },
      "outputs": [],
      "source": [
        "temp.head(3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d190e8f7-c6fd-6aae-cf78-70f76ad08168"
      },
      "outputs": [],
      "source": [
        "# Merging manager skill with training set\n",
        "\n",
        "train = train.merge(temp.reset_index(), how='left', left_on='manager_id', right_on = 'manager_id')\n",
        "train.head(3)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "d1a249b6-1463-11b4-8d47-614b0ee0533e"
      },
      "source": [
        "Creating a new variable to measure no. of listed features in the 'features' variable."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "697a75aa-4204-d2eb-88b3-39ae5c843d7d"
      },
      "outputs": [],
      "source": [
        "train['feat_len'] = train['features'].map(lambda text: len(text))\n",
        "train.feat_len.plot(bins=20, kind='hist')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "673ccc5e-3eb4-655d-7beb-fe8920d9e33d"
      },
      "source": [
        "Next, we compute the Term Frequency Inverse Document Frequency metric for the text present in the 'features' variable. The same can also be done on the 'description' variable which is probably more suitable as it contains richer text."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c5956c63-e8ca-1d20-7a9c-b83be405f9f6"
      },
      "outputs": [],
      "source": [
        "import nltk\n",
        "import string\n",
        "import os\n",
        "\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from nltk.stem.porter import PorterStemmer\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from textblob import TextBlob as tb\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem.wordnet import WordNetLemmatizer\n",
        "import nltk\n",
        "\n",
        "\n",
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt\n",
        "import csv"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7365b807-2f34-7744-db7d-c8a0ca130eca"
      },
      "outputs": [],
      "source": [
        "columns = ['new_features','new_feat_lem']\n",
        "df = pd.DataFrame(index=train.index, columns = columns)\n",
        "for i in range(len(train)):\n",
        "    df.new_features.iloc[i] = ','.join(map(str,train.features.iloc[i]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2b701255-6f60-1fa6-fa2a-4ab7bfc92d5b"
      },
      "outputs": [],
      "source": [
        "train = train.join(df.new_features)\n",
        "train['new_features'] = train['new_features'].str.lower()\n",
        "lemmatizer = WordNetLemmatizer()\n",
        "for i,w in enumerate(train.new_features):\n",
        "    df.new_feat_lem.iloc[i] = lemmatizer.lemmatize(w)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "061d7f99-9a51-896f-f709-e86b365158f3"
      },
      "outputs": [],
      "source": [
        "train_new = train.join(df.new_feat_lem)\n",
        "vectorizer = TfidfVectorizer(stop_words='english',min_df=0.01,strip_accents = ascii,norm='l2')\n",
        "transformed = vectorizer.fit_transform(train_new['new_feat_lem']).toarray()\n",
        "print(\"Num words:\", len(vectorizer.get_feature_names()))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "75fee03e-e3ae-17e9-3ac4-fb9206aed3f2"
      },
      "source": [
        "We've created additional 54 features based on TfIdf on words in the 'features' variable."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "96125880-e65d-c715-6708-0aac9b59033a"
      },
      "outputs": [],
      "source": [
        "df2 = pd.DataFrame(transformed, index=train_new.index,columns=vectorizer.get_feature_names())\n",
        "train_new = pd.concat([train_new, df2], axis=1, join_axes=[train_new.index])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "567bc829-199d-9e51-6679-6a4b7596c9d3"
      },
      "outputs": [],
      "source": [
        "# Checking if all building id's are unique\n",
        "len(train_new.building_id.unique())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "f0a3bc5d-8304-d1a3-3ce5-eed99bd104d4"
      },
      "source": [
        "Not all building id's are unique."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "94d278bd-2afe-6c12-024e-0cfc0e8ee406"
      },
      "outputs": [],
      "source": [
        "# Creating a new feature that counts the number of times a building ID appears\n",
        "\n",
        "columns = ['No_of_listings_per_build_id']\n",
        "df2 = pd.DataFrame(columns = columns)\n",
        "df2['No_of_listings_per_build_id']= train_new.building_id.value_counts()\n",
        "df2 = df2.reset_index()\n",
        "columns = {'index': 'building_id'}\n",
        "df2.rename(columns = columns, inplace=True)\n",
        "df2.head(3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ea4d4d21-4390-9da4-7326-dd6ff4f87c5e"
      },
      "outputs": [],
      "source": [
        "# Joining with training set\n",
        "train_new = train_new.merge(df2.reset_index(), how='left', left_on='building_id', right_on = 'building_id')\n",
        "del train_new['index']\n",
        "train_new.head(3)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "bf33417e-1314-e69e-a7c2-921527cc7139"
      },
      "source": [
        "Next step: Modeling with these features and analyzing results to see if Tfidf and other features help!"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}