{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "a2551e32-13ed-4529-664d-6caf2d5a0bee"
      },
      "source": [
        "In this script, I firstly use simple decision tree to find important features from feature texts, and then I  use regular expression to explore those potentially important features and generate new features for future use.\n",
        "\n",
        "This is my first post. I am new to the area, so please kindly provide me with pointers and advice. Thanks a lot.\n",
        "\n",
        "Some codes are quoted from @sudalairajkumar post."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "932f74af-cf99-6e17-e652-0dfcf147885d"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import re\n",
        "import itertools as it"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "23acd1b9-e53d-cc9e-43c5-3292cde59c2c"
      },
      "outputs": [],
      "source": [
        "train=pd.read_json('../input/train.json')\n",
        "train['listing_id']=train['listing_id'].apply(str)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "9bf6ad6d-2891-be34-c761-31b4c27f6dc8"
      },
      "source": [
        "### Gather and observe all original features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f8ecbba8-4433-46c9-87cd-67af698cfea1"
      },
      "outputs": [],
      "source": [
        "feature_total=[]\n",
        "train['features'].apply(lambda x: feature_total.append(x))\n",
        "feature_total=list(it.chain.from_iterable(feature_total))\n",
        "len(feature_total)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9ded94d3-28f6-ce7c-0c02-7bb2155d10bf"
      },
      "outputs": [],
      "source": [
        "uniq_feature_total=set(feature_total)\n",
        "len(uniq_feature_total)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1e59b3a2-ca52-e348-f1c9-9b13fda8ff8d"
      },
      "outputs": [],
      "source": [
        "list(uniq_feature_total)[:10]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "d3ebb887-d264-a4c0-d478-bf8ad69289ce"
      },
      "source": [
        "#### Findings:\n",
        "1. For many listings, the features are linked with \\*, which should be separated.\n",
        "2. The uppercase letters should be tranfered to lowercase, which will be taken care of later. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d3f15afd-ea10-e339-ac65-89cbd9bb2a67"
      },
      "outputs": [],
      "source": [
        "def feature_star_sep(feature_list):\n",
        "    '''\n",
        "    Seperate feature text with * or \u2022 as separator\n",
        "    '''\n",
        "    new_list=[]\n",
        "    for feature in feature_list:\n",
        "        if feature[:2]=='**':\n",
        "            new=feature[3:-3]\n",
        "            new_list+new.split(\" * \")\n",
        "        elif feature[:1]=='\u2022':\n",
        "            new=feature[2:]\n",
        "            new_list+new.split(\" \u2022 \")            \n",
        "        else:\n",
        "            new_list.append(feature)\n",
        "            \n",
        "    return new_list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "89bf8eaa-0f12-a81e-d291-346b33bc04f2"
      },
      "outputs": [],
      "source": [
        "train['features']=train['features'].apply(feature_star_sep)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "10f47ed1-0cb6-d6b3-7ef5-87851d4963b0"
      },
      "source": [
        "### Adopt DecisionTree on feature text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0b0800f9-8c9b-4b24-4c35-b4ca8724d72f"
      },
      "outputs": [],
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "172fadb5-c475-fbe2-21ab-daed8c6c6e24"
      },
      "outputs": [],
      "source": [
        "## Code copied from @sudalairajkumar \n",
        "vec=CountVectorizer(stop_words='english', max_features=200)\n",
        "train['features_new'] = train[\"features\"].apply(lambda y: \" \".join([\"_\".join(x.split(\" \")).lower() for x in y]))\n",
        "tr_sparse = vec.fit_transform(train[\"features_new\"])\n",
        "feature_names=vec.get_feature_names()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e5a0b0eb-d94d-733d-7248-eaad0b1cc1e3"
      },
      "outputs": [],
      "source": [
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.model_selection import StratifiedShuffleSplit\n",
        "from sklearn.metrics import log_loss"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6a673a68-0c76-441e-0999-7ccc2fd6902f"
      },
      "outputs": [],
      "source": [
        "target_num_map = {'high':0, 'medium':1, 'low':2}\n",
        "features=tr_sparse.toarray()\n",
        "labels=train['interest_level'].apply(lambda x: target_num_map[x]).as_matrix()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a0b366bb-e877-6bd0-b092-de8dc27ae1c3"
      },
      "outputs": [],
      "source": [
        "clf=DecisionTreeClassifier(max_depth=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ec1d5948-6626-f60c-b441-5350d7972058"
      },
      "outputs": [],
      "source": [
        "cv=StratifiedShuffleSplit(n_splits=3, test_size=0.3)\n",
        "\n",
        "for train_idx, test_idx in cv.split(features, labels): \n",
        "    features_train,labels_train = features[train_idx],labels[train_idx]\n",
        "    features_test,labels_test = features[test_idx],labels[test_idx]\n",
        "    clf.fit(features_train,labels_train)\n",
        "    print(\"log loss:\",(-1)*round(log_loss(labels_test,clf.predict_proba(features_test)),3))\n",
        "    \n",
        "    ## Print out features with high importance\n",
        "    print('high importance features:')\n",
        "    for idx in np.where(clf.feature_importances_>0.05)[0]:\n",
        "        print(\"  \",feature_names[idx],round(clf.feature_importances_[idx],3))\n",
        "        "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "70600506-12aa-d3a2-a884-7efb681c98d5"
      },
      "source": [
        "### Check important features"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "8a3da663-e3a6-67d2-5a12-1b0f9d56c887"
      },
      "source": [
        "Some important features:\n",
        "\n",
        "- hardwood floors\n",
        "- doorman\n",
        "- reduced fee\n",
        "- no fee\n",
        "\n",
        "Other candidates:\n",
        "\n",
        "- war\n",
        "- laundry\n",
        "- fitness/gym"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4f944ac1-33e9-b71b-91d5-02602812614a"
      },
      "outputs": [],
      "source": [
        "feature_total=[]\n",
        "train['features'].apply(lambda x: feature_total.append(x))\n",
        "feature_total=list(it.chain.from_iterable(feature_total))\n",
        "uniq_feature_total=list(set(feature_total))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "46cfdf86-26ec-5290-f66f-90cf003fda70"
      },
      "outputs": [],
      "source": [
        "def re_search(key):\n",
        "    '''\n",
        "    Present all features with specific re pattern\n",
        "    '''\n",
        "    result=[]\n",
        "    my_reg=r\"\"+key\n",
        "    for item in uniq_feature_total:\n",
        "        if re.compile(my_reg ,re.IGNORECASE).search(item)!=None:\n",
        "            result.append(item)\n",
        "    return result"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "13b0727e-c3cb-b3ec-2ca5-430a3a3a5923"
      },
      "outputs": [],
      "source": [
        "# Check all text including 'hardwood'\n",
        "re_search('hardwood')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "08fcaead-5ad5-3b2a-ca6c-24985a95a9f6"
      },
      "outputs": [],
      "source": [
        "# Check all text including 'doorman'\n",
        "re_search('doorman')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "12fcd770-6cab-6d08-a2d8-9d1d86b3a814"
      },
      "outputs": [],
      "source": [
        "# Check all text including 'fee'\n",
        "re_search('fee')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "127a39ec-695d-441f-d442-6c926b31fe87"
      },
      "outputs": [],
      "source": [
        "# Extract no fee\n",
        "re_search('no\\s*\\w*\\s*fee')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "74bfdcca-2a77-6db9-edbe-d813ddebf4b7"
      },
      "outputs": [],
      "source": [
        "# Extract low fee\n",
        "re_search('reduce|low\\sfee')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "432d7312-6bee-fd53-7274-2976d14dee8f"
      },
      "outputs": [],
      "source": [
        "# Check all text including 'laundry'\n",
        "re_search('laundry')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "25ccf90b-1fb7-5678-81c8-878818f6074f"
      },
      "outputs": [],
      "source": [
        "# Extract war and exclude other keyword with 'war' such as warmth and wardrobe\n",
        "re_search('war\\Z|war\\s')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b123e322-717d-9d90-37a2-a05f72b09ee1"
      },
      "outputs": [],
      "source": [
        "# Check all text including 'fitness' or 'gym'\n",
        "re_search('fitness|gym')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "4027154f-21fb-fb12-df12-41d4addefbc5"
      },
      "source": [
        "### Build new features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "81f8cf06-2215-8e0c-9c6f-16b806286052"
      },
      "outputs": [],
      "source": [
        "def add_feature(row):\n",
        "    if re.search(r'hardwood',row['features_new'],re.IGNORECASE)!=None:\n",
        "        row['hardwood']=1\n",
        "    else:\n",
        "        row['hardwood']=0\n",
        "        \n",
        "    if re.search(r'doorman',row['features_new'],re.IGNORECASE)!=None:\n",
        "        row['doorman']=1\n",
        "    else:\n",
        "        row['doorman']=0\n",
        "    \n",
        "    if re.search(r'no\\w*fee',row['features_new'],re.IGNORECASE)!=None:\n",
        "        row['no_fee']=1\n",
        "    else:\n",
        "        row['no_fee']=0\n",
        "    \n",
        "    if re.search(r'reduce|low\\wfee',row['features_new'],re.IGNORECASE)!=None:\n",
        "        row['reduce_fee']=1\n",
        "    else:\n",
        "        row['reduce_fee']=0\n",
        "\n",
        "    if re.search(r'laundry',row['features_new'],re.IGNORECASE)!=None:\n",
        "        row['laundry']=1\n",
        "    else:\n",
        "        row['laundry']=0\n",
        "\n",
        "    if re.search(r'war\\Z|war\\s|war_',row['features_new'],re.IGNORECASE)!=None:\n",
        "        row['war']=1\n",
        "    else:\n",
        "        row['war']=0\n",
        "\n",
        "    if re.search(r'fitness|gym',row['features_new'],re.IGNORECASE)!=None:\n",
        "        row['gym']=1\n",
        "    else:\n",
        "        row['gym']=0\n",
        "        \n",
        "    return row"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "22c5717d-dbaa-7634-ef44-64e566641175"
      },
      "outputs": [],
      "source": [
        "train=train.apply(add_feature,axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6f7f509d-b618-261b-da06-5f568a30dc3b"
      },
      "outputs": [],
      "source": [
        "train[['hardwood','doorman','no_fee','reduce_fee','laundry','war','gym']].apply(sum)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a8368e41-e227-f600-7d88-b84938959ffa"
      },
      "outputs": [],
      "source": ""
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}