{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "57687896-0a99-6fd6-6ee2-c1177b38d11f"
      },
      "source": [
        "Loading the dataset and create lists with the features."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b7bb829a-1519-e7ed-9917-dc5e8c77d7ef"
      },
      "outputs": [],
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load in \n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "import xgboost as xgb # XGBoost implementation\n",
        "\n",
        "# Input data files are available in the \"../input/\" directory.\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
        "\n",
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
        "\n",
        "# Any results you write to the current directory are saved as output.\n",
        "\n",
        "# read data\n",
        "train = pd.read_csv(\"../input/train.csv\")\n",
        "test = pd.read_csv(\"../input/test.csv\")\n",
        "\n",
        "features = [x for x in train.columns if x not in ['id','loss']]\n",
        "#print(features)\n",
        "\n",
        "cat_features = [x for x in train.select_dtypes(include=['object']).columns if x not in ['id','loss']]\n",
        "num_features = [x for x in train.select_dtypes(exclude=['object']).columns if x not in ['id','loss']]\n",
        "#print(cat_features)\n",
        "#print(num_features)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "358cc639-51de-2729-faa4-7c0cbd36b581"
      },
      "source": [
        "Now let's see if the loss follows a log-normal distribution!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "678dadbb-89e2-23d8-7cd0-aa425df85065"
      },
      "outputs": [],
      "source": [
        "from scipy.stats import norm, lognorm\n",
        "import matplotlib.mlab as mlab\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "train['log_loss'] = np.log(train['loss'])\n",
        "\n",
        "# fit the normal distribution on ln(loss)\n",
        "(mu, sigma) = norm.fit(train['log_loss'])\n",
        "\n",
        "# the histogram of the ln(loss)\n",
        "n, bins, patches = plt.hist(train['log_loss'], 60, normed=1, facecolor='green', alpha=0.75)\n",
        "\n",
        "# add the fitted line\n",
        "y = mlab.normpdf( bins, mu, sigma)\n",
        "l = plt.plot(bins, y, 'r--', linewidth=2)\n",
        "\n",
        "#plot\n",
        "plt.xlabel('Ln(loss)')\n",
        "plt.ylabel('Probability')\n",
        "plt.title(r'$\\mathrm{Histogram\\ of\\ Ln(Loss):}\\ \\mu=%.3f,\\ \\sigma=%.3f$' %(mu, sigma))\n",
        "plt.grid(True)\n",
        "\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "8bc46af2-bb33-f9ca-c6f0-7a84c20ba718"
      },
      "source": [
        "Yepp! It's not perfect, but works for me. It's important to change the target variable in this way as RMSE algorithms (default for all regressions) prefer symmetric data: asymmetric output would bias the prediction towards higher losses."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "37bb667f-61ad-d9b8-fb1b-caf7ccfd0f00"
      },
      "source": [
        "Ok, now change the categorical variables into numeric. I prefer to use only 1 column as the XGBoost algorithm doesn't really benefit from several columns. So I'm going to add the category codes to each categorical grouping value."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1963759a-72be-b3be-ee59-8803bcc59e37"
      },
      "outputs": [],
      "source": [
        "ntrain = train.shape[0]\n",
        "ntest = test.shape[0]\n",
        "train_test = pd.concat((train[features], test[features])).reset_index(drop=True)\n",
        "for c in range(len(cat_features)):\n",
        "    train_test[cat_features[c]] = train_test[cat_features[c]].astype('category').cat.codes\n",
        "\n",
        "train_x = train_test.iloc[:ntrain,:]\n",
        "test_x = train_test.iloc[ntrain:,:]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "642f8314-9b96-dbdb-4448-9224cb38a352"
      },
      "source": [
        "Now build up the XGBoost model! As I have not taken care of the ordering of the categorical variables I need to make a larger tree ensemble - let's start with 1000 trees."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "857d3686-e27e-373e-ab85-a8177ec8ca1a"
      },
      "outputs": [],
      "source": [
        "xgdmat = xgb.DMatrix(train_x, train['log_loss']) # Create our DMatrix to make XGBoost more efficient\n",
        "\n",
        "params = {'eta': 0.01, 'seed':0, 'subsample': 0.5, 'colsample_bytree': 0.5, \n",
        "             'objective': 'reg:linear', 'max_depth':6, 'min_child_weight':3} \n",
        "\n",
        "# Grid Search CV optimized settings\n",
        "num_rounds = 1000\n",
        "bst = xgb.train(params, xgdmat, num_boost_round = num_rounds)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "f03385f8-fecc-c808-2bd4-aada3fe8e705"
      },
      "source": [
        "What are the most important features?"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a4291340-6116-8a2d-e37e-f184880ffd0c"
      },
      "outputs": [],
      "source": [
        "import operator\n",
        "\n",
        "def ceate_feature_map(features):\n",
        "    outfile = open('xgb.fmap', 'w')\n",
        "    i = 0\n",
        "    for feat in features:\n",
        "        outfile.write('{0}\\t{1}\\tq\\n'.format(i, feat))\n",
        "        i = i + 1\n",
        "\n",
        "    outfile.close()\n",
        "    \n",
        "ceate_feature_map(features)\n",
        "\n",
        "importance = bst.get_fscore(fmap='xgb.fmap')\n",
        "importance = sorted(importance.items(), key=operator.itemgetter(1))\n",
        "\n",
        "df = pd.DataFrame(importance, columns=['feature', 'fscore'])\n",
        "df['fscore'] = df['fscore'] / df['fscore'].sum()\n",
        "\n",
        "plt.figure()\n",
        "df.plot()\n",
        "df.plot(kind='barh', x='feature', y='fscore', legend=False, figsize=(6, 10))\n",
        "plt.title('XGBoost Feature Importance')\n",
        "plt.xlabel('relative importance')\n",
        "plt.gcf().savefig('feature_importance_xgb.png')\n",
        "\n",
        "df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "d6978f1d-aac6-356a-d0bf-44d481843e99"
      },
      "source": [
        "It's time for making our prediction."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6bd3678b-cd08-a54b-9619-4b1e5bbfb7e7"
      },
      "outputs": [],
      "source": [
        "test_xgb = xgb.DMatrix(test_x)\n",
        "submission = pd.read_csv(\"../input/sample_submission.csv\")\n",
        "submission.iloc[:, 1] = np.exp(bst.predict(test_xgb))\n",
        "submission.to_csv('xgb_starter.sub.csv', index=None)"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}