{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c7dd05d7-ca58-4865-a356-d89b584bbcad"
      },
      "outputs": [],
      "source": [
        "import numpy as np \n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "39338028-18e9-4bd9-9599-8ceed3f509f7"
      },
      "outputs": [],
      "source": [
        "categories = pd.read_csv('../input/documents_categories.csv')\n",
        "entities = pd.read_csv('../input/documents_entities.csv')\n",
        "meta = pd.read_csv('../input/documents_meta.csv')\n",
        "topics = pd.read_csv('../input/documents_topics.csv')\n",
        "events = pd.read_csv('../input/events.csv')\n",
        "page_views = pd.read_csv('../input/page_views_sample.csv')\n",
        "promoted = pd.read_csv('../input/promoted_content.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "75c2e67f-151f-da8f-ee59-68529b6cdf1b"
      },
      "outputs": [],
      "source": [
        "# Investigating categories dataset\n",
        "categories.columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ba89e0c8-b506-21ae-4289-682b5fe178d5"
      },
      "outputs": [],
      "source": [
        "print (\"Total categories: {}\".format(categories.category_id.unique().shape[0]))\n",
        "print (\"Unique documents (web pages): {}\".format(len(set(categories.document_id))))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "68f9295f-8296-2356-9f6b-7d27df98efd9"
      },
      "outputs": [],
      "source": [
        "categories.confidence_level.hist(bins=100)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d42c2734-b4d3-6eb5-bd16-2a8c43a66b90"
      },
      "outputs": [],
      "source": [
        "categories.confidence_level.describe()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6ce4b8e7-a992-28cc-d2f2-00506b6d7a95"
      },
      "outputs": [],
      "source": [
        "# Investigating Entities"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f58a01bb-b4ff-8695-d486-3bab1be0ac8f"
      },
      "outputs": [],
      "source": [
        "entities.columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "67a2a4cf-a8b0-e34f-686f-16651e2e4b18"
      },
      "outputs": [],
      "source": [
        "print (\"Unique documents: {}\".format(entities.document_id.unique().shape[0]))\n",
        "print (\"Unique entities - derived from documents: {}\".format(entities.entity_id.unique().shape[0]))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6f546e13-17ef-b3f0-832e-49f0d7c1dfb5"
      },
      "outputs": [],
      "source": [
        "entities.confidence_level.hist(bins=100)\n",
        "plt.title(\"Confidence level of entity classification\")\n",
        "# Much better classification of entities than topics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "bf5ba384-6125-3854-b9c4-37205b9736e0"
      },
      "outputs": [],
      "source": [
        "# Meta\n",
        "meta.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "15a1e355-1621-4eab-6d1a-fa2658037f23"
      },
      "outputs": [],
      "source": [
        "# Topics\n",
        "topics.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a18c9b47-4b8f-c27b-6538-f32c92a0b699"
      },
      "outputs": [],
      "source": [
        "topics.confidence_level.hist(bins=100)\n",
        "plt.title(\"Confidence of document topic classification\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "bf6dea98-bbef-f4e7-e531-941f52353dc5"
      },
      "outputs": [],
      "source": [
        "page_views.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "84dc3cb8-e5b2-3a76-3b87-87b5854da8c8"
      },
      "outputs": [],
      "source": ""
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}