{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "03d55010-3746-0b30-192d-cd67dd5b0fbb"
      },
      "outputs": [],
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load in \n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "%matplotlib inline\n",
        "# Input data files are available in the \"../input/\" directory.\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
        "\n",
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
        "\n",
        "# Any results you write to the current directory are saved as output."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f44e15a6-10ae-708f-bdc5-155993505d92"
      },
      "outputs": [],
      "source": [
        "nrows = 1000000\n",
        "df_train = pd.read_csv('../input/clicks_train.csv',nrows=nrows)\n",
        "df_test = pd.read_csv('../input/clicks_test.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9be09415-40b9-8e7c-64eb-ef7f7ec864b9"
      },
      "outputs": [],
      "source": [
        "df_train.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2708001a-3af0-121c-8e7e-f25d133fd28e"
      },
      "outputs": [],
      "source": [
        "train_count = df_train.groupby(['display_id'])['ad_id'].count().value_counts()\n",
        "test_count = df_test.groupby(['display_id'])['ad_id'].count().value_counts()\n",
        "\n",
        "train_size = train_count / np.sum(train_count)\n",
        "test_size = test_count / np.sum(test_count)\n",
        "del train_count,test_count\n",
        "\n",
        "fig,ax = plt.subplots(figsize=(8,6))\n",
        "\n",
        "sns.barplot(train_size.index,train_size.values,color='gray',ax=ax,label='train')\n",
        "sns.barplot(test_size.index,test_size.values,color='#80cbc4',ax=ax,label='test')\n",
        "plt.legend()\n",
        "plt.xlabel('Number of ad ')\n",
        "plt.ylabel('Proportion of set')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "8c3ee62a-3ee1-0096-cc75-4e95110d3881"
      },
      "outputs": [],
      "source": [
        "ad_count = df_train.groupby(['ad_id'])['ad_id'].count()\n",
        "\n",
        "for i in [10,50,100,1000]:\n",
        "    print('Ads that appear less than {} times :{}% '.format(i,round((ad_count < i).mean() * 100,2)))\n",
        "\n",
        "plt.figure(figsize=(8,6))\n",
        "plt.hist(ad_count.values,bins=50,log=True);\n",
        "plt.title('AD Distribution')\n",
        "plt.xlabel('Numer of times ad appeared' )\n",
        "plt.ylabel('Log Count of Ad')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e6375189-1951-350c-7c35-9a8381fa05f6"
      },
      "outputs": [],
      "source": [
        "category = pd.read_csv('../input/documents_categories.csv',nrows=nrows)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0aae8ad3-ae2d-bc81-1b0c-a1c300693378"
      },
      "outputs": [],
      "source": [
        "category.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "33091b6f-2969-1585-4565-c8993680f2f8"
      },
      "outputs": [],
      "source": [
        "category_count = category.groupby('category_id')['confidence_level'].count().sort_values()\n",
        "\n",
        "for i in [5000,10000,15000,20000]:\n",
        "    print('category that appeared less than {} times: {}%'.format(i,round((category_count <i).mean() * 100,2)))\n",
        "\n",
        "plt.figure(figsize=(8,6))\n",
        "plt.hist(category_count.values,bins=50,log=True)\n",
        "plt.title('Category Distribtuion')\n",
        "plt.xlabel('Document Category')\n",
        "plt.ylabel('Total Occurrence')\n",
        "plt.show()\n",
        "del category_count"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b59044a5-1504-5b75-ba54-842f6beae805"
      },
      "outputs": [],
      "source": [
        "entity = pd.read_csv('../input/documents_entities.csv',nrows=nrows)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3914cb0b-1dbe-e2f7-7a50-01a4d63343d8"
      },
      "outputs": [],
      "source": [
        "entity.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "25e77102-d046-addd-f70a-b39c9312b5e7"
      },
      "outputs": [],
      "source": [
        "entity_count = entity.groupby('entity_id')['confidence_level'].count().sort_values()\n",
        "\n",
        "for i in [50,100,150,500]:\n",
        "    print('entity that appeared less than {} times : {}'.format(i,round((entity_count < i).mean()*100,2)))\n",
        "\n",
        "plt.figure(figsize=(8,6))\n",
        "plt.hist(entitiy_count.values,bins=50,log=True)\n",
        "plt.title('Entity Distribution')\n",
        "plt.xlabel('Document Entity')\n",
        "plt.ylabel('Total Occurrence')\n",
        "plt.show()\n",
        "del entity_count"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d0b796aa-826b-f15e-8a70-cadd5d79976b"
      },
      "outputs": [],
      "source": [
        "meta = pd.read_csv('../input/documents_meta.csv',nrows=nrows)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "bdd9f27f-451c-8c0a-8be4-7d675f66c19e"
      },
      "outputs": [],
      "source": [
        "meta.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a5fade72-6256-9a6d-5ccd-cc13b08d5bd4"
      },
      "outputs": [],
      "source": [
        "plt.title('Publisher Distribution')\n",
        "sns.distplot(meta['publisher_id'].dropna())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "243c09ea-bc7c-df2a-5f98-d7c903511b8e"
      },
      "outputs": [],
      "source": [
        "sns.distplot(meta['source_id'].dropna())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d0daf244-d578-d286-92a5-8098b1ba1183"
      },
      "outputs": [],
      "source": [
        "topics = pd.read_csv('../input/documents_topics.csv',nrows=nrows)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f2841a43-db5c-65e7-ff09-6b6662ef20c7"
      },
      "outputs": [],
      "source": [
        "topics.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "44f4de8f-563d-47b9-ae0c-cb1562bfcf4a"
      },
      "outputs": [],
      "source": [
        "topics_count = topics.groupby('topic_id')['confidence_level'].count().sort_values()\n",
        "\n",
        "for i in [1500,2000,3500,4500,6000]:\n",
        "    print('topic that appeared less than {} times : {}'.format(i,round((topics_count < i).mean() *100,2)))\n",
        "\n",
        "plt.figure(figsize=(8,6))\n",
        "plt.hist(topics_count.values,bins=50,log=True);\n",
        "plt.title('Topic Distribution')\n",
        "plt.xlabel('Topics')\n",
        "plt.ylabel('Log Count Topics')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e6ef63a2-af3e-cd7b-9074-3207a6831fd7"
      },
      "outputs": [],
      "source": [
        "event = pd.read_csv('../input/events.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "8c11f2cc-700e-b3e8-b92f-b45ff8c88aa4"
      },
      "outputs": [],
      "source": [
        "event.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3a33246b-47e7-6f90-e642-f3e3a5ce7df4"
      },
      "outputs": [],
      "source": [
        "event.platform = event.platform.astype(str)\n",
        "event_count = event.platform.value_counts()\n",
        "fig,ax = plt.subplots(figsize=(12,4))\n",
        "sns.barplot(event_count.index,event_count.values,ax=ax)\n",
        "plt.title('Platform Count')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2310a454-11e7-25eb-b59a-d9ee96e2c2db"
      },
      "outputs": [],
      "source": [
        "uuid_counts = event.groupby('uuid')['uuid'].count().sort_values()\n",
        "\n",
        "print(uuid_counts.tail())\n",
        "\n",
        "for i in [2, 5, 10]:\n",
        "    print('Users that appear less than {} times: {}%'.format(i, round((uuid_counts < i).mean() * 100, 2)))\n",
        "    \n",
        "plt.figure(figsize=(12, 4))\n",
        "plt.hist(uuid_counts.values, bins=50, log=True)\n",
        "plt.xlabel('Number of times user appeared in set', fontsize=12)\n",
        "plt.ylabel('log(Count of users)', fontsize=12)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1bfa6470-867b-1816-2aaa-9c85df28809f"
      },
      "outputs": [],
      "source": ""
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}