{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9d04bcd2-2eac-3109-b469-5a8470b79cb4"
      },
      "outputs": [],
      "source": [
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "import os\n",
        "import gc # clean memory a lot\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "%matplotlib inline\n",
        "\n",
        "p = sns.color_palette()\n",
        "\n",
        "print('#File size')\n",
        "for f in os.listdir('../input'):\n",
        "    if 'zip' not in f:\n",
        "        print(f.ljust(30) + str(round(os.path.getsize('../input/' + f) / 1000000, 2)) + 'MB')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5fda909c-a225-a512-91c3-908676f5b852"
      },
      "outputs": [],
      "source": [
        "# Clicks \n",
        "df_train = pd.read_csv('../input/clicks_train.csv')\n",
        "df_test = pd.read_csv('../input/clicks_test.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9036bb21-89f2-173f-2d85-9805dbbbec4d"
      },
      "outputs": [],
      "source": [
        "sizes_train = df_train.groupby('display_id')['ad_id'].count().value_counts()\n",
        "sizes_test = df_test.groupby('display_id')['ad_id'].count().value_counts()\n",
        "sizes_train = sizes_train / np.sum(sizes_train)\n",
        "sizes_test = sizes_test / np.sum(sizes_test)\n",
        "\n",
        "plt.figure(figsize=(12, 4))\n",
        "sns.barplot(sizes_train.index, sizes_train.values, alpha=0.8, color=p[0], label='train')\n",
        "sns.barplot(sizes_test.index, sizes_test.values, alpha=0.8, color=p[1], label='test')\n",
        "plt.legend()\n",
        "plt.xlabel('Number of Ads in display', fontsize=12)\n",
        "plt.ylabel('Proportion of set', fontsize=12)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "64efc725-7582-f4dc-65d3-5ace1dea1af2"
      },
      "outputs": [],
      "source": [
        "sizes_test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7a708a9a-6750-5b17-bc4b-314bfa9c958c"
      },
      "outputs": [],
      "source": [
        "ad_usage_train = df_train.groupby('ad_id')['ad_id'].count()\n",
        "\n",
        "for i in [2, 10, 50, 100, 1000]:\n",
        "    print('Ads that appear less than {} times: {}%'.format(i, round((ad_usage_train < i).mean() * 100, 2)))\n",
        "    \n",
        "plt.figure(figsize=(12, 6))\n",
        "plt.hist(ad_usage_train.values, bins=50, log=True)\n",
        "plt.xlabel('Number of times ad appeared', fontsize=12)\n",
        "plt.ylabel('log(Count of displays with ad)', fontsize=12)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "8ce20037-c55c-c2ef-f76b-17b796fd5982"
      },
      "outputs": [],
      "source": [
        "# Check how many ads in the test set are not in the training set\n",
        "ad_prop = len(set(df_test.ad_id.unique()).intersection(df_train.ad_id.unique())) / len(df_test.ad_id.unique())\n",
        "print('Proportion of test ads in test that are in training: {}%'.format(round(ad_prop * 100, 2)))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a923d0cf-513e-e340-0b41-2ab6a5a0b714"
      },
      "outputs": [],
      "source": [
        "# Events\n",
        "try:del df_train, df_test \n",
        "except:pass\n",
        "gc.collect()\n",
        "    \n",
        "events = pd.read_csv('../input/events.csv')\n",
        "\n",
        "events.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "26da8f20-8e9f-ab58-1ee1-2b295e1354c0"
      },
      "outputs": [],
      "source": [
        "print('Shape:', events.shape)\n",
        "print('Columns:', events.columns.tolist())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "34302d3a-61f3-0a96-001c-6274bd2e4c6e"
      },
      "outputs": [],
      "source": [
        "# \u5206\u6790Platform\n",
        "plat = events.platform.value_counts()\n",
        "print(plat)\n",
        "print('\\nUnique values of platform:', events.platform.unique())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "27cdc484-a8c9-d7f8-e117-6430b0557f8e"
      },
      "outputs": [],
      "source": [
        "events.platform = events.platform.astype(str) # \u5c06str\u578b\u7684'1','2','3'\u548c\u6574\u578b\u76841,2,3\u770b\u4f5c\u4e00\u6837\n",
        "plat = events.platform.value_counts()\n",
        "\n",
        "plt.figure(figsize=(12, 4))\n",
        "sns.barplot(plat.index, plat.values, alpha=0.8, color=p[2])\n",
        "plt.xlabel('Platform', fontsize=12)\n",
        "plt.ylabel('Occurence count', fontsize=12)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6ddc8112-b598-501b-340c-61152186a38b"
      },
      "outputs": [],
      "source": [
        "# \u5206\u6790uuid\n",
        "uuid_counts = events.groupby('uuid')['uuid'].count().sort_values()\n",
        "\n",
        "print(uuid_counts.tail())\n",
        "\n",
        "for i in [2, 5, 10]:\n",
        "    print('Users that appear less than {} times: {}%'.format(i, round((uuid_counts < i).mean() * 100, 2)))\n",
        "    \n",
        "plt.figure(figsize=(12, 4))\n",
        "plt.hist(uuid_counts.values, bins=50, log=True)\n",
        "plt.xlabel('Number of times user appeared in set', fontsize=12)\n",
        "plt.ylabel('log(Count of users)', fontsize=12)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "221dcca8-e6a3-e67a-1f25-eb0ef500e15d"
      },
      "outputs": [],
      "source": [
        "# Categorices\n",
        "try del:events\n",
        "except:pass\n",
        "gc.collect() # \u5783\u573e\u56de\u6536\uff0c\u91ca\u653e\u5185\u5b58\n",
        "\n",
        "topics = pd.read_csv('../input/documents_topics.csv')\n",
        "print('Columns:', topics.columns.tolist())\n",
        "primt('Number of unique topics:', len(topics.topics_id.unique()))\n",
        "\n",
        "topics.head()"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}