{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e390016d-6130-85d4-bdc5-c292f28f8a15"
      },
      "outputs": [],
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load in \n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "# Input data files are available in the \"../input/\" directory.\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
        "\n",
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
        "import gc\n",
        "\n",
        "# Any results you write to the current directory are saved as output."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "af8af30c-8d4b-d57f-a436-5f1ec07ded7b"
      },
      "outputs": [],
      "source": [
        "click_test_df = pd.read_csv(\"../input/clicks_test.csv\", chunksize = 10)\n",
        "clicks_train_df = pd.read_csv(\"../input/clicks_train.csv\", chunksize = 10)\n",
        "documents_categories_df = pd.read_csv(\"../input/documents_categories.csv\", chunksize = 10)\n",
        "documents_entities_df = pd.read_csv(\"../input/documents_entities.csv\", chunksize = 10)\n",
        "documents_meta_df = pd.read_csv(\"../input/documents_meta.csv\", chunksize = 10)\n",
        "documents_topics_df = pd.read_csv(\"../input/documents_topics.csv\", chunksize = 10)\n",
        "events_df = pd.read_csv(\"../input/events.csv\", chunksize = 10)\n",
        "page_views_sample_df = pd.read_csv(\"../input/page_views_sample.csv\", chunksize = 10)\n",
        "promoted_content_df = pd.read_csv(\"../input/promoted_content.csv\", chunksize = 10)\n",
        "sample_submission_df = pd.read_csv(\"../input/sample_submission.csv\", chunksize = 10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "205bc901-2ac9-f491-e160-1d9b60aa52f4"
      },
      "outputs": [],
      "source": [
        "print(\"clicks_train_df:\\t\",clicks_train_df.get_chunk().columns.tolist())\n",
        "print(\"documents_categories_df:\\t\",documents_categories_df.get_chunk().columns.tolist())\n",
        "print(\"documents_entities_df:\\t\",documents_entities_df.get_chunk().columns.tolist())\n",
        "print(\"documents_meta_df:\\t\",documents_meta_df.get_chunk().columns.tolist())\n",
        "print(\"documents_topics_df:\\t\",documents_topics_df.get_chunk().columns.tolist())\n",
        "print(\"events_df:\\t\",events_df.get_chunk().columns.tolist())\n",
        "print(\"page_views_sample_df:\\t\",page_views_sample_df.get_chunk().columns.tolist())\n",
        "print(\"promoted_content_df:\\t\",promoted_content_df.get_chunk().columns.tolist())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ac918b53-f682-be18-1a62-03ff859e6744"
      },
      "outputs": [],
      "source": [
        "#clicks_train_df = pd.read_csv(\"../input/clicks_train.csv\",chunksize=10000000)\n",
        "#promoted_content_df = pd.read_csv(\"../input/promoted_content.csv\")\n",
        "#train_promoted_df = pd.merge(clicks_train_df.get_chunk(), promoted_content_df,on = 'ad_id')\n",
        "#del clicks_train_df\n",
        "#gc.collect()\n",
        "#train_promoted_df.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cb86c3ab-fc6b-f9be-bc76-bf8dd4813da1"
      },
      "outputs": [],
      "source": [
        "#print(train_promoted_df.columns.tolist())\n",
        "#documents_categories_df = pd.read_csv(\"../input/documents_categories.csv\")\n",
        "#documents_topics_df = pd.read_csv(\"../input/documents_topics.csv\")\n",
        "#documents_entities_df = pd.read_csv(\"../input/documents_entities.csv\")\n",
        "#documents_meta_df = pd.read_csv(\"../input/documents_meta.csv\")\n",
        "events_df = pd.DataFrame()\n",
        "for chunk in pd.read_csv(\"../input/events.csv\",chunksize = 1000000):\n",
        "    events_df = events_df.append(chunk)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0e58a0fd-22da-225b-dd2c-384b795b84e2"
      },
      "outputs": [],
      "source": [
        "#del documents_topics_df,documents_entities_df\n",
        "#train_promoted_df\n",
        "#gc.collect()\n",
        "len(events_df.get_chunk())\n",
        "#print(len(set(documents_categories_df.document_id)))\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c7f8c669-e04e-43cc-961d-70007844f058"
      },
      "outputs": [],
      "source": [
        "print(len(set(documents_categories_df.document_id).intersection(set(events_df.document_id))))"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}