{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b9d18d41-b32b-453a-658a-7e717eefa569"
      },
      "outputs": [],
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load in \n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "# Input data files are available in the \"../input/\" directory.\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
        "\n",
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
        "\n",
        "# Any results you write to the current directory are saved as output."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d4c33781-1cea-01bc-f91a-b0888138aff9"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np \n",
        "\n",
        "reg = 10 # trying anokas idea of regularization\n",
        "eval = True\n",
        "\n",
        "train = pd.read_csv(\"../input/clicks_train.csv\")\n",
        "\n",
        "if eval:\n",
        "\tids = train.display_id.unique()\n",
        "\tids = np.random.choice(ids, size=len(ids)//10, replace=False)\n",
        "\n",
        "\tvalid = train[train.display_id.isin(ids)]\n",
        "\ttrain = train[~train.display_id.isin(ids)]\n",
        "\t\n",
        "\tprint (valid.shape, train.shape)\n",
        "\n",
        "cnt = train[train.clicked==1].ad_id.value_counts()\n",
        "cntall = train.ad_id.value_counts()\n",
        "del train"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4a385eb0-bbb1-582e-8702-65745290b3be"
      },
      "outputs": [],
      "source": [
        "cnt.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5f559782-0fa6-3ad4-6e5b-6c13fa2e44ef"
      },
      "outputs": [],
      "source": [
        "cntall.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "db037dfa-d172-6d73-4217-e1a94949178e"
      },
      "outputs": [],
      "source": [
        "rinds = cnt.index[1:1000]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0a804a2f-5de8-4a1b-d17e-cade57e10775"
      },
      "outputs": [],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "plt.figure(figsize=(12, 6))\n",
        "plt.scatter(cnt[rinds], [cnt[i] / (cntall[i] )for i in rinds])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e122b757-d531-847c-cb00-1befcf3c4c65"
      },
      "outputs": [],
      "source": [
        "rinds2 = cnt.index[1000:2000]\n",
        "plt.figure(figsize=(8, 6))\n",
        "plt.scatter(cnt[rinds2], [cnt[i] / (cntall[i] + 100 )for i in rinds2])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "617c623a-cf16-999a-79fa-c0db6d81d269"
      },
      "outputs": [],
      "source": ""
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}