{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "import numpy as np\nimport pandas as pd\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train = pd.read_csv('../input/train.csv',chunksize=1000000)\n\naggs = []\ndata = pd.DataFrame()\nprint('-'*38)\nfor chunk in train:\n    data = data.append(chunk.head())\n    agg = chunk.groupby(['srch_destination_id',\n                         'hotel_cluster'])['is_booking'].agg(['sum','count'])\n    agg.reset_index(inplace=True)\n    aggs.append(agg)\n    print('.',end='')\nprint('')\naggs = pd.concat(aggs, axis=0)\naggs.head()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "data.columns"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": ""
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "CLICK_WEIGHT = 0.05\nagg = aggs.groupby(['srch_destination_id','hotel_cluster']).sum().reset_index()\nagg['count'] -= agg['sum']\nagg = agg.rename(columns={'sum':'bookings','count':'clicks'})\nagg['relevance'] = agg['bookings'] + CLICK_WEIGHT * agg['clicks']\nagg.head()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "agg[agg['srch_destination_id']==1]"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "for chunk in train:\n    print(len(chunk))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "def most_popular(group, n_max=5):\n    relevance = group['relevance'].values\n    hotel_cluster = group['hotel_cluster'].values\n    most_popular = hotel_cluster[np.argsort(relevance)[::-1]][:n_max]\n    return np.array_str(most_popular)[1:-1] # remove square brackets"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "Get most popular hotel clusters for all destinations."
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "most_pop = agg.groupby(['srch_destination_id']).apply(most_popular)\nmost_pop = pd.DataFrame(most_pop).rename(columns={0:'hotel_cluster'})\nmost_pop.head()"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "### Predict for test data\nRead in the test data and merge most popular hotel clusters."
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test = pd.read_csv('../input/test.csv',\n                    dtype={'srch_destination_id':np.int32},\n                    usecols=['srch_destination_id'],)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test = test.merge(most_pop, how='left',left_on='srch_destination_id',right_index=True)\ntest.head()"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "Check hotel_cluster column in test for null values."
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test.hotel_cluster.isnull().sum()"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "Looks like there's about 14k new destinations in test. Let's fill nas with hotel clusters that are most popular overall."
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "most_pop_all = agg.groupby('hotel_cluster')['relevance'].sum().nlargest(5).index\nmost_pop_all = np.array_str(most_pop_all)[1:-1]\nmost_pop_all"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test.hotel_cluster.fillna(most_pop_all,inplace=True)"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "Save the submission."
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test.hotel_cluster.to_csv('predicted_with_pandas.csv',header=True, index_label='id')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": ""
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}