{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\n\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output."
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "from datetime import datetime\nimport time\nimport sys"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# functions\n\n# calculate difference in days between two dates represented by strings\ndef twoStrDateDifference(date_str_last, date_str_first):\n    delta = datetime.strptime(date_str_last, '%Y-%m-%d') - datetime.strptime(date_str_first, '%Y-%m-%d')\n    return delta.days\n# #######################################################\n\n"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# read all data sets\ntrain = pd.read_csv('../input/train.csv', nrows = 1000)\nprint('train is load!')\n      \ntest = pd.read_csv('../input/test.csv', nrows = 1000)\nprint('test is load!')\n\ndest = pd.read_csv('../input/destinations.csv', nrows = 1000)\nprint('destinations is load!')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# try to check how many users from train is in test and how many new users is in test\ntrain_users = pd.read_csv('../input/train.csv', usecols = ['user_id'])\nprint('train_users set is load!')\n\ntest_users = pd.read_csv('../input/test.csv',  usecols = ['user_id'])\nprint('test_users set is load!')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "nCommonUsers = len(set(test_users.values[:,0]).intersection(set(train_users.values[:,0])))\nnTestUsers = len(set(test_users.values[:,0]))\nnTrainUsers = len(set(train_users.values[:,0]))\n\nprint ('train set has ', nTrainUsers, ' users')\nprint ('test set has ', nTestUsers, ' users')\n\nprint ('test set has ', nCommonUsers, ' users from train set')\nprint ('test set has ', nTestUsers - nCommonUsers, ' new users')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train_chunk = pd.read_csv('../input/train.csv', chunksize = 100000)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test_chunk = pd.read_csv('../input/test.csv', chunksize = 100000)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "one_user_df = pd.DataFrame(columns=train.keys())\none_user_df_test = pd.DataFrame(columns=test.keys())"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "t1 = time.time()\n\nfor chunk in train_chunk:\n    one_user_df = pd.concat([one_user_df, chunk[chunk['user_id'] == 28]])\n\nfor chunk in test_chunk:\n    one_user_df_test = pd.concat([one_user_df_test, chunk[chunk['user_id'] == 28]])\n    \nprint ('one_user_df and one_user_df_test are calculated!', (time.time() - t1)/60)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "one_user_df"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "one_user_df_test"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# find all hotes clusters\nhotel_clusters = pd.read_csv('../input/train.csv', usecols=['hotel_cluster'])"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "hotel_clusters.hist()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "hotel_clusters_content = pd.read_csv('../input/train.csv', usecols=['hotel_cluster', 'srch_destination_id', 'hotel_country', 'hotel_continent','hotel_market'])"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "hotel_clusters_content.head()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "sys.getsizeof(hotel_clusters_content)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "def getCountryListForCluster(cluster_id):\n    res = np.unique(hotel_clusters_content[hotel_clusters_content['hotel_cluster'] == cluster_id]['hotel_country'].values)\n    return (res, res.size)\n\ndef getContinentListForCluster(cluster_id):\n    res = np.unique(hotel_clusters_content[hotel_clusters_content['hotel_cluster'] == cluster_id]['hotel_continent'].values)\n    return (res, res.size)\n\ndef getMarketListForCluster(cluster_id):\n    res = np.unique(hotel_clusters_content[hotel_clusters_content['hotel_cluster'] == cluster_id]['hotel_market'].values)\n    return (res, res.size)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "getMarketListForCluster(3)"
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}