{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"c10786fd-d349-4d02-8176-c1f0d8186963"},"outputs":[],"source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\n\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"2463ff64-b30c-4c5e-bee8-321ecdc3c128"},"outputs":[],"source":"from datetime import datetime\nimport time\nimport sys"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"166fef3e-f2a0-497b-9f9e-d6356dc77344"},"outputs":[],"source":"# functions\n\n# calculate difference in days between two dates represented by strings\ndef twoStrDateDifference(date_str_last, date_str_first):\n    delta = datetime.strptime(date_str_last, '%Y-%m-%d') - datetime.strptime(date_str_first, '%Y-%m-%d')\n    return delta.days\n# #######################################################\n"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"c5dcbaae-fca9-48b1-87f1-cb0c230cc86c"},"outputs":[],"source":"# read all data sets\ntrain = pd.read_csv('../input/train.csv', nrows = 1000)\nprint('train is load!')\n      \ntest = pd.read_csv('../input/test.csv', nrows = 1000)\nprint('test is load!')\n\ndest = pd.read_csv('../input/destinations.csv', nrows = 1000)\nprint('destinations is load!')"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"785a85ba-4e43-4b93-9065-fdc2eb970b8e"},"outputs":[],"source":"# try to check how many users from train is in test and how many new users is in test\ntrain_users = pd.read_csv('../input/train.csv', usecols = ['user_id'])\nprint('train_users set is load!')\n\ntest_users = pd.read_csv('../input/test.csv',  usecols = ['user_id'])\nprint('test_users set is load!')"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b78e6398-765a-40bb-b5a5-acb52cc852f2"},"outputs":[],"source":"nCommonUsers = len(set(test_users.values[:,0]).intersection(set(train_users.values[:,0])))\nnTestUsers = len(set(test_users.values[:,0]))\nnTrainUsers = len(set(train_users.values[:,0]))\n\nprint ('train set has ', nTrainUsers, ' users')\nprint ('test set has ', nTestUsers, ' users')\n\nprint ('test set has ', nCommonUsers, ' users from train set')\nprint ('test set has ', nTestUsers - nCommonUsers, ' new users')"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"f12df9a2-a62c-4983-ae9a-55226219e69f"},"outputs":[],"source":"train_chunk = pd.read_csv('../input/train.csv', chunksize = 100000)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"bbd0347f-7d77-4b80-9b6d-7e493212ff79"},"outputs":[],"source":"test_chunk = pd.read_csv('../input/test.csv', chunksize = 100000)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"be9b351c-4354-449c-834d-acf5c464dcd2"},"outputs":[],"source":"one_user_df = pd.DataFrame(columns=train.keys())\none_user_df_test = pd.DataFrame(columns=test.keys())"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"590484e0-38bc-41a3-83dd-46a8357c9289"},"outputs":[],"source":"t1 = time.time()\n\nfor chunk in train_chunk:\n    one_user_df = pd.concat([one_user_df, chunk[chunk['user_id'] == 28]])\n\nfor chunk in test_chunk:\n    one_user_df_test = pd.concat([one_user_df_test, chunk[chunk['user_id'] == 28]])\n    \nprint ('one_user_df and one_user_df_test are calculated!', (time.time() - t1)/60)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"699578d2-a14a-446f-90f6-a230595470b6"},"outputs":[],"source":"one_user_df"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"083a23d1-1514-4d43-b070-00e8cce2cb8e"},"outputs":[],"source":"one_user_df_test"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"60c93841-4f9b-477a-9795-d8412880de4b"},"outputs":[],"source":"# find all hotes clusters\nhotel_clusters = pd.read_csv('../input/train.csv', usecols=['hotel_cluster'])"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"33069632-78d0-4587-b9a5-4a2189fbb90e"},"outputs":[],"source":"hotel_clusters.hist()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"27634c77-3a1e-40cd-baa3-f0254679dac1"},"outputs":[],"source":"hotel_clusters_content = pd.read_csv('../input/train.csv', usecols=['hotel_cluster', 'srch_destination_id', 'hotel_country', 'hotel_continent','hotel_market'])"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"c6cdbe5d-a908-4d89-8abf-db3c033852f7"},"outputs":[],"source":"hotel_clusters_content.head()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"7f617098-2658-4ac6-a790-93641a72e6d9"},"outputs":[],"source":"sys.getsizeof(hotel_clusters_content)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"8efd73ce-a979-4302-b036-18a8a03782ba"},"outputs":[],"source":"def getCountryListForCluster(cluster_id):\n    res = np.unique(hotel_clusters_content[hotel_clusters_content['hotel_cluster'] == cluster_id]['hotel_country'].values)\n    return (res, res.size)\n\ndef getContinentListForCluster(cluster_id):\n    res = np.unique(hotel_clusters_content[hotel_clusters_content['hotel_cluster'] == cluster_id]['hotel_continent'].values)\n    return (res, res.size)\n\ndef getMarketListForCluster(cluster_id):\n    res = np.unique(hotel_clusters_content[hotel_clusters_content['hotel_cluster'] == cluster_id]['hotel_market'].values)\n    return (res, res.size)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"39964335-b9cc-4bdd-91e5-893c599345d4"},"outputs":[],"source":"getMarketListForCluster(3)"}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}