{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport os\nprint(os.listdir(\"../input\"))\n\n# Read input files\ndestinations = pd.read_csv(\"../input/destinations.csv\")\n# test = pd.read_csv(\"../input/test.csv\")\n# train = pd.read_csv(\"../input/train.csv\")\n\n# Read and split train data into train_new and test_new\ntrain_new = pd.DataFrame(columns=[\"date_time\", \"site_name\", \"posa_continent\", \"user_location_country\", \"user_location_region\", \"user_location_city\", \"orig_destination_distance\", \"user_id\", \"is_mobile\", \"is_package\", \"channel\", \"srch_ci\", \"srch_co\", \"srch_adults_cnt\", \"srch_children_cnt\", \"srch_rm_cnt\", \"srch_destination_id\", \"srch_destination_type_id\", \"hotel_continent\", \"hotel_country\", \"hotel_market\", \"is_booking\", \"cnt\", \"hotel_cluster\"])\ntest_new = pd.DataFrame(columns=[\"date_time\", \"site_name\", \"posa_continent\", \"user_location_country\", \"user_location_region\", \"user_location_city\", \"orig_destination_distance\", \"user_id\", \"is_mobile\", \"is_package\", \"channel\", \"srch_ci\", \"srch_co\", \"srch_adults_cnt\", \"srch_children_cnt\", \"srch_rm_cnt\", \"srch_destination_id\", \"srch_destination_type_id\", \"hotel_continent\", \"hotel_country\", \"hotel_market\", \"is_booking\", \"cnt\", \"hotel_cluster\"])\nchunksize = 10 ** 6\nfor chunk in pd.read_csv(\"../input/train.csv\", chunksize=chunksize):\n    chunk[\"date_time\"] = pd.to_datetime(chunk[\"date_time\"])\n    chunk[\"year\"] = chunk[\"date_time\"].dt.year\n    chunk[\"month\"] = chunk[\"date_time\"].dt.month\n    \n    # In the original train and test DataFrames, test contained data from 2015, and train contained data from 2013 and 2014. \n    # We split this data so that anything Jan - Aug 2013 is in train_new, and the rest of 2013 is in test_new. \n    # This gives us smaller training and testing sets with similar characteristics to train and test.\n    train_new = train_new.append(chunk[((chunk.year == 2013) & (chunk.month <= 8))], ignore_index=True)\n    test_new = test_new.append(chunk[((chunk.year == 2013) & (chunk.month > 8))], ignore_index=True)\n\nprint(train_new.shape) # train_new.shape = (7624260, 26)\nprint(test_new.shape) # test_new.shape =  (3562621, 26)\nprint(\"Done loading data\")","execution_count":10,"outputs":[]},{"metadata":{"_cell_guid":"1a5c48d0-5ab8-4d55-b65f-d7daa3f658df","_uuid":"f3fb1b677f580cd45bb62cf5a64a955293f4e447","trusted":true},"cell_type":"code","source":"import random\n\n# Pick 200000 users from train_new set and pick coresponding set of users from test_new\n# That will further downsample our dataset, and keeps the characteristic of the original data set\n# which is that all the test user ids can be found in the train\nunique_users = set(train_new.user_id.unique())\n\nsel_user_ids = random.sample(unique_users, 300000)\ntrain_new_1 = train_new[train_new.user_id.isin(sel_user_ids)]\ntest_new_1 = test_new[test_new.user_id.isin(sel_user_ids)]\n\n# Test contains only booking events, so we'll need to sample t2 to only contain bookings as well\ntest_new_1 = test_new_1[test_new_1.is_booking == True]\n\nprint(train_new_1.shape)\nprint(test_new_1.shape)\n\ntrain_new_1.to_csv('train_new.csv', sep=',')\ntest_new_1.to_csv('test_new.csv', sep=',')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"5db1a04cbeef88f8d29f3f0ef1ce220616a8a21d"},"cell_type":"code","source":"# Start your code from here. No need to rerun the first two cells of code\n# The downsample data is already saved in file train_new.csv and test_new.csv\n","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}