{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Introduction #\nThis kernel is writtern for the [Expedia Hotel Recommendations](https://www.kaggle.com/c/expedia-hotel-recommendations) competetion. If you Like the notebook and think that it helped you, <font color=\"red\"><b> please upvote</b></font>.\n\n---\n## Table of Content\n1. Data Preprocessing\n2. Modeling and Evaluation\n3. Final Prediction & Submission"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# pandas\nimport pandas as pd\nfrom pandas import Series,DataFrame\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set_style('whitegrid')\n%matplotlib inline\n\n# machine learning\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import SVC, LinearSVC\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.naive_bayes import GaussianNB\nimport xgboost as xgb\n\n# get expedia & test csv files as a DataFrame\ntrain_df = pd.read_csv('../input/train.csv', nrows=10000)\ntest_df    = pd.read_csv('../input/test.csv', nrows=10000)\ndestination = pd.read_csv('../input/destinations.csv', nrows=100000)\n\ntrain_df.info()\nprint(\"----------------------------\")\ntest_df.info()\n\n# preview the data\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Preprocessing\nThe first step is to clean and pre-process the data and perform exploratory analysis to get some interesting insights into the process of choosing a hotel.\n\n* Remove the users who did not booked the hotel\n* Identify the searches by each user belonging to a specific type of destination\n* orig_destination_distance contains Nan values\n* The check-in and check-out dates to find the duration of the stay for each of the entries in the training set.\n"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Plot \nbookings_df = train_df[train_df[\"is_booking\"] == 1]\nfig, (axis1,axis2) = plt.subplots(2,1,figsize=(15,10))\n# What are the most countries the customer travel from?\nsns.countplot('user_location_country',data=bookings_df.sort_values(by=['user_location_country']),ax=axis1,palette=\"Set3\")\n\n# What are the most countries the customer travel to?\nsns.countplot('hotel_country',data=bookings_df.sort_values(by=['hotel_country']),ax=axis2,palette=\"Set3\")\n\n# Combine both plots\n# fig, (axis1) = plt.subplots(1,1,figsize=(15,5))\n# sns.distplot(bookings_df[\"hotel_country\"], kde=False, rug=False, bins=25, ax=axis1)\n# sns.distplot(bookings_df[\"user_location_country\"], kde=False, rug=False, bins=25, ax=axis1)\n\n# Where do most of the customers from a country travel?\nuser_country_id = 66\nfig, (axis1) = plt.subplots(1,1,figsize=(15,10))\ncountry_customers = train_df[train_df[\"user_location_country\"] == user_country_id]\ncountry_customers[\"hotel_country\"].value_counts().plot(kind='bar',colormap=\"Set3\",figsize=(15,5))\n\n# Plot frequency for each hotel_clusters\ntrain_df[\"hotel_cluster\"].value_counts().plot(kind='bar',colormap=\"Set3\",figsize=(15,5))\n\n# What are the most frequent hotel clusters booked by customers from a country?\nfig, (axis1) = plt.subplots(1,1,figsize=(15,10))\ncustomer_clusters = train_df[train_df[\"user_location_country\"] == user_country_id][\"hotel_cluster\"]\ncustomer_clusters.value_counts().plot(kind='bar',colormap=\"Set3\",figsize=(15,5))\n\n# What are the most frequent hotel clusters in a country?\ncountry_id = 50\nfig, (axis1) = plt.subplots(1,1,figsize=(15,10))\ncountry_clusters = train_df[train_df[\"hotel_country\"] == country_id][\"hotel_cluster\"]\ncountry_clusters.value_counts().plot(kind='bar',colormap=\"Set3\",figsize=(15,5))\n\n# Plot post_continent & hotel_continent\nfig, ((axis1,axis2),(axis3,axis4)) = plt.subplots(2,2,figsize=(15,10))\n\n# Plot frequency for each posa_continent\nsns.countplot('posa_continent', data=train_df,order=[0,1,2,3,4],palette=\"Set3\",ax=axis1)\n\n# Plot frequency for each posa_continent decomposed by hotel_continent\nsns.countplot('posa_continent', hue='hotel_continent',data=train_df,order=[0,1,2,3,4],palette=\"Set3\",ax=axis2)\n\n# Plot frequency for each hotel_continent\nsns.countplot('hotel_continent', data=train_df,order=[0,2,3,4,5,6],palette=\"Set3\",ax=axis3)\n\n# Plot frequency for each hotel_continent decomposed by posa_continent\nsns.countplot('hotel_continent', hue='posa_continent', data=train_df, order=[0,2,3,4,5,6],palette=\"Set3\",ax=axis4)\n\n# Plot frequency of is_mobile & is_package\nfig, (axis1,axis2) = plt.subplots(1,2,figsize=(15,3))\n\n# What's the frequency of bookings through mobile?\nsns.countplot(x='is_mobile',data=bookings_df, order=[0,1], palette=\"Set3\", ax=axis1)\n\n# What's the frequency of bookings with package?\nsns.countplot(x='is_package',data=bookings_df, order=[0,1], palette=\"Set3\", ax=axis2)\n\n# What's the most impactful channel?\nfig, (axis1) = plt.subplots(1,1,figsize=(15,3))\nsns.countplot(x='channel', order=list(range(0,10)), data=train_df, palette=\"Set3\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# heatmap\nfig, ax = plt.subplots()\nfig.set_size_inches(15, 10)\nsns.heatmap(train_df.corr(),cmap='coolwarm',ax=ax,annot=True,linewidths=2)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Feature Engineering\n**Additional features from date columns**\n* stay_dur: number of duration of stay\n* no_of_days_bet_booking: number of days between the booking and\n* Cin_day: Check-in day\n* Cin_month: Check-in month\n* Cin_year: Check-out year"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Function to convert date object into relevant attributes\ndef convert_date_into_days(df):\n    df['srch_ci'] = pd.to_datetime(df['srch_ci'])\n    df['srch_co'] = pd.to_datetime(df['srch_co'])\n    df['date_time'] = pd.to_datetime(df['date_time'])\n    \n    df['stay_dur'] = (df['srch_co'] - df['srch_ci']).astype('timedelta64[D]')\n    df['no_of_days_bet_booking'] = (df['srch_ci'] - df['date_time']).astype('timedelta64[D]')\n    \n    # For hotel check-in\n    # Month, Year, Day\n    df['Cin_day'] = df[\"srch_ci\"].apply(lambda x: x.day)\n    df['Cin_month'] = df[\"srch_ci\"].apply(lambda x: x.month)\n    df['Cin_year'] = df[\"srch_ci\"].apply(lambda x: x.year)\n    \nconvert_date_into_days(train_df)\nconvert_date_into_days(test_df)\n\n# Count the bookings in each month\nfig, ax = plt.subplots()\nfig.set_size_inches(13, 8)\nsns.countplot('Cin_month',data=train_df[train_df[\"is_booking\"] == 1],order=list(range(1,13)),ax=ax)\n\n# Count the bookings as per the day\nfig, ax = plt.subplots()\nfig.set_size_inches(13, 8)\nsns.countplot('Cin_day',data=train_df[train_df[\"is_booking\"] == 1],order=list(range(1,32)),ax=ax)\n\n# Count the bookings as per the stay_duration\nfig, ax = plt.subplots()\nfig.set_size_inches(13, 8)\nsns.countplot('stay_dur',data=train_df[train_df[\"is_booking\"] == 1],ax=ax)\n\n# drop unnecessary columns, these columns won't be useful in analysis and prediction\ntest_user_id = test_df['user_id']\ncolumns = ['date_time', 'srch_ci','user_id','srch_destination_type_id','srch_destination_id', 'site_name', 'user_location_region', 'user_location_city', \n                              'user_id', 'srch_co', 'srch_adults_cnt', 'srch_children_cnt', 'srch_rm_cnt']\ntrain_df.drop(columns=columns,axis=1,inplace=True)\ntest_df.drop(columns=columns,axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check the percentage of Nan in dataset\ntotal = train_df.isnull().sum().sort_values(ascending=False)\npercent = (train_df.isnull().sum()/train_df['hotel_cluster'].count()).sort_values(ascending=False)\nmissing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_data\n\n# Fill nan with the day which has max occurence\ntrain_df['Cin_day'] = train_df['Cin_day'].fillna(26.0)\ntrain_df['Cin_month'] = train_df['Cin_month'].fillna(8.0)\ntrain_df['Cin_year'] = train_df['Cin_year'].fillna(2014.0)\ntrain_df['stay_dur'] = train_df['stay_dur'].fillna(1.0)\ntrain_df['no_of_days_bet_booking'] = train_df['no_of_days_bet_booking'].fillna(0.0)\n\n# Fill average values in place for nan, fill with mean\ntrain_df['orig_destination_distance'].fillna(train_df['orig_destination_distance'].mean(), inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction & Submission"},{"metadata":{},"cell_type":"markdown","source":"## A simple solution\nA very simple solution implemented with pandas to use the \"most popular local hotel\" as recommendation.\n\n**Step 1**\n\nRead in the train data using only the necessary columns. Specifying dtypes helps reduce memory requirements.\nThe file is read in chunks of 1 million rows each. In each chunk we count the number of rows and number of bookings for every destination-hotel cluster combination.\n\n**Step 2**\n\nNext we aggregate again to compute the total number of bookings over all chunks.\nCompute the number of clicks by subtracting the number of bookings from total row counts.\nCompute the 'relevance' of a hotel cluster with a weighted sum of bookings and clicks.\n\n**Step 3**\nRead in the test data and merge most popular hotel clusters."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Step 1\ntrain = pd.read_csv('../input/train.csv',\n                    dtype={'is_booking':bool,'srch_destination_id':np.int32, 'hotel_cluster':np.int32},\n                    usecols=['srch_destination_id','is_booking','hotel_cluster'],\n                    chunksize=1000000)\naggs = []\nprint('-'*38)\nfor chunk in train:\n    agg = chunk.groupby(['srch_destination_id',\n                         'hotel_cluster'])['is_booking'].agg(['sum','count'])\n    agg.reset_index(inplace=True)\n    aggs.append(agg)\n    print('.',end='')\nprint('')\naggs = pd.concat(aggs, axis=0)\naggs.head()\n\n# Step 2\nCLICK_WEIGHT = 0.05\nagg = aggs.groupby(['srch_destination_id','hotel_cluster']).sum().reset_index()\nagg['count'] -= agg['sum']\nagg = agg.rename(columns={'sum':'bookings','count':'clicks'})\nagg['relevance'] = agg['bookings'] + CLICK_WEIGHT * agg['clicks']\nagg.head()\n\n# Define a function to get most popular hotels for a destination group.\ndef most_popular(group, n_max=5):\n    relevance = group['relevance'].values\n    hotel_cluster = group['hotel_cluster'].values\n    most_popular = hotel_cluster[np.argsort(relevance)[::-1]][:n_max]\n    return np.array_str(most_popular)[1:-1] # remove square brackets\n\n# Get most popular hotel clusters for all destinations.\nmost_pop = agg.groupby(['srch_destination_id']).apply(most_popular)\nmost_pop = pd.DataFrame(most_pop).rename(columns={0:'hotel_cluster'})\nmost_pop.head()\n\n# Step 3: Read in the test data and merge most popular hotel clusters.\ntest = pd.read_csv('../input/test.csv',\n                    dtype={'srch_destination_id':np.int32},\n                    usecols=['srch_destination_id'],)\ntest = test.merge(most_pop, how='left',left_on='srch_destination_id',right_index=True)\ntest.head()\n\n# Check hotel_cluster column in test for null values\ntest.hotel_cluster.isnull().sum()\n\n# Let's fill nas with hotel clusters that are most popular overall.\nmost_pop_all = agg.groupby('hotel_cluster')['relevance'].sum().nlargest(5).index\nmost_pop_all = np.array_str(most_pop_all)[1:-1]\nprint(most_pop_all)\n\ntest.hotel_cluster.fillna(most_pop_all,inplace=True)\n\ntest.hotel_cluster.to_csv('submission.csv',header=True, index_label='id')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}