{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\n\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error\nfrom sklearn.model_selection import cross_val_score, train_test_split\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.preprocessing import StandardScaler\n\nfrom threading import Thread\n\n%matplotlib inline\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a8bd35c150bfe00dade8fda3e5e70984813d858c"},"cell_type":"markdown","source":"**Data Loading**"},{"metadata":{"trusted":true,"_uuid":"8893b514e93c487652b486f21854780b0f714771"},"cell_type":"code","source":"df = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"838efbf9465b94bbd78ff3585f67f291c2db566e"},"cell_type":"code","source":"df.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"22078a04cd742e8f4829fe10d1c55877d8357d78"},"cell_type":"markdown","source":"**Data Exploration **"},{"metadata":{"trusted":true,"_uuid":"9111ad47babd27d9e3f7c59abc61ab7fbbc18087"},"cell_type":"code","source":"df.groupby('vendor_id').count()['id'].plot.bar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46c33d5ebc1f3be2fd150e9d7a59275108ac77ee"},"cell_type":"code","source":"one = df.groupby('vendor_id').count()['id'].iloc[0:1]\ntwo = df.groupby('vendor_id').count()['id'].iloc[1:2]\n\nprint('Vendor_id 2 has received ' , np.subtract(two, one).values[0], 'more bookings then Vendor_id 1.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02f6e80f2ce69e1d581844f07e20f1f63f9722a9"},"cell_type":"code","source":"f, axes = plt.subplots(2,figsize=(30, 15), sharex=False, sharey = False)\nsns.distplot(df['pickup_latitude'], label = 'pickup_latitude',color=\"y\",bins = 100, ax=axes[0], hist=False).tick_params(labelsize=20)\nsns.distplot(df['pickup_longitude'], label = 'pickup_longitude',color=\"y\",bins =100, ax=axes[1], hist=False).tick_params(labelsize=20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8fd87d4573217479cbc863e9bd49175d8ea4c22b"},"cell_type":"code","source":"f, axes = plt.subplots(2,figsize=(30, 15), sharex=False, sharey = False)\nsns.distplot(df['dropoff_latitude'], label = 'dropoff_latitude',color=\"y\",bins = 100, ax=axes[0], hist=False).tick_params(labelsize=20)\nsns.distplot(df['dropoff_longitude'], label = 'dropoff_longitude',color=\"y\",bins =100, ax=axes[1], hist=False).tick_params(labelsize=20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0bd863b2eb1e75745a4333b62c8dae8d65f91086"},"cell_type":"code","source":"df['passenger_count'].value_counts()[0:6].plot(kind='pie', subplots=True, figsize=(8, 8))\n\npassenger_1_percentage = df['passenger_count'].value_counts().values[0]/df['passenger_count'].count()\nprint(passenger_1_percentage)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fc19435b3cb0671c7f197452cf1461c4450a6a4c"},"cell_type":"markdown","source":"70.86% of people travel by themselves.\n\n## Data Preprocessing"},{"metadata":{"trusted":true,"_uuid":"97de391dd6edf4935062c3bf30dbc137a14e53d5"},"cell_type":"code","source":"df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1ccbca0cea2a81c66f678140850e5395716cb7bb"},"cell_type":"markdown","source":"## Feature Engineering \n\nTravel distance based on dropoff and pickup locations: "},{"metadata":{"trusted":true,"_uuid":"5efc94935ff520ee841028e8ea1ee87435e17350"},"cell_type":"code","source":"# approximate radius of earth in km\nR = 6371.0\n\n#df\nlat1 = np.radians(df['pickup_latitude'])\nlon1 = np.radians(df['pickup_longitude'])\nlat2 = np.radians(df['dropoff_latitude']) \nlon2 = np.radians(df['dropoff_longitude'])\n\ndlon = lon2 - lon1\ndlat = lat2 - lat1\n\na = np.sin(dlat / 2)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon / 2)**2\nc = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a))\n\ndistance = R * c\n\ndistance_df = pd.DataFrame({'travel_distance_km':distance})\ndf['travel_distance_km'] = distance_df\n\nprint(np.mean(distance_df))\n\n#Test\n\nlat1 = np.radians(test['pickup_latitude'])\nlon1 = np.radians(test['pickup_longitude'])\nlat2 = np.radians(test['dropoff_latitude']) \nlon2 = np.radians(test['dropoff_longitude'])\n\ndlon = lon2 - lon1\ndlat = lat2 - lat1\n\na = np.sin(dlat / 2)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon / 2)**2\nc = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a))\n\ndistance = R * c\n\ndistance_df = pd.DataFrame({'travel_distance_km':distance})\ntest['travel_distance_km'] = distance_df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b056f38c0e9372bd7e17fa59806cb13711bce0bb"},"cell_type":"code","source":"#df\ndf['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'])\ndf['dropoff_datetime'] = pd.to_datetime(df['dropoff_datetime'])\n\ndf['hour_pickup'] = np.array(df['pickup_datetime'].dt.hour)\ndf['day_pickup'] = np.array(df['pickup_datetime'].dt.day_name())\n\n#test\ntest['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'])\n\ntest['hour_pickup'] = np.array(test['pickup_datetime'].dt.hour)\ntest['day_pickup'] = np.array(test['pickup_datetime'].dt.day_name())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a11ac4fcc640fca1e9e8d36c45623c25a8b968cb"},"cell_type":"code","source":"sns.set(rc={'figure.figsize':(15,10)})\nsns.distplot(df['travel_distance_km'],hist=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d339dc1a3e37ee9cb63e8711cc73e0dd33337399"},"cell_type":"code","source":"sns.distplot(df[ \n        (df['travel_distance_km'] >= 0) & \n        (df['travel_distance_km'] < 200)\n                                        ]['travel_distance_km'],hist=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b77c8a76a58831e3ae4b002a7e164f7bbcedc296"},"cell_type":"markdown","source":"We can conclude that most trips are between 0 to 25 km. Let's zoom in one more time :"},{"metadata":{"trusted":true,"_uuid":"b625af47c750157015e35e7a6b1e62f5dbea8a73"},"cell_type":"code","source":"category_dict = {'0-5': 0,'5-10': 0, '10-15': 0, '15-20': 0, '20-25':0 }\n\nfor x in np.array(df[(df['travel_distance_km'] >= 0) & (df['travel_distance_km'] < 25)]['travel_distance_km']): \n    \n    if x <= 5: \n        category_dict['0-5'] = category_dict['0-5'] + 1\n        \n    elif x > 5 and x <= 10 : \n        category_dict['5-10'] = category_dict['5-10'] + 1\n    \n    elif x > 10 and x <= 15 :\n        category_dict['10-15'] = category_dict['10-15'] + 1\n\n    elif x > 15 and x <= 20 :\n        category_dict['15-20'] = category_dict['15-20'] + 1\n    \n    elif x > 20 and x <= 25 :\n        category_dict['20-25'] = category_dict['20-25'] + 1\n        \npd.DataFrame(category_dict, index=[0]).T.plot(kind='pie', subplots=True, figsize=(8, 8))\nplt.title('Percentage of Trips Between 0 to 25 km')\n\nprint('% of trips between 0 to 5 km for the trip distance between 0 to 25 km dataset : ', category_dict['0-5']/sum(category_dict.values()))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"16604465b7520ad104dfb190d28b895878cc4f79"},"cell_type":"markdown","source":"- We can conclude that 82% of the trips are between 0 to 5 km only for the travel distance between 0 to 25 km. "},{"metadata":{"trusted":true,"_uuid":"b2018bf78f9b2f03cbd55c4b13be3a2940b29ee6"},"cell_type":"code","source":"df[(df['travel_distance_km'] >= 0) & (df['travel_distance_km'] < 25) & (df['trip_duration'] < 20000) ]['trip_duration'].hist(bins=300)\n\nplt.title('Duration for Trips between 0 to 25km')\nplt.xlabel('Duration in Seconds')\nplt.ylabel('Frequency')\n\nprint('mode : ', df[(df['travel_distance_km'] >= 0) & (df['travel_distance_km'] < 25) & (df['trip_duration'] < 20000) ]['trip_duration'].mode()[0])\nprint('mean :' , df[(df['travel_distance_km'] >= 0) & (df['travel_distance_km'] < 25) & (df['trip_duration'] < 20000) ]['trip_duration'].mean())\nprint('median :' , df[(df['travel_distance_km'] >= 0) & (df['travel_distance_km'] < 25) & (df['trip_duration'] < 20000) ]['trip_duration'].median())\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dd07d4d0984d25874445b9aa1a90a48d0e1ca795"},"cell_type":"markdown","source":"This is a distribution of the duration for trips only between 0 to 25 km. As we can clearly see, the distribution is skewed to the right.\n\nBelow is the log distribution including the outliers."},{"metadata":{"trusted":true,"_uuid":"4af7f1c61e1db06ff9966c9fa198beffb7008f49"},"cell_type":"code","source":"np.log(df[(df['travel_distance_km'] >= 0) & (df['travel_distance_km'] < 25)]['trip_duration']).plot.hist(bins=300)\nplt.title('log(Duration for Trips between 0 to 25km)')\nplt.xlabel('log(Duration in Seconds)')\nplt.ylabel('Frequency')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b72dddc6993e8e91a6d2f79368674728424b4dfa"},"cell_type":"code","source":"sns.scatterplot(\n    x=df[(df['travel_distance_km'] > 0) & (df['travel_distance_km'] < 25) & (df['trip_duration'] < 20000)]['trip_duration'], \n    y=df[(df['travel_distance_km'] > 0) & (df['travel_distance_km'] < 25) & (df['trip_duration'] < 20000)]['travel_distance_km'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bd5891c94ed292a4fc7183e6e1416042e4e8ef4f"},"cell_type":"markdown","source":"The scatter plot shows the relationship between the distance and the time duration. We can see that there is a correlation between trip duration and distance. In other words, the further the distance the more time the taxi will take. \n\nThe trip duration must have a correlation with traffic during peak hours so let's break down the hour and the day of the week in separate columns. \n"},{"metadata":{"trusted":true,"_uuid":"df7c49684ca6b31453c645d801905bbc0d63fe38"},"cell_type":"code","source":"df['day_pickup'] = df['day_pickup'].astype('category').cat.codes\ndf['store_and_fwd_flag'] = df['store_and_fwd_flag'].astype('category').cat.codes\n\ntest['day_pickup'] = test['day_pickup'].astype('category').cat.codes\ntest['store_and_fwd_flag'] = test['store_and_fwd_flag'].astype('category').cat.codes","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6852100b79b7154348b3cc6c9febc4893dc8e091"},"cell_type":"markdown","source":"## Model Selection"},{"metadata":{"trusted":true,"_uuid":"2c5c5879f35a0a5092043a4a0c23545078815461"},"cell_type":"code","source":"scaler = StandardScaler()\n\nscaler.fit(df[['vendor_id','passenger_count', \n        'pickup_longitude', 'pickup_latitude',\n       'dropoff_longitude', 'dropoff_latitude', 'store_and_fwd_flag',\n        'travel_distance_km', 'hour_pickup',\n       'day_pickup']])\n\nscaled_features = scaler.transform(df[['vendor_id','passenger_count', \n        'pickup_longitude', 'pickup_latitude',\n       'dropoff_longitude', 'dropoff_latitude', 'store_and_fwd_flag',\n        'travel_distance_km', 'hour_pickup',\n       'day_pickup']])\n\ndf_feat = pd.DataFrame(scaled_features, columns=['vendor_id','passenger_count', \n        'pickup_longitude', 'pickup_latitude',\n       'dropoff_longitude', 'dropoff_latitude', 'store_and_fwd_flag',\n        'travel_distance_km', 'hour_pickup',\n       'day_pickup'])\n\n\nscaler_test = StandardScaler()\n\nscaler_test.fit(test[['vendor_id','passenger_count', \n        'pickup_longitude', 'pickup_latitude',\n       'dropoff_longitude', 'dropoff_latitude', 'store_and_fwd_flag',\n        'travel_distance_km', 'hour_pickup',\n       'day_pickup']])\n\nscaled_features_test = scaler.transform(test[['vendor_id','passenger_count', \n        'pickup_longitude', 'pickup_latitude',\n       'dropoff_longitude', 'dropoff_latitude', 'store_and_fwd_flag',\n        'travel_distance_km', 'hour_pickup',\n       'day_pickup']])\n\ndf_feat_test = pd.DataFrame(scaled_features_test, columns=['vendor_id','passenger_count', \n        'pickup_longitude', 'pickup_latitude',\n       'dropoff_longitude', 'dropoff_latitude', 'store_and_fwd_flag',\n        'travel_distance_km', 'hour_pickup',\n       'day_pickup'])\n\ntest = pd.concat([test['id'],df_feat_test], axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd3798601bd66119bf8fdae91681bce9dadda762"},"cell_type":"code","source":"X = df_feat\n\ny = np.log(df['trip_duration'])\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b51cc809523047b232b8e50cfdc505771c131293"},"cell_type":"code","source":"X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0aee211a1d0832a6e82c0358455a43158cd61b35"},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"430d846cce93813009eaeb5b88cade19cefb920f"},"cell_type":"markdown","source":"## Predictions"},{"metadata":{"_uuid":"642f8b492da49d5828e05fd7457a499c44d3c373"},"cell_type":"markdown","source":" #### RandomForestRegressor"},{"metadata":{"trusted":true,"_uuid":"6652a6cbb040759e075926bd48d73119d9bf6b8e"},"cell_type":"code","source":"rmse_rfr = []\n\ndef func_rfr(num): \n    rfr = RandomForestRegressor(n_estimators=num)\n    rfr.fit(X_train, y_train)\n    pred = rfr.predict(X_valid)\n    rmse_rfr.append(np.sqrt(mean_squared_error(y_valid, pred)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08d6de40a90e2546daddc0ec1475c1d65ef2d77c"},"cell_type":"code","source":"if __name__ == '__main__':\n    Thread(target = func_rfr(20)).start()\n    Thread(target = func_rfr(50)).start()\n    Thread(target = func_rfr(100)).start()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e1298bfcc96ae9bdb77dbdd2a9e6a88be1e8ba1"},"cell_type":"code","source":"sns.lineplot(y=rmse_rfr, x=[20,50,100], markers=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4525ae678f6c3c16bcf2933b21fc9bd70332419"},"cell_type":"code","source":"rfr = RandomForestRegressor(n_estimators=100)\nrfr.fit(X_train, y_train)\npred = rfr.predict(X_valid)\nscore_rfr = cross_val_score(rfr, X_train, y_train, cv=5, scoring='neg_mean_squared_log_error')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f0b31d2058619c6fd1d37ad97b745186fd35184"},"cell_type":"code","source":"print('cross_val_score average: ', abs(np.mean(score_rfr)))\nprint('MAE: ',mean_absolute_error(y_valid, pred))\nprint('MSE: ',mean_squared_error(y_valid, pred))\nprint('RMSE: ',np.sqrt(mean_squared_error(y_valid, pred)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"885d519196e586e606e2a689de9df746b0ae8146"},"cell_type":"code","source":"pred = rfr.predict(test[['vendor_id','passenger_count',\n       'pickup_longitude', 'pickup_latitude', 'dropoff_longitude',\n       'dropoff_latitude', 'store_and_fwd_flag', 'travel_distance_km',\n       'hour_pickup', 'day_pickup']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"272c70783452f8eb139af0b5c0ac2bc2f10a157f"},"cell_type":"code","source":"arr_id = test['id']\nsubmission = pd.DataFrame({'id': arr_id, 'trip_duration': np.exp(pred)})\nprint(submission)\n\nsubmission.to_csv(\"submit_file.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}