{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd \n\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport folium\nfrom folium.plugins import HeatMap","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-10T18:04:39.535093Z","iopub.execute_input":"2022-08-10T18:04:39.535946Z","iopub.status.idle":"2022-08-10T18:04:39.546860Z","shell.execute_reply.started":"2022-08-10T18:04:39.535903Z","shell.execute_reply":"2022-08-10T18:04:39.545117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data validation, cleaning & feature engineering","metadata":{}},{"cell_type":"markdown","source":"So far I have already checked the database, to reduce the size I will load it without the Key column because it seems to be used for the unique identifier.","metadata":{}},{"cell_type":"code","source":"fields = ['pickup_datetime', 'fare_amount', 'pickup_longitude', 'pickup_latitude', \n          'dropoff_longitude', 'dropoff_latitude', 'passenger_count']\n\n\ntrain = pd.read_csv(\"/kaggle/input/new-york-city-taxi-fare-prediction/train.csv\", nrows = 1000000, \n                    skipinitialspace=True, usecols=fields, parse_dates=[\"pickup_datetime\"])\nprint(f'{train.shape} shape')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:07:35.810766Z","iopub.execute_input":"2022-08-10T18:07:35.812126Z","iopub.status.idle":"2022-08-10T18:10:05.180739Z","shell.execute_reply.started":"2022-08-10T18:07:35.812042Z","shell.execute_reply":"2022-08-10T18:10:05.179419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.info())\ntrain.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:07.833410Z","iopub.execute_input":"2022-08-10T18:10:07.834175Z","iopub.status.idle":"2022-08-10T18:10:08.201566Z","shell.execute_reply.started":"2022-08-10T18:10:07.834121Z","shell.execute_reply":"2022-08-10T18:10:08.200318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(\"/kaggle/input/new-york-city-taxi-fare-prediction/test.csv\", parse_dates=[\"pickup_datetime\"])\nprint(f'{test.shape} shape')\ntest.head() ","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:08.523015Z","iopub.execute_input":"2022-08-10T18:10:08.523493Z","iopub.status.idle":"2022-08-10T18:10:08.852201Z","shell.execute_reply.started":"2022-08-10T18:10:08.523454Z","shell.execute_reply":"2022-08-10T18:10:08.851109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:09.284412Z","iopub.execute_input":"2022-08-10T18:10:09.284861Z","iopub.status.idle":"2022-08-10T18:10:09.301075Z","shell.execute_reply.started":"2022-08-10T18:10:09.284824Z","shell.execute_reply":"2022-08-10T18:10:09.300194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:09.569419Z","iopub.execute_input":"2022-08-10T18:10:09.570540Z","iopub.status.idle":"2022-08-10T18:10:09.602023Z","shell.execute_reply.started":"2022-08-10T18:10:09.570486Z","shell.execute_reply":"2022-08-10T18:10:09.600705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:09.811733Z","iopub.execute_input":"2022-08-10T18:10:09.812768Z","iopub.status.idle":"2022-08-10T18:10:10.147268Z","shell.execute_reply.started":"2022-08-10T18:10:09.812706Z","shell.execute_reply":"2022-08-10T18:10:10.146040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"First, remove the Key column for test database too.","metadata":{}},{"cell_type":"code","source":"test = test.drop(columns=['key'])","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:10.200483Z","iopub.execute_input":"2022-08-10T18:10:10.200903Z","iopub.status.idle":"2022-08-10T18:10:10.207840Z","shell.execute_reply.started":"2022-08-10T18:10:10.200870Z","shell.execute_reply":"2022-08-10T18:10:10.206552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:10.402483Z","iopub.execute_input":"2022-08-10T18:10:10.403316Z","iopub.status.idle":"2022-08-10T18:10:10.418707Z","shell.execute_reply.started":"2022-08-10T18:10:10.403268Z","shell.execute_reply":"2022-08-10T18:10:10.417774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x='passenger_count',data=train)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:10.601797Z","iopub.execute_input":"2022-08-10T18:10:10.602221Z","iopub.status.idle":"2022-08-10T18:10:10.939701Z","shell.execute_reply.started":"2022-08-10T18:10:10.602185Z","shell.execute_reply":"2022-08-10T18:10:10.938400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, fare_amount is negative, which is illogical. Also, 208 passengers is unrealistic.\nLet's take 6 as the maximum number of passengers.","metadata":{}},{"cell_type":"code","source":"train[train['passenger_count']>6]","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:10.969021Z","iopub.execute_input":"2022-08-10T18:10:10.969429Z","iopub.status.idle":"2022-08-10T18:10:10.986600Z","shell.execute_reply.started":"2022-08-10T18:10:10.969396Z","shell.execute_reply":"2022-08-10T18:10:10.985799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.drop(train[train['passenger_count']==208].index)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:11.160688Z","iopub.execute_input":"2022-08-10T18:10:11.161388Z","iopub.status.idle":"2022-08-10T18:10:11.227152Z","shell.execute_reply.started":"2022-08-10T18:10:11.161316Z","shell.execute_reply":"2022-08-10T18:10:11.225693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.drop(train[train['fare_amount']<0].index)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:11.336452Z","iopub.execute_input":"2022-08-10T18:10:11.337067Z","iopub.status.idle":"2022-08-10T18:10:11.469045Z","shell.execute_reply.started":"2022-08-10T18:10:11.337033Z","shell.execute_reply":"2022-08-10T18:10:11.467823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In general, we know that taxi prices are greatly affected by the time period. That's why it's important to split the datetime and extract more valuable variables.\n\n","metadata":{}},{"cell_type":"code","source":"train['year'] = train['pickup_datetime'].dt.year\ntrain['month'] = train['pickup_datetime'].dt.month\ntrain['day'] = train['pickup_datetime'].dt.day\ntrain['hour'] = train['pickup_datetime'].dt.hour\ntrain['minute'] = train['pickup_datetime'].dt.minute\n\ntest['year'] = test['pickup_datetime'].dt.year\ntest['month'] = test['pickup_datetime'].dt.month\ntest['day'] = test['pickup_datetime'].dt.day\ntest['hour'] = test['pickup_datetime'].dt.hour\ntest['minute'] = test['pickup_datetime'].dt.minute","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:11.687095Z","iopub.execute_input":"2022-08-10T18:10:11.687564Z","iopub.status.idle":"2022-08-10T18:10:12.283276Z","shell.execute_reply.started":"2022-08-10T18:10:11.687524Z","shell.execute_reply":"2022-08-10T18:10:12.282044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We don't need the Datetime column anymore, so let's remove it.","metadata":{}},{"cell_type":"code","source":"train = train.drop(columns=['pickup_datetime'])\ntest = test.drop(columns=['pickup_datetime'])","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:12.285287Z","iopub.execute_input":"2022-08-10T18:10:12.285662Z","iopub.status.idle":"2022-08-10T18:10:12.355981Z","shell.execute_reply.started":"2022-08-10T18:10:12.285631Z","shell.execute_reply":"2022-08-10T18:10:12.354773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:13.933465Z","iopub.execute_input":"2022-08-10T18:10:13.933893Z","iopub.status.idle":"2022-08-10T18:10:14.452570Z","shell.execute_reply.started":"2022-08-10T18:10:13.933848Z","shell.execute_reply":"2022-08-10T18:10:14.451311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# variables = ['year', 'month', 'day', 'hour']\n\n# for var in variables:\n#     plt.figure()\n#     sns.regplot(x=var, y='fare_amount', data=train).set(title=f'Regression plot of {var} and Fare amount')","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:14.454700Z","iopub.execute_input":"2022-08-10T18:10:14.455906Z","iopub.status.idle":"2022-08-10T18:10:14.461560Z","shell.execute_reply.started":"2022-08-10T18:10:14.455858Z","shell.execute_reply":"2022-08-10T18:10:14.459863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Regplot chart does not show any significant differences, I will use also the countplot.","metadata":{}},{"cell_type":"code","source":"variables = ['year', 'month', 'day', 'hour']\n\nfor var in variables:\n    plt.figure()\n    sns.countplot(x=var, data=train).set(title=f'Count plot of {var}')","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:14.528038Z","iopub.execute_input":"2022-08-10T18:10:14.528446Z","iopub.status.idle":"2022-08-10T18:10:16.154979Z","shell.execute_reply.started":"2022-08-10T18:10:14.528411Z","shell.execute_reply":"2022-08-10T18:10:16.153773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It seems that we have data from 2009-2014 and part from 2015. And the most important thing that we see is the least traffic in 3-5AM hours and the busiest hour is 6-7PM in the evening.","metadata":{}},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:16.156850Z","iopub.execute_input":"2022-08-10T18:10:16.157290Z","iopub.status.idle":"2022-08-10T18:10:16.164675Z","shell.execute_reply.started":"2022-08-10T18:10:16.157258Z","shell.execute_reply":"2022-08-10T18:10:16.163534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are also unrealistic latitude/longitude data in the description. We can only take the latitude/longitude of New York. \n> Longitude: 71° 47' 25\" W to 79° 45' 54\" W Latitude: 40° 29' 40\" N to 45° 0' 42\" N. ","metadata":{}},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:16.165950Z","iopub.execute_input":"2022-08-10T18:10:16.166648Z","iopub.status.idle":"2022-08-10T18:10:16.675992Z","shell.execute_reply.started":"2022-08-10T18:10:16.166615Z","shell.execute_reply":"2022-08-10T18:10:16.675152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train[(train['pickup_latitude']>40)&(train['pickup_latitude']<45)]\ntrain = train[(train['dropoff_latitude']>40)&(train['dropoff_latitude']<45)]\n\ntrain = train[(train['pickup_longitude']<-71)&(train['pickup_longitude']>-79)]\ntrain = train[(train['dropoff_longitude']<-71)&(train['dropoff_longitude']>-79)]\n","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:16.678418Z","iopub.execute_input":"2022-08-10T18:10:16.682034Z","iopub.status.idle":"2022-08-10T18:10:16.969344Z","shell.execute_reply.started":"2022-08-10T18:10:16.681986Z","shell.execute_reply":"2022-08-10T18:10:16.968221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:16.970718Z","iopub.execute_input":"2022-08-10T18:10:16.971084Z","iopub.status.idle":"2022-08-10T18:10:17.458906Z","shell.execute_reply.started":"2022-08-10T18:10:16.971051Z","shell.execute_reply":"2022-08-10T18:10:17.457923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:17.723881Z","iopub.execute_input":"2022-08-10T18:10:17.724283Z","iopub.status.idle":"2022-08-10T18:10:17.731478Z","shell.execute_reply.started":"2022-08-10T18:10:17.724251Z","shell.execute_reply":"2022-08-10T18:10:17.730233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating a base map\nny_map = folium.Map(location=[40.71,-74.00], tiles='cartodbpositron', zoom_start=20)\n\nHeatMap(data=train[['pickup_latitude', 'pickup_longitude']], radius=10).add_to(ny_map)\n\nny_map","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:18.576885Z","iopub.execute_input":"2022-08-10T18:10:18.577280Z","iopub.status.idle":"2022-08-10T18:10:31.934204Z","shell.execute_reply.started":"2022-08-10T18:10:18.577247Z","shell.execute_reply":"2022-08-10T18:10:31.933241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HeatMap(data=train[['dropoff_latitude', 'dropoff_longitude']], radius=10).add_to(ny_map)\n\nny_map","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:10:31.935666Z","iopub.execute_input":"2022-08-10T18:10:31.936750Z","iopub.status.idle":"2022-08-10T18:10:50.991682Z","shell.execute_reply.started":"2022-08-10T18:10:31.936712Z","shell.execute_reply":"2022-08-10T18:10:50.988729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"These plots show that Manhatten area is the bussiest place.","metadata":{}},{"cell_type":"markdown","source":"# Modelling","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:11:24.228031Z","iopub.execute_input":"2022-08-10T18:11:24.228570Z","iopub.status.idle":"2022-08-10T18:11:24.247487Z","shell.execute_reply.started":"2022-08-10T18:11:24.228528Z","shell.execute_reply":"2022-08-10T18:11:24.246137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:11:36.110676Z","iopub.execute_input":"2022-08-10T18:11:36.111072Z","iopub.status.idle":"2022-08-10T18:11:36.127310Z","shell.execute_reply.started":"2022-08-10T18:11:36.111042Z","shell.execute_reply":"2022-08-10T18:11:36.126425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = train.loc[:, train.columns != 'fare_amount']\nx_test = test\ny = train['fare_amount'].values","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:11:37.038126Z","iopub.execute_input":"2022-08-10T18:11:37.039178Z","iopub.status.idle":"2022-08-10T18:11:37.070518Z","shell.execute_reply.started":"2022-08-10T18:11:37.039138Z","shell.execute_reply":"2022-08-10T18:11:37.069229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:11:37.372353Z","iopub.execute_input":"2022-08-10T18:11:37.373487Z","iopub.status.idle":"2022-08-10T18:11:37.563261Z","shell.execute_reply.started":"2022-08-10T18:11:37.373439Z","shell.execute_reply":"2022-08-10T18:11:37.561681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\n\nlinreg = LinearRegression()\n\nlinreg.fit(x_train,y_train)\ny_pred = linreg.predict(x_test)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:11:37.754774Z","iopub.execute_input":"2022-08-10T18:11:37.755735Z","iopub.status.idle":"2022-08-10T18:11:38.159539Z","shell.execute_reply.started":"2022-08-10T18:11:37.755668Z","shell.execute_reply":"2022-08-10T18:11:38.156699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mse = mean_squared_error(y_test, y_pred)\nrmse = np.sqrt(mse)\nprint(rmse)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:11:38.198153Z","iopub.execute_input":"2022-08-10T18:11:38.199303Z","iopub.status.idle":"2022-08-10T18:11:38.211991Z","shell.execute_reply.started":"2022-08-10T18:11:38.199234Z","shell.execute_reply":"2022-08-10T18:11:38.210214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import XGBRegressor\n\nxg = XGBRegressor()\nxg.fit(x_train, y_train)\nxg_pred = xg.predict(x_test)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:11:38.427922Z","iopub.execute_input":"2022-08-10T18:11:38.428314Z","iopub.status.idle":"2022-08-10T18:12:29.300497Z","shell.execute_reply.started":"2022-08-10T18:11:38.428283Z","shell.execute_reply":"2022-08-10T18:12:29.299483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mse = mean_squared_error(y_test, xg_pred)\nrmse = np.sqrt(mse)\nprint(rmse)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:12:29.302752Z","iopub.execute_input":"2022-08-10T18:12:29.303696Z","iopub.status.idle":"2022-08-10T18:12:29.315805Z","shell.execute_reply.started":"2022-08-10T18:12:29.303645Z","shell.execute_reply":"2022-08-10T18:12:29.314400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since the RMSE of XGBRegressor is the lowest, we will use it for prediction.","metadata":{}},{"cell_type":"code","source":"# results = pd.DataFrame({'Actual': y_test, 'Predicted': xg_pred})\n# print(results) ","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:12:49.627187Z","iopub.execute_input":"2022-08-10T18:12:49.627690Z","iopub.status.idle":"2022-08-10T18:12:49.632265Z","shell.execute_reply.started":"2022-08-10T18:12:49.627650Z","shell.execute_reply":"2022-08-10T18:12:49.631397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = xg.predict(test)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:12:50.114137Z","iopub.execute_input":"2022-08-10T18:12:50.114614Z","iopub.status.idle":"2022-08-10T18:12:50.149225Z","shell.execute_reply.started":"2022-08-10T18:12:50.114570Z","shell.execute_reply":"2022-08-10T18:12:50.142033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/new-york-city-taxi-fare-prediction/sample_submission.csv')\nsubmission['fare_amount'] = pred\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T18:12:51.752024Z","iopub.execute_input":"2022-08-10T18:12:51.752939Z","iopub.status.idle":"2022-08-10T18:12:51.808290Z","shell.execute_reply.started":"2022-08-10T18:12:51.752890Z","shell.execute_reply":"2022-08-10T18:12:51.806955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}