{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Setup**","metadata":{}},{"cell_type":"code","source":"# For data manipulation and visualization\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nplt.rcParams[\"figure.figsize\"] = (16,8)\nimport seaborn as sns\n\n# For modelling\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.linear_model import LinearRegression\nfrom xgboost import XGBRegressor\nfrom sklearn.metrics import mean_squared_error\n\n# DNN\nfrom sklearn.preprocessing import StandardScaler\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers, callbacks","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:25.258240Z","iopub.execute_input":"2022-08-10T10:55:25.258665Z","iopub.status.idle":"2022-08-10T10:55:25.266358Z","shell.execute_reply.started":"2022-08-10T10:55:25.258631Z","shell.execute_reply":"2022-08-10T10:55:25.265132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Load Data**","metadata":{}},{"cell_type":"markdown","source":"There are nearly 55 million rows in training dataset, which is too big to load on kaggle and perform operations. Therefore, I will use only 1 million rows. It should go faster.\n\n","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/new-york-city-taxi-fare-prediction/train.csv\",nrows = 1000000)\ntest_df = pd.read_csv(\"../input/new-york-city-taxi-fare-prediction/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:25.277582Z","iopub.execute_input":"2022-08-10T10:55:25.277972Z","iopub.status.idle":"2022-08-10T10:55:27.700969Z","shell.execute_reply.started":"2022-08-10T10:55:25.277937Z","shell.execute_reply":"2022-08-10T10:55:27.699678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Validate and Clean Data**","metadata":{}},{"cell_type":"markdown","source":"## **Training Data**","metadata":{}},{"cell_type":"markdown","source":"### **Check for Missing Values**","metadata":{}},{"cell_type":"code","source":"train_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:27.703043Z","iopub.execute_input":"2022-08-10T10:55:27.703499Z","iopub.status.idle":"2022-08-10T10:55:27.809775Z","shell.execute_reply.started":"2022-08-10T10:55:27.703465Z","shell.execute_reply":"2022-08-10T10:55:27.808590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dropping these small amount of missing value will not affect our analysis.","metadata":{}},{"cell_type":"code","source":"# drop NaN rows in dropoff_longitude and dropoff_latitude\ntrain_df.dropna(axis=0, subset=['dropoff_longitude', 'dropoff_latitude'], inplace=True)\ntrain_df = train_df.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:27.811076Z","iopub.execute_input":"2022-08-10T10:55:27.811407Z","iopub.status.idle":"2022-08-10T10:55:27.957720Z","shell.execute_reply.started":"2022-08-10T10:55:27.811378Z","shell.execute_reply":"2022-08-10T10:55:27.956533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Check the validity of data values**","metadata":{}},{"cell_type":"markdown","source":"#### **Coordinates**","metadata":{}},{"cell_type":"code","source":"pd.set_option('display.float_format', lambda x: '%.5f' % x)\ntrain_df.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:27.960354Z","iopub.execute_input":"2022-08-10T10:55:27.960743Z","iopub.status.idle":"2022-08-10T10:55:28.247116Z","shell.execute_reply.started":"2022-08-10T10:55:27.960697Z","shell.execute_reply":"2022-08-10T10:55:28.245941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The valid range of latitude is from -90 to 90 and Longitude is between -180 and 180. Some values are out of these ranges.","metadata":{}},{"cell_type":"code","source":"print('Number of observations out of valid range in coordinate columns:', end=\"\\n\")\n\nprint('pickup_longitude', end=': ')\nprint((train_df.pickup_longitude <-180).sum()+(train_df.pickup_longitude > 180).sum())\n\nprint('pickup_latitude', end=': ')\nprint((train_df.pickup_latitude <-90).sum()+(train_df.pickup_latitude > 90).sum())\n\nprint('dropoff_longitude', end=': ')\nprint((train_df.dropoff_longitude <-180).sum()+(train_df.dropoff_longitude > 180).sum())\n\nprint('dropoff_latitude', end=': ')\nprint((train_df.dropoff_latitude <-90).sum()+(train_df.dropoff_latitude > 90).sum())","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:28.248450Z","iopub.execute_input":"2022-08-10T10:55:28.248924Z","iopub.status.idle":"2022-08-10T10:55:28.278502Z","shell.execute_reply.started":"2022-08-10T10:55:28.248879Z","shell.execute_reply":"2022-08-10T10:55:28.277263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop these values\ntrain_df = train_df.drop(train_df[(train_df.pickup_longitude < -180) | (train_df.pickup_longitude > 180)].index, axis=0)\ntrain_df = train_df.drop(train_df[(train_df.pickup_latitude < -90) | (train_df.pickup_latitude > 90)].index, axis=0)\ntrain_df = train_df.drop(train_df[(train_df.dropoff_longitude < -180) | (train_df.dropoff_longitude > 180)].index, axis=0)\ntrain_df = train_df.drop(train_df[(train_df.dropoff_latitude < -90) | (train_df.dropoff_latitude > 90)].index, axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:28.279607Z","iopub.execute_input":"2022-08-10T10:55:28.279953Z","iopub.status.idle":"2022-08-10T10:55:28.772478Z","shell.execute_reply.started":"2022-08-10T10:55:28.279921Z","shell.execute_reply":"2022-08-10T10:55:28.771161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:28.774244Z","iopub.execute_input":"2022-08-10T10:55:28.775687Z","iopub.status.idle":"2022-08-10T10:55:29.073316Z","shell.execute_reply.started":"2022-08-10T10:55:28.775642Z","shell.execute_reply":"2022-08-10T10:55:29.072373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Latitude and longitude coordinates of New York City are: 40.730610, -73.935242. Therefore all the coordiantes should be around it in our dataset. Also, if we look at the table above closely, we can see that minimum latitude values are around -74 and maximum longitude values are around 40, which should be vice versa. Therefore these values might be switched in some rows.","metadata":{}},{"cell_type":"code","source":"train_df[(train_df.pickup_longitude>=40)]","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:29.074562Z","iopub.execute_input":"2022-08-10T10:55:29.075647Z","iopub.status.idle":"2022-08-10T10:55:29.098575Z","shell.execute_reply.started":"2022-08-10T10:55:29.075608Z","shell.execute_reply":"2022-08-10T10:55:29.097481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"These values seem to be switched","metadata":{}},{"cell_type":"code","source":"# Now let's switch these rows back\n\nindx = train_df[(train_df.pickup_longitude>=40)].index\ntrain_df.loc[indx,['dropoff_longitude','dropoff_latitude']] = train_df.loc[indx,['dropoff_latitude','dropoff_longitude']].values\ntrain_df.loc[indx,['pickup_longitude','pickup_latitude']] = train_df.loc[indx,['pickup_latitude','pickup_longitude']].values","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:29.100203Z","iopub.execute_input":"2022-08-10T10:55:29.100847Z","iopub.status.idle":"2022-08-10T10:55:29.172562Z","shell.execute_reply.started":"2022-08-10T10:55:29.100801Z","shell.execute_reply":"2022-08-10T10:55:29.171596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Also, Let's filter and remove values that are outside these latitude and longitude ranges of NYC\n\n# -75 <= pickup_longitude <= -72 \n# -75 <= dropoff_longitude <= -72\n# 40 <= pickup_latitude <= 42\n# 40 <= dropoff_latitude <= 42\n\ntrain_df = train_df.drop(train_df[(train_df.pickup_longitude<-75) | (train_df.pickup_longitude>-72)].index, axis=0)\ntrain_df = train_df.drop(train_df[(train_df.dropoff_longitude<-75) | (train_df.dropoff_longitude>-72)].index, axis=0)\ntrain_df = train_df.drop(train_df[(train_df.pickup_latitude<40) | (train_df.pickup_latitude>42)].index, axis=0)\ntrain_df = train_df.drop(train_df[(train_df.dropoff_latitude<40) | (train_df.dropoff_latitude>42)].index, axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:29.177588Z","iopub.execute_input":"2022-08-10T10:55:29.177969Z","iopub.status.idle":"2022-08-10T10:55:29.685227Z","shell.execute_reply.started":"2022-08-10T10:55:29.177934Z","shell.execute_reply":"2022-08-10T10:55:29.684093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:29.686773Z","iopub.execute_input":"2022-08-10T10:55:29.687547Z","iopub.status.idle":"2022-08-10T10:55:29.983093Z","shell.execute_reply.started":"2022-08-10T10:55:29.687500Z","shell.execute_reply":"2022-08-10T10:55:29.981821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, coordinates (even maximum and minimum) are around the real NYC latitude and longitude.","metadata":{}},{"cell_type":"markdown","source":"#### **passenger_count**","metadata":{}},{"cell_type":"code","source":"train_df.passenger_count.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:29.984342Z","iopub.execute_input":"2022-08-10T10:55:29.985168Z","iopub.status.idle":"2022-08-10T10:55:30.001312Z","shell.execute_reply.started":"2022-08-10T10:55:29.985133Z","shell.execute_reply":"2022-08-10T10:55:30.000266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Drop rows with 0 passenger count","metadata":{}},{"cell_type":"code","source":"train_df = train_df.drop(train_df[train_df.passenger_count == 0].index, axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:30.002436Z","iopub.execute_input":"2022-08-10T10:55:30.002994Z","iopub.status.idle":"2022-08-10T10:55:30.139517Z","shell.execute_reply.started":"2022-08-10T10:55:30.002960Z","shell.execute_reply":"2022-08-10T10:55:30.137977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### **fare_amount**","metadata":{}},{"cell_type":"code","source":"train_df.fare_amount.sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:30.142226Z","iopub.execute_input":"2022-08-10T10:55:30.143130Z","iopub.status.idle":"2022-08-10T10:55:30.267656Z","shell.execute_reply.started":"2022-08-10T10:55:30.143079Z","shell.execute_reply":"2022-08-10T10:55:30.266455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's remove values with the fare <=0","metadata":{}},{"cell_type":"code","source":"train_df = train_df.drop(train_df[train_df.fare_amount <= 0].index, axis=0)\ntrain_df['fare_amount'].sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:30.269032Z","iopub.execute_input":"2022-08-10T10:55:30.269364Z","iopub.status.idle":"2022-08-10T10:55:30.518597Z","shell.execute_reply.started":"2022-08-10T10:55:30.269333Z","shell.execute_reply":"2022-08-10T10:55:30.517251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Test Data**","metadata":{}},{"cell_type":"markdown","source":"### **Check for Missing Values**","metadata":{}},{"cell_type":"code","source":"test_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:30.520357Z","iopub.execute_input":"2022-08-10T10:55:30.521101Z","iopub.status.idle":"2022-08-10T10:55:30.534877Z","shell.execute_reply.started":"2022-08-10T10:55:30.521063Z","shell.execute_reply":"2022-08-10T10:55:30.533539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks like there are no missing values","metadata":{}},{"cell_type":"markdown","source":"### **Check the validity of data values**","metadata":{}},{"cell_type":"code","source":"test_df.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:30.536570Z","iopub.execute_input":"2022-08-10T10:55:30.537442Z","iopub.status.idle":"2022-08-10T10:55:30.565403Z","shell.execute_reply.started":"2022-08-10T10:55:30.537399Z","shell.execute_reply":"2022-08-10T10:55:30.564510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data values seem to be valid. Coordinates are in valid range and passenger_count is in reasonable range.","metadata":{}},{"cell_type":"markdown","source":"# **Feature Engineering**","metadata":{}},{"cell_type":"markdown","source":"#### **pickup_datetime**","metadata":{}},{"cell_type":"code","source":"# Check the data types of columns\ntrain_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:30.566833Z","iopub.execute_input":"2022-08-10T10:55:30.567436Z","iopub.status.idle":"2022-08-10T10:55:30.574713Z","shell.execute_reply.started":"2022-08-10T10:55:30.567402Z","shell.execute_reply":"2022-08-10T10:55:30.573866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Everything is good, except for pickup_datetime. It should be converted to date format. Then, we will be able to extract useful information from it, such as:\n\n* year\n* Month\n* Day\n* Weekday\n* Hour","metadata":{}},{"cell_type":"code","source":"# Convert to date type\n# training data\ntrain_df['pickup_datetime'] = pd.to_datetime(train_df['pickup_datetime'])\n\n# test data\ntest_df['pickup_datetime'] = pd.to_datetime(test_df['pickup_datetime'])\n","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:55:30.575818Z","iopub.execute_input":"2022-08-10T10:55:30.576401Z","iopub.status.idle":"2022-08-10T10:57:53.252262Z","shell.execute_reply.started":"2022-08-10T10:55:30.576365Z","shell.execute_reply":"2022-08-10T10:57:53.251013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def date_splitter(df):\n    df['Year'] = df['pickup_datetime'].dt.year\n    df['Month'] = df['pickup_datetime'].dt.month\n    df['Day'] = df['pickup_datetime'].dt.day\n    df['Weekday'] = df['pickup_datetime'].dt.dayofweek\n    df['Hour'] = df['pickup_datetime'].dt.hour\n    \n\n# extract information from pickup_datetime\ndate_splitter(train_df)\ndate_splitter(test_df)\n\n# drop unnecessary pickup_datetime column\ntrain_df.drop(['pickup_datetime'], axis=1, inplace=True)\ntest_df.drop(['pickup_datetime'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:57:53.253598Z","iopub.execute_input":"2022-08-10T10:57:53.253957Z","iopub.status.idle":"2022-08-10T10:57:53.920936Z","shell.execute_reply.started":"2022-08-10T10:57:53.253925Z","shell.execute_reply":"2022-08-10T10:57:53.919658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### **Distance**","metadata":{}},{"cell_type":"markdown","source":"Now let's create distance column by finding the distance between pickup and dropoff locations using haversine formula (that I found in google).\n\nThe formulas look like these:\n\n**a = sin²(φB - φA/2) + cos φA cos φB sin²(λB - λA/2)**\n\n**c = 2 * atan2( √a, √(1−a) )**\n\n**d = R ⋅ c**\n\nwhere:\n\n* d is the distance\n* φ is latitude\n* λ is longitude\n* R is earth’s radius (6,371km)","metadata":{}},{"cell_type":"code","source":"import math\n\ndef haversine_distance(df):\n    coord = ['pickup_latitude', \n             'pickup_longitude', \n             'dropoff_latitude', \n             'dropoff_longitude']\n    \n    phi1, lambda1, phi2, lambda2 = [df[i]*math.pi/180.0 for i in coord]\n    \n    R = 6371\n    \n    # distance between latitudes and longitudes\n    dPhi = (phi2 - phi1)\n    dLambda = (lambda2 - lambda1)\n    \n    a = np.sin(dPhi / 2.0) ** 2 + np.cos(phi1) * np.cos(phi2) * np.sin(dLambda / 2.0) ** 2\n    c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1-a))\n    d = (R * c)\n    \n    df['Distance'] = d\n    \n    \nhaversine_distance(train_df)\nhaversine_distance(test_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:57:53.922567Z","iopub.execute_input":"2022-08-10T10:57:53.923051Z","iopub.status.idle":"2022-08-10T10:57:54.057176Z","shell.execute_reply.started":"2022-08-10T10:57:53.923014Z","shell.execute_reply":"2022-08-10T10:57:54.056034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.Distance.sort_values()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:57:54.058445Z","iopub.execute_input":"2022-08-10T10:57:54.058823Z","iopub.status.idle":"2022-08-10T10:57:54.239169Z","shell.execute_reply.started":"2022-08-10T10:57:54.058789Z","shell.execute_reply":"2022-08-10T10:57:54.237875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I will drop the distance below 500 meters (0.5km). These values are suspicious. Values around 0 meter are the result of pickup and dropoff coordinates being the same. There might have been taxi orders that got cancelled after drivers had already driven several hundread meters. That's why I decided to drop the values that are less than half kilometer.","metadata":{}},{"cell_type":"code","source":"train_df = train_df.drop(train_df[train_df.Distance<0.5].index, axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:57:54.243005Z","iopub.execute_input":"2022-08-10T10:57:54.243352Z","iopub.status.idle":"2022-08-10T10:57:54.438675Z","shell.execute_reply.started":"2022-08-10T10:57:54.243323Z","shell.execute_reply":"2022-08-10T10:57:54.437290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **EDA**","metadata":{}},{"cell_type":"markdown","source":"#### **Passenger Count**","metadata":{}},{"cell_type":"code","source":"f, axes = plt.subplots(1, 2)\n\nsns.barplot(x='passenger_count', y='fare_amount', data=train_df, ax=axes[0])\nsns.scatterplot(x='passenger_count', y='fare_amount', data=train_df, ax=axes[1])","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:57:54.440414Z","iopub.execute_input":"2022-08-10T10:57:54.440812Z","iopub.status.idle":"2022-08-10T10:58:12.791678Z","shell.execute_reply.started":"2022-08-10T10:57:54.440778Z","shell.execute_reply":"2022-08-10T10:58:12.790691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks like average fare amounts are a little higher in 6 passenger rides. Though, one passenger rides have highest fare values recorded.","metadata":{}},{"cell_type":"markdown","source":"#### **Year**","metadata":{}},{"cell_type":"code","source":"f, axes = plt.subplots(1, 2)\n\nsns.barplot(x='Year', y='fare_amount', data=train_df, ax=axes[0])\nsns.scatterplot(x='Year', y='fare_amount', data=train_df, ax=axes[1])","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:58:12.792923Z","iopub.execute_input":"2022-08-10T10:58:12.793257Z","iopub.status.idle":"2022-08-10T10:58:26.633275Z","shell.execute_reply.started":"2022-08-10T10:58:12.793227Z","shell.execute_reply":"2022-08-10T10:58:26.632040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Average fare amount gets higher after 2012 and there are highest fare_amounts recorded after this time as well.","metadata":{}},{"cell_type":"markdown","source":"#### **Month**","metadata":{}},{"cell_type":"code","source":"f, axes = plt.subplots(1, 2)\n\nsns.barplot(x='Month', y='fare_amount', data=train_df, ax=axes[0])\nsns.scatterplot(x='Month', y='fare_amount', data=train_df, ax=axes[1])","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:58:26.634664Z","iopub.execute_input":"2022-08-10T10:58:26.635074Z","iopub.status.idle":"2022-08-10T10:58:40.055663Z","shell.execute_reply.started":"2022-08-10T10:58:26.635041Z","shell.execute_reply":"2022-08-10T10:58:40.054409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is quite a variation in fare amount throughout the year. Maybe it's me but it looks like average fare amount is higher in second and last quarter of the year.","metadata":{}},{"cell_type":"markdown","source":"#### **Day of the month**","metadata":{}},{"cell_type":"code","source":"f, axes = plt.subplots(1, 2)\n\nsns.barplot(x='Day', y='fare_amount', data=train_df, ax=axes[0])\nsns.scatterplot(x='Day', y='fare_amount', data=train_df, ax=axes[1])","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:58:40.057211Z","iopub.execute_input":"2022-08-10T10:58:40.057569Z","iopub.status.idle":"2022-08-10T10:58:53.270429Z","shell.execute_reply.started":"2022-08-10T10:58:40.057537Z","shell.execute_reply":"2022-08-10T10:58:53.268792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I don't see much difference in fare amount during a month. But maybe a little higher in the middle of the month.","metadata":{}},{"cell_type":"markdown","source":"#### **Day of week**","metadata":{}},{"cell_type":"code","source":"f, axes = plt.subplots(1, 2)\n\nsns.barplot(x='Weekday', y='fare_amount', data=train_df, ax=axes[0])\nsns.scatterplot(x='Weekday', y='fare_amount', data=train_df, ax=axes[1])","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:58:53.276129Z","iopub.execute_input":"2022-08-10T10:58:53.276535Z","iopub.status.idle":"2022-08-10T10:59:06.576973Z","shell.execute_reply.started":"2022-08-10T10:58:53.276499Z","shell.execute_reply":"2022-08-10T10:59:06.575713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Fare amount seems to be a little higher on sunday(puple bar). Perhaps people go to longer trips on sundays and spend more money.","metadata":{}},{"cell_type":"markdown","source":"#### **Hour**","metadata":{}},{"cell_type":"code","source":"f, axes = plt.subplots(1, 2)\n\nsns.barplot(x='Hour', y='fare_amount', data=train_df, ax=axes[0])\nsns.scatterplot(x='Hour', y='fare_amount', data=train_df, ax=axes[1])","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:59:06.578543Z","iopub.execute_input":"2022-08-10T10:59:06.579582Z","iopub.status.idle":"2022-08-10T10:59:19.831331Z","shell.execute_reply.started":"2022-08-10T10:59:06.579539Z","shell.execute_reply":"2022-08-10T10:59:19.830115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"People pay highest amount from 3 to 6 in the early morning. It's also high from 13 to 17 in the evening and at midnight from 22 to 24.","metadata":{}},{"cell_type":"markdown","source":"#### **Distance**","metadata":{}},{"cell_type":"markdown","source":"I'll create distance groups to visualize them.","metadata":{}},{"cell_type":"code","source":"train_df['DistanceGroups'] = pd.qcut(train_df['Distance'], 10)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:04:46.300633Z","iopub.execute_input":"2022-08-10T11:04:46.301116Z","iopub.status.idle":"2022-08-10T11:04:46.460583Z","shell.execute_reply.started":"2022-08-10T11:04:46.301077Z","shell.execute_reply":"2022-08-10T11:04:46.459349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, axes = plt.subplots(1, 2)\nplt.setp( axes[0].xaxis.get_majorticklabels(), rotation=70 )\nsns.barplot(x='DistanceGroups', y='fare_amount', data=train_df, ax=axes[0])\nsns.scatterplot(x='Distance', y='fare_amount', data=train_df, ax=axes[1])","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:23:07.274302Z","iopub.execute_input":"2022-08-10T11:23:07.275521Z","iopub.status.idle":"2022-08-10T11:23:20.353945Z","shell.execute_reply.started":"2022-08-10T11:23:07.275477Z","shell.execute_reply":"2022-08-10T11:23:20.350773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As expected fare amount really increases with the distance taxi travels.","metadata":{}},{"cell_type":"markdown","source":"# **Test different models and Choose the best one**","metadata":{}},{"cell_type":"markdown","source":"**I'll put every model under the *comment* and I will run only the model that gave me the best score.**","metadata":{}},{"cell_type":"code","source":"train_df.columns","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:32:03.199703Z","iopub.execute_input":"2022-08-10T11:32:03.200559Z","iopub.status.idle":"2022-08-10T11:32:03.209008Z","shell.execute_reply.started":"2022-08-10T11:32:03.200503Z","shell.execute_reply":"2022-08-10T11:32:03.207706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = train_df.columns[2:-1]\noutcome = train_df.columns[1]","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:35:32.042342Z","iopub.execute_input":"2022-08-10T11:35:32.042809Z","iopub.status.idle":"2022-08-10T11:35:32.049269Z","shell.execute_reply.started":"2022-08-10T11:35:32.042771Z","shell.execute_reply":"2022-08-10T11:35:32.048089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(train_df[['pickup_longitude', 'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude', 'passenger_count', 'Year', 'Month', 'Day', 'Weekday', 'Hour', 'Distance']], train_df['fare_amount'], test_size=0.30, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T07:35:20.111064Z","iopub.execute_input":"2022-08-10T07:35:20.111366Z","iopub.status.idle":"2022-08-10T07:35:20.375296Z","shell.execute_reply.started":"2022-08-10T07:35:20.111338Z","shell.execute_reply":"2022-08-10T07:35:20.374124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Linear Regression**","metadata":{}},{"cell_type":"markdown","source":"First I used Linear Regression which gave me the score of **5.37069**. That's in the same range of RMSE that the creators of this assignment got (5-8 dollars). Therefore, it won't work and I will have to test other models.","metadata":{}},{"cell_type":"code","source":"# linreg = LinearRegression()\n# linreg.fit(X_train, y_train)\n# y_pred = linreg.predict(X_test)\n\n# X, y = train_df[features], train_df[outcome]\n# linreg = LinearRegression()\n# linreg.fit(X, y)\n# prediction = linreg.predict(test_df[features])\n\n# # submission\n# submission = pd.DataFrame({\n#         \"key\": test_df['key'],\n#         \"fare_amount\": prediction\n# })\n\n# submission.to_csv('taxi_fare_submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T07:35:46.751215Z","iopub.execute_input":"2022-08-10T07:35:46.751624Z","iopub.status.idle":"2022-08-10T07:35:47.491292Z","shell.execute_reply.started":"2022-08-10T07:35:46.751590Z","shell.execute_reply":"2022-08-10T07:35:47.489991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **K Neighbors Regressor**","metadata":{}},{"cell_type":"markdown","source":"Next I tried K Neighbors Regression. First I split training data into training and test data and tested on several values of neighbors. Modle with 40 neighbors had the lowest score, so I ran the final model with 40 neighbors. It was a great improvement over Linear Regression, with RMSE of **3.99076**.","metadata":{}},{"cell_type":"code","source":"# def KNR(n):\n#     knr = KNeighborsRegressor(n_neighbors=n)\n#     knr.fit(X_train, y_train)\n#     y_pred = knr.predict(X_test)\n#     rmse = mean_squared_error(y_test, y_pred, squared=False)\n#     return rmse\n\n# for i in range(1,11):\n#     print(10*i, ': ', KNR(10*i))\n\n\n# It gave me the following results\n# n_neighbors    Score\n#         10  :  4.0726036722155925\n#         20  :  4.012596830258568\n#         30  :  3.999477728036027\n#         40  :  3.994987632778089\n#         50  :  3.9991219381343965\n#         60  :  4.000479797914467\n#         70  :  4.009169503938909\n#         80  :  4.01605607851281\n#         90  :  4.022168817576728\n#         100 :  4.030149453188553","metadata":{"execution":{"iopub.status.busy":"2022-08-10T08:27:18.486130Z","iopub.execute_input":"2022-08-10T08:27:18.486590Z","iopub.status.idle":"2022-08-10T08:27:18.492409Z","shell.execute_reply.started":"2022-08-10T08:27:18.486553Z","shell.execute_reply":"2022-08-10T08:27:18.491144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X, y = train_df[features], train_df[outcome]\n# knr = KNeighborsRegressor(n_neighbors=40)\n# knr.fit(X, y)\n# prediction = knr.predict(test_df[features])\n\n# # submission\n# submission = pd.DataFrame({\n#         \"key\": test_df['key'],\n#         \"fare_amount\": prediction\n# })\n\n# submission.to_csv('taxi_fare_submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T08:22:55.863222Z","iopub.execute_input":"2022-08-10T08:22:55.863636Z","iopub.status.idle":"2022-08-10T08:23:02.412193Z","shell.execute_reply.started":"2022-08-10T08:22:55.863599Z","shell.execute_reply":"2022-08-10T08:23:02.411180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Random Forest Regressor**","metadata":{}},{"cell_type":"markdown","source":"Random forest gave me even better result. Again, I tried different estimators and found that 100 estimators had best score, so I ran the final model with 100 estimators. The resulting RMSE was, **3.23022**.","metadata":{}},{"cell_type":"code","source":"# Random Forest Regressor\n# def RFR(n):\n#     rfr = RandomForestRegressor(n_estimators=n, random_state=0)\n#     rfr.fit(X_train, y_train)\n#     y_pred = rfr.predict(X_test)\n#     rmse = mean_squared_error(y_test, y_pred, squared=False)\n#     return rmse\n\n# for i in range(1,11):\n#     print(10*i, ': ', RFR(10*i))\n    \n#It gave me the following results  \n# n_neighbors    Score\n#         10  :  3.4126608405330314\n#         20  :  3.3423199591226855\n#         30  :  3.3147869330263653\n#         40  :  3.2998315520264656\n#         50  :  3.293606024843092\n#         60  :  3.290310014079794\n#         70  :  3.2822512415015654\n#         80  :  3.277728380283033\n#         90  :  3.2738638228600263\n#         100 :  3.2708099367286136","metadata":{"execution":{"iopub.status.busy":"2022-08-10T08:27:46.353573Z","iopub.execute_input":"2022-08-10T08:27:46.354011Z","iopub.status.idle":"2022-08-10T09:48:11.360991Z","shell.execute_reply.started":"2022-08-10T08:27:46.353953Z","shell.execute_reply":"2022-08-10T09:48:11.359662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X, y = train_df[features], train_df[outcome]\n# regr = RandomForestRegressor(n_estimators=100, random_state=0)\n# regr.fit(X, y)\n# prediction = regr.predict(test_df[features])\n\n# # submission\n# submission = pd.DataFrame({\n#         \"key\": test_df['key'],\n#         \"fare_amount\": prediction\n# })\n\n# submission.to_csv('taxi_fare_submission.csv',index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **XGBRegressor**","metadata":{}},{"cell_type":"markdown","source":"XGB Regressor had a little improvement over Random Forest. RMSE = **3.21471**.","metadata":{}},{"cell_type":"code","source":"my_model = XGBRegressor(n_estimators=500, learning_rate=0.05)\nmy_model.fit(X_train, y_train, \n             early_stopping_rounds=5, \n             eval_set=[(X_test, y_test)])\n\nprediction = my_model.predict(test_df[features])","metadata":{"execution":{"iopub.status.busy":"2022-08-09T19:30:50.426980Z","iopub.execute_input":"2022-08-09T19:30:50.427328Z","iopub.status.idle":"2022-08-09T19:30:50.432116Z","shell.execute_reply.started":"2022-08-09T19:30:50.427295Z","shell.execute_reply":"2022-08-09T19:30:50.430904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\n        \"key\": test_df['key'],\n        \"fare_amount\": prediction\n})\n\nsubmission.to_csv('taxi_fare_submission.csv',index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **DNN**","metadata":{}},{"cell_type":"markdown","source":"Lastly I tried DNN model. It performed well but the rmse is a little higher than it was with XGBRegressor and Random Forest. RMSE= **3.41747**. Unfortunately, I don't have any more tries left before deadline, though I believe it would generate better result with some more parameter tuning.","metadata":{}},{"cell_type":"code","source":"# scaler = StandardScaler()\n\n# scaled_train = scaler.fit_transform(X_train)\n# scaled_valid = scaler.transform(X_test)\n# scaled_test = scaler.transform(test_df[features])","metadata":{"execution":{"iopub.status.busy":"2022-08-09T21:08:48.775766Z","iopub.execute_input":"2022-08-09T21:08:48.776167Z","iopub.status.idle":"2022-08-09T21:08:49.110912Z","shell.execute_reply.started":"2022-08-09T21:08:48.776133Z","shell.execute_reply":"2022-08-09T21:08:49.109872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# early_stopping = callbacks.EarlyStopping(\n#     min_delta=0.001,\n#     patience=5,\n#     restore_best_weights=True,\n# )\n\n\n# model = keras.Sequential([\n    \n#     layers.Dense(128, activation='relu', input_dim=features.shape[0]),\n#     layers.BatchNormalization(),\n\n#     layers.Dense(64, activation='relu'),\n#     layers.BatchNormalization(),\n\n#     layers.Dense(32, activation='relu'),\n#     layers.BatchNormalization(),\n\n#     layers.Dense(8, activation='relu'),\n#     layers.BatchNormalization(),\n\n#     layers.Dense(1)\n# ])\n\n# model.compile(optimizer='adam',\n#               loss='mse',\n#               metrics=[tf.keras.metrics.RootMeanSquaredError()]\n# )","metadata":{"execution":{"iopub.status.busy":"2022-08-09T21:09:30.561175Z","iopub.execute_input":"2022-08-09T21:09:30.561619Z","iopub.status.idle":"2022-08-09T21:09:30.683738Z","shell.execute_reply.started":"2022-08-09T21:09:30.561580Z","shell.execute_reply":"2022-08-09T21:09:30.682110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history = model.fit(\n#     scaled_train,\n#     y_train,\n#     validation_data=(scaled_valid, y_test),\n#     batch_size = 256,\n#     epochs = 50,\n#     callbacks=[early_stopping]\n# )","metadata":{"execution":{"iopub.status.busy":"2022-08-09T21:10:29.518330Z","iopub.execute_input":"2022-08-09T21:10:29.518840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prediction = model.predict(scaled_test, batch_size=256, verbose=1)\n# prediction = prediction.ravel()\n\n\n# # submission\n# submission = pd.DataFrame({\n#         \"key\": test_df['key'],\n#         \"fare_amount\": prediction\n# })\n\n# submission.to_csv('taxi_fare_submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T19:34:39.145746Z","iopub.execute_input":"2022-08-09T19:34:39.146171Z","iopub.status.idle":"2022-08-09T19:34:39.611418Z","shell.execute_reply.started":"2022-08-09T19:34:39.146139Z","shell.execute_reply":"2022-08-09T19:34:39.610320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**For now XGB Regression performed the best among my models.**","metadata":{}}]}