{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport lightgbm as lgbm\n\nimport matplotlib.pyplot as plt\nfrom haversine import haversine\n%matplotlib inline\nimport gc \nimport os\nprint(os.listdir(\"../input\"))\n\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\nimport plotly.graph_objs as go\ninit_notebook_mode(connected=True)\n\n# # Class, for use in pipelines, to select certain columns from a DataFrame and convert to a numpy array\n# # From A. Geron: Hands-On Machine Learning with Scikit-Learn & TensorFlow, O'Reilly, 2017\n# # Modified by Derek Bridge to allow for casting in the same ways as pandas.DatFrame.astype\n# class DataFrameSelector(BaseEstimator, TransformerMixin):\n# def __init__(self, attribute_names, dtype=None):\n# self.attribute_names = attribute_names\n# self.dtype = dtype\n# def fit(self, X, y=None):\n# return self\n# def transform(self, X):\n# X_selected = X[self.attribute_names]\n# if self.dtype:\n# return X_selected.astype(self.dtype).values\n# return X_selected.values\n \n# # Class, for use in pipelines, to binarize nominal-valued features (while avoiding the dummy variable trap)\n# # By Derek Bridge, 2017\n# class FeatureBinarizer(BaseEstimator, TransformerMixin):\n# def __init__(self, features_values):\n# self.features_values = features_values\n# self.num_features = len(features_values)\n# self.labelencodings = [LabelEncoder().fit(feature_values) for feature_values in features_values]\n# self.onehotencoder = OneHotEncoder(sparse=False,n_values=[len(feature_values) for feature_values in features_values])\n# self.last_indexes = np.cumsum([len(feature_values) - 1 for feature_values in self.features_values])\n# def fit(self, X, y=None):\n# for i in range(0, self.num_features):\n# X[:, i] = self.labelencodings[i].transform(X[:, i])\n# return self.onehotencoder.fit(X)\n# def transform(self, X, y=None):\n# for i in range(0, self.num_features):\n# X[:, i] = self.labelencodings[i].transform(X[:, i])\n# onehotencoded = self.onehotencoder.transform(X)\n# return np.delete(onehotencoded, self.last_indexes, axis=1)\n# def fit_transform(self, X, y=None):\n# onehotencoded = self.fit(X).transform(X)\n# return np.delete(onehotencoded, self.last_indexes, axis=1)\n# def get_params(self, deep=True):\n# return {\"features_values\" : self.features_values}\n# def set_params(self, **parameters):\n# for parameter, value in parameters.items():\n# self.setattr(parameter, value)\n# return self\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"38ac811524c78255df22cf7be6abe32db95f7a5e"},"cell_type":"markdown","source":"

Import Train Set

\n

Importing data from the **train.csv** file.

"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# fields\nfields = ['fare_amount', 'pickup_datetime', 'passenger_count', 'pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude']\n\n# Use pandas to read our training set\ndf = pd.read_csv(\"../input/new-york-city-taxi-fare-prediction/train.csv\",\n skipinitialspace=True, \n parse_dates = ['pickup_datetime'],\n infer_datetime_format = True,\n usecols=fields, \n nrows= 10000000)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"4162f6bf42718d5332f0223ee58ec192df15fb9d"},"cell_type":"code","source":"df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"077a2a3631817b4f1ffdad5205ef5741ec95cad5"},"cell_type":"code","source":"df.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b9184d25663725ba1dd7bf0c3beee068dd27eed8"},"cell_type":"markdown","source":"

Feature Engineering

\n

Splitting **date_time** column into two separate columns. One column for the date and one column for the time of day (24hr).

\n

Calculating a new column for the distance between the pickup and dropoff locations using the haversine formula

"},{"metadata":{"trusted":true,"_uuid":"05c325af13186a5dc7c1199a079f1f701c1cb98c"},"cell_type":"code","source":"def calc_haversine(row):\n point1= (row['pickup_latitude'],row['pickup_longitude'])\n point2= (row['dropoff_latitude'],row['dropoff_longitude'])\n return(haversine(point1,point2))\n\ndef feature_engineer(df, train=True):\n # split the pickup_datetime into year, month, day, hour\n df['year'] = df['pickup_datetime'].dt.year\n df['month'] = df['pickup_datetime'].dt.month\n df['day'] = df['pickup_datetime'].dt.day\n df['hour'] = df['pickup_datetime'].dt.hour\n df.drop(['pickup_datetime'], axis=1, inplace=True)\n \n # converting and then downcast the new columns to smaller sizes\n df['passenger_count'] = pd.to_numeric(df['passenger_count'], downcast='integer')\n df['pickup_latitude'] = pd.to_numeric(df['pickup_latitude'], downcast='float')\n df['pickup_longitude'] = pd.to_numeric(df['pickup_longitude'], downcast='float')\n df['dropoff_latitude'] = pd.to_numeric(df['dropoff_latitude'], downcast='float')\n df['dropoff_longitude'] = pd.to_numeric(df['dropoff_longitude'], downcast='float')\n \n \n \n if(train):\n df['fare_amount'] = pd.to_numeric(df['fare_amount'], downcast='float')\n\n\n \n # calculate haversine distance between pickup and dropoff locations\n df['distance'] = df.apply(calc_haversine, axis=1)\n\n # downcast to save memory\n df['distance'] = pd.to_numeric(df['distance'], downcast='float')\n\n # checkout the new df\n df.head()\n \n return df\n\ndf = feature_engineer(df)\ndf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9348e1366c0357999f6ebd4591998f486cd0b953"},"cell_type":"code","source":"df.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f0a81d84c8c39f3855e4bf9c9a09a37209942f2d"},"cell_type":"markdown","source":"

Data Cleaning

"},{"metadata":{"trusted":true,"_uuid":"a4cf7a9d66fb49dea0a3e99436d403314e36dd1b"},"cell_type":"code","source":"def clean_data(df, train=True):\n # remove any negative fares, zero passengers and impossible coordinates\n df = df[ \n (df.passenger_count >= 1) &\n (df.passenger_count < 8) &\n (df.pickup_latitude > 40.5) &\n (df.pickup_latitude < 41) &\n (df.pickup_longitude > -75) &\n (df.pickup_longitude < -73) &\n (df.dropoff_latitude > 40.5) &\n (df.dropoff_latitude < 41) &\n (df.dropoff_longitude > -75) &\n (df.dropoff_longitude < -73) &\n (df.pickup_latitude != 0) &\n (df.pickup_longitude != 0) &\n (df.dropoff_latitude != 0) &\n (df.dropoff_latitude != 0) &\n (df.distance > 0)\n ]\n if(train):\n df = df[(df.fare_amount > 0)]\n\n # coordinates should fall within these confines \n\n #40.507754, -74.255323 # westernmost point\n #40.739021, -73.700556 # easternmost point\n #40.914862, -73.909555 # northernmost point\n #40.496218, -74.247699 # southernmost point\n return df\n\ndf = clean_data(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"4d7c99b4d47231770542c090c36c6dc907a6ff10"},"cell_type":"code","source":"df = df.reset_index(drop=True)\ndf.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea0505054c702350b683d8027c367c0cb91afc82"},"cell_type":"code","source":"# get the labels\ny = df.pop('fare_amount').values\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1dc581b8550d869ec30ff5d7ffe48095c382f2b9"},"cell_type":"code","source":"x_train = df.iloc[:, df.columns != 'fare_amount']\ny_train = y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"53a0cc971121eb9172f1cf7151608de5b3e16612"},"cell_type":"markdown","source":"

Train The Model

"},{"metadata":{"trusted":true,"_uuid":"af39e7358b0ae9040f58e46ceffcdb9e4fdf1cd6"},"cell_type":"code","source":"# Use pandas to read our training set\ntest_df = pd.read_csv(\"../input/new-york-city-taxi-fare-prediction/test.csv\",\n skipinitialspace=True, \n parse_dates = ['pickup_datetime'],\n infer_datetime_format = True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"618807c291b0d1f4f4e20d08d462f1419d2d0039"},"cell_type":"code","source":"test_df = feature_engineer(test_df, train=False)\n#test_df = clean_data(test_df, train=False)\ntest_keys = test_df.pop('key').values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"433ac8d9c086d23ba682647b77436b9a15dfe461"},"cell_type":"code","source":"test_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"213069ce680fbd8ca6c4b0ae2e5d094dca82c802"},"cell_type":"code","source":"lgbm_params = {\n 'task': 'train',\n 'boosting_type': 'gbdt',\n 'objective': 'regression',\n 'metric': 'rmse',\n 'learning_rate': 0.01,\n 'num_leaves': 31,\n 'max_depth': -1,\n 'bagging_freq': 20,\n 'colsample_bytree': 0.8,\n 'min_gain_to_split': 0.5,\n 'num_iterations': 50000,\n 'max_bin': 500\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6911174cae054db37be12880ce203892d41595c2"},"cell_type":"code","source":"x_test = test_df\n\npred_test_y = np.zeros(x_test.shape[0])\n\ntrain_set = lgbm.Dataset(x_train, y_train)\n\nmodel = lgbm.train(lgbm_params, train_set=train_set)\n\npred_test_y = model.predict(x_test, num_iteration = model.best_iteration)\n\n#len(pred_test_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80ab970a6ab1dfea6f96d068f0402f2eb0444543"},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv')\nsubmission['fare_amount'] = pred_test_y\nsubmission.to_csv('lgbm_submission.csv', index=False)\nsubmission.head(20)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bb1c272c796a74dab57beff887fa0109d4914530"},"cell_type":"markdown","source":"

Exploring NY Precipitation Data Effects

"},{"metadata":{"trusted":true,"_uuid":"a2772a049db8bc20aa373a254238982531f9db17"},"cell_type":"code","source":"p_fields = ['STATION','NAME','DATE','PRCP','SNOW','SNWD','TMAX','TMIN']\n\n# Use pandas to read our precipitation data\np_df = pd.read_csv(\"../input/ny-precipitation-data/1594710.csv\",\n skipinitialspace=True,\n infer_datetime_format = True,\n usecols=p_fields\n )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"938a65dea6a2a00190974aeb78dbe2ae7c9ddae3"},"cell_type":"code","source":"p_df = p_df[\n p_df.NAME.str.contains('NY CITY CENTRAL PARK, NY US') |\n p_df.NAME.str.contains('STATEN ISLAND 1.4 SE, NY US') |\n p_df.NAME.str.contains('STATEN ISLAND 4.5 SSE, NY US') |\n p_df.NAME.str.contains('BROOKLYN 3.1 NW, NY US') |\n #p_df.NAME.str.contains('BRONX, NY US') | removed as there are only 4 rows of data \n p_df.NAME.str.contains('JFK INTERNATIONAL AIRPORT, NY US') |\n p_df.NAME.str.contains('LA GUARDIA AIRPORT, NY US')\n]\np_df.fillna(0.0)\np_df.describe()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3fd980753863e1a18bd23b00e14b226e18957e11"},"cell_type":"code","source":"\np_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e8a3cd2a983173c1f38c6126f37ba4402b5f2fc"},"cell_type":"code","source":"n = 0\nwhile n < len(p_df.STATION.unique()):\n print(p_df.STATION.unique()[n], p_df.NAME.unique()[n])\n n = n + 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dff4a46fe92e5495b995989fcceb11896f009e90"},"cell_type":"code","source":"p_df = p_df.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"61c67e1b4329a374c26186d211dbb5ca15cefcd8"},"cell_type":"code","source":"central_park = pd.DataFrame(columns=p_fields, data=p_df[p_df.NAME.str.contains(\"NY CITY CENTRAL PARK, NY US\")])\ndata = [go.Bar(x=central_park.DATE,\n y=p_df.SNOW)]\niplot(data, filename='jupyter-basic_bar')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4f9c67f03a52f6d077c237a45c7d25e910c2e47"},"cell_type":"code","source":"\nla_guardia = pd.DataFrame(columns=p_fields, data=p_df[p_df.NAME.str.contains(\"LA GUARDIA AIRPORT, NY US\")])\ndata = [go.Bar(x=la_guardia.DATE,\n y=p_df.SNOW)]\niplot(data, filename='jupyter-basic_bar')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fcecf32e975323afaf72284fd7aa8f12cedd7199"},"cell_type":"code","source":"staten_island = pd.DataFrame(columns=p_fields, data=p_df[p_df.NAME.str.contains(\"STATEN ISLAND 1.4 SE, NY US\")])\ndata = [go.Bar(x=staten_island.DATE,\n y=p_df.SNOW)]\niplot(data, filename='jupyter-basic_bar')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8dfa877252a140b9bdbd57d16ac677b727dcc6ca"},"cell_type":"code","source":"jfk = pd.DataFrame(columns=p_fields, data=p_df[p_df.NAME.str.contains(\"JFK INTERNATIONAL AIRPORT, NY US\")])\ndata = [go.Bar(x=jfk.DATE,\n y=p_df.SNOW)]\niplot(data, filename='jupyter-basic_bar')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}