{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nimport matplotlib.pyplot as plt \nimport math\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Checking out the dtypes shows us that there are some string values in the data set","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"There are only 33 values in 5_000_000 rows.. so we just drop them\nAlso we convert those string dtypes to float in the data set","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Now we check for the outliers and other things like mean,SD and min max values for 5_000_000 * 5 rows","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_describe(chunks):\n    return chunks.describe().values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from math import radians, cos, sin, asin, sqrt\n\ndef haversine_distance(lat1, long1, lat2, long2,df):\n    data = df\n    phi1 = np.radians(df[lat1])\n    phi2 = np.radians(df[lat2])\n    R = 6371\n    delta_phi = np.radians(df[lat2]-df[lat1])\n    delta_lambda = np.radians(df[long2]-df[long1])\n    \n    #a = sin²((φB - φA)/2) + cos φA . cos φB . sin²((λB - λA)/2)\n    a = np.sin(delta_phi / 2.0) ** 2 + np.cos(phi1) * np.cos(phi2) * np.sin(delta_lambda / 2.0) ** 2\n    \n    #c = 2 * atan2( √a, √(1−a) )\n    c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1-a))\n    \n    #d = R*c\n    d = (R * c) #in kilometers\n    df['Distance'] = d\n    return d","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Function to convert dtypes of pickup date time from objects to datetime format, and dropping the null values ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_cleaning(chunks):\n    chunks.dropna(axis = 0, inplace = True)\n    chunks.fare_amount = pd.to_numeric(chunks.fare_amount, errors='coerce')\n    chunks.passenger_count = pd.to_numeric(chunks.passenger_count, errors='coerce')    \n    chunks.pickup_latitude = pd.to_numeric(chunks.pickup_latitude, errors='coerce')\n    chunks.pickup_longitude = pd.to_numeric(chunks.pickup_longitude, errors='coerce')\n    chunks.dropoff_latitude = pd.to_numeric(chunks.dropoff_latitude, errors='coerce')\n    chunks.dropoff_longitude = pd.to_numeric(chunks.dropoff_longitude, errors='coerce')\n    return chunks","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Function to plot on map","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def mapplots(df):\n    import plotly.express as px\n    fig = px.scatter_mapbox(df, lat=\"pickup_latitude\", lon=\"pickup_longitude\", hover_name=\"passenger_count\", hover_data=['passenger_count','fare_amount'],\n                        color_discrete_sequence=[\"black\"], zoom=3, height=300)\n    fig.update_layout(mapbox_style=\"open-street-map\")\n    fig.update_layout(margin={\"r\":0,\"t\":0,\"l\":0,\"b\":0})\n    fig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_new_date_time_features(dataset):\n    dataset['pickup_datetime'] = pd.to_datetime(dataset['pickup_datetime'],format=\"%Y-%m-%d %H:%M:%S UTC\")\n    dataset['hour'] = dataset.pickup_datetime.dt.hour\n    dataset['day'] = dataset.pickup_datetime.dt.day\n    dataset['month'] = dataset.pickup_datetime.dt.month\n    dataset['year'] = dataset.pickup_datetime.dt.year\n    dataset['day_of_week'] = dataset.pickup_datetime.dt.dayofweek\n    \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" dft=  pd.read_csv('/kaggle/input/iiitb2019nyctaxifare/TrainTest/train.csv', nrows=2_500_000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dft.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def map_box(dft):\n    dft.drop(index=dft[(dft.pickup_longitude <= -75) \n                         | (dft.pickup_longitude >= -72) \n                         | (dft.dropoff_longitude <= -75) \n                         | (dft.dropoff_longitude >= -72)\n                         | (dft.pickup_latitude <= 39)\n                         | (dft.pickup_latitude >= 42)\n                         | (dft.dropoff_latitude <= 39)\n                         | (dft.dropoff_latitude >= 42)].index, inplace=True)\n    return dft","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Calculating distance between two points, adding this feature as this directly effects the fare amount","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"haversine_distance('pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude',df)\ndf.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Removing the negative values in the fare column ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"old size: %d\" % len(df))\ndf = df[df.fare_amount >=0]\nprint(\"New size: %d\" % len(df))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Also the fare amount cannot be above 600, it could be because of the incorrect input values ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df[df.fare_amount>600]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As there are only 8 rows we can drop these values. Since the distance between  is of the garbage value. Better to drop these off","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"old size: %d\" % len(df))\ndf = df[df.fare_amount <=600]\nprint(\"New size: %d\" % len(df))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Passenger count cannot be more then 12 considering everyone as kid, still more then 12 is physically impossible ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df[df.passenger_count<12]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Modelling**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\n\n\nchunks = pd.read_csv(\"/kaggle/input/iiitb2019nyctaxifare/TrainTest/train.csv\", nrows = 5_00,low_memory=False)\nrf = RandomForestRegressor(warm_start=True,n_estimators= 20)\n\nchunks = data_cleaning(chunks)\nchunks = chunks.drop('key',axis=1)\nchunks = chunks[chunks.fare_amount >=0]\nchunks = chunks[chunks.fare_amount <=600]\nchunks = chunks[chunks.passenger_count<12]\nadd_new_date_time_features(chunks)\nchunks = chunks.drop('pickup_datetime',axis=1)\nchunks = map_box(chunks)\nhaversine_distance('pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude',chunks)\nX = chunks.iloc[:,chunks.columns!='fare_amount']\ny = chunks['fare_amount'].values  \nX_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=42)\nrf.fit(X_train, y_train)\npred = rf.predict(X_test)\nmse = mean_squared_error(y_test,pred)\nrmse = math.sqrt(mse)\nprint (rmse)\nprint(chunks.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import metrics\n# Model Accuracy, how often is the classifier correct?\nprint(\"Accuracy:\",rf.score(X_test,y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n\nchunks1 = 5_000\ni = 0 \nz = []\nrf = RandomForestRegressor(warm_start=True,n_estimators= 20)\nglobal testP\nglobal pred\nfor chunks in pd.read_csv(\"/kaggle/input/iiitb2019nyctaxifare/TrainTest/train.csv\", chunksize = chunks1,low_memory=False):\n    i = i+1\n    chunks = data_cleaning(chunks)\n    chunks = chunks.drop('key',axis=1)\n    chunks = chunks[chunks.fare_amount >=0]\n    chunks = chunks[chunks.fare_amount <=600]\n    chunks = chunks[chunks.passenger_count<12]\n    add_new_date_time_features(chunks)\n    chunks = chunks.drop('pickup_datetime',axis=1)\n    chunks = map_box(chunks)\n    haversine_distance('pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude',chunks)\n    X_train = chunks.iloc[:,chunks.columns!='fare_amount']\n    Y_train = chunks['fare_amount'].values    \n    \n    if(i < 10002):\n        rf.fit(X_train, Y_train)\n        rf.n_estimators += 1\n    print(\"iteration no: \"+str(i))    \n\n    if(i==10002):\n        testP = Y_train\n        pred = rf.predict(X_train)\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nimport math\nmse = mean_squared_error(testP,pred)\nrmse = math.sqrt(mse)\nprint (rmse)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model Accuracy, how often is the classifier correct?\nprint(\"Accuracy:\",rf.score(X_train,testP))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Optimisation will happen now**\nThe RMSE for our model is approx around 4.5 to 6 \nNow we need to tune our hyperparameters to get a better model ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"chunks = pd.read_csv(\"/kaggle/input/iiitb2019nyctaxifare/TrainTest/train.csv\", nrows = 500,low_memory=False)\n\nchunks.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ni = 0 \n\nestimators = [ 20 , 60 , 65 ,70 , 75, 80  ]\nchunks = pd.read_csv(\"/kaggle/input/iiitb2019nyctaxifare/TrainTest/train.csv\", nrows = 5_0000,low_memory=False)\nchunks = chunks.drop('key',axis=1)\nchunks = data_cleaning(chunks)\nchunks = chunks[chunks.fare_amount >=0]\nchunks = chunks[chunks.fare_amount <=600]\nchunks = chunks[chunks.passenger_count<12]\nadd_new_date_time_features(chunks)\nchunks = chunks.drop('pickup_datetime',axis=1)\nchunks = map_box(chunks)\nhaversine_distance('pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude',chunks)\n\ntest_results = []\nfor estimator in estimators:\n    rf = RandomForestRegressor(n_estimators=estimator, n_jobs=-1,warm_start = True)\n    \n    i = i+1\n    X = chunks.iloc[:,chunks.columns!='fare_amount']\n    y = chunks['fare_amount'].values  \n    X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=42)\n    rf.fit(X_train, y_train)\n    pred = rf.predict(X_test)\n    mse = mean_squared_error(y_test,pred)\n    rmse = math.sqrt(mse)\n    test_results.append(rmse)\n    print(\"iteration no: \"+str(i))    \n    \nax = plt.axes()   \nax.plot(estimators,test_results)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Best n estimator is around 70 ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nimport matplotlib.pyplot as plt \nimport math\ni = 0 \n\nmax_features1 = [0.1 , 0.2 , 0.3 , 0.4 , 0.5]\nchunks = pd.read_csv(\"/kaggle/input/iiitb2019nyctaxifare/TrainTest/train.csv\", nrows = 5_000_0,low_memory=False)\nchunks = chunks.drop('key',axis=1)\nchunks = data_cleaning(chunks)\nchunks = chunks[chunks.fare_amount >=0]\nchunks = chunks[chunks.fare_amount <=600]\nchunks = chunks[chunks.passenger_count<12]\nadd_new_date_time_features(chunks)\nchunks = chunks.drop('pickup_datetime',axis=1)\nchunks = map_box(chunks)\nhaversine_distance('pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude',chunks)\n\ntest_results = []\nfor features in max_features1:\n    rf = RandomForestRegressor(n_estimators= 70, n_jobs=-1,max_features = features)\n    \n    i = i+1\n    X = chunks.iloc[:,chunks.columns!='fare_amount']\n    y = chunks['fare_amount'].values  \n    X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=42)\n    rf.fit(X_train, y_train)\n    pred = rf.predict(X_test)\n    mse = mean_squared_error(y_test,pred)\n    rmse = math.sqrt(mse)\n    test_results.append(rmse)\n    print(\"iteration no: \"+str(i))    \n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ax = plt.axes()   \nax.plot(max_features1,test_results)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nimport matplotlib.pyplot as plt \nimport math\ni = 0 \n\nmin_samples_leaf1 = [1 , 2 ,3 ,10 , 40]\nchunks = pd.read_csv(\"/kaggle/input/iiitb2019nyctaxifare/TrainTest/train.csv\", nrows = 5_0,low_memory=False)\nchunks = chunks.drop('key',axis=1)\nchunks = data_cleaning(chunks)\nchunks = chunks[chunks.fare_amount >=0]\nchunks = chunks[chunks.fare_amount <=600]\nchunks = chunks[chunks.passenger_count<12]\nadd_new_date_time_features(chunks)\nchunks = chunks.drop('pickup_datetime',axis=1)\nchunks = map_box(chunks)\nhaversine_distance('pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude',chunks)\n\ntest_results = []\nfor leaves in min_samples_leaf1:\n    rf = RandomForestRegressor(n_estimators= 70, n_jobs=-1,max_features = 0.45 , min_samples_leaf = leaves)\n    \n    i = i+1\n    X = chunks.iloc[:,chunks.columns!='fare_amount']\n    y = chunks['fare_amount'].values  \n    X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=42)\n    rf.fit(X_train, y_train)\n    pred = rf.predict(X_test)\n    mse = mean_squared_error(y_test,pred)\n    rmse = math.sqrt(mse)\n    test_results.append(rmse)\n    print(\"iteration no: \"+str(i))    \n    \nax = plt.axes()   \nax.plot(min_samples_leaf1,test_results)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"tuning hyperparameter a little more","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n\nchunks1 = 50_000\ni = 0 \nz = []\nrf = RandomForestRegressor(n_estimators=80, max_features=0.5, n_jobs=-1, warm_start =True)\nglobal testP\nglobal pred\nfor chunks in pd.read_csv(\"/kaggle/input/iiitb2019nyctaxifare/TrainTest/train.csv\", chunksize = chunks1,low_memory=False):\n    i = i+1\n    chunks = data_cleaning(chunks)\n    chunks = chunks.drop('key',axis=1)\n    chunks = chunks[chunks.fare_amount >=0]\n    chunks = chunks[chunks.fare_amount <=600]\n    chunks = chunks[chunks.passenger_count<12]\n    add_new_date_time_features(chunks)\n    chunks = chunks.drop('pickup_datetime',axis=1)\n    chunks = map_box(chunks)\n    haversine_distance('pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude',chunks)\n    X_train = chunks.iloc[:,chunks.columns!='fare_amount']\n    Y_train = chunks['fare_amount'].values    \n    \n    if(i < 51):\n        rf.fit(X_train, Y_train)\n        rf.n_estimators += 1\n    print(\"iteration no: \"+str(i))    \n\n    if(i==51):\n        testP = Y_train\n        pred = rf.predict(X_train)\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgbm\nparams = {\n        'boosting_type':'gbdt',\n        'objective': 'regression',\n        'nthread': 4,\n        'num_leaves': 35,\n        'learning_rate': 0.05,\n        'max_depth': -1,\n        'subsample': 0.8,\n        'bagging_fraction' : 1,\n        'max_bin' : 5000 ,\n        'bagging_freq': 20,\n        'colsample_bytree': 0.6,\n        'metric': 'rmse',\n        'min_split_gain': 0.5,\n        'min_child_weight': 1,\n        'min_child_samples': 10,\n        'scale_pos_weight':1,\n        'zero_as_missing': True,\n        'seed':0,\n        'num_rounds':50000\n    }\n\n\ndef LGBMmodel(X_train,y_train,params):\n    matrix_train = lgbm.Dataset(X_train, y_train)\n    model=lgbm.train(params=params,\n                    train_set=matrix_train,\n                    num_boost_round=100000, \n                    early_stopping_rounds=500,\n                    verbose_eval=100)\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n\n\nparams = {\n    'objective':'regression',\n    'num_leaves':35,\n    'n_estimators':300\n}\n\nchunks1 = 50_000\ni = 0 \nz = []\nglobal testP\nglobal pred\n\nfor chunks in pd.read_csv(\"/kaggle/input/iiitb2019nyctaxifare/TrainTest/train.csv\", chunksize = chunks1,low_memory=False):\n    i = i+1\n    chunks = data_cleaning(chunks)\n    chunks = chunks.drop('key',axis=1)\n    chunks = chunks[chunks.fare_amount >=0]\n    chunks = chunks[chunks.fare_amount <=600]\n    chunks = chunks[chunks.passenger_count<12]\n    add_new_date_time_features(chunks)\n    chunks = chunks.drop('pickup_datetime',axis=1)\n    chunks = map_box(chunks)\n    haversine_distance('pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude',chunks)\n    X_train = chunks.iloc[:,chunks.columns!='fare_amount']\n    Y_train = chunks['fare_amount'].values    \n    train_data = lgb.Dataset(X_train, Y_train)\n    print('iteration = '+str(i))\n    if(i == 50):\n        testP=X_train\n        pred=Y_train\n        break;\n    if(i == 1):\n        gbm = lgb.train(params, train_data, num_boost_round = 10)\n    else:\n        gbm = lgb.train(params, train_data, num_boost_round = 10,init_model = gbm)\n\n        \ngbm.save_model('model.txt')\ny_pred = gbm.predict(testP, num_iteration=gbm.best_iteration)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gbm\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nimport math\n\nmse = mean_squared_error(testP,pred)\nrmse = math.sqrt(mse)\nprint (rmse)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model Accuracy, how often is the classifier correct?\nprint(\"Accuracy:\",rf.score(X_train,testP))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.externals import joblib\njoblib.dump(rf, 'modrf2.pkl') \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv(\"/kaggle/input/iiitb2019nyctaxifare/TrainTest/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nstart_time = time.time() \n\ndef test_box(dft):\n    dft['dropoff_latitude'].fillna(dft['dropoff_latitude'].mean(), inplace = True)\n    dft['dropoff_longitude'].fillna(dft['dropoff_longitude'].mean(), inplace = True)\n    dft['pickup_latitude'].fillna(dft['pickup_latitude'].mean(), inplace = True)\n    dft['pickup_longitude'].fillna(dft['pickup_longitude'].mean(), inplace = True)\n    ind = dft[(dft.pickup_longitude <= -75) | (dft.pickup_longitude >= -72) | (dft.dropoff_longitude <= -75) | (dft.dropoff_longitude >= -72)\n                             | (dft.pickup_latitude <= 39)\n                             | (dft.pickup_latitude >= 42)\n                             | (dft.dropoff_latitude <= 39)\n                             | (dft.dropoff_latitude >= 42)].index\n    print(len(ind))\n    k=0\n    mplat = dft['pickup_latitude'].median()\n    mplon = dft['pickup_longitude'].median()\n    mdlat = dft['dropoff_latitude'].median()\n    mdlon = dft['dropoff_longitude'].median()\n    for i in ind:\n        k=k+1\n        print(k)\n        dft.pickup_longitude.at[i] = mplon;\n        dft.pickup_latitude.at[i] = mplat\n        dft.dropoff_latitude = mdlat\n        dropoff_longitude = mdlon\n    return dft\ntest = test_box(test)\nprint(\"%s seconds\" % (time.time() - start_time))\nhaversine_distance('pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude',test)\n\nind = test[test['Distance']<=0].index\nfor i in ind:\n    test.Distance.at[i] = 2.5\n#test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp = test['key']\ntemp.to_csv(\"key.csv\", index = False , header = ['key'] )\ntest = test.drop('key',axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"add_new_date_time_features(test)\ntest = test.drop('pickup_datetime',axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = rf.predict(test)\nkey_temp = pd.read_csv(\"key.csv\")\ntest['key'] = key_temp\ntest['fare_amount'] = pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result = test.loc[:, [ 'key','fare_amount']]\nresult.to_csv(\"submit_2.csv\", index = False )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"read = pd.read_csv(\"submit_2.csv\")\nread.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"read.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"read.shape()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}