{"cells":[{"metadata":{"_uuid":"782a5276bf47329f258cd1a7d644ae508ac0fddd"},"cell_type":"markdown","source":"## 1. Data loading"},{"metadata":{"trusted":false,"_uuid":"b37486794ff0d41663bfad91c802e7f1fa4d035c"},"cell_type":"code","source":"import os\n\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\n\nimport math","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"42d4f8941101481c455a51d101a409fed1d8ee54"},"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import mean_squared_log_error as MSLE\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import cross_val_predict\nfrom sklearn.preprocessing import StandardScaler","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"9567e2b9bff212a6db0c93abec41d9ac5959e49b"},"cell_type":"code","source":"import matplotlib as mpl\nimport matplotlib.pyplot as plt\n\nmpl.rcParams['axes.titlesize']=12\nmpl.rcParams['xtick.labelsize']=12\nmpl.rcParams['ytick.labelsize']=12\n\n\n%matplotlib inline ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5316a52c41ab89a24a08090d218d7f0052661e01"},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a3dbab7fad1b37163e0ff8d90c25e9d0aeba87ff"},"cell_type":"code","source":"BASEPATH = os.path.join('../input')\nTRAIN_PATH = os.path.join(BASEPATH, 'train.csv')\nTEST_PATH = os.path.join(BASEPATH, 'test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"21f713aa709b1a539efa73c155e74726c084103c"},"cell_type":"code","source":"train = pd.read_csv(TRAIN_PATH)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8e538379b595de944595534105889473089cf045"},"cell_type":"markdown","source":"## 2. Data exploration"},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"d43b468687cf19f05714e46f9467f6da4cb9258e"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"bfdc7781fafc0de4a4fa8b536e58bc781cc99928"},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b597c5fcf5d93fbcb0b527eddd5ead6d3be1b044"},"cell_type":"code","source":"print(\"Train size :\",len(train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"17e82af5292ed4360ec891d5eab75c83e3dba866"},"cell_type":"code","source":"mpl.rcParams['figure.figsize']=(10,5)\n\nplt.hist(train['trip_duration'])\nplt.title(\"Trip Duration Distribution\");","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"4ead40b97a7e047d15d26c1536120b53fb77c0e7"},"cell_type":"code","source":"train.trip_duration[train['trip_duration'] > 3000][train['trip_duration'] < 10000].hist()\nplt.title(\"Under 50 Minutes Trips Duration Distribution\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a58949d1a4f0f4249aec6ae16560b7fd07ba7b08"},"cell_type":"code","source":"train.trip_duration[train['trip_duration'] > 9500][train['trip_duration'] < 15000].hist()\nplt.title(\"Under 50 Minutes Trips Duration Distribution\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e7912cbf2ddb1a787e6bf989ed88c6d0685c271f"},"cell_type":"code","source":"test = pd.read_csv(TEST_PATH)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"1751bee41b8e3776f246a93d2c5980b4a56fe20b"},"cell_type":"code","source":"train.dtypes, test.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e590b77336f1c681561ec1fd99bb52ba18a96a16"},"cell_type":"code","source":"NUM_VARS = ['vendor_id', 'passenger_count', 'pickup_longitude', 'pickup_latitude', 'pickup_latitude',\n           'dropoff_longitude', 'dropoff_latitude']\nCAT_VARS = ['pickup_datetime','dropoff_datetime', 'store_and_fwd_flag']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a6922b53f15be2cd87f9082542bdc476836ff9cf"},"cell_type":"markdown","source":"## 3. Data preprocessing"},{"metadata":{"_uuid":"66f537b423aef9a31d83ecee3f8f2e946bb4695a"},"cell_type":"markdown","source":"### 3.1 Outliers "},{"metadata":{"trusted":false,"_uuid":"804c66c26ab44a02854cd7c39f8f74e4a8476fc8"},"cell_type":"code","source":"train['vendor_id'].hist();","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2003627f9f0eb06f25ec934c2248d4a080458bba"},"cell_type":"code","source":"train['passenger_count'].hist();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5c9a8c6086bc200c2c8617e705e535e402224d60"},"cell_type":"markdown","source":"### Outliers : Pick-up location"},{"metadata":{"trusted":false,"_uuid":"533bb8e48d6fcf20bac9339938b4bbeac9efad94"},"cell_type":"code","source":"train['pickup_longitude'].plot.box()\nplt.title(\"Pickup Longitude Distribution\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e35fed44fc4046cd47c3fd40ae867010d11ad195"},"cell_type":"code","source":"plt.boxplot(train['pickup_latitude'])\nplt.title(\"Pickup Lattitude Distribution\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e81bf35e1458d8e77ecf02abc9e2fcab23d4115d"},"cell_type":"code","source":"plt.scatter(train['pickup_longitude'],train['pickup_latitude']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ab76967817845522beb2c6485f40210c5c2ada5e"},"cell_type":"code","source":"mpl.rcParams['figure.figsize']=(15,7)\nplt.scatter(train['pickup_longitude'],train['pickup_latitude'])\nplt.axis([-74.02,-73.92,40.7,40.82])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"34d580d121cab5b8d7c585408d610cf0b222a127"},"cell_type":"markdown","source":"On drop les quelques outliers larges sur les positions de récupération."},{"metadata":{"_uuid":"d25bf9efa4308dfe619f78f24b253eef4f5588a6"},"cell_type":"markdown","source":"#### Outliers : Pick-up location, par rapport à la longitude : "},{"metadata":{"trusted":false,"_uuid":"34e5a1b41465dc96e448594ef77878cfa063b547"},"cell_type":"code","source":"mpl.rcParams['figure.figsize']=(10,5)\nplt.scatter(train['pickup_longitude'],train['trip_duration'],color='r');","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"80e1c7465eb7067cb5d0a93d1d3a2cd86a5e33da"},"cell_type":"code","source":"train2 = train[train['pickup_longitude'] > -80][train['pickup_longitude'] < -60]\ntrain2 = train2[train2['trip_duration'] <= 500000]\nplt.scatter(train2['pickup_longitude'],train2['trip_duration'],color='r');","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a5340974417ee06fc99f4e4f15446fd2f55a3116"},"cell_type":"code","source":"train2 = train2[train2['pickup_longitude'] > -75][train2['pickup_longitude'] < -72]\nplt.scatter(train2['pickup_longitude'],train2['trip_duration'],color='r');","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8b72393fd1590be9815f893b20400b4fec60fee3"},"cell_type":"markdown","source":"#### Outliers : Pick-up location par rapport à la latitude"},{"metadata":{"trusted":false,"_uuid":"7c7efd63f4ebd9cde69a4b359771b237c0fd821e"},"cell_type":"code","source":"plt.scatter(train2['pickup_latitude'],train2['trip_duration'],color='g');","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"1d5f5d3f361fec9781e2e099da0ea8ec1b34c45d"},"cell_type":"code","source":"train2 = train2[train2['pickup_latitude'] < 42][train2['pickup_latitude'] > 39]\nplt.scatter(train2['pickup_latitude'],train2['trip_duration'],color='g');","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e03e3883208537dac8c3f1efe900d0c27cc4161d"},"cell_type":"markdown","source":"#### Finalement :"},{"metadata":{"trusted":false,"_uuid":"ac0dac5402ec4c7a7a9c8fa2a594ff3d9087e999"},"cell_type":"code","source":"plt.scatter(train2['pickup_longitude'],train2['pickup_latitude']);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"64f9f1bf960727eab19c66483314eaddd0b3b14e"},"cell_type":"markdown","source":"### Drop-off location"},{"metadata":{"trusted":false,"_uuid":"7b84c411d8cea7c90da74ad7e78920c146ec7688"},"cell_type":"code","source":"mpl.rcParams['figure.figsize']=(10,5)\nplt.boxplot(train2['dropoff_longitude']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b23288ed7901aee61a65cf1b9bf6e6627cbbffb2"},"cell_type":"code","source":"plt.boxplot(train2['dropoff_latitude']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5653da1284e328434db9db63519e2b037222e05a"},"cell_type":"code","source":"plt.scatter(train2['dropoff_longitude'],train2['dropoff_latitude']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a1e37c9dc1f24fe36ddfe409161854ec40e6c4c9"},"cell_type":"code","source":"mpl.rcParams['figure.figsize']=(15,7)\nplt.scatter(train2['dropoff_longitude'],train2['dropoff_latitude'])\nplt.axis([-74.02,-73.92,40.7,40.82])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b349e586ae68671cf7f8a5e0f2c2e497afbcce5c"},"cell_type":"markdown","source":"#### Même chose ces outliers\n#### Drop-off location et longitude"},{"metadata":{"trusted":false,"_uuid":"431e180614e4d409b670e304e3dae86b46b84e3c"},"cell_type":"code","source":"mpl.rcParams['figure.figsize']=(10,5)\nplt.scatter(train2['dropoff_longitude'],train2['trip_duration'],color='r');","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"1897dfde08abd577807fd5429a929da393af60ea"},"cell_type":"code","source":"train2 = train2[train2['dropoff_longitude'] > -76][train2['dropoff_longitude'] < -72]\nplt.scatter(train2['dropoff_longitude'],train2['trip_duration'],color='r');","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"27b63347ddcbe10a417b54a7ee4b43171d05e37b"},"cell_type":"markdown","source":"#### Drop-off location et latitude"},{"metadata":{"trusted":false,"_uuid":"eee3e276436464b46ea0d194d49774b13160a9e2"},"cell_type":"code","source":"plt.scatter(train2['dropoff_latitude'],train2['trip_duration'],color='g');","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c5350f2866ef3fcbe156400e3a87daebca223ec8"},"cell_type":"code","source":"train2 = train2[train2['dropoff_latitude'] > 40][train2['dropoff_latitude'] < 41.5]\nplt.scatter(train2['dropoff_latitude'],train2['trip_duration'],color='g');","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a5e1ca2241da3979171c9419546a878cfc9ec7b9"},"cell_type":"markdown","source":" #### Finalement :"},{"metadata":{"trusted":false,"_uuid":"abec5e837c777b2a2661e794bb00f1e75532ca26"},"cell_type":"code","source":"plt.scatter(train2['dropoff_longitude'],train2['dropoff_latitude']);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"12e3ac844347589cbd8950e0a02c1f6cd7634fc3"},"cell_type":"markdown","source":"### 3.2 Missing values "},{"metadata":{"trusted":false,"_uuid":"6c327eaff4c55d546ec5970cc73db1196c81f96c"},"cell_type":"code","source":"missing_val_count = (train2.isnull().sum())\nmissing_val_count","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a1fdd8be83b68520e0270b886017b61820edc106"},"cell_type":"markdown","source":"### 3.3 Categorical Data"},{"metadata":{"trusted":false,"_uuid":"74a65298bda40bc27d0125d4554848c9b33330d1"},"cell_type":"code","source":"# train\nfor column in CAT_VARS:\n    train2[column] = train2[column].astype('category').cat.codes\ntrain2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2a4387570b6503ef6073c5f0fecdc7d590620f27"},"cell_type":"code","source":"# test\nfor column in CAT_VARS:\n    if(column != 'dropoff_datetime'):\n        test[column] = test[column].astype('category').cat.codes\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d3b22d4087a0d715c3011f86cbcdba37a1265f11"},"cell_type":"markdown","source":"### 3.4 Log-transformation"},{"metadata":{"trusted":false,"_uuid":"025081c6c20b3b809d66aac963b0ca43126b58b7"},"cell_type":"code","source":"train2.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f9ce88e6ba5929fdea28f21724465d59f31765d4"},"cell_type":"code","source":"train3 = np.abs(train2[['vendor_id', 'pickup_datetime', 'passenger_count', 'pickup_longitude', 'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude', 'store_and_fwd_flag']]) \ntrain3 = np.log1p(train3)\ntrain3.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"500362f972b84e47eda977f7c7b88539846c7896"},"cell_type":"markdown","source":"## 4. Features"},{"metadata":{"trusted":false,"_uuid":"c5dcd82a64b9505fc158e1c063d80155e42fd2d5"},"cell_type":"code","source":"X_train = train3 #= train2[['vendor_id', 'pickup_datetime', 'passenger_count', 'pickup_longitude', 'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude', 'store_and_fwd_flag']]\nX_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e0c2a687d1c90f2bcb5feab8402232a9c9ebbf93"},"cell_type":"code","source":"y_train = np.log1p(train2['trip_duration'])\ny_train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"37be6acb5cd82d8abb3aadd39efb1054771d268d"},"cell_type":"markdown","source":"## 6. Model "},{"metadata":{"_uuid":"9240b8c28e3319395c9cea48354376ea7a74d319"},"cell_type":"markdown","source":"Scaling des données et entraînement : "},{"metadata":{"trusted":false,"_uuid":"de48510ce57861af89e552c428319eae405078e7"},"cell_type":"code","source":"t_X, val_X, t_y, val_y = train_test_split(X_train, y_train, test_size=0.2, random_state = 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"378abb026d20f27cbcfbf2f425c9b51cd6fc38d0"},"cell_type":"code","source":"scaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(t_X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ef06f6c214c3214a821d98fadc9ba200fe535656"},"cell_type":"code","source":"X_val_scaled = scaler.transform(val_X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c868c6ff79e423a706fc4f184075675e359e5814"},"cell_type":"code","source":"rf = RandomForestRegressor(random_state=1, n_jobs=-1)\nrf.fit(X_train_scaled, t_y)\npreds = rf.predict(X_val_scaled)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"57f427292c43147b9b63396cfb5ca62439d1dbdc"},"cell_type":"code","source":"print(np.sqrt(MSLE(np.exp(val_y), np.exp(preds))))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5e8c591638d9c144836dd2e9bb50df2e8d7014a3"},"cell_type":"markdown","source":"Cross Validation :"},{"metadata":{"trusted":false,"_uuid":"acacb4299d46f9480486a919f7879c2cf5cc3f16"},"cell_type":"code","source":"cv_preds = cross_val_predict(rf, X_train, y_train, cv=10, n_jobs=-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"15449399087a996d7a8e70fddf20685279a65c77"},"cell_type":"code","source":"# check cv_preds size\nprint(cv_preds)\n\nprint(np.sqrt(MSLE(np.exp(y_train), np.exp(cv_preds))))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fabef0ef9dc7be5a143edd0988a452e67b372464"},"cell_type":"markdown","source":"## 8. Predictions"},{"metadata":{"trusted":false,"_uuid":"59b19ab973353320c2f8c42685f0ccd002af9558"},"cell_type":"code","source":"test_p = np.log1p(np.abs(test[['vendor_id', 'pickup_datetime', 'passenger_count', 'pickup_longitude', 'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude', 'store_and_fwd_flag']]))\ntest_scaled = scaler.transform(test_p)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b37f9aeda429cfc7fe6cf6c42c1477e9ef8d2df7"},"cell_type":"code","source":"preds = rf.predict(test_scaled)\nnp.exp(preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d7c920f952c2b57ba20a602a78dbe234ff31409a"},"cell_type":"code","source":"sub = pd.DataFrame({'id':test.id,'trip_duration':np.exp(preds)})\nsub.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f145623e428bd24826909b866530239052d1e03f"},"cell_type":"code","source":"sub.to_csv('submission.csv', index=0)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.8"}},"nbformat":4,"nbformat_minor":1}