{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# visualiser les données\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport os\nprint(os.listdir(\"../input\"))\nfrom sklearn.preprocessing import LabelEncoder\n# package geo pour traiter les données coordinates\nfrom geopy.geocoders import Nominatim\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52b64e9c52d61311452db16e9b944eb933dd42e6"},"cell_type":"code","source":"%matplotlib inline\nsns.set({'figure.figsize':(16,8)})","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"# Data Loading"},{"metadata":{"trusted":true,"_uuid":"7958f86e205f7a1c7ef5905d8ae95bcd37349ae8"},"cell_type":"code","source":"train = pd.read_csv(\"../input/nyc-taxi-trip-duration/train.csv\")\ntest = pd.read_csv(\"../input/nyc-taxi-trip-duration/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d156370f100b3041de7bce7cd3d4e1a997e485bb"},"cell_type":"code","source":"print(f\"shape of training set{train.shape}\")\nprint(f\"shape of testing set{test.shape}\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"928bd1aff8d791d46678e024b49744786e7322de"},"cell_type":"markdown","source":"1. ### Data Training vs Data testing:"},{"metadata":{"trusted":true,"_uuid":"fcc431efb1edb4974b6b23f9448f53346707c030"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4c5569e98ea61ce4f93afd171dd484955c54f75"},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9355cd4d98c4c45e7603269d73ee51bc779687e0"},"cell_type":"code","source":"print(f\"La différence de la variable entre data training et data testing:\\\n{set(train.columns).difference(set(test.columns))}\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8d112d658083a2900a3a9cc0692e1500e7700982"},"cell_type":"markdown","source":"# Data preprocessing:"},{"metadata":{"_uuid":"9bf7c66cf8a7f92141294d130c7fe60f58e8fde9"},"cell_type":"markdown","source":" ### Valeur manquant: "},{"metadata":{"_uuid":"7dab14389a59260efaed1b9159a5e02aa1df92ef"},"cell_type":"markdown","source":"* dans data training:"},{"metadata":{"trusted":true,"_uuid":"86997a33f650cd790945b30702e68f45dfd23d45"},"cell_type":"code","source":"for i,v in zip(list(train.isnull().sum().index),list(train.isnull().sum().values)):\n    print(f\"{i} a {v} valeur(s) manquant(s)\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"abd731b77301a53f6fb9fec8a1f72e1214874c13"},"cell_type":"markdown","source":"* Dans data testing"},{"metadata":{"trusted":true,"_uuid":"3a5d4463a3e8628318bf9f8df35e5e6259a88dd7"},"cell_type":"code","source":"for i,v in zip(list(test.isnull().sum().index),list(test.isnull().sum().values)):\n    print(f\"{i} a {v} valeur(s) manquant(s)\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"41a0f0d90476f159bada4e0bc1a060966fff0729"},"cell_type":"markdown","source":"### Data duplicated"},{"metadata":{"trusted":true,"_uuid":"0c451d6f6708ac01b4597a13622596b3e9313c6c"},"cell_type":"code","source":"print(f\"Il y a {train.duplicated().sum()} données duplicates dans le train\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e0fdf810a1965e863b7cf78a822f3e8557d1941"},"cell_type":"code","source":"print(f\"Il y a {test.duplicated().sum()} données duplicates dans le test\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4fe57ee6e6e3df89a5195904ff657209fb4cd734"},"cell_type":"markdown","source":"### Analyst descriptive"},{"metadata":{"trusted":true,"_uuid":"b956f83e9fad50e865167d5e230b04aa62a733b7"},"cell_type":"code","source":"train.describe().T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a01632c344046e65276053f21b9a4dbf14689a3"},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"88477de7afe0b87e9c73ae788606e2eb6dfd0f56"},"cell_type":"markdown","source":"> Note: La data contient\n-  1 varibale en type chaîne de caractère:`id`\n- 2 variable en type date: `pickup_datetime`, `dropoff_datetime`\n- 2 varible en type catégorie: `vendor_id`, `store_and_fwd_flag`\n- 1 varible en type nombre continue: `trip_duration`\n- 1 variable en type nombre nominal: `passenger_count`\n- 4 variable en type coordinate: `pickup_longitude`, `pickup_latitude`, `dropoff_longitude`, `dropoff_latitude`"},{"metadata":{"trusted":true,"_uuid":"75cc92491882f38b5710da0a4309ede93fcc6acf"},"cell_type":"code","source":"print(f\"vendor_id ont {len(train.vendor_id.unique())} valeur: {list(train.vendor_id.unique())}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bdcec3f48c1aa95d2c37e27b8d5559a7818364dc"},"cell_type":"code","source":"print(f\"store_and_fwd_flag ont {len(train.store_and_fwd_flag.unique())} \\\nvaleur: {list(train.store_and_fwd_flag.unique())}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3463eac93badad31be9a10a6b9473924b9fbfd4"},"cell_type":"code","source":"print(f\"passenger_count ont {len(train.passenger_count.unique())} \\\nvaleur: {sorted(list(train.passenger_count.unique()))}\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1fc0cf3ccc3c08b55d28f1929eee0550862b2c71"},"cell_type":"markdown","source":"# Data Exploration"},{"metadata":{"_uuid":"4008f6d619ec1d21987f939bc8e262d0a70ce4f6"},"cell_type":"markdown","source":"## trip_duration distribution"},{"metadata":{"trusted":true,"_uuid":"f00cb44008aff7a5b1345d06ca82b255483abf9a"},"cell_type":"code","source":"train.trip_duration.hist();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c27c331bc9d4397d0215a4e210d1af6f32b487b8"},"cell_type":"markdown","source":"> Note: il nous semble que `trip_duration` contient certains valeur volumineux par rapport aux majorité de données"},{"metadata":{"trusted":true,"_uuid":"2a6b23f8582efa0dc38be2a9bc43be5276a2b73d"},"cell_type":"code","source":"len(train.trip_duration[train.trip_duration>6000].values)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ec2fffde92fbbfb5ba33c55ada22ae6c0e566f5e"},"cell_type":"markdown","source":"- Il y a 2567 trip qui ont durés plus de 6000 seconde (une heure et demi)"},{"metadata":{"trusted":true,"_uuid":"80917a741dcfd7f8264983f6b4170f38f45b7787"},"cell_type":"code","source":"train.loc[train.trip_duration<6000,\"trip_duration\"].hist(bins=100)\nplt.title(\"distribution de trip duration sans les oulieurs\");","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"99cff7c125fb2d32f3515e57fec11ad92a145ae9"},"cell_type":"markdown","source":"> Note: ça resemble la distribution log-normal"},{"metadata":{"trusted":true,"_uuid":"9c491b1181a7fff6cebfb25336d21b15b9d45f8d"},"cell_type":"code","source":"plt.hist(np.log(train.trip_duration), bins=1000, edgecolor='red');","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4190e1f5c93782d37906c59f7138d279e63a90c0"},"cell_type":"markdown","source":"la log- transformation de `trip_duration` resemble normal distribution"},{"metadata":{"trusted":true,"_uuid":"3079f1c6eb0084fa76a4e1b2fc5c61886e00be65"},"cell_type":"code","source":"train['log_trip_duration'] = np.log(train['trip_duration'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c58db00924dd66fb17ad4caec9765bf1845c1cff"},"cell_type":"markdown","source":"## Distribution de trip duration par rapport aux types de vendor"},{"metadata":{"trusted":true,"_uuid":"b0c098bd2ee280ea161afff7665f64923ede6195"},"cell_type":"code","source":"plt.hist(train.loc[train.vendor_id==1, 'log_trip_duration'], bins=100, edgecolor='red')\nplt.hist(train.loc[train.vendor_id==2, 'log_trip_duration'], bins=100, edgecolor='violet')\nplt.xlabel(\"trip duration\")\nplt.ylabel(\"frequency\")\nplt.legend(['vendor_id=1', 'vendor_id=2']);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"823ebf56cca18a57d88c0d608c93844b0c757bf7"},"cell_type":"markdown","source":"> vendor 1 a plus de `trip_duration`que vendor 2"},{"metadata":{"_uuid":"c07738e6b9d55407ace4d13b74adafaf6eb98322"},"cell_type":"markdown","source":"## passerger_count vs trip duration"},{"metadata":{"trusted":true,"_uuid":"366378865601b1b1e672279c8157ffdf01cdcbe5"},"cell_type":"code","source":"train.groupby(['vendor_id','passenger_count'])['trip_duration'].agg('mean').unstack(level=0).plot()\nplt.ylabel(\"trip duration average\")\nplt.xlabel(\"nombre de passenger\")\nplt.title(\"Trip duration by number of passenger on each vendor\");","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"efceead61be1b31ada3fba941d8c97a139170155"},"cell_type":"markdown","source":"> En moyenne: de 1 passenger à 6 passengers les deux vendors sont stables au niveau de trip duration\n\n> un remarque: même s'il n'y a pas de passenger, le trip duration de vendor 2 a 50 mn, de vendor 1 a 6 mn\n\n> vendor 1 a maximum 6 passengers, vendor 2 a plus de 6 mais le trip duration sont inférieur de 10 mn"},{"metadata":{"trusted":true,"_uuid":"79bb5e05601ba5cc248c590827e3ed2b8b2157e4"},"cell_type":"code","source":"train[['pickup_longitude', 'pickup_latitude','dropoff_longitude', 'dropoff_latitude']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b126bf10723584aa797638c1f811961aad427dc7"},"cell_type":"code","source":"fig,ax = plt.subplots(2,1)\nsns.scatterplot(x='pickup_longitude', y='pickup_latitude',data=train,ax=ax[0])\nplt.ylim([31,53]);\nsns.scatterplot(x='dropoff_longitude', y='dropoff_latitude',data=train,ax=ax[1])\nplt.ylim([31,53]);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8dfa6052176be332fbc560bd512c540a50bda85b"},"cell_type":"markdown","source":"> On a vu que la majorité des trajets est concentré au longitude: -74 latitude: 42"},{"metadata":{"trusted":true,"_uuid":"298b5e603c4cd73a4e5f3229ddddb8cbd1692d31"},"cell_type":"markdown","source":"# Features engineering"},{"metadata":{"_uuid":"f122007734ea4840c4298afe9ddfd5076049ba54"},"cell_type":"markdown","source":"## 1.Features extraction"},{"metadata":{"_uuid":"10ce5ee818f0f109ed8826a282b7004a2c824efb"},"cell_type":"markdown","source":"- Converter `store_and_fwd_flag` en chiffres"},{"metadata":{"trusted":true,"_uuid":"559279cf49d67ad5d5e10c0e97a62d29bfee7f6a"},"cell_type":"code","source":"le = LabelEncoder()\nle.fit(train['store_and_fwd_flag'])\ntrain['store_and_fwd_flag'] = le.transform(train['store_and_fwd_flag'])\ntest['store_and_fwd_flag'] = le.transform(test['store_and_fwd_flag'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6ac8068f409dd868abf86932e79693b821419e4b"},"cell_type":"markdown","source":"## - Pour les dates:"},{"metadata":{"trusted":true,"_uuid":"af89023f3d35276a0cdabd4909246f7db6a0f29d"},"cell_type":"code","source":"train['pickup_datetime'] = pd.to_datetime(train.pickup_datetime)\ntrain['month'] = train['pickup_datetime'].dt.month\ntrain['day'] = train['pickup_datetime'].dt.day\ntrain['dayofweek'] = train['pickup_datetime'].dt.dayofweek\ntrain['weekday'] = train['pickup_datetime'].dt.weekday\ntrain['hour'] = train['pickup_datetime'].dt.hour\ntrain['minute'] = train['pickup_datetime'].dt.minute\n#train['dropoff_datetime'] = pd.to_datetime(train.dropoff_datetime)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b827cce432d4bf983985185f053e0af1521d72c2"},"cell_type":"markdown","source":"> Note: trip_duration est calculé par la différence entre pickup_datetime et dropoff_datetime"},{"metadata":{"trusted":true,"_uuid":"07d7811c5421d2912679516325f80b18dcbeb016"},"cell_type":"code","source":"# petite verification\n#train['check_trip_duration'] = (train['dropoff_datetime'] - train['pickup_datetime'])\\\n#                                    .map(lambda x: x.total_seconds())\n#print(f\"il y a {sum(train['check_trip_duration'] != train.trip_duration)} ligne(s) ne pas correspondre\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"41690fe2f523b98fe2388368e45ae506badedc6a"},"cell_type":"code","source":"train.groupby(['vendor_id','dayofweek'])['trip_duration'].agg(\"mean\").unstack(level=0).plot()\nplt.xlabel(\"Journé dans la semaine (lundi=0, dimanche=6)\")\nplt.ylabel(\"Trip duration moyenne\");","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"842c9a6b548e07e559280df70d7585ecef71a30e"},"cell_type":"markdown","source":"> Dans la semaine, le vendor 2 a la moyenne de trip duration plus de vendor 1\n\n> Jeudi et Vendredi contienne de longue trajet pour le vendor 2, pour le vendor 1 est le mardi et jeudi"},{"metadata":{"trusted":true,"_uuid":"d4eaba5214491829489c4d795c749966c59549a9"},"cell_type":"code","source":"train.groupby(['vendor_id','hour'])['trip_duration'].agg(\"mean\").unstack(level=0).plot()\nplt.xlabel(\"L'heure dans la journée\")\nplt.ylabel(\"Trip duration moyenne\");","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ae37e9bfbf9e7bc0ceee793e22af97851b1aa694"},"cell_type":"markdown","source":"> De 6h du matin à 15h l'aprè midi, trip duration est augmenté"},{"metadata":{"_uuid":"9d45d9df544bcd4f656daac4eece44a8f4f467b3"},"cell_type":"markdown","source":"## - Pour la distance:"},{"metadata":{"trusted":true,"_uuid":"5ee8633e642f5ed4e175d56d4be6075507b47e3c"},"cell_type":"code","source":"train['dist_long'] = train['pickup_longitude'] - train['dropoff_longitude']\n\ntrain['dist_lat'] = train['pickup_latitude'] - train['dropoff_latitude']\n\ntrain['dist'] = np.sqrt(np.square(train['dist_long']) + np.square(train['dist_lat']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"053c518a813f1ea2b4f3af03cedf0f2b97ebe552"},"cell_type":"code","source":"#### spatial features: count and speed\ntrain['pickup_longitude_bin'] = np.round(train['pickup_longitude'], 2)\ntrain['pickup_latitude_bin'] = np.round(train['pickup_latitude'], 2)\ntrain['dropoff_longitude_bin'] = np.round(train['dropoff_longitude'], 2)\ntrain['dropoff_latitude_bin'] = np.round(train['dropoff_latitude'], 2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ec6f916525fd75f988027bd8c97c73a736dea2f"},"cell_type":"markdown","source":"## Data ingeneering on data test"},{"metadata":{"trusted":true,"_uuid":"60f5f303d710f801526a4fb9f1de8868961aabc1"},"cell_type":"code","source":"test['pickup_datetime'] = pd.to_datetime(test.pickup_datetime)\ntest['month'] = test['pickup_datetime'].dt.month\ntest['day'] = test['pickup_datetime'].dt.day\ntest['dayofweek'] = test['pickup_datetime'].dt.dayofweek\ntest['weekday'] = test['pickup_datetime'].dt.weekday\ntest['hour'] = test['pickup_datetime'].dt.hour\ntest['minute'] = test['pickup_datetime'].dt.minute","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c239f6ac6c3e63f6b0e46ef62eb85bff4a70626"},"cell_type":"code","source":"test['dist_long'] = test['pickup_longitude'] - test['dropoff_longitude']\ntest['dist_lat'] = test['pickup_latitude'] - test['dropoff_latitude']\ntest['dist'] = np.sqrt(np.square(test['dist_long']) + np.square(test['dist_lat']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50c8b8f14e8fbc2f1bb1b820020f222c4c420084"},"cell_type":"code","source":"test['pickup_longitude_bin'] = np.round(test['pickup_longitude'], 2)\ntest['pickup_latitude_bin'] = np.round(test['pickup_latitude'], 2)\ntest['dropoff_longitude_bin'] = np.round(test['dropoff_longitude'], 2)\ntest['dropoff_latitude_bin'] = np.round(test['dropoff_latitude'], 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aff8b47443807c3d3b9bc01909da65471a7c51cb"},"cell_type":"code","source":"## count features\na = pd.concat([train,test]).groupby(['pickup_longitude_bin', 'pickup_latitude_bin']).size().reset_index()\nb = pd.concat([train,test]).groupby(['dropoff_longitude_bin', 'dropoff_latitude_bin']).size().reset_index()\n\ntrain = pd.merge(train, a, on = ['pickup_longitude_bin', 'pickup_latitude_bin'], how = 'left')\ntest = pd.merge(test, a, on = ['pickup_longitude_bin', 'pickup_latitude_bin'], how = 'left')\n\ntrain = pd.merge(train, b, on = ['dropoff_longitude_bin', 'dropoff_latitude_bin'], how = 'left')\ntest = pd.merge(test, b, on = ['dropoff_longitude_bin', 'dropoff_latitude_bin'], how = 'left')\n\n## speed features\ntrain['speed'] = 100000*train['dist'] / train['trip_duration']\n\na = train[['speed', 'pickup_longitude_bin', 'pickup_latitude_bin']].groupby(['pickup_longitude_bin', 'pickup_latitude_bin']).mean().reset_index()\na = a.rename(columns = {'speed': 'ave_speed'})\nb = train[['speed', 'dropoff_longitude_bin', 'dropoff_latitude_bin']].groupby(['dropoff_longitude_bin', 'dropoff_latitude_bin']).mean().reset_index()\nb = b.rename(columns = {'speed': 'ave_speed'})\n\ntrain = pd.merge(train, a, on = ['pickup_longitude_bin', 'pickup_latitude_bin'], how = 'left')\ntest = pd.merge(test, a, on = ['pickup_longitude_bin', 'pickup_latitude_bin'], how = 'left')\n\ntrain = pd.merge(train, b, on = ['dropoff_longitude_bin', 'dropoff_latitude_bin'], how = 'left')\ntest = pd.merge(test, b, on = ['dropoff_longitude_bin', 'dropoff_latitude_bin'], how = 'left')\n\n## drop bins\ntrain = train.drop(['speed', 'pickup_longitude_bin', 'pickup_latitude_bin', 'dropoff_longitude_bin', 'dropoff_latitude_bin'], axis = 1)\ntest = test.drop(['pickup_longitude_bin', 'pickup_latitude_bin', 'dropoff_longitude_bin', 'dropoff_latitude_bin'], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff3bdf8711b7a2f2a33de2c66e3ef353e8363854"},"cell_type":"code","source":"#### weather data\nweather = pd.read_csv('../input/knycmetars2016/KNYC_Metars.csv')\nweather['Time'] = pd.to_datetime(weather['Time'])\nweather['year'] = weather['Time'].dt.year\nweather['month'] = weather['Time'].dt.month\nweather['day'] = weather['Time'].dt.day\nweather['hour'] = weather['Time'].dt.hour\nweather = weather[weather['year'] == 2016]\n\ntrain = pd.merge(train, weather[['Temp.', 'month', 'day', 'hour']], on = ['month', 'day', 'hour'], how = 'left')\ntest = pd.merge(test, weather[['Temp.', 'month', 'day', 'hour']], on = ['month', 'day', 'hour'], how = 'left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8ee9114d1ef6bc70554db19547f08c111f7c9d32"},"cell_type":"code","source":"# export data training and data testing\ntrain.to_csv(\"training_data.csv\", index=False)\ntest.to_csv(\"testing_data.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f37178d3fec0d23db4affbc642a1afbc1c38115c"},"cell_type":"code","source":"col_diff = list(set(train.columns).difference(set(test.columns)))\nprint(f\"La différence de la variable entre data training et data testing:\\\n{set(train.columns).difference(set(test.columns))}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ffd5a59ee339d30e1fb7d064caf4218a3a6713e5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}