{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"scrolled":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt # visualization library\n\n#warnings.filterwarnings('ignore')\n%matplotlib inline\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"## Data Loading"},{"metadata":{"trusted":true,"_uuid":"a727b97637e4d3742c3916771a2e96c1959df7dc"},"cell_type":"code","source":"TRAIN_PATH = os.path.join(\"..\", \"input\", \"train.csv\")\nTEST_PATH = os.path.join(\"..\", \"input\", \"test.csv\")\n\ntrain = pd.read_csv(TRAIN_PATH)\ntest = pd.read_csv(TEST_PATH)\n\nprint(f\"training set shape : {train.shape}\")\nprint(f\"testing set shape : {test.shape}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b35fdfe4ae6af8723f7e37806b14e7ff90ad5a5"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cae905ffce65b5de1c58cd358abf104e4f22100c"},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d28f1c2d9aafd4e295d76524f863de41eb1e255c"},"cell_type":"markdown","source":"## Data Exploration"},{"metadata":{"trusted":true,"_uuid":"b66d43bf7d42d34b14ec5c36070894d4ea52be3a"},"cell_type":"code","source":"train.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b479fadd2ecfdf78fc563bc12c4b8fe405da721"},"cell_type":"code","source":"train.trip_duration.min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ad832db8aaf3e3b18310db742b69226120c9f1d"},"cell_type":"code","source":"train.trip_duration.max()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bbae234eea5b34c38868a2cd93784edfb2e1d12b"},"cell_type":"markdown","source":" Pour pas fausser les résultats on décide d'enlever les trajets ou il y a un temps minimum d'une seconde et un maximum de 40jours.\nJe vais estimer qu'un temps de trajet doit être compris entre 1-5min et plusieurs heures"},{"metadata":{"_uuid":"71f997fdc8cf14a7774027b3abc5226449f1542c"},"cell_type":"markdown","source":"## Data Preprocessing\n"},{"metadata":{"_uuid":"f19f2f15f64d0501ae54c3ebd3e29e72facf4798"},"cell_type":"markdown","source":"Gestion des outliers"},{"metadata":{"_uuid":"638a12ae7e59dc721b0e3cae546e4802fa9255a5","trusted":true},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c2db43ce2bddefcb00ad74ea386276fff337fc68"},"cell_type":"markdown","source":"Il y a des valeurs aberrantes pour les trip_duration en min et max. "},{"metadata":{"trusted":true,"_uuid":"81e1c78730582090b11aac561c71eb8289c179e7"},"cell_type":"code","source":"plt.subplots(figsize=(18,6))\nplt.title(\"Visualisation des outliers\")\ntrain.boxplot();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3cf2d7d5dfd0e78486d94788ec5f210b129eef00"},"cell_type":"markdown","source":"## Features engineering"},{"metadata":{"trusted":true,"_uuid":"f75b6fae0b3cecaed27d96770927ac408e963933"},"cell_type":"code","source":"col_diff = list(set(train.columns).difference(set(test.columns)))\ncol_diff","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e6e82dc9e2f9f8cdcf53759d424a7d05b41c156"},"cell_type":"code","source":"#Calcule de la distance entre drop et pickup\ntrain['dist'] = np.sqrt((train['pickup_latitude']-train['dropoff_latitude'])**2\n                        + (train['pickup_longitude']-train['dropoff_longitude'])**2)\ntest['dist'] = np.sqrt((test['pickup_latitude']-test['dropoff_latitude'])**2\n                        + (test['pickup_longitude']-test['dropoff_longitude'])**2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a1e8479dd4619436b4a91dab28d49af0a0375d3"},"cell_type":"code","source":"#il n'est pas nécessaire d'avoir 0 passager, nous allons les enlever\ntrain = train[train['passenger_count']>= 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d1e80d41a6734ccd0d1222a78ecb0dca062b2bbd"},"cell_type":"code","source":"# La durée du voyage est comprise entre 1 sec. et 3526282 sec.\n# Nous laisserons tomber les valeurs inférieures à 1 min 30 (90sec) et supérieures à 166 min (10 000 sec).\ntrain = train[train['trip_duration']>= 1.5 ]\ntrain = train[train['trip_duration']<= 10000 ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"0bc3fbe552541104aca33a115439969fec5a7e77"},"cell_type":"code","source":"# Nous allons laisser tomber la longitude et la latitude (On drop ce qui ressemble à des valeurs aberrantes)\ntrain = train.loc[train['pickup_longitude']> -90]\ntrain = train.loc[train['pickup_latitude']< 47.5]\n\ntrain = train.loc[train['dropoff_longitude']> -90]\ntrain = train.loc[train['dropoff_latitude']> 34]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb51d6fd7b14e88eab8a3f0fb70df884adae6bd0"},"cell_type":"code","source":"#convertir le string en datetime pour avoir que l'heure \ntrain['pickup_datetime'] = pd.to_datetime(train['pickup_datetime'], format='%Y-%m-%d %H:%M:%S')\ntest['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'], format='%Y-%m-%d %H:%M:%S')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7042008b3304ca1fbe5fadba58ae70c9e470cf9b"},"cell_type":"code","source":"#ajout dans une nouvelle colonne\ntrain['hour'] = train.loc[:,'pickup_datetime'].dt.hour;\ntest['hour'] = test.loc[:,'pickup_datetime'].dt.hour;","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b2bff10696e55c28f593e9f5b4cc5a14b204f31"},"cell_type":"code","source":"#Ajout dans une nouvelle colonne à train \nX_train = train[[\"passenger_count\",\"vendor_id\", \"pickup_longitude\", \"pickup_latitude\", \"dropoff_longitude\",\"dropoff_latitude\", \"dist\", \"hour\" ]]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18cf6a4800d5b57892483fed1a17c2f889d3a320"},"cell_type":"code","source":"y_train = train[\"trip_duration\"]  # This is our target\nX_train = train[[\"passenger_count\",\"vendor_id\", \"pickup_longitude\", \"pickup_latitude\", \"dropoff_longitude\",\"dropoff_latitude\", \"dist\",\"hour\" ]]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1d502e2dce84b0478e3627153834f5a98db7e6ea"},"cell_type":"markdown","source":"## Selection du modèle\n"},{"metadata":{"trusted":true,"_uuid":"00e4a09124a17c9d2d24a9b77f7a348c3828830d"},"cell_type":"code","source":"# importer la lib pour cross valider le model\nfrom sklearn.model_selection import cross_val_score\n\n# importer la lib pour la regression de Random Forest\nfrom sklearn.ensemble import RandomForestRegressor\n\n# importer la lib pour la regression de Random Forest\nfrom sklearn.linear_model import SGDRegressor\n\nfrom sklearn.linear_model import LinearRegression\n\nfrom sklearn.model_selection import ShuffleSplit","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"653e97c642e8c8a27587c576a7bdeb333beafbbc"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, random_state=42)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e20f9a99cf12a1d7cc63708de1c7f8d5ff35fee"},"cell_type":"code","source":"sgd = SGDRegressor()\nsgd.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"719179b190d7d07d50af557108549fd30374b2bb"},"cell_type":"markdown","source":"## 6. Entrainement du ou des modèle(s) & Predictions¶\n"},{"metadata":{"trusted":true,"_uuid":"0a3e537e840f3ad637f9a380021f4d0e993df1c1"},"cell_type":"code","source":"b2o = RandomForestRegressor(n_estimators=19, min_samples_split=2, min_samples_leaf=4, max_features='auto', max_depth=80, bootstrap=True)\nb2o.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d51ffa75443dbd73d82dbe54b400d166fd6e6ef7"},"cell_type":"code","source":"cv = ShuffleSplit(n_splits=4, test_size=0.8, random_state=42)\ncv_scores = cross_val_score(b2o, X_train, y_train, cv=cv, scoring= 'neg_mean_squared_log_error')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e55a66755e9f7b76ce42bf49af63c5f98eff76c8"},"cell_type":"code","source":"cv_scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"307b0897c48fc981840ec156d3a99566b0c2d955"},"cell_type":"code","source":"for i in range(len(cv_scores)):\n    cv_scores[i] = np.sqrt(abs(cv_scores[i]))\nprint(np.mean(cv_scores))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3347df51cdefe5e748883997a3e2c20e9c7d9541"},"cell_type":"code","source":"## Prediction","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec68935b6c22c4f84b882aa09b7b9e10fa3dc11e"},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5082f279cbfaf690dadb79ccc8f7389caf9b19eb"},"cell_type":"code","source":"X_fit = test[[\"vendor_id\", \"passenger_count\",\"pickup_longitude\", \"pickup_latitude\",\"dropoff_longitude\",\"dropoff_latitude\",\"dist\",\"hour\"]]\nprediction = b2o.predict(X_fit)\nprediction","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8cdff03c4d215910294dfd9389f68de9fc7556b9"},"cell_type":"code","source":"my_submission = pd.DataFrame({'id': test.id, 'trip_duration': prediction})\nmy_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1937f4bfb3eb4cd7a471b22c7826ba629ab0c98a"},"cell_type":"code","source":"my_submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}