{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ML-based Detection of Illegal GPS Spoofing by GoJek Drivers using XGBoost Classifier Model","metadata":{"id":"8wJlhwZjb5c0"}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2021-12-07T14:39:11.905605Z","iopub.execute_input":"2021-12-07T14:39:11.905943Z","iopub.status.idle":"2021-12-07T14:39:11.936988Z","shell.execute_reply.started":"2021-12-07T14:39:11.905854Z","shell.execute_reply":"2021-12-07T14:39:11.936063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\n\nimport seaborn as sns\nimport missingno as msno\nimport folium\n\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.model_selection import cross_validate, train_test_split, StratifiedKFold\nfrom sklearn.metrics import plot_confusion_matrix, classification_report\nfrom sklearn.metrics import roc_curve, auc\nfrom xgboost import XGBClassifier\n\n!pip -q install utm\nimport utm\n\n%config InlineBackend.figure_format = 'retina'","metadata":{"id":"FgHQPoxynC9O","outputId":"516a3506-3745-4ec5-f809-5fe1faf16994","execution":{"iopub.status.busy":"2021-12-07T14:46:14.127608Z","iopub.execute_input":"2021-12-07T14:46:14.127881Z","iopub.status.idle":"2021-12-07T14:46:14.403503Z","shell.execute_reply.started":"2021-12-07T14:46:14.127853Z","shell.execute_reply":"2021-12-07T14:46:14.40265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Read training data","metadata":{"id":"gp30Rg5pbwCd"}},{"cell_type":"code","source":"# # Access dataset from repository and unzip\n# !wget 'https://github.com/yohanesnuwara/datasets/blob/master/gojek_fake_gps_detection.zip?raw=true'\n# !mv '/content/gojek_fake_gps_detection.zip?raw=true' '/content/gojek_fake_gps_detection.zip'\n# !unzip '/content/gojek_fake_gps_detection.zip'","metadata":{"id":"74JWa1bolV1e","outputId":"8a06390b-63cd-4362-999b-e96c11cb39f9"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/dsbootcamp10/train.csv')\n\ntrain","metadata":{"id":"wOiCAW0fnUTf","outputId":"24c2beb4-2f26-434d-8a4d-1914bb4a1b41","execution":{"iopub.status.busy":"2021-12-07T14:40:30.795812Z","iopub.execute_input":"2021-12-07T14:40:30.796153Z","iopub.status.idle":"2021-12-07T14:40:32.035368Z","shell.execute_reply.started":"2021-12-07T14:40:30.79611Z","shell.execute_reply":"2021-12-07T14:40:32.03428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Features description:**\n\n* order_id - an anonymous id unique to a given order number\n* service_type - service type, can be GORIDE or GOFOOD\n* driver_status - status of the driver PING, can be AVAILABLE, UNAVAILABLE, OTW_PICKUP, OTW_DROPOFF\n* hour - hour\n* seconds - seconds in linux format\n* latitude - GPS latitude\n* longitude - GPS longitude\n* altitude_in_meters - GPS Altitude\n* accuracy_in_meters - GPS Accuracy, the smaller the more accurate\n\n**Target:**\n\nlabel - label describing whether GPS is true (1) or fake (0)","metadata":{"id":"0SzMoeHCc140"}},{"cell_type":"code","source":"train.info()","metadata":{"id":"_Cb0vBUocrs1","outputId":"0fdd5cb2-2e0e-4de5-9786-dc4e45e6d4f2","execution":{"iopub.status.busy":"2021-12-07T14:40:46.839836Z","iopub.execute_input":"2021-12-07T14:40:46.84064Z","iopub.status.idle":"2021-12-07T14:40:47.120123Z","shell.execute_reply.started":"2021-12-07T14:40:46.840597Z","shell.execute_reply":"2021-12-07T14:40:47.119196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 154,403 missing values in the altitude column. My first suspection why altitude information can be missing is that fake GPS application may have intermittent or unstable altitude information. So, the missing altitude may become precious predictor (later to be discussed).","metadata":{"id":"PQrT4XwydiTP"}},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"id":"8UvajUZZdfuC","outputId":"a400cbf2-f62c-4fb2-cfda-5b8ac82ac9ef","execution":{"iopub.status.busy":"2021-12-07T14:40:50.116759Z","iopub.execute_input":"2021-12-07T14:40:50.117021Z","iopub.status.idle":"2021-12-07T14:40:50.379383Z","shell.execute_reply.started":"2021-12-07T14:40:50.116994Z","shell.execute_reply":"2021-12-07T14:40:50.378428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 7,233 zero values in hour column indicating that orders were done at 00:00 midnight. ","metadata":{"id":"ixWn4QcffB2F"}},{"cell_type":"code","source":"# Count zeros. NaN is NOT considered zero\ntrain.isin([0]).astype(int).sum(axis=0)","metadata":{"id":"nOcuftv6eKzc","outputId":"de2bcdf3-3052-420e-90cf-7a67aca27cca","execution":{"iopub.status.busy":"2021-12-07T14:40:52.964912Z","iopub.execute_input":"2021-12-07T14:40:52.96519Z","iopub.status.idle":"2021-12-07T14:40:53.789464Z","shell.execute_reply.started":"2021-12-07T14:40:52.96516Z","shell.execute_reply":"2021-12-07T14:40:53.788469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Further, I process the seconds column from Linux format to datetime format. I found out that ALL dates in the Linux seconds column do not match with the date column. I do not have an explanation for this, but I assume this is not important for our prediction task.","metadata":{"id":"62ZCmkxxidhw"}},{"cell_type":"code","source":"from datetime import datetime\n\n# Convert Linux seconds to datetime format\ntrain['linux_date'] = [datetime.utcfromtimestamp(s).strftime('%Y-%m-%d %H:%M:%S') for s in train.seconds.values]\n\n# Convert datetime to Pandas format\ntrain['linux_date'] = pd.to_datetime(train['linux_date'])\n\n# Convert datetime in date column to Pandas format\ntrain['date'] = pd.to_datetime(train['date'])\n\n# Check if date column match with Linux date column\ndf = train['linux_date'].dt.date==train['date']\nprint(df.eq(True).all())","metadata":{"id":"xDwQQ4SGgAF9","outputId":"50fffb5e-a9a0-4c85-bed1-d14c34b2d6af","execution":{"iopub.status.busy":"2021-12-07T14:40:56.62457Z","iopub.execute_input":"2021-12-07T14:40:56.625273Z","iopub.status.idle":"2021-12-07T14:41:00.018325Z","shell.execute_reply.started":"2021-12-07T14:40:56.625226Z","shell.execute_reply":"2021-12-07T14:41:00.017296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Data exploration","metadata":{"id":"raCUik0rjdbh"}},{"cell_type":"code","source":"sns.catplot(data=train, x='driver_status', y='accuracy_in_meters', hue='label', col='service_type', kind='bar')","metadata":{"id":"EkwUEFABnoST","outputId":"dbde48e5-20e3-4d19-b203-ed6bdfb82b80","execution":{"iopub.status.busy":"2021-12-07T14:43:41.144273Z","iopub.execute_input":"2021-12-07T14:43:41.144891Z","iopub.status.idle":"2021-12-07T14:43:54.082915Z","shell.execute_reply.started":"2021-12-07T14:43:41.144839Z","shell.execute_reply":"2021-12-07T14:43:54.082035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Plotting on Folium\n\nGitHub: [Link](https://github.com/lindseyberlin/Blog_FoliumMaps/blob/master/FoliumMapExamples.ipynb)\n\nArticle: [Link](https://dev.to/lberlin/folium-powerful-mapping-tool-for-absolute-beginners-1m5h)","metadata":{"id":"zp3SpRSrjfL2"}},{"cell_type":"code","source":"def plot_folium(df, order_id, lat_column, lon_column, location, zoom_start=10):\n # Select subset of dataframe by order ID\n df = df[df.order_id==order_id]\n\n # Folium plot\n my_map = folium.Map(location=location, zoom_start=zoom_start) \n\n # Define different colors for status\n for index, row in df.iterrows():\n if row.driver_status=='UNAVAILABLE':\n color = 'green'\n if row.driver_status=='AVAILABLE':\n color = 'red'\n if row.driver_status=='OTW_PICKUP':\n color = 'black'\n if row.driver_status=='OTW_DROPOFF':\n color = 'blue'\n\n # Plot coordinates on Folium\n folium.CircleMarker([row[lat_column], row[lon_column]],\n radius=5, color=color,\n fill=True).add_to(my_map)\n\n display(my_map)","metadata":{"id":"MsrrZKTPmxqR","execution":{"iopub.status.busy":"2021-12-07T14:45:44.527076Z","iopub.execute_input":"2021-12-07T14:45:44.527373Z","iopub.status.idle":"2021-12-07T14:45:44.534853Z","shell.execute_reply.started":"2021-12-07T14:45:44.527342Z","shell.execute_reply":"2021-12-07T14:45:44.533884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot on Folium\nplot_folium(train, 'RB193', 'latitude', 'longitude', [-6.920, 107.630], zoom_start=16)","metadata":{"id":"OZ1bIMbFpzkx","outputId":"c2884def-ad67-4769-ed54-f84a7a9de3f5","execution":{"iopub.status.busy":"2021-12-07T14:46:24.185702Z","iopub.execute_input":"2021-12-07T14:46:24.186009Z","iopub.status.idle":"2021-12-07T14:46:24.42527Z","shell.execute_reply.started":"2021-12-07T14:46:24.185974Z","shell.execute_reply":"2021-12-07T14:46:24.424594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot on Folium\nplot_folium(train, 'F842', 'latitude', 'longitude', [-6.920, 107.670], zoom_start=14)","metadata":{"id":"HC_xAbn0nl-d","outputId":"c453374a-b1fd-49ef-d245-fee44e33acec","execution":{"iopub.status.busy":"2021-12-07T14:46:33.098079Z","iopub.execute_input":"2021-12-07T14:46:33.098948Z","iopub.status.idle":"2021-12-07T14:46:33.437901Z","shell.execute_reply.started":"2021-12-07T14:46:33.098896Z","shell.execute_reply":"2021-12-07T14:46:33.436945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # !pip -q install contextily\n# import contextily as ctx\n\n# f, ax = plt.subplots(figsize=(10,10))\n# gdf.to_crs(epsg=3857).plot(ax=ax)\n# ctx.add_basemap(ax=ax)","metadata":{"id":"cN9JnBOzrVPa"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Feature engineering","metadata":{"id":"Xs3i0dTcjKQr"}},{"cell_type":"markdown","source":"First, I differenced latitude, longitude, and seconds columns $(x_t-x_{t-1})$ for each order ID (grouping by order ID then taking difference). ","metadata":{"id":"c79z0astiT23"}},{"cell_type":"code","source":"# Differencing some columns\ntrain['longitude_diff'] = train.groupby('order_id').longitude.diff().fillna(0)\ntrain['latitude_diff'] = train.groupby('order_id').latitude.diff().fillna(0)\ntrain['seconds_diff'] = train.groupby('order_id').seconds.diff().fillna(0)\ntrain['accuracy_diff'] = train.groupby('order_id').accuracy_in_meters.diff().fillna(0)\ntrain['altitude_diff'] = train.groupby('order_id').altitude_in_meters.diff().fillna(0)\n\ntrain","metadata":{"id":"TfyQyiqSiqIE","outputId":"92859767-7e4f-4185-b70d-c30a5fbd9185","execution":{"iopub.status.busy":"2021-12-07T14:48:34.94491Z","iopub.execute_input":"2021-12-07T14:48:34.945222Z","iopub.status.idle":"2021-12-07T14:48:38.719413Z","shell.execute_reply.started":"2021-12-07T14:48:34.945191Z","shell.execute_reply":"2021-12-07T14:48:38.718456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Then, converting the lat lon coordinates to a projected Mercator plane (UTM) coordinates. ","metadata":{"id":"lp93FbfH5Kzp"}},{"cell_type":"code","source":"# Convert lat lon to UTM\nlat, lon = train.latitude.values, train.longitude.values\nx = utm.from_latlon(lat, lon)\n\ntrain['UTMX'] = x[0]\ntrain['UTMY'] = x[1]\n\ntrain","metadata":{"id":"XI6PdX8sReAE","outputId":"487c8cf9-4d1b-4590-a30d-59614c4fb94c","execution":{"iopub.status.busy":"2021-12-07T14:48:43.208769Z","iopub.execute_input":"2021-12-07T14:48:43.209071Z","iopub.status.idle":"2021-12-07T14:48:43.340615Z","shell.execute_reply.started":"2021-12-07T14:48:43.209038Z","shell.execute_reply":"2021-12-07T14:48:43.339597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Then, calculating the step distance between two successive points. For example, at $(t-1)$, the coordinate is at $(x_{t-1}, y_{t-1})$. Then, at $t$, the coordinate is at $(x_t, y_t)$. The step distance is:\n\n$$r=\\sqrt{\\delta x^2 + \\delta y^2}$$\n\nWhere $\\delta x=x_t-x_{t-1}$ and $\\delta y=y_t-y_{t-1}$","metadata":{"id":"mxWaC-EY6NmA"}},{"cell_type":"code","source":"# Function to calculate distance between two points\ndistance = lambda x_dif, y_dif: np.sqrt(x_dif**2 + y_dif**2)","metadata":{"id":"1epSuYvS7lmm","execution":{"iopub.status.busy":"2021-12-07T14:48:48.726436Z","iopub.execute_input":"2021-12-07T14:48:48.72684Z","iopub.status.idle":"2021-12-07T14:48:48.731814Z","shell.execute_reply.started":"2021-12-07T14:48:48.726793Z","shell.execute_reply":"2021-12-07T14:48:48.730869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Differencing UTM coordinates\ntrain['UTMX_diff'] = train.groupby('order_id').UTMX.diff().fillna(0)\ntrain['UTMY_diff'] = train.groupby('order_id').UTMY.diff().fillna(0)\n\n# Calculate step distance\ntrain['distance'] = distance(train.UTMX_diff, train.UTMY_diff)\n\ntrain['distance']","metadata":{"id":"D49NRvI_jkdl","outputId":"8d3ea075-12e4-4444-ec27-751eebbdcb6b","execution":{"iopub.status.busy":"2021-12-07T14:48:53.570411Z","iopub.execute_input":"2021-12-07T14:48:53.571164Z","iopub.status.idle":"2021-12-07T14:48:55.019196Z","shell.execute_reply.started":"2021-12-07T14:48:53.571122Z","shell.execute_reply":"2021-12-07T14:48:55.01859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## `df_grouped1`\n\nCompressing 567,545 record size by grouping by order ID so that we'll have only 3,500. For each order ID, I calculate the summary stats and other things. \n\nGrouping by order ID to get the service typeEach order ID has the same service type (GoFood or GoRide) and the same label (0 or 1). ","metadata":{"id":"C5i6Fqd3J1kS"}},{"cell_type":"code","source":"# Grouping by order ID to get service type and label\ndf_grouped1 = train.groupby('order_id')[['service_type', 'label']].max()\n\ndf_grouped1","metadata":{"id":"GhHy7kS3f7us","outputId":"df5098c5-f152-4b63-d80f-6c6ce3f6a075","execution":{"iopub.status.busy":"2021-12-07T14:49:07.972771Z","iopub.execute_input":"2021-12-07T14:49:07.973474Z","iopub.status.idle":"2021-12-07T14:49:08.568935Z","shell.execute_reply.started":"2021-12-07T14:49:07.973438Z","shell.execute_reply":"2021-12-07T14:49:08.56777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_grouped1.label.value_counts().plot.pie(autopct='%.2f %%')","metadata":{"id":"6h_Qqs8quwFg","outputId":"895160eb-a1da-4e14-8cc0-eb64f5f007d9","execution":{"iopub.status.busy":"2021-12-07T14:49:12.078114Z","iopub.execute_input":"2021-12-07T14:49:12.078411Z","iopub.status.idle":"2021-12-07T14:49:12.226804Z","shell.execute_reply.started":"2021-12-07T14:49:12.078379Z","shell.execute_reply":"2021-12-07T14:49:12.225826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For each order ID, I calculate the time difference between two available status and two pickup status (the first and last PING), for this can be a useful predictor. \n\nIn this cartoon here, for example, the user's app shows that the driver is only 100 m from the pick-up spot. Normally it takes 3 minutes to reach the spot. However, the user wait for 10 minutes. This can be an indication that driver uses fake GPS.\n\n
\n\n
","metadata":{"id":"aP5G4ukdLOV5"}},{"cell_type":"code","source":"# Calculate time difference between available and otw pickup status\n\nid = list(df_grouped1.index)\n\nfor num_id, order_id in enumerate(id):\n # Select dataframe subset w.r.t. order id\n df_id = train[train.order_id==order_id]\n try:\n # Select available status\n avail = df_id[df_id.driver_status=='AVAILABLE']\n\n # Select pickup status\n pickup = df_id[df_id.driver_status=='OTW_PICKUP']\n\n # Record the first and last seconds of available and pickup\n t_avail0 = avail.seconds.values[0]\n t_avail1 = avail.seconds.values[1]\n t_pickup0 = pickup.seconds.values[0]\n t_pickup1 = pickup.seconds.values[-1]\n\n # Calculate time difference of available and pickup last and first seconds\n avail_sec_diff = t_avail1 - t_avail0\n pickup_sec_diff = t_pickup1 - t_pickup0\n\n except:\n # Set time difference to Null of there is no available/pickup status \n avail_sec_diff = np.nan\n pickup_sec_diff = np.nan\n\n # Record time difference to df_grouped1\n df_grouped1.loc[order_id, 'avail_sec_diff'] = avail_sec_diff\n df_grouped1.loc[order_id, 'pickup_sec_diff'] = pickup_sec_diff\n\n # Logger of every 100 ids\n if num_id%100==0:\n print('Finish ID:', num_id)","metadata":{"id":"FqzaFAe5B6bS","outputId":"11357e5b-40f6-4ed5-dd1a-95e6e7e89ca2","execution":{"iopub.status.busy":"2021-12-07T14:49:37.562762Z","iopub.execute_input":"2021-12-07T14:49:37.563291Z","iopub.status.idle":"2021-12-07T14:54:39.362727Z","shell.execute_reply.started":"2021-12-07T14:49:37.563246Z","shell.execute_reply":"2021-12-07T14:54:39.361757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_grouped1","metadata":{"id":"CCOGHuBXI-da","outputId":"07274cf9-da72-4da8-80b3-300d3988356d","execution":{"iopub.status.busy":"2021-12-07T14:54:39.364341Z","iopub.execute_input":"2021-12-07T14:54:39.364576Z","iopub.status.idle":"2021-12-07T14:54:39.383902Z","shell.execute_reply.started":"2021-12-07T14:54:39.364547Z","shell.execute_reply":"2021-12-07T14:54:39.382992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## `df_grouped2`\n\nNext, I group by order ID to calculate summary stats (mean, min, max, IQR, and range).","metadata":{"id":"5DLugVpSPV-J"}},{"cell_type":"code","source":"train = train[['order_id', 'service_type', 'driver_status', 'distance', 'hour',\n 'accuracy_in_meters', 'accuracy_diff', 'altitude_in_meters', \n 'altitude_diff', 'longitude_diff', 'latitude_diff', 'UTMX_diff', \n 'UTMY_diff', 'seconds_diff', 'label']]\n\ntrain","metadata":{"id":"pb4Zse_99r9M","outputId":"fead4c38-5978-4e48-8a45-85ad95a916c1","execution":{"iopub.status.busy":"2021-12-07T14:58:31.171648Z","iopub.execute_input":"2021-12-07T14:58:31.171943Z","iopub.status.idle":"2021-12-07T14:58:31.226782Z","shell.execute_reply.started":"2021-12-07T14:58:31.171912Z","shell.execute_reply":"2021-12-07T14:58:31.225608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Interquartile and range function\niqr = lambda x: np.percentile(x, 75) - np.percentile(x, 25)\nrange = lambda x: np.max(x) - np.min(x)\n\n# Drop the last 1 column: label\ndf_grouped2 = train.iloc[:,:-1]\n\n# Calculate summary statistics\ndf_grouped2 = df_grouped2.groupby('order_id').aggregate([np.mean, np.min, np.max, np.std, iqr, range])\n\n# Reduce multi-index\ndf_grouped2.columns = ['_'.join(col).strip() for col in df_grouped2.columns.values]\n\ndf_grouped2","metadata":{"id":"rcy2S2asZebf","outputId":"e36c3a84-1a12-4e22-e144-a871cae3147b","execution":{"iopub.status.busy":"2021-12-07T14:58:35.892064Z","iopub.execute_input":"2021-12-07T14:58:35.892341Z","iopub.status.idle":"2021-12-07T14:58:50.164335Z","shell.execute_reply.started":"2021-12-07T14:58:35.892312Z","shell.execute_reply":"2021-12-07T14:58:50.163462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Replace column name to IQR and to range \ncol_groupby2 = df_grouped2.columns\ncol_groupby2 = [w.replace('', 'IQR') for w in col_groupby2]\ncol_groupby2 = [w.replace('', 'range') for w in col_groupby2]\n\n# Update names of columns\ndf_grouped2.columns = col_groupby2\n\ndf_grouped2","metadata":{"id":"xkF5C_ulHlJN","outputId":"ee4effc6-dff4-4a91-d713-83fa93e8caa9","execution":{"iopub.status.busy":"2021-12-07T14:58:59.774675Z","iopub.execute_input":"2021-12-07T14:58:59.774973Z","iopub.status.idle":"2021-12-07T14:58:59.818104Z","shell.execute_reply.started":"2021-12-07T14:58:59.774941Z","shell.execute_reply":"2021-12-07T14:58:59.817222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## `df_grouped3`\n\nMake dummies from driver status, then for each order ID groups, I count the PINGS of each status.","metadata":{"id":"PcOU7EwZRUH_"}},{"cell_type":"code","source":"# Get dummies of driver status\ntrain = pd.get_dummies(train, columns=['driver_status'])\n\ntrain","metadata":{"id":"Un23em4CC295","outputId":"68dd0069-6404-49c0-bbcc-d62566b89082","execution":{"iopub.status.busy":"2021-12-07T14:59:11.57536Z","iopub.execute_input":"2021-12-07T14:59:11.576052Z","iopub.status.idle":"2021-12-07T14:59:11.71336Z","shell.execute_reply.started":"2021-12-07T14:59:11.576007Z","shell.execute_reply":"2021-12-07T14:59:11.712435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Count number of PING by driver status\ndf_grouped3 = train.groupby('order_id')[['driver_status_AVAILABLE', 'driver_status_OTW_DROPOFF',\n 'driver_status_OTW_PICKUP','driver_status_UNAVAILABLE']].sum()\n\ndf_grouped3","metadata":{"id":"aJhFu9YNFCUX","outputId":"7f8a1d10-3457-4348-c0df-5a77d6ed99ca","execution":{"iopub.status.busy":"2021-12-07T14:59:16.262088Z","iopub.execute_input":"2021-12-07T14:59:16.262429Z","iopub.status.idle":"2021-12-07T14:59:16.389494Z","shell.execute_reply.started":"2021-12-07T14:59:16.262395Z","shell.execute_reply":"2021-12-07T14:59:16.388649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## `df_grouped4`\n\nAs mentioned before, altitude contains many missing values. I presumed that it is possible that a fake GPS give unstable altitude, thus more missing altitudes.\n\nHere, I counted the missing altitudes of each order ID groups, and use as new feature.","metadata":{"id":"BFnxje_rR2iT"}},{"cell_type":"code","source":"df_grouped4 = train[['order_id', 'altitude_in_meters']]\n\n# Check for each row if altitude is Null\ndf_grouped4['altitude_isnan'] = df_grouped4.altitude_in_meters.isnull()\n\ndf_grouped4 = df_grouped4.groupby('order_id')[['altitude_isnan']].sum()\n\ndf_grouped4","metadata":{"id":"RxUq_VGhdGmo","outputId":"3c3398b6-f993-49e7-a6bf-b48decc4176f","execution":{"iopub.status.busy":"2021-12-07T14:59:21.135171Z","iopub.execute_input":"2021-12-07T14:59:21.135473Z","iopub.status.idle":"2021-12-07T14:59:21.218114Z","shell.execute_reply.started":"2021-12-07T14:59:21.135435Z","shell.execute_reply":"2021-12-07T14:59:21.217372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Combined all new features\n\n**After feature engineering, the new size of dataset is (3500, 74) where 3500=number of order ID and 74=number of new features.**","metadata":{"id":"0RG-FxsWSnsG"}},{"cell_type":"code","source":"# Merge all grouped dataframe\ndf = pd.concat((df_grouped1, df_grouped2, df_grouped3, df_grouped4), axis=1)\n\n# Encode service_type\nservice_label = {'service_type': {'GO_FOOD': 0, 'GO_RIDE': 1}}\ndf = df.replace(service_label)\n\ndf","metadata":{"id":"AWBhpMX2GIs5","outputId":"bb2fbf34-3053-4166-9f7f-758315e53cbd","execution":{"iopub.status.busy":"2021-12-07T14:59:26.595349Z","iopub.execute_input":"2021-12-07T14:59:26.596108Z","iopub.status.idle":"2021-12-07T14:59:26.650933Z","shell.execute_reply.started":"2021-12-07T14:59:26.596067Z","shell.execute_reply":"2021-12-07T14:59:26.650076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculate correlation with target\n\nFrom the bar plot, we get top 10 features with highest correlation with target, as follows:\n\ndistance IQR, altitude difference IQR, UTMX UTMY difference IQR, latitude longitude difference IQR, accuracy difference IQR, min of distance, and missing altitude counts ","metadata":{"id":"lgeLsSqyS72H"}},{"cell_type":"code","source":"# Correlation bar plot\ndf.corr()['label'][2:].sort_values(ascending=True).plot.bar(figsize=(14,5))","metadata":{"id":"CwZtt8MpGf3g","outputId":"2d84cd3f-dc88-4286-e045-a33b2697f2eb","execution":{"iopub.status.busy":"2021-12-07T14:59:33.789489Z","iopub.execute_input":"2021-12-07T14:59:33.789816Z","iopub.status.idle":"2021-12-07T14:59:35.754286Z","shell.execute_reply.started":"2021-12-07T14:59:33.789782Z","shell.execute_reply":"2021-12-07T14:59:35.75364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.scatter(train.longitude, train.latitude, c=train.label, s=5)\n# plt.xlim(107.4,107.9)\n# plt.ylim(-7.1,-6.7)","metadata":{"id":"HOS3R6udsryn"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Machine learning - training and evaluation","metadata":{"id":"JfmytPg5Q054"}},{"cell_type":"markdown","source":"An XGBoost classifier model is built to classify if the GPS is true (1) or fake (0).","metadata":{"id":"7bqODekguPIN"}},{"cell_type":"code","source":"# Feature and target\nX = df.drop(columns=['label'])\ny = df.label\n\n# Train test split\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n\n# Pipeline\npipe = make_pipeline(StandardScaler(), XGBClassifier())\n\n# Define multiple scoring metrics\nscoring = {\n 'acc': 'accuracy',\n 'prec_macro': 'precision_macro',\n 'rec_macro': 'recall_macro',\n 'f1_macro': 'f1_macro'\n}\n\n# Stratified K-Fold\nstratkfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\n# Cross-validation.Ignore the warning\ncv_scores = cross_validate(pipe, X_train, y_train, cv=stratkfold, scoring=scoring)","metadata":{"id":"-n7dW7gcRH5G","outputId":"b257d4a3-2348-43ab-f40c-e58de6d402a0","execution":{"iopub.status.busy":"2021-12-07T15:01:58.737333Z","iopub.execute_input":"2021-12-07T15:01:58.737637Z","iopub.status.idle":"2021-12-07T15:02:06.700506Z","shell.execute_reply.started":"2021-12-07T15:01:58.737606Z","shell.execute_reply":"2021-12-07T15:02:06.699836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The CV scores show all mean precision, recall, and F1-score of 79%. ","metadata":{}},{"cell_type":"code","source":"# Print scoring results from dictionary\nfor metric_name, metric_value in cv_scores.items():\n mean = np.mean(metric_value)\n print(f'{metric_name}: {np.round(metric_value, 4)}, Mean: {np.round(mean, 4)}')","metadata":{"execution":{"iopub.status.busy":"2021-12-07T15:02:15.288214Z","iopub.execute_input":"2021-12-07T15:02:15.28848Z","iopub.status.idle":"2021-12-07T15:02:15.296775Z","shell.execute_reply.started":"2021-12-07T15:02:15.288452Z","shell.execute_reply":"2021-12-07T15:02:15.295791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit pipeline to train set\npipe.fit(X_train, y_train)\n\n# Predict on test set\ny_pred = pipe.predict(X_test)","metadata":{"id":"-b_W0_8Bob5C","execution":{"iopub.status.busy":"2021-12-07T15:04:59.859948Z","iopub.execute_input":"2021-12-07T15:04:59.86028Z","iopub.status.idle":"2021-12-07T15:05:01.821763Z","shell.execute_reply.started":"2021-12-07T15:04:59.860244Z","shell.execute_reply":"2021-12-07T15:05:01.821077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save pipeline into pickle\nimport joblib\njoblib.dump(pipe, './gojek_xgboost.pkl')","metadata":{"id":"dIyoOiI0JzAY","outputId":"f61d4ffe-48e5-46a5-d258-c1008dff864e","execution":{"iopub.status.busy":"2021-12-07T15:05:15.575593Z","iopub.execute_input":"2021-12-07T15:05:15.575909Z","iopub.status.idle":"2021-12-07T15:05:15.597709Z","shell.execute_reply.started":"2021-12-07T15:05:15.575874Z","shell.execute_reply":"2021-12-07T15:05:15.597065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Confusion matrix of test set\nplot_confusion_matrix(pipe, X_test, y_test, values_format='.5g') \nplt.show()","metadata":{"id":"FOxDgBoTogND","outputId":"d7441124-5119-4842-c0f5-82bf00036423","execution":{"iopub.status.busy":"2021-12-07T15:05:25.730133Z","iopub.execute_input":"2021-12-07T15:05:25.73045Z","iopub.status.idle":"2021-12-07T15:05:26.020611Z","shell.execute_reply.started":"2021-12-07T15:05:25.730418Z","shell.execute_reply":"2021-12-07T15:05:26.019791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Classification report\nprint(classification_report(y_test, y_pred))","metadata":{"id":"4cMvK_3DpTnE","outputId":"065fb3b3-0f6a-48da-91ca-cef813a8b5e8","execution":{"iopub.status.busy":"2021-12-07T15:05:30.279613Z","iopub.execute_input":"2021-12-07T15:05:30.28076Z","iopub.status.idle":"2021-12-07T15:05:30.292656Z","shell.execute_reply.started":"2021-12-07T15:05:30.280708Z","shell.execute_reply":"2021-12-07T15:05:30.291625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate class membership probabilities\ny_pred_probs = pipe.predict_proba(X_test)\n\nclasses = [0,1]\n\n# For each class\nfor i, clas in enumerate(classes):\n # Calculate False Positive Rate, True Negative Rate\n fpr, tpr, thresholds = roc_curve(y_test, y_pred_probs[:,i], \n pos_label = clas) \n \n # Calculate AUC\n auroc = auc(fpr, tpr)\n \n # Plot ROC AUC curve for each class\n plt.plot(fpr, tpr, label=f'{clas}, AUC: {auroc:.2f}')\n plt.plot([0, 1], [0, 1], 'k--')\n\nplt.title('ROC AUC')\nplt.xlabel('FPR'); plt.ylabel('TPR')\nplt.xlim(0,1); plt.ylim(0,1)\nplt.legend()\nplt.show()","metadata":{"id":"WYduyZ7Upii4","outputId":"aa006ea4-4df5-4e93-db5e-7c59b2309328","execution":{"iopub.status.busy":"2021-12-07T15:05:34.692843Z","iopub.execute_input":"2021-12-07T15:05:34.693277Z","iopub.status.idle":"2021-12-07T15:05:34.97127Z","shell.execute_reply.started":"2021-12-07T15:05:34.693241Z","shell.execute_reply":"2021-12-07T15:05:34.970352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a pd.Series of features importances\nfimp = pipe.steps[1][1].feature_importances_\nimportances = pd.Series(data=fimp,\n index= X_train.columns)\n\n# Sort importances\nimportances_sorted = importances.sort_values()[-15:]\n\n# Draw a horizontal barplot of importances_sorted\nimportances_sorted.plot(kind='barh', color='red')\nplt.title('Features Importances')\nplt.show()","metadata":{"id":"iHM8ivPmvbYm","outputId":"97b6591d-471a-46dc-8e6f-5ca914692e87","execution":{"iopub.status.busy":"2021-12-07T15:05:39.352924Z","iopub.execute_input":"2021-12-07T15:05:39.353236Z","iopub.status.idle":"2021-12-07T15:05:39.694965Z","shell.execute_reply.started":"2021-12-07T15:05:39.3532Z","shell.execute_reply":"2021-12-07T15:05:39.694092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict on test set","metadata":{"id":"6Y1ceCp4xOQe"}},{"cell_type":"markdown","source":"Creating a function to transform the test data by grouping by each order ID and engineer 74 new features.","metadata":{}},{"cell_type":"code","source":"def gojek_data_transform(df):\n # Differencing some columns\n df['longitude_diff'] = df.groupby('order_id').longitude.diff().fillna(0)\n df['latitude_diff'] = df.groupby('order_id').latitude.diff().fillna(0)\n df['seconds_diff'] = df.groupby('order_id').seconds.diff().fillna(0)\n df['accuracy_diff'] = df.groupby('order_id').accuracy_in_meters.diff().fillna(0)\n df['altitude_diff'] = df.groupby('order_id').altitude_in_meters.diff().fillna(0)\n\n # Convert lat lon to UTM\n lat, lon = df.latitude.values, df.longitude.values\n x = utm.from_latlon(lat, lon)\n\n df['UTMX'] = x[0]\n df['UTMY'] = x[1]\n\n # Function to calculate distance between two points\n distance = lambda x_dif, y_dif: np.sqrt(x_dif**2 + y_dif**2)\n\n # Differencing UTM coordinates\n df['UTMX_diff'] = df.groupby('order_id').UTMX.diff().fillna(0)\n df['UTMY_diff'] = df.groupby('order_id').UTMY.diff().fillna(0)\n\n # Calculate step distance\n df['distance'] = distance(df.UTMX_diff, df.UTMY_diff)\n\n # Grouping by order ID to get service type and label\n df_grouped1 = df.groupby('order_id')[['service_type']].max()\n\n # Calculate time difference between available and otw pickup status\n id = list(df_grouped1.index)\n\n for num_id, order_id in enumerate(id):\n # Select dataframe subset w.r.t. order id\n df_id = df[df.order_id==order_id]\n try:\n # Select available status\n avail = df_id[df_id.driver_status=='AVAILABLE']\n\n # Select pickup status\n pickup = df_id[df_id.driver_status=='OTW_PICKUP']\n\n # Record the first and last seconds of available and pickup\n t_avail0 = avail.seconds.values[0]\n t_avail1 = avail.seconds.values[1]\n t_pickup0 = pickup.seconds.values[0]\n t_pickup1 = pickup.seconds.values[-1]\n\n # Calculate time difference of available and pickup last and first seconds\n avail_sec_diff = t_avail1 - t_avail0\n pickup_sec_diff = t_pickup1 - t_pickup0\n\n except:\n # Set time difference to Null of there is no available/pickup status \n avail_sec_diff = np.nan\n pickup_sec_diff = np.nan\n\n # Record time difference to df_grouped1\n df_grouped1.loc[order_id, 'avail_sec_diff'] = avail_sec_diff\n df_grouped1.loc[order_id, 'pickup_sec_diff'] = pickup_sec_diff\n\n df = df[['order_id', 'service_type', 'driver_status', 'distance', 'hour',\n 'accuracy_in_meters', 'accuracy_diff', 'altitude_in_meters', \n 'altitude_diff', 'longitude_diff', 'latitude_diff', 'UTMX_diff', \n 'UTMY_diff', 'seconds_diff']]\n\n # Interquartile and range function\n iqr = lambda x: np.percentile(x, 75) - np.percentile(x, 25)\n range = lambda x: np.max(x) - np.min(x)\n\n # Calculate summary statistics\n df_grouped2 = df.groupby('order_id').aggregate([np.mean, np.min, np.max, np.std, iqr, range])\n\n # Reduce multi-index\n df_grouped2.columns = ['_'.join(col).strip() for col in df_grouped2.columns.values]\n\n # Replace column name to IQR and to range \n col_groupby2 = df_grouped2.columns\n col_groupby2 = [w.replace('', 'IQR') for w in col_groupby2]\n col_groupby2 = [w.replace('', 'range') for w in col_groupby2]\n\n # Update names of columns\n df_grouped2.columns = col_groupby2\n\n # Get dummies of driver status\n df = pd.get_dummies(df, columns=['driver_status'])\n\n # Count number of PING by driver status\n df_grouped3 = df.groupby('order_id')[['driver_status_AVAILABLE', 'driver_status_OTW_DROPOFF',\n 'driver_status_OTW_PICKUP','driver_status_UNAVAILABLE']].sum()\n df_grouped4 = df[['order_id', 'altitude_in_meters']]\n\n # Check for each row if altitude is Null\n df_grouped4['altitude_isnan'] = df_grouped4.altitude_in_meters.isnull()\n\n df_grouped4 = df_grouped4.groupby('order_id')[['altitude_isnan']].sum()\n\n # Merge all grouped dataframe\n df = pd.concat((df_grouped1, df_grouped2, df_grouped3, df_grouped4), axis=1)\n\n # Encode service_type\n service_label = {'service_type': {'GO_FOOD': 0, 'GO_RIDE': 1}}\n df = df.replace(service_label) \n\n return df","metadata":{"id":"hNm4tpoh9H1s","execution":{"iopub.status.busy":"2021-12-07T15:11:04.420771Z","iopub.execute_input":"2021-12-07T15:11:04.421106Z","iopub.status.idle":"2021-12-07T15:11:04.445284Z","shell.execute_reply.started":"2021-12-07T15:11:04.421068Z","shell.execute_reply":"2021-12-07T15:11:04.444008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read test set\ntest = pd.read_csv('../input/dsbootcamp10/test.csv')\n\ntest","metadata":{"id":"ZLjfobSgxdub","outputId":"8c1a5ca5-f0ee-45f7-86af-5c00fa51f4c7","execution":{"iopub.status.busy":"2021-12-07T15:11:17.882622Z","iopub.execute_input":"2021-12-07T15:11:17.882909Z","iopub.status.idle":"2021-12-07T15:11:18.085982Z","shell.execute_reply.started":"2021-12-07T15:11:17.882879Z","shell.execute_reply":"2021-12-07T15:11:18.085013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After transformation, the size of test set is (500, 74) where 500=number of order ID and 74=number of new features.","metadata":{}},{"cell_type":"code","source":"# Transform test set to produce 74 features\ntest_ready = gojek_data_transform(test)\n\ntest_ready","metadata":{"id":"gs8rW7Pz9PbS","outputId":"d3b9f209-fe8a-4264-ad0f-98c9fe2fec21","execution":{"iopub.status.busy":"2021-12-07T15:11:23.502712Z","iopub.execute_input":"2021-12-07T15:11:23.502985Z","iopub.status.idle":"2021-12-07T15:11:33.574631Z","shell.execute_reply.started":"2021-12-07T15:11:23.502956Z","shell.execute_reply":"2021-12-07T15:11:33.573601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict on test set\ny_pred = pipe.predict(test_ready)\n\n# Print the first 20 predictions\nprint(y_pred[:20])","metadata":{"id":"ekVzd7dX-IM1","outputId":"864ff123-2f25-4d17-feb3-c3253e98a2ff","execution":{"iopub.status.busy":"2021-12-07T15:11:42.855459Z","iopub.execute_input":"2021-12-07T15:11:42.856427Z","iopub.status.idle":"2021-12-07T15:11:42.879146Z","shell.execute_reply.started":"2021-12-07T15:11:42.856389Z","shell.execute_reply":"2021-12-07T15:11:42.877708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make submission dataframe\nsubmission = test_ready.reset_index().iloc[:,:1]\nsubmission['label'] = y_pred\n\nsubmission","metadata":{"id":"SpEie5psR9Z4","outputId":"cab678ad-db1c-4139-80d1-425908e17512","execution":{"iopub.status.busy":"2021-12-07T15:11:50.584092Z","iopub.execute_input":"2021-12-07T15:11:50.584393Z","iopub.status.idle":"2021-12-07T15:11:50.601126Z","shell.execute_reply.started":"2021-12-07T15:11:50.584359Z","shell.execute_reply":"2021-12-07T15:11:50.600277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submission to csv\nsubmission.to_csv('./GoJek_submission.csv', index=False)","metadata":{"id":"cgD0QSu4MIv3","execution":{"iopub.status.busy":"2021-12-07T15:12:01.001971Z","iopub.execute_input":"2021-12-07T15:12:01.002855Z","iopub.status.idle":"2021-12-07T15:12:01.010746Z","shell.execute_reply.started":"2021-12-07T15:12:01.002817Z","shell.execute_reply":"2021-12-07T15:12:01.009766Z"},"trusted":true},"execution_count":null,"outputs":[]}]}