{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Special thank you to the authors of the next notebooks:\n\nhttps://www.kaggle.com/robikscube/indoor-navigation-snap-to-grid-post-processing\n\nhttps://www.kaggle.com/joelqv/where-are-your-predictions-located-shapely\n\nand to everyone who took a part in the next discussion:\n\nhttps://www.kaggle.com/c/indoor-location-navigation/discussion/237776","metadata":{}},{"cell_type":"code","source":"# Helper Functions\nimport pandas as pd\nimport numpy as np\n\nimport json\nimport matplotlib.pylab as plt\nimport shapely\nimport matplotlib\n\nfrom shapely.geometry import Point\nfrom shapely.geometry.polygon import Polygon\nimport shapely.ops as so\n\nfrom shapely.affinity import scale\nfrom shapely.ops import transform\n\nimport copy as copy\nimport gc\n\ndef split_col(df):\n    df = pd.concat([\n        df['site_path_timestamp'].str.split('_', expand=True) \\\n        .rename(columns={0:'site',\n                         1:'path',\n                         2:'timestamp'}),\n        df\n    ], axis=1).copy()\n    return df\n\nfloor_map = {\"B2\":-2, \"B1\":-1, \"F1\":0, \"F2\": 1, \"F3\":2,\n             \"F4\":3, \"F5\":4, \"F6\":5, \"F7\":6,\"F8\":7,\"F9\":8,\n             \"1F\":0, \"2F\":1, \"3F\":2, \"4F\":3, \"5F\":4, \"6F\":5,\n             \"7F\":6, \"8F\": 7, \"9F\":8}\n\n\ndef plot_preds(\n    site,\n    floorNo,\n    sub=None,\n    true_locs=None,\n    base=\"../input/indoor-location-navigation\",\n    show_train=True,\n    show_preds=True,\n    fix_labels=True,\n    map_floor=None\n):\n    \"\"\"\n    Plots predictions on floorplan map.\n    \n    map_floor : use a different floor's map\n    \"\"\"\n    if map_floor is None:\n        map_floor = floorNo\n    # Prepare width_meter & height_meter (taken from the .json file)\n    floor_plan_filename = f\"{base}/metadata/{site}/{map_floor}/floor_image.png\"\n    json_plan_filename = f\"{base}/metadata/{site}/{map_floor}/floor_info.json\"\n    with open(json_plan_filename) as json_file:\n        json_data = json.load(json_file)\n\n    width_meter = json_data[\"map_info\"][\"width\"]\n    height_meter = json_data[\"map_info\"][\"height\"]\n\n    floor_img = plt.imread(f\"{base}/metadata/{site}/{map_floor}/floor_image.png\")\n\n    fig, ax = plt.subplots(figsize=(12, 12))\n    plt.imshow(floor_img)\n\n    if show_train:\n        true_locs = true_locs.query('site == @site and floorNo == @map_floor').copy()\n        true_locs[\"x_\"] = true_locs[\"x\"] * floor_img.shape[0] / height_meter\n        true_locs[\"y_\"] = (\n            true_locs[\"y\"] * -1 * floor_img.shape[1] / width_meter\n        ) + floor_img.shape[0]\n        true_locs.query(\"site == @site and floorNo == @map_floor\").groupby(\"path\").plot(\n            x=\"x_\",\n            y=\"y_\",\n            style=\"+\",\n            ax=ax,\n            label=\"train waypoint location\",\n            color=\"grey\",\n            alpha=0.5,\n        )\n\n    if show_preds:\n        sub = sub.query('site == @site and floorNo == @floorNo').copy()\n        sub[\"x_\"] = sub[\"x\"] * floor_img.shape[0] / height_meter\n        sub[\"y_\"] = (\n            sub[\"y\"] * -1 * floor_img.shape[1] / width_meter\n        ) + floor_img.shape[0]\n        for path, path_data in sub.query(\n            \"site == @site and floorNo == @floorNo\"\n        ).groupby(\"path\"):\n            path_data.plot(\n                x=\"x_\",\n                y=\"y_\",\n                style=\".-\",\n                ax=ax,\n                title=f\"{site} - floor - {floorNo}\",\n                alpha=1,\n                label=path,\n            )\n    if fix_labels:\n        handles, labels = ax.get_legend_handles_labels()\n        by_label = dict(zip(labels, handles))\n        plt.legend(\n            by_label.values(), by_label.keys(), loc=\"center left\", bbox_to_anchor=(1, 0.5)\n        )\n    return fig, ax\n\ndef sub_process(sub, train_waypoints):\n    train_waypoints['isTrainWaypoint'] = True\n    sub = split_col(sub[['site_path_timestamp','floor','x','y']]).copy()\n    sub = sub.merge(train_waypoints[['site','floorNo','floor']].drop_duplicates(), how='left')\n    sub = sub.merge(\n        train_waypoints[['x','y','site','floor','isTrainWaypoint']].drop_duplicates(),\n        how='left',\n        on=['site','x','y','floor']\n             )\n    sub['isTrainWaypoint'] = sub['isTrainWaypoint'].fillna(False)\n    return sub.copy()\n\nimport shapely as sh\n\ndef get_coords_from_polygon(shape):\n    coords = set()\n\n    if isinstance(shape, sh.geometry.Polygon):\n        coords.update(shape.exterior.coords[:-1])\n        for linearring in shape.interiors:\n            coords.update(linearring.coords[:-1])\n    elif isinstance(shape, sh.geometry.MultiPolygon):\n        for polygon in shape:\n            coords.update(get_coords_from_polygon(polygon))\n\n    return coords","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_waypoints = pd.read_csv('../input/indoor-location-train-waypoints/train_waypoints.csv')\ntrain_waypoints['isTrainWaypoint'] = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Step 1: Identify training waypoints","metadata":{}},{"cell_type":"code","source":"# Blend Subs\nsub1 = split_col(pd.read_csv('../input/indoor-location-train-waypoints/6.771LB_submission.csv'))\nsub2 = split_col(pd.read_csv('../input/indoor-location-train-waypoints/7.274LB_submission.csv'))\nsub3 = split_col(pd.read_csv('../input/indoor-location-train-waypoints/7.518LB_submission_LSTM.csv'))\nsub4 = split_col(pd.read_csv('../input/indoor-location-train-waypoints/7.661LB_LSTM_submission.csv'))\nsub5 = split_col(pd.read_csv('../input/indoor-location-train-waypoints/7.745LB_submission.csv'))\n\n# Blend\nsub = sub1.merge(sub2[['site_path_timestamp','floor','x','y']],\n           on=['site_path_timestamp','floor'],\n           how='left',\n           suffixes=('_s1','_s2')).copy()\nsub['x'] = sub1['x'] \nsub['y'] = sub1['y'] \n\nsub = sub.merge(sub3[['site_path_timestamp','floor','x','y']],\n           on=['site_path_timestamp','floor'],\n           how='left',\n           suffixes=('','_s3')).copy()\nsub = sub.merge(sub4[['site_path_timestamp','floor','x','y']],\n           on=['site_path_timestamp','floor'],\n           how='left',\n           suffixes=('','_s4')).copy()\nsub = sub.merge(sub5[['site_path_timestamp','floor','x','y']],\n           on=['site_path_timestamp','floor'],\n           how='left',\n           suffixes=('','_s5')).copy()\n\nsub['x'] = sub[['x_s1','x_s2','x_s3','x_s4','x_s5']].mean(axis=1)\nsub['y'] = sub[['y_s1','y_s2','y_s3','y_s4','y_s5']].mean(axis=1)\n\nfloors = np.unique(list(train_waypoints['floorNo']))\nsites = np.unique(list(train_waypoints['site']))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For example, let us choose the site and the floor:","metadata":{}},{"cell_type":"code","source":"train_waypoints = pd.read_csv('../input/indoor-location-train-waypoints/train_waypoints.csv')\ntrain_waypoints['isTrainWaypoint'] = True\n# sub = sub_process(pd.read_csv('../input/indoor-location-train-waypoints/6.578LB_submission.csv'),\n#                  train_waypoints)\nsub = sub_process(sub, train_waypoints)\n# Plot the training Data For an example Floor\n#Here we choose the site and the floor for the example\nexample_site = sites[15]\nexample_floorNo = floors[11]\n\nplot_preds(example_site, example_floorNo, sub,\n           train_waypoints, show_preds=False)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets get the polygons:","metadata":{}},{"cell_type":"code","source":"with open(f\"../input/indoor-location-navigation/metadata/{sites[15]}/{floors[11]}/geojson_map.json\") as json_file:\n    \n    geofloor_data = json.load(json_file)\n\ntype_poly = geofloor_data['features'][0]['geometry']['type']\n\nif type_poly == 'Polygon':\n    \n    polygon = np.array(geofloor_data['features'][0]['geometry']['coordinates'][0])\n\nelse:\n    \n    polygon = np.array(geofloor_data['features'][0]['geometry']['coordinates'][0][0])\n\nfloor_polygons = Polygon(polygon)\n\nstore_polygons_l = [Polygon(features['geometry']['coordinates'][0]) for features in geofloor_data['features'][1:]]\n\nstore_polygons = so.unary_union(store_polygons_l)\n\nsafe_area_polygons = floor_polygons.difference(store_polygons)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get the centers of the polygons\nx_mean = (polygon[:, 0].max() - polygon[:, 0].min())*0.5 + polygon[:, 0].min()\ny_mean = (polygon[:, 1].max() - polygon[:, 1].min())*0.5 + polygon[:, 1].min()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x,y = floor_polygons.exterior.xy\nplt.plot(x,y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x,y = safe_area_polygons[0].exterior.xy\nplt.plot(x,y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For some reason grid generator code below tends to mirror the polygons, so we are going to mirror them in advance to get the normal ones\nfor the processing","metadata":{}},{"cell_type":"code","source":"floor_polygons1 = scale(floor_polygons, xfact = -1, origin = (x_mean, y_mean))\nfloor_polygons1 = scale(floor_polygons1, yfact = -1, origin = (x_mean, y_mean))\n\nsafe_area_polygons1 = scale(safe_area_polygons, xfact = -1, origin = (x_mean, y_mean))\nsafe_area_polygons1 = scale(safe_area_polygons1, yfact = -1, origin = (x_mean, y_mean))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x,y = floor_polygons1.exterior.xy\nplt.plot(x,y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x,y = floor_polygons1.exterior.xy\nplt.plot(x,y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x,y = safe_area_polygons1[0].exterior.xy\nplt.plot(x,y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xy_comb = train_waypoints[['site', 'floorNo']].values\nxy_comb_unique = [list(x) for x in set(tuple(x) for x in xy_comb)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(xy_comb_unique)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_waypoints = pd.read_csv('../input/indoor-location-train-waypoints/train_waypoints.csv')\ntrain_waypoints['isTrainWaypoint'] = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#distance between points on the grid\ndense_step = 3.0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for j in range(0, len(sites)):\n    \n    print(f'site is number {j} out of a {len(sites)}')\n    \n    for i in range(0, len(floors)):\n        \n        print(f'floor is number {i} out of a {len(floors)}')\n        \n        site = sites[j]\n        floorNo = floors[i]\n        \n        if [sites[j], floors[i]] in xy_comb_unique:\n            \n            print('now we count')\n            \n    \n            with open(f\"../input/indoor-location-navigation/metadata/{site}/{floorNo}/floor_info.json\") as json_file0:\n\n                json_data = json.load(json_file0)\n                \n            width_meter = json_data[\"map_info\"][\"width\"]\n            height_meter = json_data[\"map_info\"][\"height\"]\n        \n            with open(f\"../input/indoor-location-navigation/metadata/{site}/{floorNo}/geojson_map.json\") as json_file:\n\n                geofloor_data = json.load(json_file)\n\n            type_poly = geofloor_data['features'][0]['geometry']['type']\n\n            if type_poly == 'Polygon':\n\n                polygon = np.array(geofloor_data['features'][0]['geometry']['coordinates'][0])\n\n            else:\n\n                polygon = np.array(geofloor_data['features'][0]['geometry']['coordinates'][0][0])\n\n            floor_polygons = Polygon(polygon)\n\n            store_polygons_l = [Polygon(features['geometry']['coordinates'][0]) for features in geofloor_data['features'][1:]]\n\n            store_polygons = so.unary_union(store_polygons_l)\n\n            safe_area_polygons = floor_polygons.difference(store_polygons)\n\n            x_max = polygon[:, 0].max()\n            y_max = polygon[:, 1].max()\n            \n            y_min = polygon[:,1].min()\n            x_min = polygon[:,0].min()\n            \n            x_mean = (polygon[:, 0].max() - polygon[:, 0].min())*0.5 + polygon[:, 0].min()\n            y_mean = (polygon[:, 1].max() - polygon[:, 1].min())*0.5 + polygon[:, 1].min()\n            \n            floor_polygons = scale(floor_polygons, xfact = -1, origin = (x_mean, y_mean))\n            floor_polygons = scale(floor_polygons, yfact = -1, origin = (x_mean, y_mean))\n\n            safe_area_polygons = scale(safe_area_polygons, xfact = -1, origin = (x_mean, y_mean))\n            safe_area_polygons = scale(safe_area_polygons, yfact = -1, origin = (x_mean, y_mean))\n\n            temp_df = train_waypoints.loc[(train_waypoints['site'] == sites[j]) & (train_waypoints['floorNo'] == floors[i])]\n\n            min_x = np.min(temp_df['x'])\n            max_x = np.max(temp_df['x'])\n\n            min_y = np.min(temp_df['y'])\n            max_y = np.max(temp_df['y'])\n\n            min_x_moving = copy.copy(min_x)\n            min_y_moving = copy.copy(min_y)\n\n            max_x_moving = copy.copy(max_x)\n            max_y_moving = copy.copy(max_y)\n\n            y_values = [min_y_moving]\n            x_values = [min_x_moving]\n\n            while min_x_moving <= max_x_moving:\n\n                min_x_moving = min_x_moving + dense_step\n                x_values.append(min_x_moving)\n\n            x_values.append(max_x_moving)\n\n            while min_y_moving <= max_y_moving:\n\n                min_y_moving = min_y_moving + dense_step\n                y_values.append(min_y_moving)\n\n            y_values.append(max_y_moving)\n\n            for x_iter in range(0, len(x_values)):\n\n                for y_iter in range(0, len(y_values)):\n\n                    x_scaled = x_max - x_values[x_iter]*(x_max - x_min)/width_meter\n                    y_scaled = y_max - y_values[y_iter]*(y_max - y_min)/height_meter\n                    \n                    InSafe = safe_area_polygons.contains(Point(x_scaled, y_scaled))\n\n                    if InSafe == True:\n\n                        df2 = pd.DataFrame([['TYPE_WAYPOINT', x_values[x_iter],  y_values[y_iter], 1573187328401, site, floorNo, 666, \n                                            '5dc4f878171e610006b5d8ad', True]],\n                                           columns=['type_name','x','y', 'timestamp', 'site', 'floorNo', 'floor', 'path', 'isTrainWaypoint'])\n\n                        train_waypoints = pd.concat([df2, train_waypoints]).reset_index(drop=True)            ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_waypoints.loc[(train_waypoints['site'] == sites[15]) & (train_waypoints['floorNo'] == floors[11])]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_df = train_waypoints.loc[(train_waypoints['site'] == sites[15]) & (train_waypoints['floorNo'] == floors[11]) & (train_waypoints['isTrainWaypoint'] == True) ]\ntemp_df_o = train_waypoints.loc[(train_waypoints['site'] == sites[15]) & (train_waypoints['floorNo'] == floors[11]) & (train_waypoints['floor'] != 666) ]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_df\nx = temp_df['x']\ny = temp_df['y']\nmatplotlib.pyplot.scatter(x,y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = temp_df_o['x']\ny = temp_df_o['y']\nmatplotlib.pyplot.scatter(x,y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_full = train_waypoints.loc[(train_waypoints['site'] == sites[15]) & (train_waypoints['floorNo'] == floors[11])]\nx = temp_full['x']\ny = temp_full['y']\nmatplotlib.pyplot.scatter(x,y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Generated grid seems to match the original one, so let us go further","metadata":{}},{"cell_type":"markdown","source":"## Step 2: Find the closest \"grid\" point for each prediction.\n\nWe can find the closest \"grid\" point to our predictions using the `cdist` function in scipy.","metadata":{}},{"cell_type":"code","source":"#train_waypoints = pd.read_csv('../input/indoor-location-train-waypoints/train_waypoints.csv')\n#train_waypoints['isTrainWaypoint'] = True\n# sub = sub_process(pd.read_csv('../input/indoor-location-train-waypoints/6.578LB_submission.csv'),\n#                  train_waypoints)\nsub = sub_process(sub, train_waypoints)\n# Plot the training Data For an example Floor\nexample_site = sites[15]\nexample_floorNo = floors[11]\n\nplot_preds(example_site, example_floorNo, sub,\n           train_waypoints, show_preds=False)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks dense enough","metadata":{}},{"cell_type":"code","source":"from scipy.spatial.distance import cdist\n\ndef add_xy(df):\n    df['xy'] = [(x, y) for x,y in zip(df['x'], df['y'])]\n    return df\n\ndef closest_point(point, points):\n    \"\"\" Find closest point from a list of points. \"\"\"\n    return points[cdist([point], points).argmin()]\n\nsub = add_xy(sub)\ntrain_waypoints = add_xy(train_waypoints)\n\nds = []\nfor (site, myfloor), d in sub.groupby(['site','floorNo']):\n    true_floor_locs = train_waypoints.loc[(train_waypoints['floorNo'] == myfloor) &\n                                          (train_waypoints['site'] == site)] \\\n        .reset_index(drop=True)\n    if len(true_floor_locs) == 0:\n        print(f'Skipping {site} {myfloor}')\n        continue\n    d['matched_point'] = [closest_point(x, list(true_floor_locs['xy'])) for x in d['xy']]\n    d['x_'] = d['matched_point'].apply(lambda x: x[0])\n    d['y_'] = d['matched_point'].apply(lambda x: x[1])\n    ds.append(d)\n\nsub = pd.concat(ds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example of raw predictions\nplot_preds(example_site, example_floorNo, sub,\n           train_waypoints, show_preds=True)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Step 3: Apply a Threshold and \"Snap to Grid\"\n\nLet us take the thr = 3","metadata":{}},{"cell_type":"code","source":"def snap_to_grid(sub, threshold):\n    \"\"\"\n    Snap to grid if within a threshold.\n    \n    x, y are the predicted points.\n    x_, y_ are the closest grid points.\n    _x_, _y_ are the new predictions after post processing.\n    \"\"\"\n    sub['_x_'] = sub['x']\n    sub['_y_'] = sub['y']\n    sub.loc[sub['dist'] < threshold, '_x_'] = sub.loc[sub['dist'] < threshold]['x_']\n    sub.loc[sub['dist'] < threshold, '_y_'] = sub.loc[sub['dist'] < threshold]['y_']\n    return sub.copy()\n\n# Calculate the distances\nsub['dist'] = np.sqrt( (sub.x-sub.x_)**2 + (sub.y-sub.y_)**2)\n\nsub_pp = snap_to_grid(sub, threshold=3)\n\nsub_pp = sub_pp[['site_path_timestamp','floor','_x_','_y_','site','path','floorNo']] \\\n    .rename(columns={'_x_':'x', '_y_':'y'})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets take a look at the predictions after post processing.","metadata":{}},{"cell_type":"code","source":"# Plot example after post processing\nplot_preds(example_site, example_floorNo, sub_pp,\n           train_waypoints, show_preds=True)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluate The Change in Predictions","metadata":{}},{"cell_type":"code","source":"sub['dist_pp_change'] = np.sqrt(((sub['x'] - sub['_x_']) ** 2) + ((sub['y'] - sub['_y_']) ** 2))\nfig, axs = plt.subplots(1, 2, figsize=(15, 5))\nsub['dist_pp_change'].plot(kind='hist', bins=30,\n                           ax=axs[0],\n                           title='Distance Changed by Post Processing')\nsub.query('dist_pp_change > 0.1')['dist_pp_change'] \\\n    .plot(kind='hist', bins=30, ax=axs[1],\n          title='Distance Changed (Excluding <0.1 Change)')\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.groupby(['site','floorNo'])['dist_pp_change'].mean() \\\n    .reset_index() \\\n    .sort_values('dist_pp_change') \\\n    .set_index(['site','floorNo']).head(20).plot(kind='barh')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Final Step: Save Post Processed Submission And New Waypoints.","metadata":{}},{"cell_type":"code","source":"sub_pp[['site_path_timestamp','floor','x','y']] \\\n    .to_csv('submission_snap_to_grid.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_waypoints.to_csv('new_grid.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}