{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**V1**: Basic EDA  \n**V3**: Added GIF","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd\n\nimport glob\nimport itertools\n\nimport matplotlib.pyplot as plt\nimport numpy as np \nimport pandas as pd\nimport plotly.express as px\nimport seaborn\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\n%matplotlib inline\n\nDATA_PATH = \"../input/smartphone-decimeter-2022/\"","metadata":{"execution":{"iopub.status.busy":"2022-05-04T09:39:07.331746Z","iopub.execute_input":"2022-05-04T09:39:07.332116Z","iopub.status.idle":"2022-05-04T09:39:10.013521Z","shell.execute_reply.started":"2022-05-04T09:39:07.332085Z","shell.execute_reply":"2022-05-04T09:39:10.012547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv(DATA_PATH + \"sample_submission.csv\")\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-04T09:39:10.015423Z","iopub.execute_input":"2022-05-04T09:39:10.015738Z","iopub.status.idle":"2022-05-04T09:39:10.166278Z","shell.execute_reply.started":"2022-05-04T09:39:10.015705Z","shell.execute_reply":"2022-05-04T09:39:10.165455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_sample_trail = pd.read_csv(DATA_PATH + \"train/2020-05-15-US-MTV-1/GooglePixel4XL/device_gnss.csv\")\ndf_sample_trail_gt = pd.read_csv(DATA_PATH + \"train/2020-05-15-US-MTV-1/GooglePixel4XL/ground_truth.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-05-04T09:39:10.167691Z","iopub.execute_input":"2022-05-04T09:39:10.168637Z","iopub.status.idle":"2022-05-04T09:39:10.194717Z","shell.execute_reply.started":"2022-05-04T09:39:10.168586Z","shell.execute_reply":"2022-05-04T09:39:10.193616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sample_trail_gt.head(6)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T09:39:10.196670Z","iopub.execute_input":"2022-05-04T09:39:10.196992Z","iopub.status.idle":"2022-05-04T09:39:10.214475Z","shell.execute_reply.started":"2022-05-04T09:39:10.196962Z","shell.execute_reply":"2022-05-04T09:39:10.213546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FILE_NAME = '2020-05-15-US-MTV-1'","metadata":{"execution":{"iopub.status.busy":"2022-05-04T09:39:10.216063Z","iopub.execute_input":"2022-05-04T09:39:10.216696Z","iopub.status.idle":"2022-05-04T09:39:10.226977Z","shell.execute_reply.started":"2022-05-04T09:39:10.216646Z","shell.execute_reply":"2022-05-04T09:39:10.226169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_trafic(df, center, zoom=8):\n    fig = px.scatter_mapbox(df,\n                            zoom=zoom,\n                            center=center,\n                            \n                            lat=\"LatitudeDegrees\",\n                            lon=\"LongitudeDegrees\",\n                            color=\"MessageType\",\n                            labels='Provider',\n                            \n                            height=600,\n                            width=800)\n    fig.update_layout(mapbox_style='stamen-terrain')\n    fig.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-04T09:39:10.229772Z","iopub.execute_input":"2022-05-04T09:39:10.230023Z","iopub.status.idle":"2022-05-04T09:39:10.241911Z","shell.execute_reply.started":"2022-05-04T09:39:10.229996Z","shell.execute_reply":"2022-05-04T09:39:10.240571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sample_trail_gt = pd.read_csv(DATA_PATH + \"train/{}/GooglePixel4XL/ground_truth.csv\".format(FILE_NAME))\n\ncenter = {\"lat\":37.42, \"lon\":-122.1}\nvisualize_trafic(df_sample_trail_gt, center)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T09:39:10.243643Z","iopub.execute_input":"2022-05-04T09:39:10.244230Z","iopub.status.idle":"2022-05-04T09:39:11.697423Z","shell.execute_reply.started":"2022-05-04T09:39:10.244190Z","shell.execute_reply":"2022-05-04T09:39:11.696636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Borrowed from prev. competition notebook\ndef add_lack_data(data, phoneNames, collectionNames):\n    result = []\n    for collectionName in collectionNames:\n        for phoneName in phoneNames:\n            if [collectionName, phoneName] in data:\n                result.append([collectionName, phoneName, 1])\n            else:\n                result.append([collectionName, phoneName, 0])\n    return result","metadata":{"execution":{"iopub.status.busy":"2022-05-04T09:39:11.698707Z","iopub.execute_input":"2022-05-04T09:39:11.698963Z","iopub.status.idle":"2022-05-04T09:39:11.705735Z","shell.execute_reply.started":"2022-05-04T09:39:11.698933Z","shell.execute_reply":"2022-05-04T09:39:11.704584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_phones = glob.glob(\"../input/smartphone-decimeter-2022/train/*/*\")\ntest_phones = glob.glob(\"../input/smartphone-decimeter-2022/test/*/*\")\ntrain_data = [item.split(\"/\")[-2:] for item in train_phones]\ntest_data = [item.split(\"/\")[-2:] for item in test_phones]","metadata":{"execution":{"iopub.status.busy":"2022-05-04T09:39:11.706985Z","iopub.execute_input":"2022-05-04T09:39:11.707250Z","iopub.status.idle":"2022-05-04T09:39:11.975038Z","shell.execute_reply.started":"2022-05-04T09:39:11.707211Z","shell.execute_reply":"2022-05-04T09:39:11.973788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"phoneNames = [item.split(\"/\")[-1:] for item in train_phones] + [item.split(\"/\")[-1:] for item in test_phones]\nphoneNames = set(list(itertools.chain.from_iterable(phoneNames)))\ncollectionNames = [item.split(\"/\")[-2] for item in train_phones] + [item.split(\"/\")[-2] for item in test_phones]","metadata":{"execution":{"iopub.status.busy":"2022-05-04T09:39:11.977387Z","iopub.execute_input":"2022-05-04T09:39:11.977999Z","iopub.status.idle":"2022-05-04T09:39:11.985319Z","shell.execute_reply.started":"2022-05-04T09:39:11.977951Z","shell.execute_reply":"2022-05-04T09:39:11.984236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = add_lack_data(train_data, phoneNames, collectionNames)\ntest_data = add_lack_data(test_data, phoneNames, collectionNames)\ntrain_data = pd.DataFrame(train_data, columns=[\"collectionNames\", \"phoneNames\", \"count\"])\ntest_data = pd.DataFrame(test_data, columns=[\"collectionNames\", \"phoneNames\", \"count\"])","metadata":{"execution":{"iopub.status.busy":"2022-05-04T09:39:11.986882Z","iopub.execute_input":"2022-05-04T09:39:11.987348Z","iopub.status.idle":"2022-05-04T09:39:12.014318Z","shell.execute_reply.started":"2022-05-04T09:39:11.987313Z","shell.execute_reply":"2022-05-04T09:39:12.013586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for phoneName in phoneNames:\n    fig, axes = plt.subplots(1, 2, figsize=(20,30))\n    plt.subplots_adjust(wspace=0.3, hspace=0.5)\n    g1 = seaborn.barplot(data=train_data[train_data[\"phoneNames\"] == phoneName],\n                         x=\"count\", y=\"collectionNames\", ax=axes[0])\n    g1.set_title(f\"{phoneName} in train data\")\n    g2 = seaborn.barplot(data=test_data[test_data[\"phoneNames\"] == phoneName],\n                         x=\"count\", y=\"collectionNames\", ax=axes[1])\n    g2.set_title(f\"{phoneName} in test data\")","metadata":{"execution":{"iopub.status.busy":"2022-05-04T09:39:12.016056Z","iopub.execute_input":"2022-05-04T09:39:12.016597Z","iopub.status.idle":"2022-05-04T09:39:59.722167Z","shell.execute_reply.started":"2022-05-04T09:39:12.016550Z","shell.execute_reply":"2022-05-04T09:39:59.720925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from shapely.geometry import Point, shape\nfrom geopandas import GeoDataFrame\nfrom matplotlib import animation\ndef create_gif_track(df, git_path):\n    \"\"\" Create git animation of phone track.\n    \"\"\"\n\n    fig, ax = plt.subplots()\n\n    imgs = []\n    df[\"geometry\"] = [Point(lngDeg, latDeg) for lngDeg, latDeg in zip(df[\"LongitudeDegrees\"], df[\"LatitudeDegrees\"])]\n    gdf = GeoDataFrame(df)\n    gdf.plot(color=\"lightskyblue\", ax=ax)\n    \n    for i in range(0, len(gdf), 10):\n        p = ax.plot(gdf.iloc[i][\"LongitudeDegrees\"], gdf.iloc[i][\"LatitudeDegrees\"], \n                    color = 'dodgerblue', marker = 'o', markersize = 10)\n        imgs.append(p)\n\n    # Create animation and saving it\n    ani = animation.ArtistAnimation(fig, imgs, interval=200)\n    ani.save(git_path, writer='imagemagick', dpi = 300)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T10:44:40.171546Z","iopub.execute_input":"2022-05-04T10:44:40.171866Z","iopub.status.idle":"2022-05-04T10:44:40.182257Z","shell.execute_reply.started":"2022-05-04T10:44:40.171832Z","shell.execute_reply":"2022-05-04T10:44:40.181152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ngif_gt = pd.read_csv(DATA_PATH + \"train/2021-04-26-US-SVL-2/SamsungGalaxyS20Ultra/ground_truth.csv\")\n\ncreate_gif_track(gif_gt, \"./2021_04_29_US_SJC_2_Pixel4_gt.gif\")","metadata":{"execution":{"iopub.status.busy":"2022-05-04T10:53:42.172773Z","iopub.execute_input":"2022-05-04T10:53:42.173247Z","iopub.status.idle":"2022-05-04T10:59:37.207941Z","shell.execute_reply.started":"2022-05-04T10:53:42.173212Z","shell.execute_reply":"2022-05-04T10:59:37.207075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](./2021_04_29_US_SJC_2_Pixel4_gt.gif)","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}