{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Verification phoneName difference","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom pathlib import Path\nfrom tqdm import tqdm\nimport os\nfrom glob import glob\n\nimport matplotlib.pyplot as plt\nplt.style.use('seaborn')\n\nimport folium\nimport branca","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-26T14:17:19.841659Z","iopub.execute_input":"2021-05-26T14:17:19.842222Z","iopub.status.idle":"2021-05-26T14:17:20.109578Z","shell.execute_reply.started":"2021-05-26T14:17:19.842190Z","shell.execute_reply":"2021-05-26T14:17:20.108799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data read","metadata":{}},{"cell_type":"code","source":"trainfile = pd.read_csv(\"../input/google-smartphone-decimeter-challenge/baseline_locations_train.csv\")\ntestfile = pd.read_csv(\"../input/google-smartphone-decimeter-challenge/baseline_locations_test.csv\")\nsubmission = pd.read_csv(\"../input/google-smartphone-decimeter-challenge/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:18:36.008878Z","iopub.execute_input":"2021-05-26T14:18:36.009382Z","iopub.status.idle":"2021-05-26T14:18:36.718786Z","shell.execute_reply.started":"2021-05-26T14:18:36.009350Z","shell.execute_reply":"2021-05-26T14:18:36.717424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datapath = Path(\"../input/google-smartphone-decimeter-challenge\")\ntruths = (datapath / 'train').rglob('ground_truth.csv')\n\ncols = ['collectionName', 'phoneName', 'millisSinceGpsEpoch', 'latDeg', 'lngDeg']\ntruth_arr =[]\nfor filepath in tqdm(truths, total=73):\n    df_buf = pd.read_csv(filepath, usecols=cols)\n    truth_arr.append(df_buf)\n    \ndf_truth = pd.concat(truth_arr, ignore_index=True)\ndf_train = pd.merge(trainfile, df_truth, on=['collectionName', 'phoneName', 'millisSinceGpsEpoch'], suffixes=(\"_current\", \"_truth\"))","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:20:07.807384Z","iopub.execute_input":"2021-05-26T14:20:07.807727Z","iopub.status.idle":"2021-05-26T14:20:09.564817Z","shell.execute_reply.started":"2021-05-26T14:20:07.807700Z","shell.execute_reply":"2021-05-26T14:20:09.563948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.sample(3)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:20:27.383781Z","iopub.execute_input":"2021-05-26T14:20:27.384249Z","iopub.status.idle":"2021-05-26T14:20:27.426782Z","shell.execute_reply.started":"2021-05-26T14:20:27.384219Z","shell.execute_reply":"2021-05-26T14:20:27.425880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Latitude / longitude difference and simple distance","metadata":{}},{"cell_type":"code","source":"df_train['lat_dif'] = df_train['latDeg_truth'] - df_train['latDeg_current']\ndf_train['lng_dif'] = df_train['lngDeg_truth'] - df_train['lngDeg_current']\ndf_train['dummy_dist'] = df_train['lat_dif']**2 + df_train['lng_dif']**2","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:26:15.732822Z","iopub.execute_input":"2021-05-26T14:26:15.733194Z","iopub.status.idle":"2021-05-26T14:26:15.746620Z","shell.execute_reply.started":"2021-05-26T14:26:15.733164Z","shell.execute_reply":"2021-05-26T14:26:15.745668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pnames = df_train['phoneName'].unique()\npnames","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:21:20.539481Z","iopub.execute_input":"2021-05-26T14:21:20.539868Z","iopub.status.idle":"2021-05-26T14:21:20.556670Z","shell.execute_reply.started":"2021-05-26T14:21:20.539837Z","shell.execute_reply":"2021-05-26T14:21:20.555600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.groupby('phoneName').std()[['lat_dif','lng_dif']]","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:21:34.389809Z","iopub.execute_input":"2021-05-26T14:21:34.390167Z","iopub.status.idle":"2021-05-26T14:21:34.443120Z","shell.execute_reply.started":"2021-05-26T14:21:34.390137Z","shell.execute_reply":"2021-05-26T14:21:34.441834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=3, ncols=3, figsize=(15, 15))\nax = axes.flatten()\nfor i, pn in enumerate(pnames):\n    df_g = df_train[df_train['phoneName']==pn]\n    ax[i].scatter(df_g['lng_dif'], df_g['lat_dif'], s=20, c='blue', alpha=0.5)\n    ax[i].spines['left'].set(position=('data', 0.0))\n    ax[i].spines['bottom'].set(position=('data', 0.0))\n    ax[i].set_title(pn)\n    if pn == 'Mi8': continue\n    if pn == 'Pixel4':\n        ax[i].set_xlim((-0.015, 0.015))\n        ax[i].set_ylim((-0.015, 0.015))\n    else:\n        ax[i].set_xlim((-0.006, 0.003))\n        ax[i].set_ylim((-0.006, 0.003))\n        \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:21:47.883294Z","iopub.execute_input":"2021-05-26T14:21:47.883646Z","iopub.status.idle":"2021-05-26T14:21:49.598538Z","shell.execute_reply.started":"2021-05-26T14:21:47.883616Z","shell.execute_reply":"2021-05-26T14:21:49.597445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=3, ncols=3, figsize=(15, 15))\nax = axes.flatten()\nfor i, pn in enumerate(pnames):\n    df_g = df_train[df_train['phoneName']==pn]\n    ax[i].hist(df_g['dummy_dist'], bins=20)\n    ax[i].set_yscale('log')\n    ax[i].set_title(pn)\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:37:56.712034Z","iopub.execute_input":"2021-05-26T14:37:56.712433Z","iopub.status.idle":"2021-05-26T14:38:00.152182Z","shell.execute_reply.started":"2021-05-26T14:37:56.712401Z","shell.execute_reply":"2021-05-26T14:38:00.151283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The variation of Mi8 is remarkable.","metadata":{}},{"cell_type":"markdown","source":"### Check sample collectionName.","metadata":{}},{"cell_type":"code","source":"cname = df_train['collectionName'].unique()[4]\ndf_cn = df_train[df_train['collectionName']==cname][['phoneName','latDeg_current','lngDeg_current','latDeg_truth','lngDeg_truth','dummy_dist']]\nprint('ex.', cname)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:27:55.159691Z","iopub.execute_input":"2021-05-26T14:27:55.160100Z","iopub.status.idle":"2021-05-26T14:27:55.198673Z","shell.execute_reply.started":"2021-05-26T14:27:55.160069Z","shell.execute_reply":"2021-05-26T14:27:55.197556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cm = branca.colormap.LinearColormap(['blue','lime','red'], vmin=0, vmax=np.quantile(df_cn['dummy_dist'], 0.95))\ncm","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:27:58.815615Z","iopub.execute_input":"2021-05-26T14:27:58.815986Z","iopub.status.idle":"2021-05-26T14:27:58.837938Z","shell.execute_reply.started":"2021-05-26T14:27:58.815955Z","shell.execute_reply":"2021-05-26T14:27:58.836897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"center = df_cn[['latDeg_current','lngDeg_current']].mean().tolist()\nm = folium.Map(location=center, zoom_start=10)\nfor pn, df_pn in df_cn.groupby('phoneName'):\n    fg = folium.FeatureGroup(name=pn)\n    cm = branca.colormap.LinearColormap(['blue','lime','red'], vmin=0, vmax=np.quantile(df_pn['dummy_dist'], 0.95))\n    for pn, latc, lonc, latt, lont, dist in df_pn.values:\n#         folium.Circle(location=[latc,lonc], radius=2, color='tomato').add_to(fg)\n#         folium.Circle(location=[latt,lont], radius=2, color='blue').add_to(fg)\n        folium.Circle(location=[latt,lont], radius=8, color=cm(dist), fill=True).add_to(fg)\n    folium.Marker(location=df_pn[['latDeg_current','lngDeg_current']].values[0].tolist(), popup=pn).add_to(fg)\n    fg.add_to(m)\n\nfolium.LayerControl(collapsed=False).add_to(m)\nm","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:28:00.548745Z","iopub.execute_input":"2021-05-26T14:28:00.549110Z","iopub.status.idle":"2021-05-26T14:28:04.903552Z","shell.execute_reply.started":"2021-05-26T14:28:00.549080Z","shell.execute_reply":"2021-05-26T14:28:04.901750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Although there are differences in variation, the points that do not deviate are similar. ","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}