{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-06T09:24:15.704442Z","iopub.execute_input":"2022-05-06T09:24:15.704869Z","iopub.status.idle":"2022-05-06T09:24:17.841275Z","shell.execute_reply.started":"2022-05-06T09:24:15.704768Z","shell.execute_reply":"2022-05-06T09:24:17.840379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**After examining the data, a discrepancy was found in the test and training datasets. Let's try to conduct a comparative analysis of data on the same smartphone model - Samsung Galaxy S20 Ultra**","metadata":{}},{"cell_type":"markdown","source":"**First, read the data. The data in dataset \"device_gnss\"is more suitable, since the dataset \"device_imu\" contains data only about the position of the smartphone in space**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n# read the test and train dataset in the Samsung Galaxy S20 Ultra\ndata2020 = pd.read_csv ('../input/smartphone-decimeter-2022/test/2021-04-28-US-MTV-2/SamsungGalaxyS20Ultra/device_gnss.csv')\ndata2020.head (10)","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:17.842705Z","iopub.execute_input":"2022-05-06T09:24:17.842928Z","iopub.status.idle":"2022-05-06T09:24:18.852125Z","shell.execute_reply.started":"2022-05-06T09:24:17.842900Z","shell.execute_reply":"2022-05-06T09:24:18.851123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data2021 = pd.read_csv ('../input/smartphone-decimeter-2022/train/2021-08-04-US-SJC-1/SamsungGalaxyS20Ultra/device_gnss.csv')\ndata2021.head (10)","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:18.854024Z","iopub.execute_input":"2022-05-06T09:24:18.854431Z","iopub.status.idle":"2022-05-06T09:24:19.746541Z","shell.execute_reply.started":"2022-05-06T09:24:18.854391Z","shell.execute_reply":"2022-05-06T09:24:19.745683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Clean data in data2020 and data2021**","metadata":{}},{"cell_type":"code","source":"data2020.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:19.749241Z","iopub.execute_input":"2022-05-06T09:24:19.749930Z","iopub.status.idle":"2022-05-06T09:24:19.759229Z","shell.execute_reply.started":"2022-05-06T09:24:19.749880Z","shell.execute_reply":"2022-05-06T09:24:19.758407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data2020.isnull ().sum ()","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:19.760469Z","iopub.execute_input":"2022-05-06T09:24:19.760756Z","iopub.status.idle":"2022-05-06T09:24:19.788834Z","shell.execute_reply.started":"2022-05-06T09:24:19.760717Z","shell.execute_reply":"2022-05-06T09:24:19.787973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data2021.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:19.790281Z","iopub.execute_input":"2022-05-06T09:24:19.791291Z","iopub.status.idle":"2022-05-06T09:24:19.799906Z","shell.execute_reply.started":"2022-05-06T09:24:19.791244Z","shell.execute_reply":"2022-05-06T09:24:19.799020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data2021.isnull ().sum ()","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:19.800878Z","iopub.execute_input":"2022-05-06T09:24:19.801308Z","iopub.status.idle":"2022-05-06T09:24:19.829816Z","shell.execute_reply.started":"2022-05-06T09:24:19.801275Z","shell.execute_reply":"2022-05-06T09:24:19.829187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data2020.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:19.830962Z","iopub.execute_input":"2022-05-06T09:24:19.831390Z","iopub.status.idle":"2022-05-06T09:24:19.836086Z","shell.execute_reply.started":"2022-05-06T09:24:19.831359Z","shell.execute_reply":"2022-05-06T09:24:19.835368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data2021.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:19.837292Z","iopub.execute_input":"2022-05-06T09:24:19.837488Z","iopub.status.idle":"2022-05-06T09:24:19.850692Z","shell.execute_reply.started":"2022-05-06T09:24:19.837464Z","shell.execute_reply":"2022-05-06T09:24:19.850115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Datasets are almost the same size**","metadata":{}},{"cell_type":"code","source":"data2020.describe ()","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:19.854123Z","iopub.execute_input":"2022-05-06T09:24:19.854485Z","iopub.status.idle":"2022-05-06T09:24:20.101522Z","shell.execute_reply.started":"2022-05-06T09:24:19.854441Z","shell.execute_reply":"2022-05-06T09:24:20.100689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data2021.describe ()","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:20.102746Z","iopub.execute_input":"2022-05-06T09:24:20.103605Z","iopub.status.idle":"2022-05-06T09:24:20.329988Z","shell.execute_reply.started":"2022-05-06T09:24:20.103556Z","shell.execute_reply":"2022-05-06T09:24:20.328957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The column \"ConstellationType\" in data2020 and data2021 gives a hint about the type of satellite. In the future, this will help determine satellites that provide more accurate geodata.**","metadata":{}},{"cell_type":"code","source":"data2020.isnull ().sum (axis = 1).loc [:47]","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:20.331082Z","iopub.execute_input":"2022-05-06T09:24:20.331393Z","iopub.status.idle":"2022-05-06T09:24:20.353467Z","shell.execute_reply.started":"2022-05-06T09:24:20.331362Z","shell.execute_reply":"2022-05-06T09:24:20.352524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data2021.isnull ().sum (axis = 1).loc [:47]","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:20.354783Z","iopub.execute_input":"2022-05-06T09:24:20.355077Z","iopub.status.idle":"2022-05-06T09:24:20.378200Z","shell.execute_reply.started":"2022-05-06T09:24:20.355038Z","shell.execute_reply":"2022-05-06T09:24:20.377183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Create dummy columns**","metadata":{}},{"cell_type":"code","source":"data2020 = pd.get_dummies (data2020)\ndata2020.columns","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:20.379327Z","iopub.execute_input":"2022-05-06T09:24:20.379628Z","iopub.status.idle":"2022-05-06T09:24:20.425308Z","shell.execute_reply.started":"2022-05-06T09:24:20.379586Z","shell.execute_reply":"2022-05-06T09:24:20.424455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data2021 = pd.get_dummies (data2021)\ndata2021.columns","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:20.426474Z","iopub.execute_input":"2022-05-06T09:24:20.427160Z","iopub.status.idle":"2022-05-06T09:24:20.474734Z","shell.execute_reply.started":"2022-05-06T09:24:20.427120Z","shell.execute_reply":"2022-05-06T09:24:20.473819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data2020 = pd.get_dummies (data2020, drop_first = True) \ndata2020.columns","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:20.476199Z","iopub.execute_input":"2022-05-06T09:24:20.476432Z","iopub.status.idle":"2022-05-06T09:24:20.519959Z","shell.execute_reply.started":"2022-05-06T09:24:20.476404Z","shell.execute_reply":"2022-05-06T09:24:20.519062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data2021 = pd.get_dummies (data2021, drop_first = True)\ndata2021.columns","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:20.521787Z","iopub.execute_input":"2022-05-06T09:24:20.522325Z","iopub.status.idle":"2022-05-06T09:24:20.658744Z","shell.execute_reply.started":"2022-05-06T09:24:20.522283Z","shell.execute_reply":"2022-05-06T09:24:20.657893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Create a dataframe (X) with the features and a series (y) with the labels**","metadata":{}},{"cell_type":"code","source":"# create a consistent sample from data2020 and data2021\ndata2020S = data2020.sample (n = 3000)\n\ndata2021S = data2021.sample (n = 3000)","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:20.659824Z","iopub.execute_input":"2022-05-06T09:24:20.660045Z","iopub.status.idle":"2022-05-06T09:24:20.705364Z","shell.execute_reply.started":"2022-05-06T09:24:20.660020Z","shell.execute_reply":"2022-05-06T09:24:20.704477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = data2020S [['HardwareClockDiscontinuityCount',\n       'Svid', 'TimeOffsetNanos', 'State', 'ReceivedSvTimeNanos',\n        'AccumulatedDeltaRangeMeters',\n       'CarrierFrequencyHz',\n       'MultipathIndicator', 'ConstellationType',\n       'ChipsetElapsedRealtimeNanos', 'ArrivalTimeNanosSinceGpsEpoch',\n       'RawPseudorangeMeters', 'RawPseudorangeUncertaintyMeters',\n       'SvPositionXEcefMeters',\n       'SvPositionYEcefMeters', 'SvPositionZEcefMeters', 'SvElevationDegrees',\n       'SvAzimuthDegrees',\n       'SvClockBiasMeters', 'SvClockDriftMetersPerSecond',\n       'WlsPositionXEcefMeters', 'WlsPositionYEcefMeters',\n       'WlsPositionZEcefMeters', 'CodeType_C', 'CodeType_I', 'CodeType_X',\n       'SignalType_BDS_B1I', 'SignalType_GAL_E1', 'SignalType_GAL_E5A',\n       'SignalType_GLO_G1', 'SignalType_GPS_L1', 'SignalType_GPS_L5']]\n\nX = data2021S [['HardwareClockDiscontinuityCount',\n       'Svid', 'TimeOffsetNanos', 'State', 'ReceivedSvTimeNanos',\n        'AccumulatedDeltaRangeMeters',\n       'CarrierFrequencyHz',\n       'MultipathIndicator', 'ConstellationType',\n       'ChipsetElapsedRealtimeNanos', 'ArrivalTimeNanosSinceGpsEpoch',\n       'RawPseudorangeMeters', 'RawPseudorangeUncertaintyMeters',\n       'SvPositionXEcefMeters',\n       'SvPositionYEcefMeters', 'SvPositionZEcefMeters', 'SvElevationDegrees',\n       'SvAzimuthDegrees',\n       'SvClockBiasMeters', 'SvClockDriftMetersPerSecond',\n       'WlsPositionXEcefMeters', 'WlsPositionYEcefMeters',\n       'WlsPositionZEcefMeters', 'CodeType_C', 'CodeType_I', 'CodeType_X',\n       'SignalType_BDS_B1I', 'SignalType_GAL_E1', 'SignalType_GAL_E5A',\n       'SignalType_GLO_G1', 'SignalType_GPS_L1', 'SignalType_GPS_L5']]","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:20.706768Z","iopub.execute_input":"2022-05-06T09:24:20.707000Z","iopub.status.idle":"2022-05-06T09:24:20.717939Z","shell.execute_reply.started":"2022-05-06T09:24:20.706972Z","shell.execute_reply":"2022-05-06T09:24:20.717060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Divide the dataset into test and train**","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_test, y_train, y_test = train_test_split (X, y, test_size = 0.3, random_state = 42)","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:20.719251Z","iopub.execute_input":"2022-05-06T09:24:20.719480Z","iopub.status.idle":"2022-05-06T09:24:21.746473Z","shell.execute_reply.started":"2022-05-06T09:24:20.719453Z","shell.execute_reply":"2022-05-06T09:24:21.745761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Exploring data**","metadata":{}},{"cell_type":"code","source":"X.describe ()","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:24:57.818734Z","iopub.execute_input":"2022-05-06T09:24:57.819564Z","iopub.status.idle":"2022-05-06T09:24:57.916991Z","shell.execute_reply.started":"2022-05-06T09:24:57.819511Z","shell.execute_reply":"2022-05-06T09:24:57.916122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the histogram\nimport matplotlib.pyplot as plt\n\nfig, ax = plt.subplots (figsize = (16, 14))\nX.ConstellationType.plot (kind = \"hist\", ax = ax)","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:29:36.973780Z","iopub.execute_input":"2022-05-06T09:29:36.974539Z","iopub.status.idle":"2022-05-06T09:29:37.207515Z","shell.execute_reply.started":"2022-05-06T09:29:36.974489Z","shell.execute_reply":"2022-05-06T09:29:37.206665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Scatter plot\nfig, ax = plt.subplots (figsize = (16, 14))\nX.plot.scatter (x = \"Svid\", y = \"RawPseudorangeMeters\", ax = ax, alpha = 0.3)","metadata":{"execution":{"iopub.status.busy":"2022-05-06T09:39:30.949961Z","iopub.execute_input":"2022-05-06T09:39:30.950830Z","iopub.status.idle":"2022-05-06T09:39:31.205892Z","shell.execute_reply.started":"2022-05-06T09:39:30.950794Z","shell.execute_reply":"2022-05-06T09:39:31.205164Z"},"trusted":true},"execution_count":null,"outputs":[]}]}