{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        os.path.join(dirname, filename)\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-30T06:12:01.207163Z","iopub.execute_input":"2021-06-30T06:12:01.207638Z","iopub.status.idle":"2021-06-30T06:12:02.082220Z","shell.execute_reply.started":"2021-06-30T06:12:01.207539Z","shell.execute_reply":"2021-06-30T06:12:02.081184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\ntrainfile = pd.read_csv(\"../input/google-smartphone-decimeter-challenge/baseline_locations_train.csv\")\ntestfile = pd.read_csv(\"../input/google-smartphone-decimeter-challenge/baseline_locations_test.csv\")\ndatapath = Path(\"../input/google-smartphone-decimeter-challenge\")\ntruths = (datapath/'train').rglob('ground_truth.csv')\ncols1 = ['collectionName', 'phoneName', 'millisSinceGpsEpoch', 'latDeg',\n       'lngDeg']\ntruthlist = []\nfor fpath in truths:\n    fcsv = pd.read_csv(fpath, usecols = cols1)\n    truthlist.append(fcsv)\n\ntruthdata = pd.concat(truthlist, ignore_index = True)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:12:14.287923Z","iopub.execute_input":"2021-06-30T06:12:14.288264Z","iopub.status.idle":"2021-06-30T06:12:15.815008Z","shell.execute_reply.started":"2021-06-30T06:12:14.288233Z","shell.execute_reply":"2021-06-30T06:12:15.813963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols2 = ['collectionName','phoneName','millisSinceGpsEpoch','constellationType','svid','signalType','receivedSvTimeInGpsNanos','xSatPosM','ySatPosM','zSatPosM']\nlogs = (datapath/'train').rglob('*derived.csv')\n\nlogslist = []\nfor fpath in logs:\n    logcsv = pd.read_csv(fpath, usecols = cols2)\n    logslist.append(logcsv)\n\nlogsdata = pd.concat(logslist, ignore_index = True)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:12:21.139992Z","iopub.execute_input":"2021-06-30T06:12:21.140358Z","iopub.status.idle":"2021-06-30T06:12:38.785004Z","shell.execute_reply.started":"2021-06-30T06:12:21.140324Z","shell.execute_reply":"2021-06-30T06:12:38.783994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logsdata_svtime = logsdata.groupby(['collectionName','phoneName','millisSinceGpsEpoch']).receivedSvTimeInGpsNanos.apply(lambda x: np.min(x)).reset_index()\nlogsdata_xsat = logsdata.groupby(['collectionName','phoneName','millisSinceGpsEpoch']).xSatPosM.apply(lambda x: np.average(x)).reset_index()\nlogsdata_ysat = logsdata.groupby(['collectionName','phoneName','millisSinceGpsEpoch']).ySatPosM.apply(lambda x: np.average(x)).reset_index()\nlogsdata_zsat = logsdata.groupby(['collectionName','phoneName','millisSinceGpsEpoch']).zSatPosM.apply(lambda x: np.average(x)).reset_index()\nlogsdata_svid = logsdata.groupby(['collectionName','phoneName','millisSinceGpsEpoch']).svid.apply(lambda x: x.value_counts().index[0]).reset_index()\n#logsdata_constellation = logsdata.groupby(['collectionName','phoneName','millisSinceGpsEpoch']).constellationType.apply(lambda x:x.value_counts().index[0]).reset_index()\nlogsdata_signaltype = logsdata.groupby(['collectionName','phoneName','millisSinceGpsEpoch']).signalType.apply(lambda x: x.value_counts().index[0]).reset_index()\n\nlogsdata1 = logsdata_svtime.merge(logsdata_xsat, on = ['collectionName','phoneName','millisSinceGpsEpoch'])\nlogsdata2 = logsdata1.merge(logsdata_ysat, on = ['collectionName','phoneName','millisSinceGpsEpoch'])\nlogsdata3 = logsdata2.merge(logsdata_zsat, on = ['collectionName','phoneName','millisSinceGpsEpoch'])\nlogsdata4 = logsdata3.merge(logsdata_svid, on = ['collectionName','phoneName','millisSinceGpsEpoch'])\n#logsdata5 = logsdata4.merge(logsdata_constellation, on = ['collectionName','phoneName','millisSinceGpsEpoch'])\nlogsdata_final = logsdata4.merge(logsdata_signaltype, on = ['collectionName','phoneName','millisSinceGpsEpoch'])\n#logsdata_final['constellationType'] = logsdata_final.constellationType.apply(lambda x: \"one\" if x == 1 else(\"two\" if x == 2 else(\"three\" if x == 3 else(\"four\" if x == 4 else(\"five\" if x == 5 else(\"six\" if x == 6 else \"zero\"))))))                                                                                                 \nlogsdata_final.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:12:46.652605Z","iopub.execute_input":"2021-06-30T06:12:46.652981Z","iopub.status.idle":"2021-06-30T06:15:24.683488Z","shell.execute_reply.started":"2021-06-30T06:12:46.652938Z","shell.execute_reply":"2021-06-30T06:15:24.682564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(logsdata_final))\nprint(len(trainfile))\ntraindata1 = trainfile.merge(truthdata, on = cols1[:3], suffixes=(\"_current\",\"_truth\")) # merge train file with observed data\ntraindata = pd.merge(traindata1, logsdata_final, how = 'left', on = ['collectionName','phoneName','millisSinceGpsEpoch'])\ntraindata.sort_values(by = ['phone','millisSinceGpsEpoch'])\n#traindata = trainfile.merge(truthdata, on = cols1[:3], suffixes=(\"_current\",\"_truth\")) # merge train file with observed data\n#traindata = traindata.merge(logsdata, on = cols1[:3]) # add columns from gnss logs to the traindata\ntraindata.head()\ntraindata = traindata.fillna(0)\ntraindata['latDeg_prev'] = traindata['latDeg_current'].shift(1).where(traindata['phone'].eq(traindata['phone'].shift(1)))\ntraindata['lngDeg_prev'] = traindata['lngDeg_current'].shift(1).where(traindata['phone'].eq(traindata['phone'].shift(1)))    \ntraindata['latDeg_next'] = traindata['latDeg_current'].shift(-1).where(traindata['phone'].eq(traindata['phone'].shift(-1)))\ntraindata['lngDeg_next'] = traindata['lngDeg_current'].shift(-1).where(traindata['phone'].eq(traindata['phone'].shift(-1)))\ntraindata['time_prev'] = traindata['millisSinceGpsEpoch'].shift(1).where(traindata['phone'].eq(traindata['phone'].shift(1)))\ntraindata['time_next'] = traindata['millisSinceGpsEpoch'].shift(-1).where(traindata['phone'].eq(traindata['phone'].shift(-1)))\ntraindata['x_prev'] = traindata['xSatPosM'].shift(1).where(traindata['phone'].eq(traindata['phone'].shift(1)))\ntraindata['x_next'] = traindata['xSatPosM'].shift(-1).where(traindata['phone'].eq(traindata['phone'].shift(-1)))\ntraindata['y_prev'] = traindata['ySatPosM'].shift(1).where(traindata['phone'].eq(traindata['phone'].shift(1)))\ntraindata['y_next'] = traindata['ySatPosM'].shift(-1).where(traindata['phone'].eq(traindata['phone'].shift(-1)))\ntraindata['z_prev'] = traindata['zSatPosM'].shift(1).where(traindata['phone'].eq(traindata['phone'].shift(1)))\ntraindata['z_next'] = traindata['zSatPosM'].shift(-1).where(traindata['phone'].eq(traindata['phone'].shift(-1)))\n\n#traindata['svid'] = pd.Categorical(traindata.svid)\n#traindata['constellationType'] = pd.Categorical(traindata.constellationType) \nfor i in traindata.index:\n    if pd.isna(traindata['latDeg_prev'][i]):\n        traindata['latDeg_prev'][i] = traindata['latDeg_current'][i]\n        traindata['lngDeg_prev'][i] = traindata['lngDeg_current'][i]\n        traindata['time_prev'][i] = traindata['millisSinceGpsEpoch'][i]\n        traindata['x_prev'][i] = traindata['xSatPosM'][i]\n        traindata['y_prev'][i] = traindata['ySatPosM'][i]\n        traindata['z_prev'][i] = traindata['zSatPosM'][i]\n    if pd.isna(traindata['latDeg_next'][i]):\n        traindata['latDeg_next'][i] = traindata['latDeg_current'][i]\n        traindata['lngDeg_next'][i] = traindata['lngDeg_current'][i]\n        traindata['time_next'][i] = traindata['millisSinceGpsEpoch'][i]\n        traindata['x_prev'][i] = traindata['xSatPosM'][i]\n        traindata['y_prev'][i] = traindata['ySatPosM'][i]\n        traindata['z_prev'][i] = traindata['zSatPosM'][i]\n    \ntraindata['lat_del_prev'] = (traindata['latDeg_current'] - traindata['latDeg_prev'])*10**6\ntraindata['lat_del_truth'] = (traindata['latDeg_truth'] - traindata['latDeg_prev'])*10**6\ntraindata['lng_del_prev'] = (traindata['lngDeg_current'] - traindata['lngDeg_prev'])* 10**6\ntraindata['lng_del_truth'] = (traindata['lngDeg_truth'] - traindata['lngDeg_prev']) * 10**6\ntraindata['lat_vel'] = ((abs(traindata['latDeg_next'] - traindata['latDeg_prev'])/(traindata['time_next'] - traindata['time_prev'])) * 10**6)\ntraindata['lng_vel'] = ((abs(traindata['lngDeg_next'] - traindata['lngDeg_prev'])/(traindata['time_next'] - traindata['time_prev'])) * 10**6)\ntraindata['time_since_last_read'] = traindata['millisSinceGpsEpoch'] - traindata['time_prev']\n    #if traindata['time_prev'][i] == traindata['millisSinceGpsEpoch'][i]:\n        #traindata['vel_lat'] = 0\n        #traindata['vel_lng'] = 0\n    #else:\n        #traindata['vel_lat'] = (traindata['latDeg_current'][i] - traindata['latDeg_prev'][i])/(traindata['millisSinceGpsEpoch'][i] - traindata['time_prev'][i])\n        #traindata['vel_lng'] = (traindata['lngDeg_current'][i] - traindata['lngDeg_prev'][i])/(traindata['millisSinceGpsEpoch'][i] - traindata['time_prev'][i])\n\nprint(len(traindata))\n#print(len(trainfile))\ntraindata.head(1)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:46:21.120141Z","iopub.execute_input":"2021-06-30T06:46:21.120538Z","iopub.status.idle":"2021-06-30T06:46:24.907747Z","shell.execute_reply.started":"2021-06-30T06:46:21.120487Z","shell.execute_reply":"2021-06-30T06:46:24.906696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#traindata = traindata.fillna(0)\ntraindata.to_csv('./traindata1.csv', sep = ',', index = False)","metadata":{"execution":{"iopub.status.busy":"2021-06-26T07:30:52.73578Z","iopub.execute_input":"2021-06-26T07:30:52.73616Z","iopub.status.idle":"2021-06-26T07:31:00.932652Z","shell.execute_reply.started":"2021-06-26T07:30:52.736129Z","shell.execute_reply":"2021-06-26T07:31:00.931572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#traindata.columns\nfrom sklearn.model_selection import train_test_split\ntraindata = traindata.fillna(0)\nxcols = [c for c in traindata.columns if c not in ('latDeg_truth','lngDeg_truth','lat_del_truth','lng_del_truth','collectionName','phone','phoneName','millisSinceGpsEpoch','svid',\n                                                  'signalType','time_prev','time_next','xSatPosM','ySatPosM','zSatPosM','x_prev','x_next','y_prev','y_next','z_prev','z_next')]\nycols = ['latDeg_truth','lngDeg_truth','lat_del_truth','lng_del_truth']\nxdata = traindata[xcols]\nydata = traindata[ycols]\n#ydata_lat = traindata[['latDeg_truth']]\n\nxtrain, xtest, ytrain, ytest = train_test_split(xdata, ydata, test_size = 0.3, random_state = 144)\nytrain_lat = ytrain[['lat_del_truth']]\nytrain_lng = ytrain[['lng_del_truth']]\nytest_lat = ytest[['lat_del_truth']]\nytest_lng = ytest[['lng_del_truth']]\n\n#xtrain.to_csv('./xtrain.csv', sep = ',', index = False)\n#xtest.to_csv('./xtest.csv', sep = ',', index = False)\n#ytrain.to_csv('./ytrain.csv', sep = ',', index = False)\n#ytest.to_csv('./ytest.csv', sep = ',', index = False)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:47:02.154167Z","iopub.execute_input":"2021-06-30T06:47:02.154545Z","iopub.status.idle":"2021-06-30T06:47:02.306213Z","shell.execute_reply.started":"2021-06-30T06:47:02.154489Z","shell.execute_reply":"2021-06-30T06:47:02.305179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(xdata.columns)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:34:23.330865Z","iopub.execute_input":"2021-06-30T06:34:23.331202Z","iopub.status.idle":"2021-06-30T06:34:23.337160Z","shell.execute_reply.started":"2021-06-30T06:34:23.331172Z","shell.execute_reply":"2021-06-30T06:34:23.335892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.wrappers.scikit_learn import KerasRegressor\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.pipeline import Pipeline\n\ndef baseline_model():\n    model = Sequential()\n    model.add(Dense(13, input_dim=13, kernel_initializer='normal', activation='relu'))\n    model.add(Dense(6, kernel_initializer = 'normal', activation = 'relu'))\n    model.add(Dense(6, kernel_initializer = 'normal', activation = 'relu'))\n    model.add(Dense(1, kernel_initializer='normal'))\n    model.compile(loss = 'mean_squared_error', optimizer = 'adam', metrics=['accuracy'])\n    return model\n\nkfold = KFold(n_splits=5)\nestimators = []\nestimators.append(('standardize', StandardScaler()))\nestimators.append(('mlp', KerasRegressor(build_fn=baseline_model, epochs=1800, batch_size=25)))\n#estimators.append(('cv', kfold))\n#estimator = KerasRegressor(build_fn=baseline_model, epochs=100, batch_size=5, verbose=10)\n\npipeline = Pipeline(estimators)\nytest['phone'] = traindata['phone']\nytest['millisSinceGpsEpoch'] = traindata['millisSinceGpsEpoch']\nytest['latDeg_prev'] = traindata['latDeg_prev']\nytest['lngDeg_prev'] = traindata['lngDeg_prev']\nest_lat = pipeline.fit(xtrain, ytrain_lat)\n#results = cross_val_score(pipeline, xtrain, ytrain_lat, cv=kfold)\n#print(\"Baseline: %.5f (%.5f) MSE\" % (results.mean(), results.std()))\nypred_lat = est_lat.predict(xtest)\nypred_lat_final = est_lat.predict(testdata_x)\nlat_preds_final = testdata[['phone','millisSinceGpsEpoch','latDeg_prev']]\nlat_preds_final['lat_del_pred'] = pd.Series(ypred_lat_final)\nytest.reset_index(inplace = True)\nytest.drop(['index'], axis = 1)\nytest['lat_del_pred'] = pd.Series(ypred_lat)\nytest['latDeg_pred'] = ytest['latDeg_prev'] + (ytest['lat_del_pred'] / 10**6)\nlat_preds_final['latDeg'] = lat_preds_final['latDeg_prev'] + (lat_preds_final['lat_del_pred']/10**6) \nytest.to_csv('./lat_preds_data_dum_test.csv', sep = ',', index = False)\nlat_preds_final.to_csv('./lat_preds_final.csv')","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-06-30T06:49:02.280850Z","iopub.execute_input":"2021-06-30T06:49:02.281210Z","iopub.status.idle":"2021-06-30T08:51:47.766040Z","shell.execute_reply.started":"2021-06-30T06:49:02.281181Z","shell.execute_reply":"2021-06-30T08:51:47.764956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdata_x.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-27T11:03:05.076091Z","iopub.execute_input":"2021-06-27T11:03:05.076649Z","iopub.status.idle":"2021-06-27T11:03:05.109439Z","shell.execute_reply.started":"2021-06-27T11:03:05.076601Z","shell.execute_reply":"2021-06-27T11:03:05.108259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytest.head()\nytest.to_csv('./ytest_1.csv', sep = ',')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.wrappers.scikit_learn import KerasRegressor\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.pipeline import Pipeline\n\ndef baseline_model():\n    model = Sequential()\n    model.add(Dense(13, input_dim=13, kernel_initializer='normal', activation='relu'))\n    #model.add(Dense(12, kernel_initializer = 'normal', activation = 'relu'))\n    model.add(Dense(6, kernel_initializer = 'normal', activation = 'relu'))\n    model.add(Dense(6, kernel_initializer = 'normal', activation = 'relu'))\n    model.add(Dense(1, kernel_initializer='normal'))\n    model.compile(loss = 'mean_squared_error', optimizer = 'adam', metrics=['accuracy'])\n    return model\n\nkfold = KFold(n_splits=5)\nestimators = []\nestimators.append(('standardize', StandardScaler()))\nestimators.append(('mlp', KerasRegressor(build_fn=baseline_model, epochs=2000, batch_size=32)))\n#estimators.append(('cv', kfold))\n#estimator = KerasRegressor(build_fn=baseline_model, epochs=100, batch_size=5, verbose=10)\n\npipeline = Pipeline(estimators)\nest_lng = pipeline.fit(xtrain, ytrain_lng)\nypred_lng = est_lng.predict(xtest)\nypred_lng_final = est_lng.predict(testdata_x)\nytest_final = testdata[['phone','millisSinceGpsEpoch','lngDeg_prev']]\nytest['phone'] = traindata['phone']\nytest['millisSinceGpsEpoch'] = traindata['millisSinceGpsEpoch']\nytest['latDeg_prev'] = traindata['latDeg_prev']\nytest['lngDeg_prev'] = traindata['lngDeg_prev']\nytest.reset_index(inplace = True)\nytest.drop(['index'], axis = 1)\nytest['lng_del_pred'] = pd.Series(ypred_lng)\nytest['lngDeg_pred'] = ytest['lngDeg_prev'] + (ytest['lng_del_pred'] / 10**6)\nytest_final['lng_del_pred'] = pd.Series(ypred_lng_final)\nytest_final['lngDeg_pred'] = ytest_final['lngDeg_prev'] + (ytest_final['lng_del_pred']/ 10**6)\nytest.to_csv('./lng_preds_data_dum_test_v1.csv', sep = ',', index = False)\nytest_final.to_csv('./lng_preds_final_v1.csv', sep = ',', index = False)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T08:57:29.777705Z","iopub.execute_input":"2021-06-30T08:57:29.778050Z","iopub.status.idle":"2021-06-30T10:45:45.148986Z","shell.execute_reply.started":"2021-06-30T08:57:29.778019Z","shell.execute_reply":"2021-06-30T10:45:45.147813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(\"Baseline: %.2f (%.2f) MSE\" % (results.mean(), results.std()))\n#ytest.head()\n#ytest['lngDeg_pred'] = ytest['lngDeg_truth']\nlat_pred = pd.read_csv('/kaggle/input/predictions_v1/lat_preds_final_v1.csv', sep = ',')\nlat_pred.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-30T11:02:38.705413Z","iopub.execute_input":"2021-06-30T11:02:38.705785Z","iopub.status.idle":"2021-06-30T11:02:38.793996Z","shell.execute_reply.started":"2021-06-30T11:02:38.705745Z","shell.execute_reply":"2021-06-30T11:02:38.791847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from math import radians, cos, sin, asin, sqrt\ndef haversine(df):\n    dist = []\n    for i in df.index:\n        lat1 = df['latDeg_truth'][i]\n        lng1 = df['lngDeg_truth'][i]\n        lat2 = df['latDeg_pred'][i]\n        lng2 = df['lngDeg_pred'][i]\n        # convert decimal degrees to radians \n        lat1,lng1,lat2,lng2 = map(radians, [lat1, lng1, lat2, lng2])\n\n        # haversine formula \n        dlon = lng2 - lng1 \n        dlat = lat2 - lat1 \n        a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2\n        c = 2 * asin(sqrt(a)) \n        r = 6371000 # Radius of earth in kilometers. Use 3956 for miles\n        d = c * r\n        dist.append(d)\n    return dist","metadata":{"execution":{"iopub.status.busy":"2021-06-30T10:59:34.877662Z","iopub.execute_input":"2021-06-30T10:59:34.878006Z","iopub.status.idle":"2021-06-30T10:59:34.886406Z","shell.execute_reply.started":"2021-06-30T10:59:34.877977Z","shell.execute_reply":"2021-06-30T10:59:34.884976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytest['dist'] = haversine(ytest)\ndist_by_phone = ytest.groupby(['phone']).dist.apply(lambda x: (np.percentile(x,50) + np.percentile(x,95))/2).reset_index()\ndist_by_phone.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-26T16:54:54.67621Z","iopub.execute_input":"2021-06-26T16:54:54.676784Z","iopub.status.idle":"2021-06-26T16:54:56.343782Z","shell.execute_reply.started":"2021-06-26T16:54:54.67674Z","shell.execute_reply":"2021-06-26T16:54:56.342797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dist_by_phone['dist'].mean()","metadata":{"execution":{"iopub.status.busy":"2021-06-26T16:55:03.305016Z","iopub.execute_input":"2021-06-26T16:55:03.305578Z","iopub.status.idle":"2021-06-26T16:55:03.312331Z","shell.execute_reply.started":"2021-06-26T16:55:03.305543Z","shell.execute_reply":"2021-06-26T16:55:03.311405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols2 = ['collectionName','phoneName','millisSinceGpsEpoch','constellationType','svid','signalType','receivedSvTimeInGpsNanos','xSatPosM','ySatPosM','zSatPosM']\nlogs = (datapath/'test').rglob('*derived.csv')\n\nlogslist = []\nfor fpath in logs:\n    logcsv = pd.read_csv(fpath, usecols = cols2)\n    logslist.append(logcsv)\n\nlogsdata_test = pd.concat(logslist, ignore_index = True)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:23.794098Z","iopub.execute_input":"2021-06-30T06:16:23.794451Z","iopub.status.idle":"2021-06-30T06:16:35.000350Z","shell.execute_reply.started":"2021-06-30T06:16:23.794414Z","shell.execute_reply":"2021-06-30T06:16:34.999652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logsdata_svtime = logsdata_test.groupby(['collectionName','phoneName','millisSinceGpsEpoch']).receivedSvTimeInGpsNanos.apply(lambda x: np.min(x)).reset_index()\nlogsdata_xsat = logsdata_test.groupby(['collectionName','phoneName','millisSinceGpsEpoch']).xSatPosM.apply(lambda x: np.average(x)).reset_index()\nlogsdata_ysat = logsdata_test.groupby(['collectionName','phoneName','millisSinceGpsEpoch']).ySatPosM.apply(lambda x: np.average(x)).reset_index()\nlogsdata_zsat = logsdata_test.groupby(['collectionName','phoneName','millisSinceGpsEpoch']).zSatPosM.apply(lambda x: np.average(x)).reset_index()\nlogsdata_svid = logsdata_test.groupby(['collectionName','phoneName','millisSinceGpsEpoch']).svid.apply(lambda x: x.value_counts().index[0]).reset_index()\n#logsdata_constellation = logsdata_test.groupby(['collectionName','phoneName','millisSinceGpsEpoch']).constellationType.apply(lambda x:x.value_counts().index[0]).reset_index()\nlogsdata_signaltype = logsdata_test.groupby(['collectionName','phoneName','millisSinceGpsEpoch']).signalType.apply(lambda x: x.value_counts().index[0]).reset_index()\n\nlogsdata1 = logsdata_svtime.merge(logsdata_xsat, on = ['collectionName','phoneName','millisSinceGpsEpoch'])\nlogsdata2 = logsdata1.merge(logsdata_ysat, on = ['collectionName','phoneName','millisSinceGpsEpoch'])\nlogsdata3 = logsdata2.merge(logsdata_zsat, on = ['collectionName','phoneName','millisSinceGpsEpoch'])\nlogsdata4 = logsdata3.merge(logsdata_svid, on = ['collectionName','phoneName','millisSinceGpsEpoch'])\n#logsdata5 = logsdata4.merge(logsdata_constellation, on = ['collectionName','phoneName','millisSinceGpsEpoch'])\nlogsdata_test_final = logsdata4.merge(logsdata_signaltype, on = ['collectionName','phoneName','millisSinceGpsEpoch'])\n#logsdata_final['constellationType'] = logsdata_final.constellationType.apply(lambda x: \"one\" if x == 1 else(\"two\" if x == 2 else(\"three\" if x == 3 else(\"four\" if x == 4 else(\"five\" if x == 5 else(\"six\" if x == 6 else \"zero\"))))))                                                                                                 \nlogsdata_test_final.head()\n","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:59.422697Z","iopub.execute_input":"2021-06-30T06:16:59.423209Z","iopub.status.idle":"2021-06-30T06:18:49.168061Z","shell.execute_reply.started":"2021-06-30T06:16:59.423176Z","shell.execute_reply":"2021-06-30T06:18:49.166903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdata = pd.DataFrame()\nprint(len(testfile))\nprint(len(logsdata_test_final))\ntestdata = pd.merge(testfile, logsdata_test_final, how = 'left', on = ['collectionName','phoneName','millisSinceGpsEpoch'])\nprint(len(testdata))\ntestdata.tail()","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:47:24.256392Z","iopub.execute_input":"2021-06-30T06:47:24.256797Z","iopub.status.idle":"2021-06-30T06:47:24.373644Z","shell.execute_reply.started":"2021-06-30T06:47:24.256759Z","shell.execute_reply":"2021-06-30T06:47:24.372835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdata.rename(columns = {'latDeg': 'latDeg_current', 'lngDeg': 'lngDeg_current'}, inplace = True)\ntestdata.sort_values(by = ['phone','millisSinceGpsEpoch'])\n#testdata = testdata.fillna(0)\ntestdata = testdata.fillna(0)\ntestdata['latDeg_prev'] = testdata['latDeg_current'].shift(1).where(testdata['phone'].eq(testdata['phone'].shift(1)))\ntestdata['lngDeg_prev'] = testdata['lngDeg_current'].shift(1).where(testdata['phone'].eq(testdata['phone'].shift(1)))    \ntestdata['latDeg_next'] = testdata['latDeg_current'].shift(-1).where(testdata['phone'].eq(testdata['phone'].shift(-1)))\ntestdata['lngDeg_next'] = testdata['lngDeg_current'].shift(-1).where(testdata['phone'].eq(testdata['phone'].shift(-1)))\ntestdata['time_prev'] = testdata['millisSinceGpsEpoch'].shift(1).where(testdata['phone'].eq(testdata['phone'].shift(1)))\ntestdata['time_next'] = testdata['millisSinceGpsEpoch'].shift(-1).where(testdata['phone'].eq(testdata['phone'].shift(-1)))\ntestdata['x_prev'] = testdata['xSatPosM'].shift(1).where(testdata['phone'].eq(testdata['phone'].shift(1)))\ntestdata['x_next'] = testdata['xSatPosM'].shift(-1).where(testdata['phone'].eq(testdata['phone'].shift(-1)))\ntestdata['y_prev'] = testdata['ySatPosM'].shift(1).where(testdata['phone'].eq(testdata['phone'].shift(1)))\ntestdata['y_next'] = testdata['ySatPosM'].shift(-1).where(testdata['phone'].eq(testdata['phone'].shift(-1)))\ntestdata['z_prev'] = testdata['zSatPosM'].shift(1).where(testdata['phone'].eq(testdata['phone'].shift(1)))\ntestdata['z_next'] = testdata['zSatPosM'].shift(-1).where(testdata['phone'].eq(testdata['phone'].shift(-1)))\n\n#traindata['svid'] = pd.Categorical(traindata.svid)\n#traindata['constellationType'] = pd.Categorical(traindata.constellationType) \nfor i in testdata.index:\n    if pd.isna(testdata['latDeg_prev'][i]):\n        testdata['latDeg_prev'][i] = testdata['latDeg_current'][i]\n        testdata['lngDeg_prev'][i] = testdata['lngDeg_current'][i]\n        testdata['time_prev'][i] = testdata['millisSinceGpsEpoch'][i]\n        testdata['x_prev'][i] = testdata['xSatPosM'][i]\n        testdata['y_prev'][i] = testdata['ySatPosM'][i]\n        testdata['z_prev'][i] = testdata['zSatPosM'][i]\n    if pd.isna(testdata['latDeg_next'][i]):\n        testdata['latDeg_next'][i] = testdata['latDeg_current'][i]\n        testdata['lngDeg_next'][i] = testdata['lngDeg_current'][i]\n        testdata['time_next'][i] = testdata['millisSinceGpsEpoch'][i]\n        testdata['x_prev'][i] = testdata['xSatPosM'][i]\n        testdata['y_prev'][i] = testdata['ySatPosM'][i]\n        testdata['z_prev'][i] = testdata['zSatPosM'][i]\n    \ntestdata['lat_del_prev'] = (testdata['latDeg_current'] - testdata['latDeg_prev'])*10**6\ntestdata['lng_del_prev'] = (testdata['lngDeg_current'] - testdata['lngDeg_prev'])* 10**6\ntestdata['lat_vel'] = ((abs(testdata['latDeg_next'] - testdata['latDeg_prev'])/(testdata['time_next'] - testdata['time_prev'])) * 10**6)\ntestdata['lng_vel'] = ((abs(testdata['lngDeg_next'] - testdata['lngDeg_prev'])/(testdata['time_next'] - testdata['time_prev'])) * 10**6)\ntestdata['time_since_last_read'] = testdata['millisSinceGpsEpoch'] - testdata['time_prev']\n    #if traindata['time_prev'][i] == traindata['millisSinceGpsEpoch'][i]:\n        #traindata['vel_lat'] = 0\n        #traindata['vel_lng'] = 0\n    #else:\n        #traindata['vel_lat'] = (traindata['latDeg_current'][i] - traindata['latDeg_prev'][i])/(traindata['millisSinceGpsEpoch'][i] - traindata['time_prev'][i])\n        #traindata['vel_lng'] = (traindata['lngDeg_current'][i] - traindata['lngDeg_prev'][i])/(traindata['millisSinceGpsEpoch'][i] - traindata['time_prev'][i])\n\nprint(len(testdata))\n#print(len(trainfile))\ntestdata.head(1)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:47:52.825009Z","iopub.execute_input":"2021-06-30T06:47:52.825339Z","iopub.status.idle":"2021-06-30T06:47:55.241454Z","shell.execute_reply.started":"2021-06-30T06:47:52.825311Z","shell.execute_reply":"2021-06-30T06:47:55.240512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdata = testdata.fillna(0)\nxcols = [c for c in traindata.columns if c not in ('latDeg_truth','lngDeg_truth','lat_del_truth','lng_del_truth','collectionName','phone','phoneName','millisSinceGpsEpoch','svid',\n                                                  'signalType','time_prev','time_next','xSatPosM','ySatPosM','zSatPosM','x_prev','x_next','y_prev','y_next','z_prev','z_next')]\n\ntestdata_x = testdata[xcols]\ntestdata_x.head(1)\ntestdata_x.to_csv('./testdata.csv', sep = ',')","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:48:07.216729Z","iopub.execute_input":"2021-06-30T06:48:07.217108Z","iopub.status.idle":"2021-06-30T06:48:09.201663Z","shell.execute_reply.started":"2021-06-30T06:48:07.217075Z","shell.execute_reply":"2021-06-30T06:48:09.200337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdata['zSatPosM'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:27:48.519698Z","iopub.execute_input":"2021-06-30T06:27:48.520055Z","iopub.status.idle":"2021-06-30T06:27:48.534746Z","shell.execute_reply.started":"2021-06-30T06:27:48.520025Z","shell.execute_reply":"2021-06-30T06:27:48.533533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nlat_preds = pd.read_csv('../input/predictions/lat_preds_final.csv', sep = ',')\nlng_preds = pd.read_csv('../input/predictions/lng_preds_final_1.csv', sep = ',')\npreds_df = pd.merge(lat_preds, lng_preds, on = ['phone','millisSinceGpsEpoch'])\nfor i in preds_df.index:\n    if pd.isna(preds_df['latDeg'][i]):\n        preds_df.at[i,'latDeg'] = preds_df['latDeg_prev'][i]\n    if pd.isna(preds_df['lngDeg_pred'][i]):\n        preds_df.at[i,'lngDeg_pred'] = preds_df['lngDeg_prev'][i]\n        \nfinal_df = preds_df[['phone','millisSinceGpsEpoch','latDeg','lngDeg_pred']]\nlen(final_df)\nfinal_df.to_csv('./predictions_v10.csv', sep = ',')","metadata":{"execution":{"iopub.status.busy":"2021-06-29T14:09:59.75371Z","iopub.execute_input":"2021-06-29T14:09:59.754122Z","iopub.status.idle":"2021-06-29T14:10:02.724387Z","shell.execute_reply.started":"2021-06-29T14:09:59.754087Z","shell.execute_reply":"2021-06-29T14:10:02.723157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_df['latDeg'][3487]","metadata":{"execution":{"iopub.status.busy":"2021-06-29T14:07:09.70899Z","iopub.execute_input":"2021-06-29T14:07:09.709369Z","iopub.status.idle":"2021-06-29T14:07:09.71902Z","shell.execute_reply.started":"2021-06-29T14:07:09.709336Z","shell.execute_reply":"2021-06-29T14:07:09.717702Z"},"trusted":true},"execution_count":null,"outputs":[]}]}