{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install pymap3d","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-03T06:23:49.017334Z","iopub.execute_input":"2023-05-03T06:23:49.017743Z","iopub.status.idle":"2023-05-03T06:24:00.023828Z","shell.execute_reply.started":"2023-05-03T06:23:49.017714Z","shell.execute_reply":"2023-05-03T06:24:00.022673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport pymap3d as pm\nimport random\n# !pip uninstall numpy\n# !pip -v install numpy==1.22.0\nimport tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2023-05-03T07:39:06.2532Z","iopub.execute_input":"2023-05-03T07:39:06.25384Z","iopub.status.idle":"2023-05-03T07:39:06.275811Z","shell.execute_reply.started":"2023-05-03T07:39:06.25379Z","shell.execute_reply":"2023-05-03T07:39:06.26957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def filter_by_signal_type(df:pd.DataFrame, signal_type:list):    \n    \"\"\"Frilters data frame to contain only the signal types that in the signal type list\n\n    Args:\n        df (pd.DataFrame): Data frame of all the metadata\n        signal_type (list): List of signal types to remain in the data frame optinal values:\n        [GPS_L1, GPS_L5, GAL_E1, GAL_E5A, GLO_G1, BDS_B1I, BDS_B1C, BDS_B2A, QZS_J1, QZS_J5]\n\n    Returns:\n        df_out (pd.DataFrame): Filtered data frame\n    \"\"\"\n    \n    df_out = df[df['SignalType'].isin(signal_type)]\n    \n    return df_out\n\ndef add_imu_data(df:pd.DataFrame, df_imu:pd.DataFrame):    \n    \"\"\"Add the imu data to the primery data frame\n\n    Args:\n        df (pd.DataFrame): Primery data frame \n        df_imu (pd.DataFrame):Imu data frame \n    \"\"\"\n    \n#     messType = ['UncalMag', 'UncalAccel', 'UncalGyro']\n    messType = ['UncalAccel']\n    for mess in messType:\n        df_temp = df_imu[df_imu['MessageType'].isin([mess])]\n        df[mess + 'X'] = df_temp.groupby(df_temp.index // 10)['MeasurementX'].mean()   \n        df[mess + 'Y'] = df_temp.groupby(df_temp.index // 10)['MeasurementY'].mean()\n        df[mess + 'Z'] = df_temp.groupby(df_temp.index // 10)['MeasurementZ'].mean()   \n\ndef filter_by_column_name(df:pd.DataFrame, columns_names:list):\n    \"\"\"Frilters data frame to contain only the columns that in the columns names type list\n\n    Args:\n        df (pd.DataFrame): Data frame of all the metadata\n        columns_names (list): List of columns to keep in the data frame\n\n    Returns:\n        df_out (pd.DataFrame): Filtered data frame\n    \"\"\"\n    df_out = df.filter(items=columns_names)\n    \n    return df_out\n\n\ndef convert_ecef_to_geodetic(df:pd.DataFrame):\n    \"\"\"Convert the fix coordinates from ECEF to Geodetic and group the fixes according to the time.\n\n    Args:\n        df_in (pd.DataFrame): GNSS data frame\n\n    Returns:\n        df_out (pd.DataFrame): Converted data frame\n    \"\"\"\n    df_out = pd.DataFrame()\n    position_arr = df.groupby('utcTimeMillis')[['WlsPositionXEcefMeters', 'WlsPositionYEcefMeters', 'WlsPositionZEcefMeters']].mean().to_numpy()\n    geodetic_rsualt = np.array(pm.ecef2geodetic(position_arr[:, 0], position_arr[:, 1], position_arr[:, 2])).T\n    df_out[['PositionX', 'PositionY']] = geodetic_rsualt[:,:2]\n    return df_out\n\ndef df_to_ds(df_data:pd.DataFrame, df_GT:pd.DataFrame):\n    \"\"\"Convert data frame to data set\n\n    Args:\n        df_data (pd.DataFrame): Data frame that contain the training data\n        df_GT (pd.DataFrame): Data frame that contain the ground truth\n\n    Returns:\n        ds (tf.data.Dataset): Data set for Trensorflow training\n    \"\"\"\n    ds = tf.data.Dataset.from_tensor_slices((df_data.values,df_GT.values))\n    return ds\n\ndef create_dataset(num_of_csv_sheets_train:int, num_of_csv_sheets_test:int, val_press:int, time_step:int):\n    folder_path = \"/kaggle/input/smartphone-decimeter-2022/train\" \n\n    subfolder_names = ['GooglePixel4','GooglePixel4XL', 'GooglePixel5','XiaomiMi8', 'SamsungGalaxyS20Ultra', 'GooglePixel5Pro']\n    \n    subfolders = []\n\n    for rootdir, dirs, files in os.walk(folder_path):\n        for subdir in dirs:\n            dir_path = os.path.join(rootdir, subdir)\n            if (dir_path.split(\"/\")[-1] in subfolder_names):\n                subfolders.append(dir_path)\n    \n    selected_dirs_train = random.sample(subfolders, num_of_csv_sheets_train)\n    selected_dirs_test = random.sample([x for x in subfolders if x not in selected_dirs_train], num_of_csv_sheets_test)\n    \n    df = pd.DataFrame()\n    df_GT = pd.DataFrame()\n    for dir in selected_dirs_train:\n        gnss_df = pd.read_csv(dir + '/device_gnss.csv')\n        imu_df = pd.read_csv(dir + '/device_imu.csv')\n        df_temp = filter_by_column_name(df=gnss_df, columns_names=['utcTimeMillis', 'WlsPositionXEcefMeters', 'WlsPositionYEcefMeters', 'WlsPositionZEcefMeters'])\n        df_temp = convert_ecef_to_geodetic(df=df_temp)\n        add_imu_data(df=df_temp, df_imu=imu_df)\n        df = pd.concat([df,df_temp], ignore_index=True)\n        df_GT_temp = pd.read_csv(dir + '/ground_truth.csv')\n        df_GT_temp = filter_by_column_name(df=df_GT_temp, columns_names=['LatitudeDegrees', 'LongitudeDegrees'])\n        df_GT = pd.concat([df_GT, df_GT_temp], ignore_index=True)\n    \n    n = (df.shape[0]*val_press) // 100\n\n    df_train = df.iloc[:n]\n    df_val = df.iloc[n:]\n\n    df_GT_train = df_GT.iloc[:n]\n    df_GT_val = df_GT.iloc[n:]\n\n    train_ds = df_train.values\n    val_ds = df_val.values\n\n    GT_train_ds = df_GT_train.values\n    GT_val_ds = df_GT_val.values\n\n\n    train_ds = np.expand_dims(train_ds, axis=1)\n    val_ds = np.expand_dims(val_ds, axis=1)\n    GT_train_ds = np.expand_dims(GT_train_ds, axis=1)\n    GT_val_ds = np.expand_dims(GT_val_ds, axis=1)\n    \n    train_ds.reshape(train_ds.shape[0]//time_step, time_step, train_ds.shape[2])\n    val_ds.reshape(val_ds.shape[0]//time_step, time_step, val_ds.shape[2])\n    GT_train_ds.reshape(GT_train_ds.shape[0]//time_step, time_step, GT_train_ds.shape[2])\n    GT_val_ds.reshape(GT_val_ds.shape[0]//time_step, time_step, GT_val_ds.shape[2])\n    \n    \n    df = pd.DataFrame()\n    df_GT = pd.DataFrame()\n    for dir in selected_dirs_test:\n        gnss_df = pd.read_csv(dir + '/device_gnss.csv')\n        imu_df = pd.read_csv(dir + '/device_imu.csv')\n        df_temp = filter_by_column_name(df=gnss_df, columns_names=['utcTimeMillis', 'WlsPositionXEcefMeters', 'WlsPositionYEcefMeters', 'WlsPositionZEcefMeters'])\n        df_temp = convert_ecef_to_geodetic(df=df_temp)\n        add_imu_data(df=df_temp, df_imu=imu_df)\n        df = pd.concat([df,df_temp], ignore_index=True)\n        df_GT_temp = pd.read_csv(dir + '/ground_truth.csv')\n        df_GT_temp = filter_by_column_name(df=df_GT_temp, columns_names=['LatitudeDegrees', 'LongitudeDegrees'])\n        df_GT = pd.concat([df_GT, df_GT_temp], ignore_index=True)\n    \n    test_ds = df.values\n    GT_test_ds = df_GT.values\n\n    test_ds = np.expand_dims(test_ds, axis=1)\n    GT_test_ds = np.expand_dims(GT_test_ds, axis=1)\n    \n    test_ds.reshape(test_ds.shape[0]//time_step, time_step, test_ds.shape[2])\n    GT_test_ds.reshape(GT_test_ds.shape[0]//time_step, time_step, GT_test_ds.shape[2])\n    \n    return train_ds, GT_train_ds , val_ds , GT_val_ds, test_ds, GT_test_ds\n        \n        \n    \n","metadata":{"execution":{"iopub.status.busy":"2023-05-03T07:48:35.562534Z","iopub.execute_input":"2023-05-03T07:48:35.563081Z","iopub.status.idle":"2023-05-03T07:48:35.585478Z","shell.execute_reply.started":"2023-05-03T07:48:35.563053Z","shell.execute_reply":"2023-05-03T07:48:35.584528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds, GT_train_ds , val_ds , GT_val_ds, test_ds, GT_test_ds = create_dataset(num_of_csv_sheets_train=20, num_of_csv_sheets_test=5, val_press=80, time_step=2)\nprint(test_ds)\nprint(GT_test_ds)","metadata":{"execution":{"iopub.status.busy":"2023-05-03T07:49:43.394023Z","iopub.execute_input":"2023-05-03T07:49:43.394414Z","iopub.status.idle":"2023-05-03T07:50:12.555248Z","shell.execute_reply.started":"2023-05-03T07:49:43.394387Z","shell.execute_reply":"2023-05-03T07:50:12.552005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_shape=train_ds.shape[2]\nbatch_size = 10\ntime_step = 2","metadata":{"execution":{"iopub.status.busy":"2023-05-03T07:41:38.387321Z","iopub.execute_input":"2023-05-03T07:41:38.38774Z","iopub.status.idle":"2023-05-03T07:41:38.395119Z","shell.execute_reply.started":"2023-05-03T07:41:38.387711Z","shell.execute_reply":"2023-05-03T07:41:38.394109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = tf.keras.Sequential([\n    tf.keras.layers.LSTM(50,return_sequences=True, input_shape=[time_step, data_shape]),            #, input_shape=(batch_size ,1,data_shape)),\n    tf.keras.layers.Dropout(rate=0.25),\n    tf.keras.layers.LSTM(30),\n    tf.keras.layers.Dense(2)\n])\nmodel.summary()\nmodel.compile(optimizer='adam', loss='mean_squared_error',metrics = ['accuracy'])\n\n","metadata":{"execution":{"iopub.status.busy":"2023-05-03T07:41:41.284729Z","iopub.execute_input":"2023-05-03T07:41:41.285154Z","iopub.status.idle":"2023-05-03T07:41:41.963992Z","shell.execute_reply.started":"2023-05-03T07:41:41.285127Z","shell.execute_reply":"2023-05-03T07:41:41.956088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(x=train_ds,\n                    y=GT_train_ds,\n                    epochs=20,\n                    validation_data=(val_ds , GT_val_ds),  \n                    batch_size=batch_size)","metadata":{"execution":{"iopub.status.busy":"2023-05-03T07:41:47.327556Z","iopub.execute_input":"2023-05-03T07:41:47.32794Z","iopub.status.idle":"2023-05-03T07:42:55.866609Z","shell.execute_reply.started":"2023-05-03T07:41:47.327913Z","shell.execute_reply":"2023-05-03T07:42:55.863806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predication = model.predict(val_ds)\nprint(predication)","metadata":{"execution":{"iopub.status.busy":"2023-05-02T15:33:49.777117Z","iopub.execute_input":"2023-05-02T15:33:49.777857Z","iopub.status.idle":"2023-05-02T15:33:51.102625Z","shell.execute_reply.started":"2023-05-02T15:33:49.777818Z","shell.execute_reply":"2023-05-02T15:33:51.10145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"converter = tf.lite.TFLiteConverter.from_keras_model(model)\nconverter.optimizations = [tf.lite.Optimize.DEFAULT]\nconverter.experimental_new_converter=True\nconverter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS,\ntf.lite.OpsSet.SELECT_TF_OPS]\n\ntflite_model = converter.convert()\nopen(\"kakipoop_model.tflite\", \"wb\").write(tflite_model)","metadata":{"execution":{"iopub.status.busy":"2023-05-02T16:41:33.829209Z","iopub.execute_input":"2023-05-02T16:41:33.829601Z","iopub.status.idle":"2023-05-02T16:41:50.944299Z","shell.execute_reply.started":"2023-05-02T16:41:33.829572Z","shell.execute_reply":"2023-05-02T16:41:50.943206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install xxd if it is not available\n!apt-get -qq install xxd\n# Save the file as a C source file\n!xxd -i kakipoop_model.tflite > kakipoop_model.cc\n# Print the source file\n!cat kakipoop_model.cc","metadata":{"execution":{"iopub.status.busy":"2023-05-02T16:42:47.064041Z","iopub.execute_input":"2023-05-02T16:42:47.064844Z","iopub.status.idle":"2023-05-02T16:42:55.849469Z","shell.execute_reply.started":"2023-05-02T16:42:47.064806Z","shell.execute_reply":"2023-05-02T16:42:55.8483Z"},"trusted":true},"execution_count":null,"outputs":[]}]}