{"cells":[{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Data Modeling: Long Short Term Memory (LSTM) Neural Network Implementation"},{"metadata":{},"cell_type":"markdown","source":"We'll be using Keras through Tensorflow to implement a LSTM Neural Network. We hope to get a good prediction of the true location of the smartphones. We'll use a different model for floor predictions."},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns \nfrom pathlib import Path\nimport random\nimport os\nimport sys\nimport glob\nimport pickle\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\n\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M\nimport tensorflow.keras.backend as K\nimport tensorflow_addons as tfa\nfrom tensorflow_addons.layers import WeightNormalization\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping\n\nfrom datetime import datetime\n\nfrom dataclasses import dataclass\n\nfrom sklearn.preprocessing import MinMaxScaler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# copy from https://github.com/location-competition/indoor-location-competition-20/blob/master/io_f.py\n\n@dataclass\nclass ReadData:\n    acce: np.ndarray\n    acce_uncali: np.ndarray\n    gyro: np.ndarray\n    gyro_uncali: np.ndarray\n    magn: np.ndarray\n    magn_uncali: np.ndarray\n    ahrs: np.ndarray\n    wifi: np.ndarray\n    ibeacon: np.ndarray\n    waypoint: np.ndarray\n\n\ndef read_data_file(data_filename):\n    acce = []\n    acce_uncali = []\n    gyro = []\n    gyro_uncali = []\n    magn = []\n    magn_uncali = []\n    ahrs = []\n    wifi = []\n    ibeacon = []\n    waypoint = []\n\n    with open(data_filename, 'r', encoding='utf-8') as file:\n        lines = file.readlines()\n\n    for line_data in lines:\n        line_data = line_data.strip()\n        if not line_data or line_data[0] == '#':\n            continue\n\n        line_data = line_data.split('\\t')\n\n        if line_data[1] == 'TYPE_WAYPOINT':\n            waypoint.append([int(line_data[0]), float(line_data[2]), float(line_data[3])])\n            continue\n       \n        if line_data[1] == 'TYPE_ACCELEROMETER':\n            acce.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n        \n        if line_data[1] == 'TYPE_ACCELEROMETER_UNCALIBRATED':\n            acce_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n        \n        if line_data[1] == 'TYPE_GYROSCOPE':\n            gyro.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_GYROSCOPE_UNCALIBRATED':\n            gyro_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n        \n        if line_data[1] == 'TYPE_MAGNETIC_FIELD':\n            magn.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_MAGNETIC_FIELD_UNCALIBRATED':\n            magn_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_ROTATION_VECTOR':\n            ahrs.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_WIFI':\n            sys_ts = line_data[0]\n            ssid = line_data[2]\n            bssid = line_data[3]\n            rssi = line_data[4]\n            lastseen_ts = line_data[6]\n            wifi_data = [sys_ts, ssid, bssid, rssi, lastseen_ts]\n            wifi.append(wifi_data)\n            continue\n\n        if line_data[1] == 'TYPE_BEACON':\n            ts = line_data[0]\n            uuid = line_data[2]\n            major = line_data[3]\n            minor = line_data[4]\n            rssi = line_data[6]\n            ibeacon_data = [ts, '_'.join([uuid, major, minor]), rssi]\n            ibeacon.append(ibeacon_data)\n            continue\n        \n    \n    acce = np.array(acce)\n    acce_uncali = np.array(acce_uncali)\n    gyro = np.array(gyro)\n    gyro_uncali = np.array(gyro_uncali)\n    magn = np.array(magn)\n    magn_uncali = np.array(magn_uncali)\n    ahrs = np.array(ahrs)\n    wifi = np.array(wifi)\n    ibeacon = np.array(ibeacon)\n    waypoint = np.array(waypoint)\n    \n    return ReadData(acce, acce_uncali, gyro, gyro_uncali, magn, magn_uncali, ahrs, wifi, ibeacon, waypoint)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_cols = { # Dictionary containing the names of the cols of each dataframe\n    'acce': ['ts', 'x_acce', 'y_acce', 'z_acce'],\n    'acce_uncali': ['ts', 'x_acce_uncali', 'y_acce_uncali', 'z_acce_uncali'],\n    'gyro': ['ts', 'x_gyro', 'y_gyro', 'z_gyro'],\n    'gyro_uncali': ['ts', 'x_gyro_uncali', 'y_gyro_uncali', 'z_gyro_uncali'],\n    'magn': ['ts', 'x_magn', 'y_magn', 'z_magn'],\n    'magn_uncali': ['ts', 'x_magn_uncali', 'y_magn_uncali', 'z_magn_uncali'],\n    'ahrs': ['ts', 'x_ahrs', 'y_ahrs', 'z_ahrs'],\n    'wifi': ['ts', 'ssid', 'bssid', 'rssi_wifi', 'lastseen_ts'],\n    'ibeacon': ['ts', 'uuid_major_minor', 'rssi_ibeacon'],\n    'waypoint': ['ts', 'x', 'y']\n}\n\ndef make_unified_csv(file_name):\n    df_dict = {}\n    file = read_data_file(file_name)\n    dataset_names = [attr for attr in dir(file) if not attr.startswith('__')] \n    for dataset in dataset_names:\n        try:\n            cols = dataset_cols[dataset] # Getting the colnames for dataset\n            vals = getattr(file, dataset) # Getting values for dataset\n#             print(f'\\n\\n{dataset} dataframe:') \n#             print(cols)\n#             print(vals)\n            df_dict[dataset] = pd.DataFrame(data=vals, columns=cols) # Creating dataset using cols and vals\n            df_dict[dataset].drop_duplicates('ts', inplace=True) # Dropping duplicate timestamps\n            df_dict[dataset]['ts'] = pd.to_numeric(df_dict[dataset]['ts']) # Converting timestamp to numeric type\n            df_dict[dataset] = df_dict[dataset].set_index('ts') # Setting timestamp as index\n            # print(f'\\n\\n{dataset} dataframe:') \n            # display(df_dict[dataset].head())\n        except:\n            df_dict[dataset] = pd.DataFrame(columns=cols)\n            df_dict[dataset] = df_dict[dataset].set_index('ts')\n    df_dict['wifi']['lastseen_ts_datetime'] = pd.to_numeric(df_dict['wifi']['lastseen_ts'])\n    df_dict['wifi']['lastseen_ts_datetime'] = pd.to_datetime(df_dict['wifi']['lastseen_ts_datetime'], unit='ms')\n\n    # Merging df_dict to create master dataframe\n    merged=pd.concat(df_dict,axis=1)\n    # 'Flattening' datafram column names\n    master = merged.reset_index()\n    master.columns = [i[1] if i != ('ts','') else i[0] for i in master.columns]\n\n    # Converting rssi columns to numeric\n    master['rssi_wifi'] = pd.to_numeric(master['rssi_wifi'])\n    master['rssi_ibeacon'] = pd.to_numeric(master['rssi_ibeacon'])\n\n    # Creating separate column with timestamps as \n    master['ts_datetime'] = pd.to_numeric(master['ts'])\n    master['ts_datetime'] = pd.to_datetime(master['ts_datetime'], unit='ms')\n\n    # calculated column: difference of time between timestamps\n    master['ts_diff'] = master['ts'].diff()\n    \n    # set index to ts_datetime\n    master.set_index('ts_datetime', inplace=True)\n    \n    # interpolate x and y\n    master['x'] = master['x'].interpolate(method='time')\n    master['y'] = master['y'].interpolate(method='time')\n    \n    # set path (metadata)\n    master['path'] = file_name.split('/')[-1][:-4]\n        \n    return master\n    \n\nsample_file = make_unified_csv('../input/indoor-location-navigation/train/5cd56c0ce2acfd2d33b6ab27/B1/5d09a625bd54340008acddb9.txt')\n# sample_file = read_data_file('../input/indoor-location-navigation/train/5cd56c0ce2acfd2d33b6ab27/B1/5d09a625bd54340008acddb9.txt')\n\nsample_file","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## I. Loading Data and Visualizing Paths\n\nCode courtesy of [@titeriks](https://www.kaggle.com/titericz), from his notebook '[EDA - Loading Data and Visualizing Paths](https://www.kaggle.com/titericz/eda-loading-data-and-visualizing-paths)';  and"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!ls ../input/indoor-location-navigation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainfiles = glob.glob('../input/indoor-location-navigation/train/*/*/*')\nlen(trainfiles), trainfiles[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!head ../input/indoor-location-navigation/train/5cd56c0ce2acfd2d33b6ab27/B1/5d09a625bd54340008acddb9.txt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dict( filename ):\n    with open(filename) as f:\n        lines = f.readlines()\n\n    proc = {}\n    for l in lines:\n        l = l.replace('\\n','')\n        \n        if l[0]!='#':\n            val = l.split('\\t')\n            if val[0] not in proc:\n                proc[val[0]] = {}\n            proc[val[0]][val[1]] = val[2:]\n\n    return proc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = load_dict( trainfiles[0] )\n\nlen(path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path['1560913374746']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_values( path, var='TYPE_ACCELEROMETER', cumsum=False ):\n    x = [ float(path[p][var][0]) for p in path if var in path[p] ]\n    y = [ float(path[p][var][1]) for p in path if var in path[p] ]\n    if cumsum:\n        plt.plot( np.cumsum(x), np.cumsum(y) )\n    else:\n        plt.plot( x, y )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_values( path, var='TYPE_ACCELEROMETER', cumsum=True  )\nplot_values( path, var='TYPE_ACCELEROMETER_UNCALIBRATED', cumsum=True  )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_values( path, var='TYPE_MAGNETIC_FIELD', cumsum=False  )\nplot_values( path, var='TYPE_MAGNETIC_FIELD_UNCALIBRATED', cumsum=False  )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_values( path, var='TYPE_GYROSCOPE', cumsum=True  )\nplot_values( path, var='TYPE_GYROSCOPE_UNCALIBRATED', cumsum=True  )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_values( path, var='TYPE_ROTATION_VECTOR', cumsum=False  )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# II. Creating and Implementing the LSTM "},{"metadata":{},"cell_type":"markdown","source":"### Part 0.1: Prep the data (Sensors - acce, gyro, magne, ahrs)\n\nUsing functions borrowed from our second EDA notebook"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Getting test and train files\ntrain_files = glob.glob('../input/indoor-location-navigation/train/*/*/*')\ntest_files = glob.glob('../input/indoor-location-navigation/test/*')\n\n# Sampling train files\nsampled_train = random.sample(train_files, 10)#len(train_files)//100) # sampling without replacement, grabbing 10% of the data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(sampled_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"master_sensor_train = pd.DataFrame()\n\nfor i, train_file in enumerate(sampled_train):\n    #print(i)\n    train = make_unified_csv(train_file)\n    train.dropna(subset=['x_acce', 'y_acce'], inplace=True)\n    master_sensor_train = pd.concat([master_sensor_train, train])\n\nmaster_sensor_train.head(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Part 0.2: Prep the data (Wifi features)\n\nCode couretsy of [@kokitanisaka](https://www.kaggle.com/kokitanisaka)'s [awesome work](https://www.kaggle.com/kokitanisaka/lstm-by-keras-with-unified-wi-fi-feats).\n\n(Ignore for now)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# options\n\nN_SPLITS = 10\n\nSEED = 2021\n\nNUM_FEATS = 20 # number of features that we use. there are 100 feats but we don't need to use all of them\n\nbase_path = '/kaggle'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    session_conf = tf.compat.v1.ConfigProto(\n        intra_op_parallelism_threads=1,\n        inter_op_parallelism_threads=1\n    )\n    sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph(), config=session_conf)\n    tf.compat.v1.keras.backend.set_session(sess)\n    \ndef comp_metric(xhat, yhat, fhat, x, y, f):\n    intermediate = np.sqrt(np.power(xhat-x, 2) + np.power(yhat-y, 2)) + 15 * np.abs(fhat-f)\n    return intermediate.sum()/xhat.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# wifi_dir = f\"{base_path}/input/indoorunifiedwifids\"\n# wifi_train_files = sorted(glob.glob(os.path.join(wifi_dir, '*_train.csv')))\n# wifi_test_files = sorted(glob.glob(os.path.join(wifi_dir, '*_test.csv')))\n# subm = pd.read_csv(f'{base_path}/input/indoor-location-navigation/sample_submission.csv', index_col=0)\n\n# master_wifi_train = pd.DataFrame()\n\n# for i, train_file in enumerate(wifi_train_files):\n#     #print(i)\n#     train = pd.read_csv(train_file)\n#     master_wifi_train = pd.concat([master_wifi_train, train])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Part 1: Pre-process (scale) the data\n\nWe might want to scale the data to normalize it before plugging it in into a neural network.\n\nHere's a list of scalers available in sklearn:\n\n![](https://www.kdnuggets.com/wp-content/uploads/sklearn-scalers.png)"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Part 2: Split the model for internal train and test\n\nA common way is 75% train and 25% test"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Part 3: Build the arquitecture of the model"},{"metadata":{},"cell_type":"markdown","source":"From [datacamp.com](https://www.datacamp.com/community/tutorials/lstm-python-stock-market):\n\nLong Short-Term Memory models are extremely powerful time-series models. They can predict an arbitrary number of steps into the future. An LSTM module (or cell) has 5 essential components which allows it to model both long-term and short-term data.\n\n* Cell state (ct) - This represents the internal memory of the cell which stores both short term memory and long-term memories\n* Hidden state (ht) - This is output state information calculated w.r.t. current input, previous hidden state and current cell input which you eventually use to predict the target. Additionally, the hidden state can decide to only retrive the short or long-term or both types of memory stored in the cell state to make the next prediction.\n* Input gate (it) - Decides how much information from current input flows to the cell state\n* Forget gate (ft) - Decides how much information from the current input and the previous cell state flows into the current cell state\n* Output gate (ot) - Decides how much information from the current cell state flows into the hidden state, so that if needed LSTM can only pick the long-term memories or short-term memories and long-term memories\n\n![](http://res.cloudinary.com/dyd911kmh/image/upload/f_auto,q_auto:best/v1523953369/lstm_xszk4d.png)"},{"metadata":{},"cell_type":"markdown","source":"#### Data Generation and Augmentation\n\nYou are first going to implement a data generator to train your model. This data generator will have a method called .unroll_batches(...) which will output a set of num_unrollings batches of input data obtained sequentially, where a batch of data is of size [batch_size, 1]. Then each batch of input data will have a corresponding output batch of data.\n\nAlso to make your model robust you will not make the output for x_t always x_t+1. Rather you will randomly sample an output from the set x_t+1,x_t+2,…,xt+N where N is a small window size.\n\nAssumption: x_t+1,x_t+2,…,xt+N will not be very far from each other\n\n![](http://res.cloudinary.com/dyd911kmh/image/upload/f_auto,q_auto:best/v1523953369/batch_pno02e.png)","attachments":{}},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass DataGeneratorSeq(object):\n\n    def __init__(self,feats,batch_size,num_unroll):\n        self._feats = feats\n        self._feats_length = len(self._feats) - num_unroll\n        self._batch_size = batch_size\n        self._num_unroll = num_unroll\n        self._segments = self._feats_length //self._batch_size\n        self._cursor = [offset * self._segments for offset in range(self._batch_size)]\n\n    def next_batch(self):\n\n        batch_data = np.zeros((self._batch_size),dtype=np.float32)\n        batch_labels = np.zeros((self._batch_size),dtype=np.float32)\n\n        for b in range(self._batch_size):\n            if self._cursor[b]+1>=self._feats_length:\n                #self._cursor[b] = b * self._segments\n                self._cursor[b] = np.random.randint(0,(b+1)*self._segments)\n\n            batch_data[b] = self._feats[self._cursor[b]]\n            batch_labels[b]= self._feats[self._cursor[b]+np.random.randint(0,5)]\n\n            self._cursor[b] = (self._cursor[b]+1)%self._feats_length\n\n        return batch_data,batch_labels\n\n    def unroll_batches(self):\n\n        unroll_data,unroll_labels = [],[]\n        init_data, init_label = None,None\n        for ui in range(self._num_unroll):\n\n            data, labels = self.next_batch()    \n\n            unroll_data.append(data)\n            unroll_labels.append(labels)\n\n        return unroll_data, unroll_labels\n\n    def reset_indices(self):\n        for b in range(self._batch_size):\n            self._cursor[b] = np.random.randint(0,min((b+1)*self._segments,self._feats_length-1))\n\n\n\ndg = DataGeneratorSeq(master_sensor_train,5,5)\nu_data, u_labels = dg.unroll_batches()\n\nfor ui,(dat,lbl) in enumerate(zip(u_data,u_labels)):   \n    print('\\n\\nUnrolled index %d'%ui)\n    dat_ind = dat\n    lbl_ind = lbl\n    print('\\tInputs: ',dat )\n    print('\\n\\tOutput:',lbl)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Define Hyperparameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"D = 0 # Dimensionality of the data. Since your data is 1-D this would be 1\nnum_unrollings = 0 # Number of time steps you look into the future.\nbatch_size = 0 # Number of samples in a batch\nnum_nodes = [0,0,0] # Number of hidden nodes in each layer of the deep LSTM stack we're using\nn_layers = len(num_nodes) # number of layers\ndropout = 0.0 # dropout amount\n\ntf.reset_default_graph() # This is important in case you run this multiple times","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Define Inputs and Outputs\n\nNext you define placeholders for training inputs and labels. This is very straightforward as you have a list of input placeholders, where each placeholder contains a single batch of data. And the list has num_unrollings placeholders, that will be used at once for a single optimization step."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_inputs, train_outputs = [],[]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Defining Parameters of the LSTM and Regression layer"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Loss Calculation and Optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Part 4: Running the LSTM"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Part 5: Visualization and Prediction "},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# IV. Predicting Floor Data\n\n(Don't worry about this, we'll use another notebook for floor predictions)."},{"metadata":{},"cell_type":"markdown","source":"# V. Submitting"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}