{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport glob\nimport time\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\nfrom io import StringIO","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_one_file(fn, naming_df):\n    allTypes = naming_df.index\n    with open(fn) as f:\n        A = f.readlines()\n    A_str = ''.join([L for L in A if L[0]!='#'])\n    B = StringIO(A_str)\n    df = pd.read_csv(B,sep='\\t',header=None,names=range(8),index_col=0)\n    X_df = pd.DataFrame()\n    for col_str in allTypes:  #  such as 'TYPE_WAYPOINT'\n        temp = df[df[1]==col_str].iloc[:,1:].dropna(axis=1,how='all')\n        temp.columns = [f'{col_str}_{naming_df.loc[col_str,i]}' for i in range(temp.shape[1])]\n        X_df = X_df.merge(temp,left_index=True,right_index=True,how='outer')\n    X_df = X_df.sort_index()\n    return X_df\n\n\ndef find_beacon_min_max(df,my_min,my_max):\n    beacon_df = df[df[1]=='TYPE_BEACON']\n    if beacon_df.shape[0]>1:\n        C = sorted(beacon_df[9])\n    else:\n        C = [np.nan, np.nan]\n    my_min = min(C[0], my_min)\n    my_max = max(C[-1], my_max)\n    return (my_min, my_max)\n\n\ndef fn_to_df_and_id(fn):\n    with open(fn) as f:\n        A = f.readlines()\n        A_str = ''.join([L for L in A if L[0]!='#'])\n        B = StringIO(A_str)\n        df = pd.read_csv(B,sep='\\t',header=None,names=range(12),index_col=0)\n        siteID_str = A[1].split('SiteID:')[-1].split('\\t')[0]\n        floor_str = A[1].split('FloorName:')[-1].split('\\n')[0]\n    return df, siteID_str, floor_str\n\n\ndef print_header(fn):\n    with open(fn) as f:\n        A = f.readlines()\n        A_str = ''.join([L for L in A if L[0]=='#'])\n        print(fn)\n        print(A_str)\n    return None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"naming_df = pd.read_csv('/kaggle/input/namingcsv/naming.csv').dropna(how='all')\nnaming_df\nfor i in range(naming_df.shape[0]//2):\n    naming_df.iloc[i*2,2:] = naming_df.iloc[i*2+1,2:]\n\nnaming_df = naming_df[~naming_df['Time'].isnull()].set_index('Data Type').iloc[:,1:]\nnaming_df.columns = range(naming_df.shape[1])\n\ntestPathSiteMapping_ser = pd.read_csv('/kaggle/input/namingcsv/test_path_and_site_mapping.csv',index_col='pathID').iloc[:,0]\ntestPathSiteMapping_ser","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /kaggle/input/indoor-pca-paths","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Read and reformat data"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Find a representative site-floor for experiment purpose\\n')\nprint('Test data\\'s most used sites:\\n')\nprint(testPathSiteMapping_ser.value_counts()[:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"chosenSite = '5d2709e003f801723c32d896'\nprint(f'Let\\'s use the second floor of this one:\\n\\t{chosenSite}')\nprint('How many path files are there for this floor?')\nprint('\\t%d'%len(glob.glob(f'/kaggle/input/indoor-location-navigation/train/{chosenSite}/F2/*.txt')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t0 = time.time()\noneFloorAllPath_df = pd.DataFrame()\nfor i,fn in enumerate(glob.glob(f'/kaggle/input/indoor-location-navigation/train/{chosenSite}/F2/*.txt')):\n    df = read_one_file(fn, naming_df)\n    pathID = fn.split('/')[-1].split('.txt')[0]\n    df1 = df.filter(regex='WIFI').dropna(how='all')    # focus on Wi-Fi signals only\n    df1 = df1.reset_index()\n    df1 = df1[~df1.duplicated(subset=[0,'TYPE_WIFI_bssid'],keep='last')].set_index(0)\n    df2 = df1.pivot(columns='TYPE_WIFI_bssid',values='TYPE_WIFI_RSSI')  # make Wi-Fi ID's the columns\n    # df2 = df2.reindex(np.arange(df2.index[0],df2.index[-1]))\n#     df2.columns = range(df2.shape[1])           #  don't need the name of the Wi-Fi ID's\n    df2 = df2[df2.columns[(df2.nunique(axis=0)>1)]]    # if there are only 2 or less unique values, drop this feature\n#     df_temp = df2.copy()\n#     df2 = df2.interpolate(axis=0,limit_direction='both')   # fill the NaN's by interpolation\n    # df2 = df2.fillna(-99)\n    waypoint_df = df.filter(regex='WAYPOINT').dropna()\n    waypoint_df.columns = ['x','y']\n    df3 = df2.merge(waypoint_df,how='outer',left_index=True,right_index=True)\n    df3['pathID'] = pathID\n    print(fn,df1.shape,df1.index.unique().shape,df2.shape,df3.shape,end='\\r')\n    oneFloorAllPath_df = pd.concat([oneFloorAllPath_df,df3],axis=0)\nt1 = time.time()\nprint(f'\\nFinished. Total reading time {t1-t0}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Convert data into various normalised matrix form"},{"metadata":{"trusted":true},"cell_type":"code","source":"oneFloorAllPath_df = oneFloorAllPath_df.sort_index()\nonlyWIFI_df = oneFloorAllPath_df.drop(columns=['pathID','x','y'])\nonlyWIFI_df = onlyWIFI_df.astype(float)\nprint(f'RSSI range ({onlyWIFI_df.max().max()}, {onlyWIFI_df.min().min()})')\nprint('This dataset has %d training examples and %d Wi-Fi features'%(onlyWIFI_df.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"normA_1 = (onlyWIFI_df.fillna(-99).values/99) + 1\nnormA_2 = ((onlyWIFI_df-onlyWIFI_df.mean(axis=0))/onlyWIFI_df.std(axis=0)).interpolate(axis=0,limit_direction='both').values ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## PCA"},{"metadata":{"trusted":true},"cell_type":"code","source":"U,S,V = np.linalg.svd(normA_2) # equivalent implementation:  np.linalg.eig(A@A.T)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"R = np.zeros_like(normA_2)    # initialise the recovery matrix\ntop_sigmas = 12   # how many principal components to keep?\nfor j in range(top_sigmas):   # fill the recovery matrix with few singular values \n    R[j,j] = S[j]\nrecoveredA_2 = (U@R@V)     # complete recovery process","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig81 = plt.figure(81,figsize=(20,10))\nax81 = fig81.subplots(ncols=1,nrows=3)\nax81[0].plot(S,color='red',marker='.')\nax81[0].plot([top_sigmas-1]*2,[1, 500],ls='--',c='gray')\nax81[0].annotate('#%d component'%top_sigmas,xy=(top_sigmas-1+0.5,100),c='gray')\nax81[0].set_title('Singular values $\\sigma$ of matrix A',color='red')\n\nfor i in [7, 9, 18]:\n    ax81[1].plot(normA_2[:,i],lw=1.5,c=cm.tab20(i))\n    ax81[1].plot(recoveredA_2[:,i],lw=1.5,ls='--',c=cm.tab20(i))\nax81[1].set_title(f'Some Wi-Fi signals and the recovery with {top_sigmas}-top components',color='blue')\nax81[2].plot(U[:,:top_sigmas],lw=1)\nax81[2].set_title('%d most influential singular vectors'%top_sigmas,color='blue')\nfor stuff in ax81.ravel():\n    stuff.margins(0.01)\n    stuff.grid(True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## data preparation"},{"metadata":{"trusted":true},"cell_type":"code","source":"label_df = oneFloorAllPath_df[['x','y']].astype(float).dropna()\nlabel_mean = label_df.mean(axis=0)\nlabel_std = label_df.std(axis=0)\nlabel_df = (label_df - label_mean)/label_std\n# feature_df = pd.DataFrame(A,index=temp_df.index).reindex(label_df.index)\n# feature_df = (temp_df.fillna(-99)/99+1).reindex(label_df.index)\n# feature_df = (temp_df.interpolate(axis=0,limit_direction='both')/99+1).reindex(label_df.index)\n# feature_df = pd.DataFrame(U[:,:top_sigmas],index=temp_df.index).reindex(label_df.index)\n# feature_df = pd.DataFrame(A_recovery,index=temp_df.index).reindex(label_df.index)\n\n\n# feature_df = pd.DataFrame(normA_1,index=onlyWIFI_df.index)\n# feature_df = pd.DataFrame(normA_2,index=onlyWIFI_df.index)\nfeature_df = pd.DataFrame((V[:top_sigmas,:]@(normA_2[:,:].T)).T,index=onlyWIFI_df.index)\nfeature_df = feature_df.reindex(label_df.index)\nfeature_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ratio = 0.2\nm = feature_df.shape[0]\n_ = feature_df.sample(m)\nfeature_train, feature_test = _.iloc[:-int(m*test_ratio)], _.iloc[-int(m*test_ratio):]\nlabel_train = label_df.reindex(feature_train.index)\nlabel_test = label_df.reindex(feature_test.index)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Application - NN"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nbatch_size = feature_df.shape[0]\nnFeatures = feature_df.shape[1]\nmodel1 = tf.keras.Sequential()\nmodel1.add(tf.keras.layers.Dense(10,activation='relu',input_shape=(batch_size,nFeatures)))\nmodel1.add(tf.keras.layers.Dense(50,activation='relu'))\n# model1.add(tf.keras.layers.Dense(30,activation='relu'))\nmodel1.add(tf.keras.layers.Dense(2,activation='relu'))\n# model1(feature_df.values[0,:].reshape(-1))\n\n# optimizer = tf.keras.optimizers.SGD(lr=1e-3,momentum=0.9)\noptimizer = tf.keras.optimizers.Adam(lr=1e-3)\nmodel1.compile(loss='mse',optimizer=optimizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model1.fit(feature_df,label_df.iloc[:,:].values.reshape(-1,2),epochs=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"((((label_df - labelHat_df)*np.array(label_std))**2).sum(axis=1)**0.5).describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labelHat_df = pd.DataFrame(model1.predict(feature_df).reshape(-1,2),index=label_df.index)\nlabelHat_df.columns = ['x','y']\n# labelHat_df\n\nfig2 = plt.figure(2,figsize=(16,7))\nax2 = fig2.subplots(ncols=2,nrows=1)\nax2[0].scatter(labelHat_df.iloc[:,0],label_df.iloc[:,0],s=50,alpha=0.5)\nax2[1].scatter(labelHat_df.iloc[:,1],label_df.iloc[:,1],s=50,alpha=0.5)\nfor stuff in ax2.ravel():\n    stuff.grid(True)\n    stuff.set_xlabel('predict')\n    stuff.set_ylabel('actual')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Application: lightgbm"},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"collection_dict = {}\nparams = {'objective': 'regression',\n      'metric': 'l2',\n      'num_iterations':500,\n      'num_leaves':96,\n      'verbosity':-1,\n      'learning_rate':0.1,'max_bin':20000\n      }\ntest_ratio = 0.2\nm = feature_df.shape[0]\n\nsample_ind = feature_df.sample(m).index\ntrain_ind, test_ind = sample_ind[:-int(m*test_ratio)], sample_ind[-int(m*test_ratio):]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.concatenate([np.arange(2,50,4),np.arange(50,480,20)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for top_sigmas in np.concatenate([np.arange(2,50,4),np.arange(50,480,20)]):\n    feature_df = pd.DataFrame((V[:top_sigmas,:]@(normA_2[:,:].T)).T,index=onlyWIFI_df.index)\n    feature_df = feature_df.reindex(label_df.index)\n\n\n    feature_train = feature_df.reindex(train_ind)\n    feature_test = feature_df.reindex(test_ind)\n    label_train = label_df.reindex(train_ind)\n    label_test = label_df.reindex(test_ind)\n    \n    evals_result_x = {}\n    data_train_x = lgb.Dataset(feature_train,label_train['x'])\n    data_val_x = [lgb.Dataset(feature_train,label_train['x']), lgb.Dataset(feature_test,label_test['x'])]\n    boostingModel_x = lgb.train(params,data_train_x,valid_sets=data_val_x,valid_names=['train','test'],evals_result=evals_result_x,verbose_eval=False,early_stopping_rounds=150)\n\n    evals_result_y = {}\n    data_train_y = lgb.Dataset(feature_train,label_train['y'])\n    data_val_y = [lgb.Dataset(feature_train,label_train['y']), lgb.Dataset(feature_test,label_test['y'])]\n    boostingModel_y = lgb.train(params,data_train_y,valid_sets=data_val_y,valid_names=['train','test'],evals_result=evals_result_y,verbose_eval=False,early_stopping_rounds=150)\n    error_x = (boostingModel_x.predict(feature_test)-label_test['x'])*label_std['x']\n    error_y = (boostingModel_y.predict(feature_test)-label_test['y'])*label_std['y']\n    res = (error_x**2).mean()**0.5,(error_y**2).mean()**0.5,((error_x**2 + error_y**2)**0.5).mean()\n    collection_dict[top_sigmas] = res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"collection_df = pd.DataFrame(collection_dict).T\ncollection_df.index.name = 'nPCA'\ncollection_df.columns = ['RMSE(x)','RMSE(y)','MAE(position)']\ncollection_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig4 = plt.figure(4,figsize=(16,10))\nax4 = fig4.subplots(nrows=2,ncols=1)\nax4[0].plot(evals_result_x['train']['l2'],label='training set')\nax4[0].plot(evals_result_x['test']['l2'],label='test set')\nax4[1].plot(evals_result_y['train']['l2'],label='training set')\nax4[1].plot(evals_result_y['test']['l2'],label='test set')\n\nfor stuff in ax4.ravel():\n    stuff.grid(True)\n    stuff.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boostingModel_x = lgb.LGBMRegressor(n_estimators = 1000)#, max_depth=5,num_leaves=20)\nboostingModel_y = lgb.LGBMRegressor(n_estimators = 1000)#, max_depth=5,num_leaves=20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boostingModel_x.fit(feature_train,label_train['x'])\nboostingModel_y.fit(feature_train,label_train['y'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"error_x = (boostingModel_x.predict(feature_test)-label_test['x'])*label_std['x']\nerror_y = (boostingModel_y.predict(feature_test)-label_test['y'])*label_std['y']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(error_x**2).mean()**0.5,(error_y**2).mean()**0.5,((error_x**2 + error_y**2)**0.5).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wrong_mask = labelHat_df.abs().sum(axis=1)==0\nwrong_ind = label_df[wrong_mask].index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wrong_ind","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"normA_1_df = pd.DataFrame(normA_1,index=onlyWIFI_df.index)\nnormA_2_df = pd.DataFrame(normA_2,index=onlyWIFI_df.index)\n\n# .reindex(wrong_ind)#.sum(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# normA_2_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig3 = plt.figure(3, figsize=(20,10))\nax3 = fig3.subplots(ncols=1,nrows=2)\nax3[0].plot(normA_2_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"onlyWIFI_df.reindex(wrong_ind).sum(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Draft"},{"metadata":{"trusted":true},"cell_type":"code","source":"import sklearn.datasets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\n\n\ndef window_data(data, win_size, batch_size):\n    A = tf.data.Dataset.from_tensor_slices(data)\n    A = A.window(win_size+1, shift=1, drop_remainder=True)\n    A = A.flat_map(lambda window:window.batch(win_size+1))  # need to flatten for next line\n    A = A.map(lambda window: (window[:-1],window[-1:]))\n    A = A.shuffle(buffer_size=10)        # optional\n    A = A.batch(batch_size).prefetch(1)  #  some batch technique\n    return A\n\n\ndef model_forecast(model, data, win_size):\n    B = tf.data.Dataset.from_tensor_slices(data)\n    B = B.window(win_size,shift=1, drop_remainder=True)\n    B = B.flat_map(lambda w:w.batch(win_size))\n    B = B.batch(32).prefetch(1)\n    forecast = model.predict(B)\n    return forecast","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp = sklearn.datasets.load_boston()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = temp['data']\ny = temp['target']\n\nX = (X-X.mean(axis=0))/X.std(axis=0)\ny = (y-y.mean())/y.std()\nX_train,X_test = X[:400,:],X[400:,:]\ny_train,y_test = y[:400],y[400:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.Series(temp['target']).describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"timeLength = X.shape[0]\nnp.random.seed(2)\nt = np.linspace(1,100,timeLength).reshape(timeLength,-1)\n\n# X = np.random.random([timeLength,2])/10+0.5\n# X = np.hstack([X, np.cos(t*10)/2+0.5])\ndata = pd.DataFrame(X,index=t.squeeze())\ndata['y'] = y\n# data = data.fillna(0.5)\n\n# data.iloc[:50,:].plot(figsize=(15,5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"win_size = 3\nbatch_size = 64\ndata_processed = window_data(data, win_size, batch_size)\n\nmy_model = tf.keras.Sequential()\nmy_model.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(16,return_sequences=True,input_shape=(batch_size, win_size, 3))))\nmy_model.add(tf.keras.layers.Dense(30,activation='relu'))\nmy_model.add(tf.keras.layers.Dense(1,activation='relu'))\n# my_model.add(tf.keras.layers.Lambda(lambda x: x*3))\n\n\noptimizer = tf.keras.optimizers.SGD(lr=1e-2,momentum=0.9)\nmy_model.compile(loss='mae',optimizer=optimizer)\nmy_model.fit(data_processed,epochs=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"win_size = 5\nbatch_size = 128\n# data_processed = window_data(data, win_size, batch_size)\n\nMLP_model = tf.keras.Sequential()\nMLP_model.add(tf.keras.layers.Dense(30,activation='relu'))\nMLP_model.add(tf.keras.layers.Dense(1,activation='relu'))\n\noptimizer = tf.keras.optimizers.SGD(lr=1e-4)\nMLP_model.compile(loss='mae',optimizer=optimizer)\nMLP_model.fit(X_train,y_train,epochs=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntemp = np.arange(20)\na = temp**1.1\nb = temp**1.1\na[::2] = np.nan\nc = a.copy()\nc[np.isnan(c)] = -99\nb[1::2] = np.nan\nplt.figure(figsize=(15,5))\nplt.scatter(temp,a)\nplt.scatter(temp,b)\n# plt.scatter(temp,c)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}