{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is demonstration code of training and inference part for the [Indoor Location & Navigation](https://www.kaggle.com/c/indoor-location-navigation) competition.  \nThe following changes have been made from the code executed by google colab.\n- Changed the variable \"inference_only\" from False to True\n\n\nHere are links to a great notebook of my teammates.\n- [Minh Tri Phan](https://www.kaggle.com/shinomoriaoshi)'s training, inference  \n  https://www.kaggle.com/shinomoriaoshi/iln-transformer-train-oof  \n  https://www.kaggle.com/shinomoriaoshi/iln-transformer-inference  \n- [Kouki](https://www.kaggle.com/kokitanisaka)'s training, inference, and post process   \n  https://www.kaggle.com/kokitanisaka/self-attentintive-lstm-by-keras  \n  https://www.kaggle.com/kokitanisaka/fix-snapped-waypoints  \n  https://www.kaggle.com/kokitanisaka/create-arrayed-map\n- [darich](https://www.kaggle.com/daaariiich)'s post process     \n  https://www.kaggle.com/daaariiich/nonlinear-cost-minimization-with-geojson\n  ","metadata":{}},{"cell_type":"markdown","source":"# Overview of the model","metadata":{}},{"cell_type":"code","source":"from IPython.display import Image\nImage(filename=\"../input/iln-dataset/mymodel-1.png\",\n      format='png')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%pip install einops","metadata":{"id":"O8HLPd2NldF4","outputId":"f03667ac-b82a-4556-bffe-8df1b0594716","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport os, random, pickle, gc, time, json, math, copy\nfrom tqdm import tqdm\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom matplotlib import pyplot as plt\n\nimport torch\nfrom torch import nn, optim\nfrom torch.nn.parameter import Parameter\nfrom torch.nn.utils import rnn\nfrom einops import rearrange","metadata":{"id":"ulaWLiy6ldF5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)","metadata":{"id":"OR-AyQuGldF6","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dat_ver = 'ILN_631dat'\nmodel_ver = 'ILN_632'\nsubm_file = '../input/indoor-location-navigation/sample_submission.csv'\n\nN_SPLITS = 10\nNUM_FEATS = 40\nSEED = 1605\nbatch_size = 128\ninference_only = True\n\nif inference_only:\n    n_epoch = 1\n    data_dir = '../input/iln-dataset/ILN_train_results/'\nelse:\n    n_epoch = 400\n    data_dir = ''\n\nset_seed(SEED)\n    \nID_FEATS = [f'id_{i}' for i in range(NUM_FEATS)]\nSTRG_FEATS  = [f'strength_{i}' for i in range(NUM_FEATS)]\nIMU_FEATS = ['gyro_x_mean', 'gyro_y_mean', 'gyro_z_mean',\n             'gyro_x_std', 'gyro_y_std', 'gyro_z_std',\n             'gyro_x_max', 'gyro_y_max', 'gyro_z_max',\n             'gyro_x_min', 'gyro_y_min', 'gyro_z_min',\n             'gyro_x_skew', 'gyro_y_skew', 'gyro_z_skew',\n             'acce_x_mean', 'acce_y_mean', 'acce_z_mean',\n             'acce_x_std', 'acce_y_std', 'acce_z_std',\n             'acce_x_max', 'acce_y_max', 'acce_z_max',\n             'acce_x_min', 'acce_y_min', 'acce_z_min',\n             'acce_x_skew', 'acce_y_skew', 'acce_z_skew',\n             'ahrs_x_mean', 'ahrs_y_mean', 'ahrs_z_mean',\n             'ahrs_x_std', 'ahrs_y_std', 'ahrs_z_std',\n             'ahrs_x_max', 'ahrs_y_max', 'ahrs_z_max',\n             'ahrs_x_min', 'ahrs_y_min', 'ahrs_z_min',\n             'ahrs_x_skew', 'ahrs_y_skew', 'ahrs_z_skew',\n             'head_magn_x_mean', 'head_magn_y_mean',\n             'head_magn_x_std', 'head_magn_y_std',\n             'head_magn_x_max', 'head_magn_y_max',\n             'head_magn_x_min', 'head_magn_y_min',\n             'head_magn_x_skew', 'head_magn_y_skew',\n             'magn_z_mean', 'magn_z_std',\n             'magn_z_max', 'magn_z_min', 'magn_z_skew']","metadata":{"id":"RCo1n8J7ldF6","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"floor_map = {\"B2\":-2, \"B1\":-1,\n             \"F1\":0, \"F2\":1, \"F3\":2, \"F4\":3, \"F5\":4,\n             \"F6\":5, \"F7\":6, \"F8\":7, \"F9\":8,\n             \"1F\":0, \"2F\":1, \"3F\":2, \"4F\":3, \"5F\":4,\n             \"6F\":5, \"7F\":6, \"8F\":7, \"9F\":8}\nsite_list = ['5d2709a003f801723c3251bf','5a0546857ecc773753327266',\n             '5c3c44b80379370013e0fd2b','5d2709b303f801723c327472',\n             '5d2709bb03f801723c32852c','5d2709c303f801723c3299ee',\n             '5d2709d403f801723c32bd39','5d2709e003f801723c32d896',\n             '5d27075f03f801723c2e360f','5d27096c03f801723c31e5e0',\n             '5d27097f03f801723c320d97','5d27099f03f801723c32511d',\n             '5da138b74db8ce0c98bd4774','5da958dd46f8266d0737457b',\n             '5da1382d4db8ce0c98bbe92e','5da1383b4db8ce0c98bc11ab',\n             '5da1389e4db8ce0c98bd0547','5da138274db8ce0c98bbd3d2',\n             '5da138314db8ce0c98bbf3a0','5da138364db8ce0c98bc00f1',\n             '5da138754db8ce0c98bca82f','5da138764db8ce0c98bcaa46',\n             '5dbc1d84c1eb61796cf7c010','5dc8cea7659e181adb076a3f']","metadata":{"id":"P7truUoQldF7","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\"df_TestTimeLag\" in the below cell is a time lag table for each path in the test data, and created with [this notebook](https://www.kaggle.com/horsek/ilnpre2-create-test-ts-lag).   \n  \n\"ILN_631dat_df_wifi_all.pkl\" in the below cell is signal sequence data created with [this notebook](https://www.kaggle.com/horsek/iln-preprocess-create-dataset-public).","metadata":{"id":"HUJvBCkNldF7"}},{"cell_type":"code","source":"df_TestTimeLag = pd.read_csv('../input/iln-preprocess-time-lag-table-of-test-data/test_ts_lag.csv',index_col=0)\nwith open(f'../input/iln-preprocess-create-dataset-public/{dat_ver}_df_wifi_all.pkl', 'rb') as f:\n    data = pickle.load(f)","metadata":{"id":"vPnfd3g2ldF8","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n- Cutoff based on signal strength\n  If \"thresh\" is set less than or equals to -100, cutoff is not performed.\n- Divide long sequence into short ones (maximum length is max_seq_len)\n  by GPU memory limitation.\n'''\nmin_seq_len = 10\nmax_seq_len = 200\ntmp = []\nfor gid, grp in tqdm(data.groupby('path'), ncols=60):\n    if grp.shape[0] > min_seq_len:\n        thresh = np.sort(grp['strength_0'].values)[-min_seq_len]\n        thresh = min([-100, thresh])\n        grp = grp[grp['strength_0']>=thresh].copy()\n    grp['seq_label'] = grp['path'].str.cat((np.arange(grp.shape[0])\n                                            //max_seq_len).astype(str),\n                                           sep='_')\n    tmp.append(grp)\ndata = pd.concat(tmp)\ndata.reset_index(drop=True, inplace=True)\ndel tmp; gc.collect()\nprint(f'total num of path data: {data.shape[0]}')","metadata":{"id":"cIhfecGwldF8","outputId":"04406afb-e91b-45de-ee13-b04747709d76","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for s in data.columns:\n#     tmp = [a for a in data.columns if a==s]\n#     if len(tmp)>=2:\n#         print(tmp)\n#     # print((data[s].iloc[:,0]==data[s].iloc[:,1]).all())","metadata":{"id":"6MCjdlQy2CTM","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tslabel ='sys_ts'\ndata['site_path_timestamp'] = (data['site']\n                               .str.cat(data['path'], sep='_'))\n# timestamp is subtract with time lag value (test data only)\nmask = data['train/test']=='train'\ndata.loc[ mask,'timestamp'] = data.loc[mask,tslabel]\ndata.loc[~mask,'timestamp'] = (data.loc[~mask,tslabel]\n                              -df_TestTimeLag.loc[data.loc[~mask,'path']]\n                              .values.reshape(-1))\ndata['site_path_timestamp'] = (data['site_path_timestamp']\n                               .str.cat(data['timestamp']\n                                        .astype('int64').astype(str)\n                                        .str.zfill(13), sep='_'))","metadata":{"id":"9nICVOE3ldF9","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OTHER_FEATS = [itm for itm in data.columns\n               if (itm[:3]!='id_')&(itm[:9]!='strength_')&\n                  (itm[-5:]!='_mean')&(itm[-4:]!='_std')&\n                  (itm[-4:]!='_max')&(itm[-4:]!='_min')&\n                  (itm[-5:]!='_skew')]\ndata = data[OTHER_FEATS+ID_FEATS+STRG_FEATS+IMU_FEATS]","metadata":{"id":"zkmrIfWuldF9","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# signal ID vocabulary for embedding layer\nID_vocab = sorted(list(set(data[ID_FEATS].values.reshape(-1))))\nprint(f'ID vocabulary: {len(ID_vocab)}')","metadata":{"id":"zaqQ2ujgldF-","outputId":"5a399cf8-a25d-4534-a50e-d702f0c28d67","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' label encoding '''\nle_id = LabelEncoder()\nle_id.fit(ID_vocab)\nle_site = LabelEncoder()\nle_site.fit(data['site'].unique())\nfor i in ID_FEATS:\n    data.loc[:,i] = le_id.transform(data.loc[:,i])+1\ndata.loc[:, 'site'] = le_site.transform(data.loc[:, 'site'])\n \n# strength=nan -> token=0 (for zero padding)\ndata[ID_FEATS] *= (~(data[STRG_FEATS].isna()).values)\n \ndata[STRG_FEATS] = data[STRG_FEATS].fillna(-100)+100","metadata":{"id":"ypluiyNkldF-","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' signal strength normalization (path wise) '''\ndata.sort_index(inplace=True)\ntmp = []\nfor _, gdf in tqdm(data.groupby('path',sort=False),\n                   ncols=60):\n    arr = gdf[STRG_FEATS].values.flatten()\n    gdf[STRG_FEATS] = (gdf[STRG_FEATS]-arr.min())/(arr.max()-arr.min())\n    tmp.append(gdf[STRG_FEATS])\ntmp = pd.concat(tmp)\ntmp.sort_index(inplace=True)\ndata[STRG_FEATS] = tmp.values\ndel arr,tmp; gc.collect()","metadata":{"id":"9pBya4_bldF-","outputId":"db3b7162-f001-4c1f-e13e-a652b14c0dc6","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def replace_outlier(series, thresh=0.25, bias=1.5):\n    q1 = series.quantile(thresh)\n    q3 = series.quantile(1-thresh)\n    iqr = q3 - q1\n\n    outlier_min = q1 - (iqr) * bias\n    outlier_max = q3 + (iqr) * bias\n\n    series = series.clip(outlier_min, outlier_max)\n    return series","metadata":{"id":"yAHWSrddD7uZ","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' IMU data standardization '''\ndata[['acce_z_mean','acce_z_max','acce_z_min']] = \\\n    data[['acce_z_mean','acce_z_max','acce_z_min']]-9.80665\nfor sens in['gyro','acce','ahrs','magn']:\n    ss = StandardScaler()\n    if sens=='magn':\n        tmpFEATS = [sens+'_z_mean']\n        ss.fit(data[tmpFEATS].values.reshape(-1,1))\n    else: \n        tmpFEATS = [sens+'_x_mean', sens+'_y_mean', sens+'_z_mean']\n        ss.fit(np.tile(data[tmpFEATS].values.reshape(-1,1),(1,3)))\n    data[tmpFEATS] = ss.transform(data[tmpFEATS])\n    for itm in tmpFEATS:\n        data[itm]=replace_outlier(data[itm])    \n\n    ss = StandardScaler()\n    if sens=='magn':\n        tmpFEATS = [sens+'_z_std']\n        ss.fit(data[tmpFEATS].values.reshape(-1,1))\n    else: \n        tmpFEATS = [sens+'_x_std', sens+'_y_std', sens+'_z_std']\n        ss.fit(np.tile(data[tmpFEATS].values.reshape(-1,1),(1,3)))\n    data[tmpFEATS] = ss.transform(data[tmpFEATS])\n    for itm in tmpFEATS:\n        data[itm]=replace_outlier(data[itm])    \n\n    ss = StandardScaler()\n    if sens=='magn':\n        tmpFEATS = [sens+'_z_skew']\n        ss.fit(data[tmpFEATS].values.reshape(-1,1))\n    else: \n        tmpFEATS = [sens+'_x_skew', sens+'_y_skew', sens+'_z_skew']\n        ss.fit(np.tile(data[tmpFEATS].values.reshape(-1,1),(1,3)))\n    data[tmpFEATS] = ss.transform(data[tmpFEATS])\n    for itm in tmpFEATS:\n        data[itm]=replace_outlier(data[itm])    \n\n    ss = StandardScaler()\n    if sens=='magn':\n        tmpFEATS = [sens+'_z_max', sens+'_z_min']\n        ss.fit(np.tile(data[tmpFEATS].values.reshape(-1,1),(1,2)))\n    else: \n        tmpFEATS = [sens+'_x_max', sens+'_x_min',\n                    sens+'_y_max', sens+'_y_min',\n                    sens+'_z_max', sens+'_z_min']\n        ss.fit(np.tile(data[tmpFEATS].values.reshape(-1,1),(1,6)))\n    data[tmpFEATS] = ss.transform(data[tmpFEATS])\n    for itm in tmpFEATS:\n        data[itm]=replace_outlier(data[itm])","metadata":{"id":"4RraUnQBXnFC","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' relative position '''\nDELTA_FEATS = ['delta_x_hat', 'delta_y_hat',\n               'delta_x_mag', 'delta_y_mag']\ntmp = []\nfor gid, grp in tqdm(data.groupby('seq_label'), ncols=60):\n    grp[DELTA_FEATS[0]] = grp['rel_x'].diff().fillna(0)\n    grp[DELTA_FEATS[1]] = grp['rel_y'].diff().fillna(0)\n    tmp.append(grp[DELTA_FEATS[0:2]])\nassert (pd.concat(tmp).index==data.index).all()\ndata[DELTA_FEATS[0:2]] = pd.concat(tmp)\n\ndelta_l = np.sqrt(np.square(data[DELTA_FEATS[0:2]]).sum(axis=1))\ndata[DELTA_FEATS[2]] = delta_l * data['head_magn_x_mean']\ndata[DELTA_FEATS[3]] = delta_l * data['head_magn_y_mean']\n\ndel tmp, delta_l; gc.collect()\nss = StandardScaler()\nss.fit(np.tile(data[DELTA_FEATS].values.reshape(-1,1),(1,4)))\ndata[DELTA_FEATS] = ss.transform(data[DELTA_FEATS])\nfor itm in DELTA_FEATS:\n    data[itm]=replace_outlier(data[itm])","metadata":{"id":"KadHeXs0orFX","outputId":"a654bc23-ab52-43af-fec3-7e65c7bcb105","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for col in IMU_FEATS:\n#     print(col)\n#     print(data[col].describe())\n#     data[col].hist(bins=50)\n#     plt.show()","metadata":{"id":"my9vcNm_VvLe","outputId":"881025cb-fb08-473c-eb4a-d49076b0faa6","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for col in DELTA_FEATS:\n#     print(col)\n#     print(data[col].describe())\n#     data[col].hist(bins=50)\n#     plt.show()","metadata":{"id":"9R9GUHcIXnHz","outputId":"ca7f7f55-2c38-4937-a768-df9f4a0c8205","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' DataFrame split '''\ntrvl_data = data[data['train/test']=='train'].copy()\ntest_data = data[data['train/test']=='test'].copy()\ndel data; gc.collect()","metadata":{"id":"yqbAN0CyldF_","outputId":"557b9e2e-8fd2-4379-8ae7-bc6d5de3d779","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' train val split '''\ngrb = trvl_data[['seq_label','site','floor']].groupby('seq_label')\ndf_stra = grb.first()\ndf_stra['seq_len'] = grb['seq_label'].count().values\ndf_stra['seq_len_cut'] = pd.qcut(df_stra['seq_len'],4)\ndf_stra['stratify'] = df_stra.apply(lambda x:\n                                    str(x['seq_len_cut'])+'_'+\n                                    str(x['site'])+'_'+\n                                    str(x['floor']),\n                                    axis=1)\n\nskf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True,\n                      random_state=SEED)\nfor i, (_, val_idx) in enumerate(skf.split(df_stra['stratify'],\n                                           df_stra['stratify'])):\n    tmp = np.zeros(df_stra.shape[0]).astype(bool)\n    tmp[val_idx] = True\n    df_stra[f'fold_{i}'] = tmp\ndf_stra.shape","metadata":{"id":"v_J34DnuldGB","outputId":"16210e78-d266-47bb-8156-87d612cb209c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\nprint('using device:', device)","metadata":{"id":"4tQsP70FldGC","outputId":"279f26a4-6036-4ec6-dfd2-bc03618b1072","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' create tensor for train data '''\ntr_id, tr_strg, tr_dlt, tr_imu, tr_site, tr_pos, tr_xy, tr_mask = \\\n    [],[],[],[],[],[],[],[]\nfor gid, grp in tqdm(trvl_data.groupby('seq_label'), ncols=60):\n    tr_id.append(torch.from_numpy(grp[ID_FEATS].values).long())\n    tr_strg.append(torch.from_numpy(grp[STRG_FEATS].values).float())\n    tr_dlt.append(torch.from_numpy(grp[DELTA_FEATS].values).float())\n    tr_imu.append(torch.from_numpy(grp[IMU_FEATS].values).float())\n    tr_site.append(torch.from_numpy(grp['site'].values).long())\n    tr_pos.append(torch.from_numpy(grp['len_pos'].values).float())\n    tr_xy.append(torch.from_numpy(grp[['x','y']].values).float())\n    ''' tr_mask: the tensor for padding '''\n    tr_mask.append(tr_id[-1].sum(dim=-1)==0)\ntr_id = rnn.pad_sequence(tr_id, batch_first=True).to(device)\ntr_strg = rnn.pad_sequence(tr_strg, batch_first=True).to(device)\ntr_dlt = rnn.pad_sequence(tr_dlt, batch_first=True).to(device)\ntr_imu = rnn.pad_sequence(tr_imu, batch_first=True).to(device)\ntr_site = rnn.pad_sequence(tr_site, batch_first=True).to(device)\ntr_pos = rnn.pad_sequence(tr_pos, batch_first=True).to(device)\ntr_xy = rnn.pad_sequence(tr_xy, batch_first=True).to(device)\ntr_mask = rnn.pad_sequence(tr_mask, batch_first=True,\n                           padding_value=True).to(device)","metadata":{"id":"4ky96RAwldGC","outputId":"d920ae30-faab-4e4b-f3d1-c86b81c5f6e0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ILNdatasets(torch.utils.data.Dataset):\n    def __init__(self, x_id, x_strg, x_dlt, x_imu, x_site, x_pos, x_mask,\n                 y, train=True):\n        self.train = train\n        self.x_id = x_id\n        self.x_strg = x_strg\n        self.x_dlt = x_dlt\n        self.x_imu = x_imu\n        self.x_site = x_site\n        self.x_pos = x_pos\n        self.x_mask = x_mask\n        self.y = y\n        self.datanum = len(y)\n    \n    def __len__(self):\n        return self.datanum\n    \n    def __getitem__(self, idx):\n        return (self.x_id[idx], self.x_strg[idx], self.x_dlt[idx], self.x_imu[idx],\n                self.x_site[idx], self.x_pos[idx], self.x_mask[idx],\n                self.y[idx])","metadata":{"id":"4VlODkArldGC","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PositionalEncoding(nn.Module):\n\n    def __init__(self, d_model, dropout=0.1):\n        super(PositionalEncoding, self).__init__()\n        self.dropout = nn.Dropout(p=dropout)\n        self.div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n\n    def forward(self, x, position, mask):\n        if position is None:\n            position = torch.cumsum(mask.logical_not(), dim=1)-1\n        else:\n            position *= mask.logical_not()\n        position = rearrange(position, 'bn seq -> bn seq 1')\n        \n        pe = torch.zeros(x.shape).to(x.device)\n        div_term = self.div_term.to(x.device)\n        pe[:,:,0::2] = torch.sin(position * div_term)\n        pe[:,:,1::2] = torch.cos(position * div_term)\n        x = x + pe\n        \n        return self.dropout(x)","metadata":{"id":"GAbcv9I68o_S","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' Special Thanks to \n    https://www.kaggle.com/shinomoriaoshi/iln-transformer-train-k1?scriptVersionId=59645688 '''\n\ndef clones(module, N):\n    \"Produce N identical layers.\"\n    return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])\n\ndef attention(query, key, value, key_padding_mask=None, attn_weight=None, dropout=None):\n    \"Compute 'Scaled Dot Product Attention'\"\n    d_k = query.size(-1)\n    scores = torch.matmul(query, key.transpose(-2, -1)) \\\n             / math.sqrt(d_k)\n    if key_padding_mask is not None:\n        scores = scores.masked_fill(key_padding_mask.unsqueeze(1).unsqueeze(2), -1e9)\n    if attn_weight is not None:\n        if len(attn_weight.shape) != len(scores.shape):\n            attn_weight = attn_weight.unsqueeze(-3)\n        scores -= attn_weight\n    p_attn = nn.functional.softmax(scores, dim=-1)\n    if dropout is not None:\n        p_attn = dropout(p_attn)\n    return torch.matmul(p_attn, value), p_attn\n\nclass MultiHeadedAttention(nn.Module):\n    def __init__(self, d_model, nhead, dropout=0.1):\n        \"Take in model size and number of heads.\"\n        super(MultiHeadedAttention, self).__init__()\n        assert d_model % nhead == 0\n        # We assume d_v always equals d_k\n        self.d_k = d_model // nhead\n        self.nhead = nhead\n        self.linears = clones(nn.Linear(d_model, d_model, bias=False), 4) # Q, K, V, last\n        self.attn = None\n        self.dropout = nn.Dropout(p=dropout)\n\n    def forward(self, query, key, value, key_padding_mask=None, attn_weight=None):\n        \"Implements Figure 2\"\n        nbatches = query.size(0)\n\n        # 1) Do all the linear projections in batch from d_model => h x d_k\n        query, key, value = \\\n            [l(x).view(nbatches, -1, self.nhead, self.d_k).transpose(1, 2)\n             for l, x in zip(self.linears, (query, key, value))]\n\n        # 2) Apply attention on all the projected vectors in batch.\n        x, self.attn = attention(query, key, value, key_padding_mask=key_padding_mask,\n                                 attn_weight=attn_weight,\n                                 dropout=self.dropout)\n\n        # 3) \"Concat\" using a view and apply a final linear.\n        x = x.transpose(1, 2).contiguous() \\\n            .view(nbatches, -1, self.nhead * self.d_k)\n        return self.linears[-1](x)\n\n\nclass PositionwiseFeedForward(nn.Module):\n    \"Implements FFN equation.\"\n    def __init__(self, d_model, d_ff, dropout=0.1):\n        super(PositionwiseFeedForward, self).__init__()\n        self.w_1 = nn.Linear(d_model, d_ff)\n        self.w_2 = nn.Linear(d_ff, d_model)\n        self.dropout = nn.Dropout(dropout)\n\n    def forward(self, x):\n        return self.w_2(self.dropout(nn.functional.relu(self.w_1(x))))\n    \nclass CustomEncoderLayer(nn.Module):\n    \"\"\"\n    Single Encoder block of SAINT\n    \"\"\"\n    def __init__(self, d_model, nhead, dim_feedforward = 1024, dropout = 0.1):\n        super().__init__()\n        self._self_attn = MultiHeadedAttention(d_model, nhead, dropout)\n        self._ffn = PositionwiseFeedForward(d_model, dim_feedforward, dropout)\n        self._layernorms = clones(nn.LayerNorm(d_model, eps=1e-6), 2)\n        self._dropout = nn.Dropout(dropout)\n\n    def forward(self, src, key_padding_mask = None, attn_weight = None):\n        \"\"\"\n        query: question embeddings\n        key: interaction embeddings\n        \"\"\"\n        # self-attention block\n        src2 = self._self_attn(query=src, key=src, value=src, key_padding_mask=key_padding_mask,\n                               attn_weight=attn_weight)\n        src = src + self._dropout(src2)\n        src = self._layernorms[0](src)\n        src2 = self._ffn(src)\n        src = src + self._dropout(src2)\n        src = self._layernorms[1](src)\n        return src","metadata":{"id":"DHVks4bwWTcW","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class CorrectWithDelta(nn.Module):\n\n#     def __init__(self):\n#         super(CorrectWithDelta, self).__init__()\n\n#     def forward(self, x, c, x_dlt, x_pos, x_mask):\n#         x_dlt = (x_dlt - x_dlt[:,0:1,:]) * x_mask.unsqueeze(-1).logical_not()\n#         x_rel = x_dlt.cumsum(dim=1)\n#         XX = (rearrange(x_rel, 'bn seq d -> bn seq 1 d')\n#              -rearrange(x_rel, 'bn seq d -> bn 1 seq d'))\n#         XX += rearrange(x, 'bn seq d -> bn 1 seq d')\n#         attn = rearrange(c, 'bn seq -> bn 1 seq 1')\n#         attn_wt = (rearrange(x_pos, 'bn seq -> bn seq 1 1')\n#                   -rearrange(x_pos, 'bn seq -> bn 1 seq 1')).abs()\n#         attn = attn - torch.log(1.0 + attn_wt)\n#         attn -= (1e8 * rearrange(x_mask, 'bn seq -> bn 1 seq 1'))\n#         x = (XX * nn.functional.softmax(attn, dim=2)).sum(dim=2)\n#         return x","metadata":{"id":"_0DBcYwKLQZo","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' Learning Model '''\nclass ILNnet(nn.Module):\n    def __init__(self, input_dim, id_num_embd, id_embd_dim,\n                 strg_dim, site_num_embd, imu_dim,\n                 ninp, nhead, nhid, nlayers, dropout):\n        super(ILNnet, self).__init__()\n \n        ''' embedding and concat '''\n        self.id_embd = nn.Embedding(id_num_embd, id_embd_dim,\n                                    padding_idx=0)\n        self.strg_ln = nn.LayerNorm(input_dim)\n        self.strg_lin = nn.Linear(input_dim, strg_dim)\n        self.site_embd = nn.Embedding(site_num_embd, ninp)\n        cat_dim = input_dim*id_embd_dim + strg_dim\n        self.main_seq1 = nn.Sequential(nn.Linear(cat_dim, ninp),\n                                       nn.LayerNorm(ninp),\n                                       nn.ReLU(),\n                                       nn.Dropout(dropout))\n        self.dltimu_lin = nn.Linear(4+imu_dim, ninp)\n \n        ''' main stream(1) TRANSFORMER with positional decay '''\n        self.pos_encoder = PositionalEncoding(ninp, dropout)\n        # encoder_layers = nn.TransformerEncoderLayer(ninp, nhead, nhid, dropout)\n        # self.transformer_encoder = nn.TransformerEncoder(encoder_layers, nlayers)\n        self.attn_wt_coef = nn.Parameter(torch.clamp(torch.randn(nhead), min=1e-3))\n        self.customTR1 = CustomEncoderLayer(ninp, nhead, nhid, dropout)\n        self.customTR2 = CustomEncoderLayer(ninp, nhead, nhid, dropout)\n \n        ''' main stream(2) LSTM '''\n        self.lstm1 = nn.LSTM(batch_first=True,\n                             bidirectional=True, num_layers=1,\n                             input_size=ninp, hidden_size=ninp//2)\n        self.lstm2 = nn.LSTM(batch_first=True,\n                             bidirectional=True, num_layers=1,\n                             input_size=ninp, hidden_size=ninp//2)\n        # self.lstm3 = nn.LSTM(batch_first=True,\n        #                      bidirectional=True, num_layers=1,\n        #                      input_size=ninp, hidden_size=ninp//2)\n        self.main_seq2 = nn.Sequential(nn.Linear(ninp,ninp),\n                                       nn.LayerNorm(ninp),\n                                       nn.Dropout(dropout),\n                                       nn.Linear(ninp,2))\n        self.init_weights()\n        # self.correct = CorrectWithDelta()\n    \n    def init_weights(self):\n        self.id_embd.weight.data.zero_()\n        self.site_embd.weight.data.zero_()\n    \n    def forward(self, x_id, x_strg, x_dlt, x_imu, x_site, x_pos, x_mask):\n        \n        ''' embedding and concat '''\n        x_id = self.id_embd(x_id)\n        x_id = rearrange(x_id, 'bn seq d1 d2 -> bn seq (d1 d2)')\n        x_strg = self.strg_ln(x_strg)\n        x_strg = torch.relu(self.strg_lin(x_strg))\n        x_site = self.site_embd(x_site)\n        x = torch.cat((x_id, x_strg), dim=-1)\n        x = self.main_seq1(x) + x_site\n \n        ''' concat delta & IMU '''\n        x_dltimu = torch.cat((x_dlt, x_imu),-1)\n        x_dltimu = torch.relu(self.dltimu_lin(x_dltimu))\n        x = x + x_dltimu\n        \n        ''' main stream(1) TRANSFORMER with positional decay '''\n        xt = self.pos_encoder(x, None, x_mask)\n        attn_wt = (rearrange(x_pos, 'bn seq -> bn 1 seq 1')\n                  -rearrange(x_pos, 'bn seq -> bn 1 1 seq')).abs()\n        attn_coef = rearrange(self.attn_wt_coef, 'nh -> 1 nh 1 1')\n        xt = self.customTR1(xt, x_mask, torch.log(1.0 + attn_wt) * attn_coef)\n        xt = self.customTR2(xt, x_mask, torch.log(1.0 + attn_wt) * attn_coef)\n        x = x + xt\n        \n        ''' main stream(2) LSTM '''\n        lengths = x_mask.logical_not().sum(dim=1).to('cpu')\n        x = rnn.pack_padded_sequence(x, lengths, batch_first=True,\n                                     enforce_sorted=False)\n        x,_ = self.lstm1(x)\n        x,_ = self.lstm2(x)\n        # x,_ = self.lstm3(x)\n        x = rnn.pad_packed_sequence(x, batch_first=True)[0]\n        if x.size(1)!=x_mask.size(1):\n            pad_len = x_mask.size(1)-x.size(1)\n            x = torch.cat([x, torch.zeros(x.size(0),pad_len,x.size(2)).to(device)],\n                          dim=1)\n            assert x.size(1)==x_mask.size(1)\n        x = self.main_seq2(x)\n \n        # ''' correct with delta '''\n        # c = torch.relu(x[:,:,2])\n        # x = x[:,:,0:2]\n        # x = self.correct(x, c, x_dlt, x_pos, x_mask)\n \n        return x","metadata":{"id":"O4b3yBsYldGD","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ILN_loss(output, filter_, delta, label):\n    \n    metr1 = (output[:,:,:2]-label[:,:,:2]).square()\n    metr1 = metr1.sum(dim=-1)+1e-6\n    metr1 = metr1.sqrt().sum()\n \n    # metr2 = ((output[:,1:,:2]-output[:,:-1,:2])\n    #          -delta[:,1:,:])**2\n    # metr2 = metr2.sum(dim=-1)+1e-6\n    # metr2 = (metr2.sqrt()*filter_[:,1:]).sum()\n \n    # metr3 = ((output[:,1:,:2]-output[:,:-1,:2]).square().sum(dim=-1)+1e-6).sqrt() \\\n    #         -(delta[:,1:,:].square().sum(dim=-1)+1e-6).sqrt()\n    # metr3 = (metr3 * filter_[:,1:]).abs().sum()\n \n    # loss = (metr1 + 10.0 * metr3)/filter_.sum()\n    metric = metr1/filter_.sum()\n    loss = metric\n \n    return loss, metric","metadata":{"id":"_CGA2HMLldGF","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LearningRateScheduler:\n    def __init__(self, lr:list, \n                 switch_epoch: list):\n        self.lr = lr\n        self.switch_epoch = switch_epoch\n \n    def __call__(self, epoch:int):\n        idx = [i>epoch for i\n               in self.switch_epoch+[1e9]].index(True)\n        return self.lr[idx]","metadata":{"id":"6YMB_92RldGH","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' This function plays the same role as model.summary() of keras '''\ndef param_count(model, print_all=False):\n    if print_all:\n        print('-'*80)\n    psum=0\n    for n,p in model.named_parameters():\n        if p.requires_grad:\n            if print_all:\n                print(f'{n}:')\n                print(f'     params:{p.numel():,},  {p.shape}')\n#                 print(p)\n            psum += p.numel()\n    print(f'Total params: {psum:,}')\n    if print_all:\n        print('-'*80)","metadata":{"id":"9vcNN9PtldGH","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_val(net, trainloader, valloader, bsz):\n    ''' train '''\n    loss_sum, metric_sum = 0., 0.\n    net.train()\n    optimizer.zero_grad()\n    # note: \"prefix\" and \"descpost\" are the variable just for tqdm\n    with tqdm(trainloader, ncols=80) as pbar:\n        prefix = f\"epoch {i_epoch+1} train\"\n        pbar.set_description(prefix)\n        descpost = None\n        for i, (tr_id, tr_strg, tr_dlt, tr_imu, tr_site, tr_pos,\n                tr_mask, tr_xy) in enumerate(pbar):\n\n            output = net(tr_id, tr_strg, tr_dlt, tr_imu, tr_site, tr_pos, tr_mask)\n            filter_ = tr_mask.logical_not()\n            output *= filter_.unsqueeze(-1)\n            \n            loss, metric = ILN_loss(output, filter_, tr_dlt, tr_xy)\n            loss.backward()\n            optimizer.step()\n            optimizer.zero_grad()\n                \n            loss_sum += loss.item()\n            metric_sum += metric.item()\n            descpost = f'{loss_sum/(i+1):.2f}/{metric_sum/(i+1):.2f}'\n            pbar.set_postfix({\"l/m\":descpost})\n        history['epoch'].append(i_epoch+1)\n        history['train_loss'].append(loss_sum/(i+1))\n        history['train_metric'].append(metric_sum/(i+1))\n\n    ''' validation '''\n    with torch.no_grad():\n        loss_sum, metric_sum = 0., 0.\n        net.eval()\n        with tqdm(valloader, ncols=80) as pbar:\n            prefix = ' '*(len(prefix)-10)+'validation'\n            pbar.set_description(prefix)\n            descpost = None\n            for i, (vl_id, vl_strg, vl_dlt, vl_imu, vl_site, vl_pos, vl_mask, vl_xy)\\\n              in enumerate(pbar):\n\n                output = net(vl_id, vl_strg, vl_dlt, vl_imu, vl_site, vl_pos, vl_mask)\n                filter_ = vl_mask.logical_not()\n                output *= filter_.unsqueeze(-1)\n                \n                loss, metric = ILN_loss(output, filter_, vl_dlt, vl_xy)\n                loss_sum += loss.item()\n                metric_sum += metric.item()\n                descpost = f'{loss_sum/(i+1):.2f}/{metric_sum/(i+1):.2f}'\n                pbar.set_postfix({\"l/m\":descpost})\n    history['val_loss'].append(loss_sum/(i+1))\n    history['val_metric'].append(metric_sum/(i+1))\n    lr_scheduler.step()\n    \n    return net, history['val_metric'][-1]","metadata":{"id":"uXlscXnmldGH","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not inference_only:\n    for i_fold in range(N_SPLITS):\n        print('='*20 + f' FOLD {i_fold} ' + '='*20)\n\n        ''' model comopile '''\n        net = ILNnet(NUM_FEATS, len(ID_vocab)+1, 128, 1280, 24, len(IMU_FEATS),\n                     ninp=256, nhead=2, nhid=512, nlayers=2, dropout=0.1)\n        net = net.to(device)\n    #     param_count(net, print_all=True)\n\n        mask = df_stra[f'fold_{i_fold}'].values\n    #     trn_idx = np.arange(df_stra.shape[0]) # train with all data\n        trn_idx = np.where(~mask)[0]\n        val_idx = np.where(mask)[0]\n\n        trainvaldata = ILNdatasets(tr_id, tr_strg, tr_dlt, tr_imu, tr_site,\n                                   tr_pos, tr_mask, tr_xy)\n        traindata = torch.utils.data.dataset.Subset(trainvaldata,\n                                                    trn_idx)\n        valdata = torch.utils.data.dataset.Subset(trainvaldata,\n                                                  val_idx)\n        trainloader = torch.utils.data.DataLoader(traindata, \n                                                  # pin_memory=True,\n                                                  shuffle=True,\n                                                  batch_size=batch_size)\n        valloader = torch.utils.data.DataLoader(valdata, \n                                                # pin_memory=True,\n                                                shuffle=True,\n                                                batch_size=batch_size)\n\n        optimizer = optim.Adam(net.parameters(), lr=5e-4)\n        lr_scheduler_func = LearningRateScheduler([1.0, 0.1],\n                                                  [int(0.75*n_epoch)])\n        lr_scheduler = optim.lr_scheduler.LambdaLR(optimizer,\n                           lr_lambda=lr_scheduler_func)\n\n        history = {'epoch':[], 'train_loss': [], 'val_loss': [],\n                   'train_metric': [], 'val_metric': []}\n\n        ''' train and validation '''\n        time.sleep(1)\n        early_stop = 0\n        min_metric = 1000.\n        for i_epoch in range(n_epoch):\n            net, es_metric = train_val(net, trainloader, valloader, batch_size)\n\n            ''' history json dump '''\n            with open(f'_history_{model_ver}_fold{i_fold}_latest.json', 'w') as f:\n                json.dump(history, f, indent=4)\n\n    #         ''' early stopping '''\n    #         es_metric = round(es_metric,2)\n    #         if es_metric<min_metric:\n    #             min_metric=es_metric\n    #             early_stop=0\n    #         else:\n    #             early_stop+=1 \n    #         if early_stop>10 and i_epoch>int(n_epoch*0.5):\n    #             print('early stopping.')\n    #             break\n\n        i_epoch += 1\n\n        ''' model output '''\n        model_path = f'{model_ver}_fold{i_fold}_epoch{i_epoch}.pth'\n        torch.save(net.state_dict(), model_path)","metadata":{"id":"mQyObkT1ldGH","outputId":"469122a7-565b-4f31-f384-8c5888e7737b","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' create tensor for test data '''\nte_id, te_strg, te_dlt, te_imu, te_site, te_pos, te_mask = \\\n    [],[],[],[],[],[],[]\nfor gid, grp in tqdm(test_data.groupby('seq_label'), ncols=60):\n    te_id.append(torch.from_numpy(grp[ID_FEATS].values).long())\n    te_strg.append(torch.from_numpy(grp[STRG_FEATS].values).float())\n    te_dlt.append(torch.from_numpy(grp[DELTA_FEATS].values).float())\n    te_imu.append(torch.from_numpy(grp[IMU_FEATS].values).float())\n    te_site.append(torch.from_numpy(grp['site'].values).long())\n    te_pos.append(torch.from_numpy(grp['len_pos'].values).float())\n    ''' te_mask: the tensor for padding '''\n    te_mask.append(te_id[-1].sum(dim=-1)==0)\nte_id = rnn.pad_sequence(te_id, batch_first=True).to(device)\nte_strg = rnn.pad_sequence(te_strg, batch_first=True).to(device)\nte_dlt = rnn.pad_sequence(te_dlt, batch_first=True).to(device)\nte_imu = rnn.pad_sequence(te_imu, batch_first=True).to(device)\nte_site = rnn.pad_sequence(te_site, batch_first=True).to(device)\nte_pos = rnn.pad_sequence(te_pos, batch_first=True).to(device)\nte_mask = rnn.pad_sequence(te_mask, batch_first=True,\n                           padding_value=True).to(device)","metadata":{"id":"1Uf9NWQbldGI","outputId":"88c2cc41-50d8-4b36-90f0-3b90433d428e","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.set_index('site_path_timestamp', inplace=True)\n''' model comopile '''\nnet = ILNnet(NUM_FEATS, len(ID_vocab)+1, 128, 1280, 24, len(IMU_FEATS),\n             ninp=256, nhead=2, nhid=512, nlayers=2, dropout=0.1)\nnet = net.to(device)\n\n''' prediction '''\nxy_all = []\nfor i_fold in range(N_SPLITS):\n    print('='*20 + f' FOLD {i_fold} ' + '='*20)\n    time.sleep(1)\n\n    ''' model load '''\n    with open(f'{data_dir}_history_{model_ver}_fold{i_fold}_latest.json','r') as f:\n        history = json.load(f)\n    i_epoch = history['epoch'][-1]\n    model_path = f'{data_dir}{model_ver}_fold{i_fold}_epoch{i_epoch}.pth'\n    net.load_state_dict(torch.load(model_path))\n    \n    net.eval()\n    pred = []\n    with torch.no_grad():\n        for te_id_ch, te_strg_ch, te_dlt_ch, te_imu_ch, \\\n            te_site_ch, te_pos_ch, te_mask_ch in \\\n            (zip(tqdm(te_id.split(batch_size),ncols=60),\n                 te_strg.split(batch_size), te_dlt.split(batch_size), \n                 te_imu.split(batch_size), te_site.split(batch_size), \n                 te_pos.split(batch_size), te_mask.split(batch_size))):\n            output = net(te_id_ch, te_strg_ch, te_dlt_ch, te_imu_ch, te_site_ch,\n                         te_pos_ch, te_mask_ch)\n            output  = rearrange(output, 'bn seq d -> (bn seq) d')\n            filter_ = rearrange(te_mask_ch.logical_not(),\n                                'bn seq -> (bn seq)')\n            pred.append(output[filter_])\n    test_data[['x','y']] = torch.cat(pred, dim=0).to('cpu').numpy()\n    test_data['floor']=0\n    \n    subm = pd.read_csv(subm_file, index_col=0)\n    subm.loc[:,:]=np.nan\n    all_preds = pd.concat([test_data[subm.columns].copy() ,subm])\n    all_preds.sort_index(inplace=True)\n    df_tmp = pd.Series(all_preds.index).str.split('_', expand=True)\n    df_tmp.index = all_preds.index\n    df_tmp.columns = ['site','path','timestamp']\n    all_preds = pd.concat([all_preds, df_tmp],axis=1)\n    del df_tmp; gc.collect()\n    all_preds['timestamp'] = all_preds['timestamp'].astype('int')\n\n    tmp = []\n    for gid, gdf in tqdm(all_preds.groupby('path'),ncols=60):\n        gdf.reset_index(drop=False, inplace=True)\n        gdf.set_index('timestamp', inplace=True)\n        gdf.sort_index(inplace=True)\n#         for itm in ['x','y']:\n#             gdf[itm].interpolate('nearest', inplace=True)\n#             gdf[itm].fillna(method='bfill', inplace=True)\n#             gdf[itm].fillna(method='ffill', inplace=True)\n        gdf[['x','y']] = gdf[['x','y']].interpolate(limit_direction='both',\n                                                    method='index')\n        gdf.set_index('site_path_timestamp', inplace=True)\n        tmp.append(gdf[['floor','x','y']])\n\n    all_preds = pd.concat(tmp).groupby(level=0).mean().reindex(subm.index)\n    \n    simple_accurate_99 = pd.read_csv('../input/simple-99-accurate-floor-model/submission.csv')\n    all_preds['floor'] = simple_accurate_99['floor'].values\n    \n    all_preds.to_csv(f'{model_ver}_fold{i_fold}_submission.csv')\n    xy_all.append(all_preds[['x','y']].values)\n    \nall_preds[['x','y']] = np.stack(xy_all, axis=-1).mean(axis=-1)\nall_preds.to_csv(f'./{model_ver}_fold_all_submission.csv')","metadata":{"id":"FHKuJJcdldGI","outputId":"bae25d94-54f6-4141-e4da-db2ff32000f0","trusted":true},"execution_count":null,"outputs":[]}]}