{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ============================\n# library\n# ============================\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nfrom torch.nn import LayerNorm,TransformerEncoder\nfrom torch.utils.data import DataLoader, TensorDataset, Dataset\nimport pickle\nfrom tqdm import tqdm\nimport gc\nimport glob\nfrom sklearn.preprocessing import StandardScaler,RobustScaler","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-07T13:25:00.701474Z","iopub.execute_input":"2023-05-07T13:25:00.702081Z","iopub.status.idle":"2023-05-07T13:25:00.70899Z","shell.execute_reply.started":"2023-05-07T13:25:00.702043Z","shell.execute_reply":"2023-05-07T13:25:00.707584Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============================\n# constant\n# ============================\nSUB_PATH = \"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/sample_submission.csv\"\nDEFOG_DATA_PATH = \"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/test/defog/*.csv\"\nTDCSFOG_DATA_PATH = \"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/test/tdcsfog/*.csv\"","metadata":{"execution":{"iopub.status.busy":"2023-05-07T13:25:01.701049Z","iopub.execute_input":"2023-05-07T13:25:01.701442Z","iopub.status.idle":"2023-05-07T13:25:01.707809Z","shell.execute_reply.started":"2023-05-07T13:25:01.701409Z","shell.execute_reply":"2023-05-07T13:25:01.705972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============================\n# settings\n# ============================\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nbs = 32","metadata":{"execution":{"iopub.status.busy":"2023-05-07T13:25:01.822813Z","iopub.execute_input":"2023-05-07T13:25:01.823185Z","iopub.status.idle":"2023-05-07T13:25:01.829928Z","shell.execute_reply.started":"2023-05-07T13:25:01.823153Z","shell.execute_reply":"2023-05-07T13:25:01.828647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============================\n# settings\n# ============================\ntdcsfog_path1 = [f\"/kaggle/input/fog-ex143/ex143_{i}.pth\" for i in range(5)] # len 3000 cv TdcsfogRnnModel cv\ntdcsfog_path3_1 = [f\"/kaggle/input/fog-ex145/ex145_{i}.pth\" for i in range(5)] # len 3000 TdcsfogRnnModel  \ntdcsfog_path3_2 = [f\"/kaggle/input/fog-ex146/ex146_{i}.pth\" for i in range(5)] # len 3000 TdcsfogRnnModel \ntdcsfog_path3_3 = [f\"/kaggle/input/fog-ex147/ex147_{i}.pth\" for i in range(5)] # len 3000 TdcsfogRnnModel cv\ntdcsfog_path4_1 = [f\"/kaggle/input/fog-ex182/ex182_{i}.pth\" for i in range(5)] # len 3000 TdcsfogRnnModel  \ntdcsfog_path4_2 = [f\"/kaggle/input/fog-ex183/ex183_{i}.pth\" for i in range(5)] # len 3000 TdcsfogRnnModel \ntdcsfog_path4_3 = [f\"/kaggle/input/fog-ex184/ex184_{i}.pth\" for i in range(5)] # len 3000 TdcsfogRnnModel cv\n\ndefog_path2 = [f\"/kaggle/input/fog-ex153/ex153_{i}.pth\" for i in range(5)] # len 30000 defog1 \ndefog_path4 = [f\"/kaggle/input/fog-ex179/ex179_{i}.pth\" for i in range(5)] # len 30000 defog1\ndefog_path5 = [f\"/kaggle/input/fog-ex185/ex185_{i}.pth\" for i in range(5)] # len 30000 defog1\ndefog_path6 = [f\"/kaggle/input/fog-ex204/ex204_{i}.pth\" for i in range(5)] # len 30000 defog2\ndefog_path7 = [f\"/kaggle/input/pd-exp238/fold{i}_best.pth\" for i in [0, 1, 2, 3, 4]]  # len 30000 Defog3Model","metadata":{"execution":{"iopub.status.busy":"2023-05-07T13:25:02.100998Z","iopub.execute_input":"2023-05-07T13:25:02.10251Z","iopub.status.idle":"2023-05-07T13:25:02.111959Z","shell.execute_reply.started":"2023-05-07T13:25:02.10246Z","shell.execute_reply":"2023-05-07T13:25:02.110131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============================\n# Functions\n# ============================\n\ndef preprocess(numerical_array, \n               mask_array,\n               ):\n    \n    attention_mask = mask_array == 0\n\n    return {\n        'input_data_numerical_array': numerical_array,\n        'input_data_mask_array': mask_array,\n        'attention_mask': attention_mask,\n    }\n\nclass FogDataset(Dataset):\n    def __init__(self, numerical_array, \n                 mask_array,\n                 train = True, y = None):\n        self.numerical_array = numerical_array\n        self.mask_array = mask_array\n        self.train = train\n        self.y = y\n    \n    def __len__(self):\n        return len(self.numerical_array)\n\n    def __getitem__(self, item):\n        data = preprocess(\n            self.numerical_array[item],\n            self.mask_array[item],\n            \n        )\n\n        # Return the processed data where the lists are converted to `torch.tensor`s\n        if self.train : \n            return {\n              'input_data_numerical_array': torch.tensor(data['input_data_numerical_array'],dtype=torch.float32),\n              'input_data_mask_array':torch.tensor(data['input_data_mask_array'], dtype=torch.long),  \n              'attention_mask': torch.tensor(data[\"attention_mask\"], dtype=torch.bool),\n              \"y\":torch.tensor(self.y[item], dtype=torch.float32)\n               }\n        else:\n            return {\n             'input_data_numerical_array': torch.tensor(data['input_data_numerical_array'],dtype=torch.float32),\n              'input_data_mask_array':torch.tensor(data['input_data_mask_array'], dtype=torch.long),  \n              'attention_mask': torch.tensor(data[\"attention_mask\"], dtype=torch.bool),\n               }\n        \n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-05-07T13:25:02.266476Z","iopub.execute_input":"2023-05-07T13:25:02.266919Z","iopub.status.idle":"2023-05-07T13:25:02.294798Z","shell.execute_reply.started":"2023-05-07T13:25:02.266884Z","shell.execute_reply":"2023-05-07T13:25:02.293581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ================================\n# tdcsfog\n# ================================\nclass TdcsfogRnnModel(nn.Module):\n    def __init__(\n        self, dropout=0.2,\n        input_numerical_size=12,\n        numeraical_linear_size = 64,\n        model_size = 128,\n        linear_out = 128,\n        out_size=3):\n        super(TdcsfogRnnModel, self).__init__()\n        self.numerical_linear  = nn.Sequential(\n                nn.Linear(input_numerical_size, numeraical_linear_size),\n                nn.LayerNorm(numeraical_linear_size)\n            )\n        \n        self.rnn = nn.GRU(numeraical_linear_size, model_size,\n                            num_layers = 2, \n                            batch_first=True,\n                            bidirectional=True)\n        self.linear_out  = nn.Sequential(\n                nn.Linear(model_size*2, \n                          linear_out),\n                nn.LayerNorm(linear_out),\n                nn.ReLU(),\n                nn.Dropout(dropout),\n                nn.Linear(linear_out, \n                          out_size))\n        self._reinitialize()\n        \n    def _reinitialize(self):\n        \"\"\"\n        Tensorflow/Keras-like initialization\n        \"\"\"\n        for name, p in self.named_parameters():\n            if 'rnn' in name:\n                if 'weight_ih' in name:\n                    nn.init.xavier_uniform_(p.data)\n                elif 'weight_hh' in name:\n                    nn.init.orthogonal_(p.data)\n                elif 'bias_ih' in name:\n                    p.data.fill_(0)\n                    # Set forget-gate bias to 1\n                    n = p.size(0)\n                    p.data[(n // 4):(n // 2)].fill_(1)\n                elif 'bias_hh' in name:\n                    p.data.fill_(0)\n    \n    def forward(self, numerical_array,\n                mask_array,\n                attention_mask):\n        \n        numerical_embedding = self.numerical_linear(numerical_array)\n        output,_ = self.rnn(numerical_embedding)\n        output = self.linear_out(output)\n        return output\n    \n    \nclass TdcsfogRnnModel2(nn.Module):\n    def __init__(\n        self, dropout=0.2,\n        input_numerical_size=12,\n        numeraical_linear_size = 64,\n        model_size = 128,\n        linear_out = 128,\n        out_size=3):\n        super(TdcsfogRnnModel2, self).__init__()\n        self.numerical_linear  = nn.Sequential(\n                nn.Linear(input_numerical_size, numeraical_linear_size),\n                nn.LayerNorm(numeraical_linear_size)\n            )\n        \n        self.rnn = nn.GRU(numeraical_linear_size, model_size,\n                            num_layers = 2, \n                            batch_first=True,\n                            bidirectional=True)\n        self.linear_out  = nn.Sequential(\n                nn.Linear(model_size*2, \n                          linear_out),\n                nn.LayerNorm(linear_out),\n                nn.ReLU(),\n                nn.Dropout(dropout))\n        self.out1 = nn.Linear(linear_out, \n                          out_size)\n        self.out2 = nn.Linear(linear_out, \n                          out_size)\n\n        \n        self._reinitialize()\n        \n    def _reinitialize(self):\n        \"\"\"\n        Tensorflow/Keras-like initialization\n        \"\"\"\n        for name, p in self.named_parameters():\n            if 'rnn' in name:\n                if 'weight_ih' in name:\n                    nn.init.xavier_uniform_(p.data)\n                elif 'weight_hh' in name:\n                    nn.init.orthogonal_(p.data)\n                elif 'bias_ih' in name:\n                    p.data.fill_(0)\n                    # Set forget-gate bias to 1\n                    n = p.size(0)\n                    p.data[(n // 4):(n // 2)].fill_(1)\n                elif 'bias_hh' in name:\n                    p.data.fill_(0)\n    \n    def forward(self, numerical_array,\n                mask_array,\n                attention_mask):\n        \n        numerical_embedding = self.numerical_linear(numerical_array)\n        output,_ = self.rnn(numerical_embedding)\n        output = self.linear_out(output)\n        output1 = self.out1(output)\n        output2 = self.out2(output)\n        return output1,output2","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DefogRnnModel(nn.Module):\n    def __init__(\n        self, dropout=0.2,\n        input_numerical_size=9,\n        numeraical_linear_size = 64,\n        model_size = 128,\n        linear_out = 128,\n        out_size=3):\n        super(DefogRnnModel, self).__init__()\n        self.numerical_linear  = nn.Sequential(\n                nn.Linear(input_numerical_size, numeraical_linear_size),\n                nn.LayerNorm(numeraical_linear_size)\n            )\n        \n        self.rnn = nn.GRU(numeraical_linear_size, model_size,\n                            num_layers = 2, \n                            batch_first=True,\n                            bidirectional=True)\n        self.linear_out  = nn.Sequential(\n                nn.Linear(model_size*2, \n                          linear_out),\n                nn.LayerNorm(linear_out),\n                nn.ReLU(),\n                nn.Dropout(dropout),\n                nn.Linear(linear_out, \n                          out_size))\n        self._reinitialize()\n        \n    def _reinitialize(self):\n        \"\"\"\n        Tensorflow/Keras-like initialization\n        \"\"\"\n        for name, p in self.named_parameters():\n            if 'rnn' in name:\n                if 'weight_ih' in name:\n                    nn.init.xavier_uniform_(p.data)\n                elif 'weight_hh' in name:\n                    nn.init.orthogonal_(p.data)\n                elif 'bias_ih' in name:\n                    p.data.fill_(0)\n                    # Set forget-gate bias to 1\n                    n = p.size(0)\n                    p.data[(n // 4):(n // 2)].fill_(1)\n                elif 'bias_hh' in name:\n                    p.data.fill_(0)\n    \n    def forward(self, numerical_array,\n                mask_array,\n                attention_mask):\n        \n        numerical_embedding = self.numerical_linear(numerical_array)\n        output,_ = self.rnn(numerical_embedding)\n        output = self.linear_out(output)\n        return output\n    \n    \nclass DefogRnnModel2(nn.Module):\n    def __init__(\n        self, dropout=0.2,\n        input_numerical_size=9,\n        numeraical_linear_size = 96,\n        model_size = 256,\n        linear_out = 256,\n        out_size=3):\n        super(DefogRnnModel2, self).__init__()\n        self.numerical_linear  = nn.Sequential(\n                nn.Linear(input_numerical_size, numeraical_linear_size),\n                nn.LayerNorm(numeraical_linear_size)\n            )\n        \n        self.rnn = nn.GRU(numeraical_linear_size, model_size,\n                            num_layers = 2, \n                            batch_first=True,\n                            bidirectional=True)\n        self.linear_out  = nn.Sequential(\n                nn.Linear(model_size*2, \n                          linear_out),\n                nn.LayerNorm(linear_out),\n                nn.ReLU(),\n                nn.Dropout(dropout),\n                nn.Linear(linear_out, \n                          out_size))\n        self._reinitialize()\n        \n    def _reinitialize(self):\n        \"\"\"\n        Tensorflow/Keras-like initialization\n        \"\"\"\n        for name, p in self.named_parameters():\n            if 'rnn' in name:\n                if 'weight_ih' in name:\n                    nn.init.xavier_uniform_(p.data)\n                elif 'weight_hh' in name:\n                    nn.init.orthogonal_(p.data)\n                elif 'bias_ih' in name:\n                    p.data.fill_(0)\n                    # Set forget-gate bias to 1\n                    n = p.size(0)\n                    p.data[(n // 4):(n // 2)].fill_(1)\n                elif 'bias_hh' in name:\n                    p.data.fill_(0)\n    \n    def forward(self, numerical_array,\n                mask_array,\n                attention_mask):\n        \n        numerical_embedding = self.numerical_linear(numerical_array)\n        output,_ = self.rnn(numerical_embedding)\n        output = self.linear_out(output)\n        return output\n    \n    \nclass Defog3Model(nn.Module):\n    def __init__(\n        self,\n        dropout=0.2,\n        input_numerical_size=9,\n        numeraical_linear_size=64,\n        model_size=128,\n        linear_out=128,\n        out_size=3,\n    ):\n        super(Defog3Model, self).__init__()\n        self.numerical_linear = nn.Sequential(\n                nn.Linear(input_numerical_size, numeraical_linear_size),\n                nn.LayerNorm(numeraical_linear_size)\n            )\n        self.lstm = nn.GRU(\n            numeraical_linear_size,\n            model_size,\n            num_layers=2,\n            batch_first=True,\n            bidirectional=True,\n        )\n        self.linear_out = nn.Sequential(\n            nn.Linear(model_size*2, linear_out),\n            nn.LayerNorm(linear_out),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(linear_out, out_size),\n        )\n        self._reinitialize()\n\n    def _reinitialize(self):\n        \"\"\"\n        Tensorflow/Keras-like initialization\n        \"\"\"\n        for name, p in self.named_parameters():\n            if 'lstm' in name:\n                if 'weight_ih' in name:\n                    nn.init.xavier_uniform_(p.data)\n                elif 'weight_hh' in name:\n                    nn.init.orthogonal_(p.data)\n                elif 'bias_ih' in name:\n                    p.data.fill_(0)\n                    # Set forget-gate bias to 1\n                    n = p.size(0)\n                    p.data[(n // 4):(n // 2)].fill_(1)\n                elif 'bias_hh' in name:\n                    p.data.fill_(0)\n\n    def forward(self, numerical_array, mask_array, attention_mask):\n        numerical_embedding = self.numerical_linear(numerical_array)\n        output, _ = self.lstm(numerical_embedding)\n        output = self.linear_out(output)\n        return output","metadata":{"execution":{"iopub.status.busy":"2023-05-07T13:25:02.321945Z","iopub.execute_input":"2023-05-07T13:25:02.322312Z","iopub.status.idle":"2023-05-07T13:25:02.351413Z","shell.execute_reply.started":"2023-05-07T13:25:02.32228Z","shell.execute_reply":"2023-05-07T13:25:02.350095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_pred(test_loader,model):\n    test_preds = []\n    tk0 = tqdm(test_loader, total=len(test_loader))\n    with torch.no_grad():  # Do not calculate gradient since we are only predicting\n        # Predicting on validation set\n        for d in tk0:\n            input_data_numerical_array = d['input_data_numerical_array'].to(device)\n            input_data_mask_array = d['input_data_mask_array'].to(device)\n            attention_mask = d['attention_mask'].to(device)\n            output = model(input_data_numerical_array, \n                       input_data_mask_array,\n                       attention_mask)\n            test_preds.append(output.sigmoid().cpu().numpy())\n    test_preds = np.concatenate(test_preds,axis=0)\n    return test_preds\n\n\ndef make_pred2(test_loader,model):\n    test_preds = []\n    tk0 = tqdm(test_loader, total=len(test_loader))\n    with torch.no_grad():  # Do not calculate gradient since we are only predicting\n        # Predicting on validation set\n        for d in tk0:\n            input_data_numerical_array = d['input_data_numerical_array'].to(device)\n            input_data_mask_array = d['input_data_mask_array'].to(device)\n            attention_mask = d['attention_mask'].to(device)\n            output,_ = model(input_data_numerical_array, \n                       input_data_mask_array,\n                       attention_mask)\n            test_preds.append(output.sigmoid().cpu().numpy())\n    test_preds = np.concatenate(test_preds,axis=0)\n    return test_preds","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# =======================\n# main\n# =======================\nsub = pd.read_csv(SUB_PATH)","metadata":{"execution":{"iopub.status.busy":"2023-05-07T13:25:02.401456Z","iopub.execute_input":"2023-05-07T13:25:02.401768Z","iopub.status.idle":"2023-05-07T13:25:02.597178Z","shell.execute_reply.started":"2023-05-07T13:25:02.40174Z","shell.execute_reply":"2023-05-07T13:25:02.596009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ===========================\n# tdcsfog model\n# ===========================\ntdcsfog_model_list1 = []\nfor i in tdcsfog_path1:\n    model = TdcsfogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    tdcsfog_model_list1.append(model)\n\ntdcsfog_model_list3_1 = []\nfor i in tdcsfog_path3_1:\n    model = TdcsfogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    tdcsfog_model_list3_1.append(model)\n    \ntdcsfog_model_list3_2 = []\nfor i in tdcsfog_path3_2:\n    model = TdcsfogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    tdcsfog_model_list3_2.append(model)\n    \ntdcsfog_model_list3_3 = []\nfor i in tdcsfog_path3_3:\n    model = TdcsfogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    tdcsfog_model_list3_3.append(model)\n    \ntdcsfog_model_list4_1 = []\nfor i in tdcsfog_path4_1:\n    model = TdcsfogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    tdcsfog_model_list4_1.append(model)\n    \ntdcsfog_model_list4_2 = []\nfor i in tdcsfog_path4_2:\n    model = TdcsfogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    tdcsfog_model_list4_2.append(model)\n    \ntdcsfog_model_list4_3 = []\nfor i in tdcsfog_path4_3:\n    model = TdcsfogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    tdcsfog_model_list4_3.append(model)\n    \n\n# ===========================\n# defog model\n# ===========================\ndefog_model_list2 = []\nfor i in defog_path2:\n    model = DefogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    defog_model_list2.append(model)\n    \ndefog_model_list4 = []\nfor i in defog_path4:\n    model = DefogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    defog_model_list4.append(model)\n    \ndefog_model_list5 = []\nfor i in defog_path5:\n    model = DefogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    defog_model_list5.append(model)\n    \ndefog_model_list6 = []\nfor i in defog_path6:\n    model = DefogRnnModel2()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    defog_model_list6.append(model)\n\ndefog_model_list7 = []\nfor path in defog_path7:\n    model = Defog3Model()\n    state = torch.load(path, map_location=torch.device(\"cpu\"))\n    model.load_state_dict(state[\"model\"])\n    model = model.to(device)\n    model.eval()\n    defog_model_list7.append(model)\n    print(f\"load weights from {path}\")","metadata":{"execution":{"iopub.status.busy":"2023-05-07T13:25:02.59952Z","iopub.execute_input":"2023-05-07T13:25:02.599944Z","iopub.status.idle":"2023-05-07T13:25:04.427281Z","shell.execute_reply.started":"2023-05-07T13:25:02.599904Z","shell.execute_reply":"2023-05-07T13:25:04.425498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all = []","metadata":{"execution":{"iopub.status.busy":"2023-05-07T13:25:04.432064Z","iopub.execute_input":"2023-05-07T13:25:04.432526Z","iopub.status.idle":"2023-05-07T13:25:04.44581Z","shell.execute_reply.started":"2023-05-07T13:25:04.43248Z","shell.execute_reply":"2023-05-07T13:25:04.444414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# tdcsfog","metadata":{}},{"cell_type":"code","source":"# =========================\n# tdcsfog1\n# =========================\nth_len = 5000\nw = 0.20\n\ntdcsfog_list = glob.glob(TDCSFOG_DATA_PATH)\ncols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = ['AccV', 'AccML', 'AccAP', \n       'AccV_lag_diff', 'AccV_lead_diff', 'AccV_cumsum', 'AccML_lag_diff',\n       'AccML_lead_diff', 'AccML_cumsum', 'AccAP_lag_diff', 'AccAP_lead_diff',\n       'AccAP_cumsum']\nfor p in tqdm(tdcsfog_list):\n    id_values = p.split(\"/\")[-1].split(\".\")[0]\n    df = pd.read_csv(p)\n    \n    if len(df) > th_len:\n        seq_len = 5000\n        shift = 2500\n        offset = 1250\n    else:\n        seq_len = 3000\n        shift = 1500\n        offset = 750\n        \n    batch = (len(df)-1) // shift\n    if batch == 0:\n        batch = 1\n    for c in cols:\n        df[f\"{c}_lag_diff\"] = df[c].diff()\n        df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n        df[f\"{c}_cumsum\"] = df[c].cumsum()\n    sc = RobustScaler()\n    df[num_cols] = sc.fit_transform(df[num_cols].values)\n    df[num_cols] = df[num_cols].fillna(0)\n    num = df[num_cols].values\n    time = df[\"Time\"].values\n    \n    num_array = np.zeros([batch,seq_len,12])\n    mask_array = np.zeros([batch,seq_len],dtype=int)\n    time_array = np.zeros([batch,seq_len],dtype=int)\n    pred_use_array = np.zeros([batch,seq_len],dtype=int)\n    \n    if len(df) <= seq_len:\n        b = 0\n        num_ = num.copy()\n        time_ = time.copy()\n        num_len = len(num_)\n\n        num_array[b,:num_len,:] = num_\n        time_array[b,:num_len] = time_\n        mask_array[b,:num_len] = 1\n        pred_use_array[b,:num_len] = 1\n    else:\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                time_ = time[b*shift : ]\n                num_len = len(num_)\n\n                num_array[b,:num_len,:] = num_\n                time_array[b,:num_len] = time_\n                mask_array[b,:num_len] = 1\n                pred_use_array[b,offset:num_len] = 1\n            elif b == 0:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,:shift+offset] = 1\n            else:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,offset:shift+offset] = 1\n            \n            \n\n    \n    \n    \n    test_ = FogDataset(num_array,\n                       mask_array,\n                       train=False)\n    test_loader = DataLoader(dataset=test_, \n                        batch_size=bs, \n                        shuffle = False)\n    for n,m in enumerate(tdcsfog_model_list1):\n        if n == 0:\n            pred = make_pred(test_loader,m) / len(tdcsfog_model_list1)\n        else:\n            pred += make_pred(test_loader,m) / len(tdcsfog_model_list1)\n    pred_list = []\n    for i in range(batch):\n        mask_ = pred_use_array[i]\n        pred_ = pred[i,mask_ == 1,:]\n        time_ = time_array[i, mask_ == 1]\n        df_ = pd.DataFrame()\n        df_[\"StartHesitation\"] = pred_[:,0] * w\n        df_[\"Turn\"] = pred_[:,1] * w\n        df_[\"Walking\"] = pred_[:,2] * w\n        df_[\"Time\"] = time_\n        df_[\"Id\"] = id_values\n        df_[\"Id\"] = df_[\"Id\"].astype(str) + \"_\" + df_[\"Time\"].astype(str)\n        pred_list.append(df_)\n    pred = pd.concat(pred_list).reset_index(drop=True)\n    df_all.append(pred)","metadata":{"execution":{"iopub.status.busy":"2023-05-07T13:25:04.448594Z","iopub.execute_input":"2023-05-07T13:25:04.449128Z","iopub.status.idle":"2023-05-07T13:25:07.092854Z","shell.execute_reply.started":"2023-05-07T13:25:04.448987Z","shell.execute_reply":"2023-05-07T13:25:07.091721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# =========================\n# tdcsfog3\n# =========================\nth_len = 5000\nw = 0.40\n\ntdcsfog_list = glob.glob(TDCSFOG_DATA_PATH)\ncols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = ['AccV', 'AccML', 'AccAP', \n       'AccV_lag_diff', 'AccV_lead_diff', 'AccV_cumsum', 'AccML_lag_diff',\n       'AccML_lead_diff', 'AccML_cumsum', 'AccAP_lag_diff', 'AccAP_lead_diff',\n       'AccAP_cumsum']\nfor p in tqdm(tdcsfog_list):\n    id_values = p.split(\"/\")[-1].split(\".\")[0]\n    df = pd.read_csv(p)\n    if len(df) > th_len:\n        seq_len = 5000\n        shift = 2500\n        offset = 1250\n    else:\n        seq_len = 3000\n        shift = 1500\n        offset = 750\n    batch = (len(df)-1) // shift\n    if batch == 0:\n        batch = 1\n    for c in cols:\n        df[f\"{c}_lag_diff\"] = df[c].diff()\n        df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n        df[f\"{c}_cumsum\"] = df[c].cumsum()\n    sc = RobustScaler()\n    df[num_cols] = sc.fit_transform(df[num_cols].values)\n    df[num_cols] = df[num_cols].fillna(0)\n    num = df[num_cols].values\n    time = df[\"Time\"].values\n    \n    num_array = np.zeros([batch,seq_len,12])\n    mask_array = np.zeros([batch,seq_len],dtype=int)\n    time_array = np.zeros([batch,seq_len],dtype=int)\n    pred_use_array = np.zeros([batch,seq_len],dtype=int)\n    \n    if len(df) <= seq_len:\n        b = 0\n        num_ = num.copy()\n        time_ = time.copy()\n        num_len = len(num_)\n\n        num_array[b,:num_len,:] = num_\n        time_array[b,:num_len] = time_\n        mask_array[b,:num_len] = 1\n        pred_use_array[b,:num_len] = 1\n    else:\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                time_ = time[b*shift : ]\n                num_len = len(num_)\n\n                num_array[b,:num_len,:] = num_\n                time_array[b,:num_len] = time_\n                mask_array[b,:num_len] = 1\n                pred_use_array[b,offset:num_len] = 1\n            elif b == 0:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,:shift+offset] = 1\n            else:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,offset:shift+offset] = 1\n    \n    \n    \n    test_ = FogDataset(num_array,\n                       mask_array,\n                       train=False)\n    test_loader = DataLoader(dataset=test_, \n                        batch_size=bs, \n                        shuffle = False)\n    for n,m in enumerate(tdcsfog_model_list3_1):\n        if n == 0:\n            pred1 = make_pred(test_loader,m) / len(tdcsfog_model_list3_1)\n        else:\n            pred1 += make_pred(test_loader,m) / len(tdcsfog_model_list3_1)\n    for n,m in enumerate(tdcsfog_model_list3_2):\n        if n == 0:\n            pred2 = make_pred(test_loader,m) / len(tdcsfog_model_list3_2)\n        else:\n            pred2 += make_pred(test_loader,m) / len(tdcsfog_model_list3_2)\n    for n,m in enumerate(tdcsfog_model_list3_3):\n        if n == 0:\n            pred3 = make_pred(test_loader,m) / len(tdcsfog_model_list3_3)\n        else:\n            pred3 += make_pred(test_loader,m) / len(tdcsfog_model_list3_3)\n    pred = pred1.copy()\n    pred[:,:,1] = pred2[:,:,1]\n    pred[:,:,2] = pred3[:,:,2]\n    \n    \n    pred_list = []\n    for i in range(batch):\n        mask_ = pred_use_array[i]\n        pred_ = pred[i,mask_ == 1,:]\n        time_ = time_array[i, mask_ == 1]\n        df_ = pd.DataFrame()\n        df_[\"StartHesitation\"] = pred_[:,0] * w\n        df_[\"Turn\"] = pred_[:,1] * w\n        df_[\"Walking\"] = pred_[:,2] * w\n        df_[\"Time\"] = time_\n        df_[\"Id\"] = id_values\n        df_[\"Id\"] = df_[\"Id\"].astype(str) + \"_\" + df_[\"Time\"].astype(str)\n        pred_list.append(df_)\n    pred = pd.concat(pred_list).reset_index(drop=True)\n    df_all.append(pred)","metadata":{"execution":{"iopub.status.busy":"2023-05-07T13:25:07.572411Z","iopub.execute_input":"2023-05-07T13:25:07.57502Z","iopub.status.idle":"2023-05-07T13:25:08.62357Z","shell.execute_reply.started":"2023-05-07T13:25:07.574974Z","shell.execute_reply":"2023-05-07T13:25:08.622452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# =========================\n# tdcsfog4\n# =========================\nth_len = 5000\nw = 0.40\n\ntdcsfog_list = glob.glob(TDCSFOG_DATA_PATH)\ncols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = ['AccV', 'AccML', 'AccAP', \n       'AccV_lag_diff', 'AccV_lead_diff', 'AccV_cumsum', 'AccML_lag_diff',\n       'AccML_lead_diff', 'AccML_cumsum', 'AccAP_lag_diff', 'AccAP_lead_diff',\n       'AccAP_cumsum']\nfor p in tqdm(tdcsfog_list):\n    id_values = p.split(\"/\")[-1].split(\".\")[0]\n    df = pd.read_csv(p)\n    if len(df) > th_len:\n        seq_len = 5000\n        shift = 2500\n        offset = 1250\n    else:\n        seq_len = 3000\n        shift = 1500\n        offset = 750\n    batch = (len(df)-1) // shift\n    if batch == 0:\n        batch = 1\n    for c in cols:\n        df[f\"{c}_lag_diff\"] = df[c].diff()\n        df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n        df[f\"{c}_cumsum\"] = df[c].cumsum()\n    sc = RobustScaler()\n    df[num_cols] = sc.fit_transform(df[num_cols].values)\n    df[num_cols] = df[num_cols].fillna(0)\n    num = df[num_cols].values\n    time = df[\"Time\"].values\n    \n    num_array = np.zeros([batch,seq_len,12])\n    mask_array = np.zeros([batch,seq_len],dtype=int)\n    time_array = np.zeros([batch,seq_len],dtype=int)\n    pred_use_array = np.zeros([batch,seq_len],dtype=int)\n    \n    if len(df) <= seq_len:\n        b = 0\n        num_ = num.copy()\n        time_ = time.copy()\n        num_len = len(num_)\n\n        num_array[b,:num_len,:] = num_\n        time_array[b,:num_len] = time_\n        mask_array[b,:num_len] = 1\n        pred_use_array[b,:num_len] = 1\n    else:\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                time_ = time[b*shift : ]\n                num_len = len(num_)\n\n                num_array[b,:num_len,:] = num_\n                time_array[b,:num_len] = time_\n                mask_array[b,:num_len] = 1\n                pred_use_array[b,offset:num_len] = 1\n            elif b == 0:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,:shift+offset] = 1\n            else:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,offset:shift+offset] = 1\n    \n    \n    \n    test_ = FogDataset(num_array,\n                       mask_array,\n                       train=False)\n    test_loader = DataLoader(dataset=test_, \n                        batch_size=bs, \n                        shuffle = False)\n    for n,m in enumerate(tdcsfog_model_list4_1):\n        if n == 0:\n            pred1 = make_pred(test_loader,m) / len(tdcsfog_model_list4_1)\n        else:\n            pred1 += make_pred(test_loader,m) / len(tdcsfog_model_list4_1)\n    for n,m in enumerate(tdcsfog_model_list4_2):\n        if n == 0:\n            pred2 = make_pred(test_loader,m) / len(tdcsfog_model_list4_2)\n        else:\n            pred2 += make_pred(test_loader,m) / len(tdcsfog_model_list4_2)\n    for n,m in enumerate(tdcsfog_model_list4_3):\n        if n == 0:\n            pred3 = make_pred(test_loader,m) / len(tdcsfog_model_list4_3)\n        else:\n            pred3 += make_pred(test_loader,m) / len(tdcsfog_model_list4_3)\n    pred = pred1.copy()\n    pred[:,:,0] = pred1[:,:,0]*0.5 + pred2[:,:,0]*0.5\n    pred[:,:,1] = pred1[:,:,1]*0.5 + pred3[:,:,1]*0.5\n    pred[:,:,2] = pred2[:,:,2]*0.5 + pred3[:,:,2]*0.5\n    \n    \n    pred_list = []\n    for i in range(batch):\n        mask_ = pred_use_array[i]\n        pred_ = pred[i,mask_ == 1,:]\n        time_ = time_array[i, mask_ == 1]\n        df_ = pd.DataFrame()\n        df_[\"StartHesitation\"] = pred_[:,0] * w\n        df_[\"Turn\"] = pred_[:,1] * w\n        df_[\"Walking\"] = pred_[:,2] * w\n        df_[\"Time\"] = time_\n        df_[\"Id\"] = id_values\n        df_[\"Id\"] = df_[\"Id\"].astype(str) + \"_\" + df_[\"Time\"].astype(str)\n        pred_list.append(df_)\n    pred = pd.concat(pred_list).reset_index(drop=True)\n    df_all.append(pred)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# defog","metadata":{}},{"cell_type":"code","source":"# =========================\n# defog2\n# =========================\nth_len = 200000\nw = 0.35\n\ndefog_list = glob.glob(DEFOG_DATA_PATH)\ncols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = [\"AccV\",\"AccML\",\"AccAP\",'AccV_lag_diff',\n            'AccV_lead_diff', 'AccML_lag_diff', 'AccML_lead_diff',\n            'AccAP_lag_diff', 'AccAP_lead_diff']\nfor p in tqdm(defog_list):\n    id_values = p.split(\"/\")[-1].split(\".\")[0]\n    df = pd.read_csv(p)\n    if len(df) > th_len:\n        seq_len = 30000\n        shift = 15000\n        offset = 7500\n    else:\n        seq_len = 15000\n        shift = 7500\n        offset = 3750\n    batch = (len(df)-1) // shift\n    if batch == 0:\n        batch = 1\n    for c in cols:\n        df[f\"{c}_lag_diff\"] = df[c].diff()\n        df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n    sc = StandardScaler()\n    df[num_cols] = sc.fit_transform(df[num_cols].values)\n    df[num_cols] = df[num_cols].fillna(0)\n    num = df[num_cols].values\n    time = df[\"Time\"].values\n    \n    num_array = np.zeros([batch,seq_len,9])\n    mask_array = np.zeros([batch,seq_len],dtype=int)\n    time_array = np.zeros([batch,seq_len],dtype=int)\n    pred_use_array = np.zeros([batch,seq_len],dtype=int)\n    \n    if len(df) <= seq_len:\n        b = 0\n        num_len = len(num)\n        num_array[b,:num_len,:] = num\n        time_array[b,:num_len] = time\n        mask_array[b,:num_len] = 1\n        pred_use_array[b,:num_len] = 1\n    else:\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                time_ = time[b*shift : ]\n                num_len = len(num_)\n\n                num_array[b,:num_len,:] = num_\n                time_array[b,:num_len] = time_\n                mask_array[b,:num_len] = 1\n                pred_use_array[b,offset:num_len] = 1\n            elif b == 0:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,:shift+offset] = 1\n            else:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,offset:shift+offset] = 1  \n    \n    test_ = FogDataset(num_array,\n                       mask_array,\n                       train=False)\n    test_loader = DataLoader(dataset=test_, \n                        batch_size=bs, \n                        shuffle = False)\n    for n,m in enumerate(defog_model_list2):\n        if n == 0:\n            pred = make_pred(test_loader,m) / len(defog_model_list2)\n        else:\n            pred += make_pred(test_loader,m) / len(defog_model_list2)\n    pred_list = []\n    for i in range(batch):\n        mask_ = pred_use_array[i]\n        pred_ = pred[i,mask_ == 1,:]\n        time_ = time_array[i, mask_ == 1]\n        df_ = pd.DataFrame()\n        df_[\"StartHesitation\"] = pred_[:,0] * w\n        df_[\"Turn\"] = pred_[:,1] * w\n        df_[\"Walking\"] = pred_[:,2] * w\n        df_[\"Time\"] = time_\n        df_[\"Id\"] = id_values\n        df_[\"Id\"] = df_[\"Id\"].astype(str) + \"_\" + df_[\"Time\"].astype(str)\n        pred_list.append(df_)\n    pred = pd.concat(pred_list).reset_index(drop=True)\n    df_all.append(pred)","metadata":{"execution":{"iopub.status.busy":"2023-05-07T13:25:19.231231Z","iopub.execute_input":"2023-05-07T13:25:19.232193Z","iopub.status.idle":"2023-05-07T13:25:26.930424Z","shell.execute_reply.started":"2023-05-07T13:25:19.232143Z","shell.execute_reply":"2023-05-07T13:25:26.929281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# =========================\n# defog4\n# =========================\nth_len = 200000\nw = 0.25\n\ndefog_list = glob.glob(DEFOG_DATA_PATH)\ncols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = [\"AccV\",\"AccML\",\"AccAP\",'AccV_lag_diff',\n            'AccV_lead_diff', 'AccML_lag_diff', 'AccML_lead_diff',\n            'AccAP_lag_diff', 'AccAP_lead_diff']\nfor p in tqdm(defog_list):\n    id_values = p.split(\"/\")[-1].split(\".\")[0]\n    df = pd.read_csv(p)\n    if len(df) > th_len:\n        seq_len = 30000\n        shift = 15000\n        offset = 7500\n    else:\n        seq_len = 15000\n        shift = 7500\n        offset = 3750\n    batch = (len(df)-1) // shift\n    if batch == 0:\n        batch = 1\n    for c in cols:\n        df[f\"{c}_lag_diff\"] = df[c].diff()\n        df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n    sc = StandardScaler()\n    df[num_cols] = sc.fit_transform(df[num_cols].values)\n    df[num_cols] = df[num_cols].fillna(0)\n    num = df[num_cols].values\n    time = df[\"Time\"].values\n    \n    num_array = np.zeros([batch,seq_len,9])\n    mask_array = np.zeros([batch,seq_len],dtype=int)\n    time_array = np.zeros([batch,seq_len],dtype=int)\n    pred_use_array = np.zeros([batch,seq_len],dtype=int)\n    \n    if len(df) <= seq_len:\n        b = 0\n        num_len = len(num)\n        num_array[b,:num_len,:] = num\n        time_array[b,:num_len] = time\n        mask_array[b,:num_len] = 1\n        pred_use_array[b,:num_len] = 1\n    else:\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                time_ = time[b*shift : ]\n                num_len = len(num_)\n\n                num_array[b,:num_len,:] = num_\n                time_array[b,:num_len] = time_\n                mask_array[b,:num_len] = 1\n                pred_use_array[b,offset:num_len] = 1\n            elif b == 0:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,:shift+offset] = 1\n            else:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,offset:shift+offset] = 1  \n    \n    test_ = FogDataset(num_array,\n                       mask_array,\n                       train=False)\n    test_loader = DataLoader(dataset=test_, \n                        batch_size=bs, \n                        shuffle = False)\n    for n,m in enumerate(defog_model_list4):\n        if n == 0:\n            pred = make_pred(test_loader,m) / len(defog_model_list4)\n        else:\n            pred += make_pred(test_loader,m) / len(defog_model_list4)\n    pred_list = []\n    for i in range(batch):\n        mask_ = pred_use_array[i]\n        pred_ = pred[i,mask_ == 1,:]\n        time_ = time_array[i, mask_ == 1]\n        df_ = pd.DataFrame()\n        df_[\"StartHesitation\"] = pred_[:,0] * w\n        df_[\"Turn\"] = pred_[:,1] * w\n        df_[\"Walking\"] = pred_[:,2] * w\n        df_[\"Time\"] = time_\n        df_[\"Id\"] = id_values\n        df_[\"Id\"] = df_[\"Id\"].astype(str) + \"_\" + df_[\"Time\"].astype(str)\n        pred_list.append(df_)\n    pred = pd.concat(pred_list).reset_index(drop=True)\n    df_all.append(pred)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# =========================\n# defog5\n# =========================\nth_len = 200000\nw = 0.25\n\ndefog_list = glob.glob(DEFOG_DATA_PATH)\ncols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = [\"AccV\",\"AccML\",\"AccAP\",'AccV_lag_diff',\n            'AccV_lead_diff', 'AccML_lag_diff', 'AccML_lead_diff',\n            'AccAP_lag_diff', 'AccAP_lead_diff']\nfor p in tqdm(defog_list):\n    id_values = p.split(\"/\")[-1].split(\".\")[0]\n    df = pd.read_csv(p)\n    if len(df) > th_len:\n        seq_len = 30000\n        shift = 15000\n        offset = 7500\n    else:\n        seq_len = 15000\n        shift = 7500\n        offset = 3750\n    batch = (len(df)-1) // shift\n    if batch == 0:\n        batch = 1\n    for c in cols:\n        df[f\"{c}_lag_diff\"] = df[c].diff()\n        df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n    sc = StandardScaler()\n    df[num_cols] = sc.fit_transform(df[num_cols].values)\n    df[num_cols] = df[num_cols].fillna(0)\n    num = df[num_cols].values\n    time = df[\"Time\"].values\n    \n    num_array = np.zeros([batch,seq_len,9])\n    mask_array = np.zeros([batch,seq_len],dtype=int)\n    time_array = np.zeros([batch,seq_len],dtype=int)\n    pred_use_array = np.zeros([batch,seq_len],dtype=int)\n    \n    if len(df) <= seq_len:\n        b = 0\n        num_len = len(num)\n        num_array[b,:num_len,:] = num\n        time_array[b,:num_len] = time\n        mask_array[b,:num_len] = 1\n        pred_use_array[b,:num_len] = 1\n    else:\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                time_ = time[b*shift : ]\n                num_len = len(num_)\n\n                num_array[b,:num_len,:] = num_\n                time_array[b,:num_len] = time_\n                mask_array[b,:num_len] = 1\n                pred_use_array[b,offset:num_len] = 1\n            elif b == 0:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,:shift+offset] = 1\n            else:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,offset:shift+offset] = 1  \n    \n    test_ = FogDataset(num_array,\n                       mask_array,\n                       train=False)\n    test_loader = DataLoader(dataset=test_, \n                        batch_size=bs, \n                        shuffle = False)\n    for n,m in enumerate(defog_model_list5):\n        if n == 0:\n            pred = make_pred(test_loader,m) / len(defog_model_list5)\n        else:\n            pred += make_pred(test_loader,m) / len(defog_model_list5)\n    pred_list = []\n    for i in range(batch):\n        mask_ = pred_use_array[i]\n        pred_ = pred[i,mask_ == 1,:]\n        time_ = time_array[i, mask_ == 1]\n        df_ = pd.DataFrame()\n        df_[\"StartHesitation\"] = pred_[:,0] * w\n        df_[\"Turn\"] = pred_[:,1] * w\n        df_[\"Walking\"] = pred_[:,2] * w\n        df_[\"Time\"] = time_\n        df_[\"Id\"] = id_values\n        df_[\"Id\"] = df_[\"Id\"].astype(str) + \"_\" + df_[\"Time\"].astype(str)\n        pred_list.append(df_)\n    pred = pd.concat(pred_list).reset_index(drop=True)\n    df_all.append(pred)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# =========================\n# defog6\n# =========================\nth_len = 200000\nw = 0.10\n\ndefog_list = glob.glob(DEFOG_DATA_PATH)\ncols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = [\"AccV\",\"AccML\",\"AccAP\",'AccV_lag_diff',\n            'AccV_lead_diff', 'AccML_lag_diff', 'AccML_lead_diff',\n            'AccAP_lag_diff', 'AccAP_lead_diff']\nfor p in tqdm(defog_list):\n    id_values = p.split(\"/\")[-1].split(\".\")[0]\n    df = pd.read_csv(p)\n    if len(df) > th_len:\n        seq_len = 30000\n        shift = 15000\n        offset = 7500\n    else:\n        seq_len = 15000\n        shift = 7500\n        offset = 3750\n    batch = (len(df)-1) // shift\n    if batch == 0:\n        batch = 1\n    for c in cols:\n        df[f\"{c}_lag_diff\"] = df[c].diff()\n        df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n    sc = StandardScaler()\n    df[num_cols] = sc.fit_transform(df[num_cols].values)\n    df[num_cols] = df[num_cols].fillna(0)\n    num = df[num_cols].values\n    time = df[\"Time\"].values\n    \n    num_array = np.zeros([batch,seq_len,9])\n    mask_array = np.zeros([batch,seq_len],dtype=int)\n    time_array = np.zeros([batch,seq_len],dtype=int)\n    pred_use_array = np.zeros([batch,seq_len],dtype=int)\n    \n    if len(df) <= seq_len:\n        b = 0\n        num_len = len(num)\n        num_array[b,:num_len,:] = num\n        time_array[b,:num_len] = time\n        mask_array[b,:num_len] = 1\n        pred_use_array[b,:num_len] = 1\n    else:\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                time_ = time[b*shift : ]\n                num_len = len(num_)\n\n                num_array[b,:num_len,:] = num_\n                time_array[b,:num_len] = time_\n                mask_array[b,:num_len] = 1\n                pred_use_array[b,offset:num_len] = 1\n            elif b == 0:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,:shift+offset] = 1\n            else:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,offset:shift+offset] = 1  \n    \n    test_ = FogDataset(num_array,\n                       mask_array,\n                       train=False)\n    test_loader = DataLoader(dataset=test_, \n                        batch_size=bs, \n                        shuffle = False)\n    for n,m in enumerate(defog_model_list6):\n        if n == 0:\n            pred = make_pred(test_loader,m) / len(defog_model_list6)\n        else:\n            pred += make_pred(test_loader,m) / len(defog_model_list6)\n    pred_list = []\n    for i in range(batch):\n        mask_ = pred_use_array[i]\n        pred_ = pred[i,mask_ == 1,:]\n        time_ = time_array[i, mask_ == 1]\n        df_ = pd.DataFrame()\n        df_[\"StartHesitation\"] = pred_[:,0] * w\n        df_[\"Turn\"] = pred_[:,1] * w\n        df_[\"Walking\"] = pred_[:,2] * w\n        df_[\"Time\"] = time_\n        df_[\"Id\"] = id_values\n        df_[\"Id\"] = df_[\"Id\"].astype(str) + \"_\" + df_[\"Time\"].astype(str)\n        pred_list.append(df_)\n    pred = pd.concat(pred_list).reset_index(drop=True)\n    df_all.append(pred)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# =========================\n# defog7\n# =========================\nth_len = 200000\nw = 0.05\n\ndefog_list = glob.glob(DEFOG_DATA_PATH)\ncols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = [\"AccV\",\"AccML\",\"AccAP\",'AccV_lag_diff',\n            'AccV_lead_diff', 'AccML_lag_diff', 'AccML_lead_diff',\n            'AccAP_lag_diff', 'AccAP_lead_diff']\nfor p in tqdm(defog_list):\n    id_values = p.split(\"/\")[-1].split(\".\")[0]\n    df = pd.read_csv(p)\n    if len(df) > th_len:\n        seq_len = 30000\n        shift = 15000\n        offset = 7500\n    else:\n        seq_len = 15000\n        shift = 7500\n        offset = 3750\n    batch = (len(df)-1) // shift\n    if batch == 0:\n        batch = 1\n    for c in cols:\n        df[f\"{c}_lag_diff\"] = df[c].diff()\n        df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n    sc = StandardScaler()\n    df[num_cols] = sc.fit_transform(df[num_cols].values)\n    df[num_cols] = df[num_cols].fillna(0)\n    num = df[num_cols].values\n    time = df[\"Time\"].values\n    \n    num_array = np.zeros([batch,seq_len,9])\n    mask_array = np.zeros([batch,seq_len],dtype=int)\n    time_array = np.zeros([batch,seq_len],dtype=int)\n    pred_use_array = np.zeros([batch,seq_len],dtype=int)\n    \n    if len(df) <= seq_len:\n        b = 0\n        num_len = len(num)\n        num_array[b,:num_len,:] = num\n        time_array[b,:num_len] = time\n        mask_array[b,:num_len] = 1\n        pred_use_array[b,:num_len] = 1\n    else:\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                time_ = time[b*shift : ]\n                num_len = len(num_)\n\n                num_array[b,:num_len,:] = num_\n                time_array[b,:num_len] = time_\n                mask_array[b,:num_len] = 1\n                pred_use_array[b,offset:num_len] = 1\n            elif b == 0:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,:shift+offset] = 1\n            else:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,offset:shift+offset] = 1  \n    \n    test_ = FogDataset(num_array,\n                       mask_array,\n                       train=False)\n    test_loader = DataLoader(dataset=test_, \n                        batch_size=bs, \n                        shuffle = False)\n    for n,m in enumerate(defog_model_list7):\n        if n == 0:\n            pred = make_pred(test_loader,m) / len(defog_model_list7)\n        else:\n            pred += make_pred(test_loader,m) / len(defog_model_list7)\n    pred_list = []\n    for i in range(batch):\n        mask_ = pred_use_array[i]\n        pred_ = pred[i,mask_ == 1,:]\n        time_ = time_array[i, mask_ == 1]\n        df_ = pd.DataFrame()\n        df_[\"StartHesitation\"] = pred_[:,0] * w\n        df_[\"Turn\"] = pred_[:,1] * w\n        df_[\"Walking\"] = pred_[:,2] * w\n        df_[\"Time\"] = time_\n        df_[\"Id\"] = id_values\n        df_[\"Id\"] = df_[\"Id\"].astype(str) + \"_\" + df_[\"Time\"].astype(str)\n        pred_list.append(df_)\n    pred = pd.concat(pred_list).reset_index(drop=True)\n    df_all.append(pred)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all = pd.concat(df_all).reset_index(drop=True)\ndf_all = df_all.groupby(by=\"Id\")[['StartHesitation', 'Turn', 'Walking']].sum().reset_index()\ndf_all[['Id', 'StartHesitation', 'Turn', 'Walking']].to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-07T13:25:35.194117Z","iopub.execute_input":"2023-05-07T13:25:35.194986Z","iopub.status.idle":"2023-05-07T13:25:36.549021Z","shell.execute_reply.started":"2023-05-07T13:25:35.194944Z","shell.execute_reply":"2023-05-07T13:25:36.547921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all","metadata":{"execution":{"iopub.status.busy":"2023-05-07T13:25:36.552095Z","iopub.execute_input":"2023-05-07T13:25:36.552446Z","iopub.status.idle":"2023-05-07T13:25:36.575261Z","shell.execute_reply.started":"2023-05-07T13:25:36.552414Z","shell.execute_reply":"2023-05-07T13:25:36.574123Z"},"trusted":true},"execution_count":null,"outputs":[]}]}