{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ============================\n# library\n# ============================\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nfrom torch.nn import LayerNorm,TransformerEncoder\nfrom torch.utils.data import DataLoader, TensorDataset, Dataset\nimport pickle\nfrom tqdm.auto import tqdm\nimport gc\nimport glob\nfrom sklearn.preprocessing import StandardScaler,RobustScaler","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-04T16:07:46.166870Z","iopub.execute_input":"2023-07-04T16:07:46.167468Z","iopub.status.idle":"2023-07-04T16:07:49.746477Z","shell.execute_reply.started":"2023-07-04T16:07:46.167434Z","shell.execute_reply":"2023-07-04T16:07:49.744391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog_fold_df = pd.read_csv(\"/kaggle/input/fog-fold/defog.csv\")\ntdcsfog_fold_df = pd.read_csv(\"/kaggle/input/fog-fold/tdcsfog.csv\")\n\ntdcsfog_list = tdcsfog_fold_df[tdcsfog_fold_df[\"fold\"]==0][\"Id\"].map(lambda x:f\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/train/tdcsfog/{x}.csv\").tolist()\ndefog_list = defog_fold_df[defog_fold_df[\"fold\"]==0][\"Id\"].map(lambda x:f\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/train/defog/{x}.csv\").tolist()","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:07:49.754259Z","iopub.execute_input":"2023-07-04T16:07:49.754912Z","iopub.status.idle":"2023-07-04T16:07:49.806411Z","shell.execute_reply.started":"2023-07-04T16:07:49.754863Z","shell.execute_reply":"2023-07-04T16:07:49.805370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_list = tdcsfog_list+defog_list\nprint(data_list[:5])","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:07:49.811139Z","iopub.execute_input":"2023-07-04T16:07:49.813675Z","iopub.status.idle":"2023-07-04T16:07:49.824540Z","shell.execute_reply.started":"2023-07-04T16:07:49.813637Z","shell.execute_reply":"2023-07-04T16:07:49.823345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============================\n# settings\n# ============================\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nbs = 32","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:07:49.829793Z","iopub.execute_input":"2023-07-04T16:07:49.833260Z","iopub.status.idle":"2023-07-04T16:07:49.903838Z","shell.execute_reply.started":"2023-07-04T16:07:49.833220Z","shell.execute_reply":"2023-07-04T16:07:49.902468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============================\n# settings\n# ============================\ntdcsfog_path = [f\"/kaggle/input/fog-ex143/ex143_{i}.pth\" for i in range(1)] # tdcsfog model\ndefog_path = [f\"/kaggle/input/fog-ex185/ex185_{i}.pth\" for i in range(1)] # defog model","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:07:49.906321Z","iopub.execute_input":"2023-07-04T16:07:49.907186Z","iopub.status.idle":"2023-07-04T16:07:49.916150Z","shell.execute_reply.started":"2023-07-04T16:07:49.907144Z","shell.execute_reply":"2023-07-04T16:07:49.915171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============================\n# Functions\n# ============================\n\ndef preprocess(numerical_array, \n               mask_array,\n               ):\n    \n    attention_mask = mask_array == 0\n\n    return {\n        'input_data_numerical_array': numerical_array,\n        'input_data_mask_array': mask_array,\n        'attention_mask': attention_mask,\n    }\n\nclass FogDataset(Dataset):\n    def __init__(self, numerical_array, \n                 mask_array,\n                 train = True, y = None):\n        self.numerical_array = numerical_array\n        self.mask_array = mask_array\n        self.train = train\n        self.y = y\n    \n    def __len__(self):\n        return len(self.numerical_array)\n\n    def __getitem__(self, item):\n        data = preprocess(\n            self.numerical_array[item],\n            self.mask_array[item],\n            \n        )\n\n        # Return the processed data where the lists are converted to `torch.tensor`s\n        if self.train : \n            return {\n              'input_data_numerical_array': torch.tensor(data['input_data_numerical_array'],dtype=torch.float32),\n              'input_data_mask_array':torch.tensor(data['input_data_mask_array'], dtype=torch.long),  \n              'attention_mask': torch.tensor(data[\"attention_mask\"], dtype=torch.bool),\n              \"y\":torch.tensor(self.y[item], dtype=torch.float32)\n               }\n        else:\n            return {\n             'input_data_numerical_array': torch.tensor(data['input_data_numerical_array'],dtype=torch.float32),\n              'input_data_mask_array':torch.tensor(data['input_data_mask_array'], dtype=torch.long),  \n              'attention_mask': torch.tensor(data[\"attention_mask\"], dtype=torch.bool),\n               }","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:07:49.918269Z","iopub.execute_input":"2023-07-04T16:07:49.919314Z","iopub.status.idle":"2023-07-04T16:07:49.935419Z","shell.execute_reply.started":"2023-07-04T16:07:49.919271Z","shell.execute_reply":"2023-07-04T16:07:49.934321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ================================\n# tdcsfog\n# ================================\nclass TdcsfogRnnModel(nn.Module):\n    def __init__(\n        self, dropout=0.2,\n        input_numerical_size=12,\n        numeraical_linear_size = 64,\n        model_size = 128,\n        linear_out = 128,\n        out_size=3):\n        super(TdcsfogRnnModel, self).__init__()\n        self.numerical_linear  = nn.Sequential(\n                nn.Linear(input_numerical_size, numeraical_linear_size),\n                nn.LayerNorm(numeraical_linear_size)\n            )\n        \n        self.rnn = nn.GRU(numeraical_linear_size, model_size,\n                            num_layers = 2, \n                            batch_first=True,\n                            bidirectional=True)\n        self.linear_out  = nn.Sequential(\n                nn.Linear(model_size*2, \n                          linear_out),\n                nn.LayerNorm(linear_out),\n                nn.ReLU(),\n                nn.Dropout(dropout),\n                nn.Linear(linear_out, \n                          out_size))\n        self._reinitialize()\n        \n    def _reinitialize(self):\n        \"\"\"\n        Tensorflow/Keras-like initialization\n        \"\"\"\n        for name, p in self.named_parameters():\n            if 'rnn' in name:\n                if 'weight_ih' in name:\n                    nn.init.xavier_uniform_(p.data)\n                elif 'weight_hh' in name:\n                    nn.init.orthogonal_(p.data)\n                elif 'bias_ih' in name:\n                    p.data.fill_(0)\n                    # Set forget-gate bias to 1\n                    n = p.size(0)\n                    p.data[(n // 4):(n // 2)].fill_(1)\n                elif 'bias_hh' in name:\n                    p.data.fill_(0)\n    \n    def forward(self, numerical_array,\n                mask_array,\n                attention_mask):\n        \n        numerical_embedding = self.numerical_linear(numerical_array)\n        output,_ = self.rnn(numerical_embedding)\n        output = self.linear_out(output)\n        return output","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:07:49.937381Z","iopub.execute_input":"2023-07-04T16:07:49.938231Z","iopub.status.idle":"2023-07-04T16:07:49.956717Z","shell.execute_reply.started":"2023-07-04T16:07:49.938193Z","shell.execute_reply":"2023-07-04T16:07:49.955295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DefogRnnModel(nn.Module):\n    def __init__(\n        self, dropout=0.2,\n        input_numerical_size=9,\n        numeraical_linear_size = 64,\n        model_size = 128,\n        linear_out = 128,\n        out_size=3):\n        super(DefogRnnModel, self).__init__()\n        self.numerical_linear  = nn.Sequential(\n                nn.Linear(input_numerical_size, numeraical_linear_size),\n                nn.LayerNorm(numeraical_linear_size)\n            )\n        \n        self.rnn = nn.GRU(numeraical_linear_size, model_size,\n                            num_layers = 2, \n                            batch_first=True,\n                            bidirectional=True)\n        self.linear_out  = nn.Sequential(\n                nn.Linear(model_size*2, \n                          linear_out),\n                nn.LayerNorm(linear_out),\n                nn.ReLU(),\n                nn.Dropout(dropout),\n                nn.Linear(linear_out, \n                          out_size))\n        self._reinitialize()\n        \n    def _reinitialize(self):\n        \"\"\"\n        Tensorflow/Keras-like initialization\n        \"\"\"\n        for name, p in self.named_parameters():\n            if 'rnn' in name:\n                if 'weight_ih' in name:\n                    nn.init.xavier_uniform_(p.data)\n                elif 'weight_hh' in name:\n                    nn.init.orthogonal_(p.data)\n                elif 'bias_ih' in name:\n                    p.data.fill_(0)\n                    # Set forget-gate bias to 1\n                    n = p.size(0)\n                    p.data[(n // 4):(n // 2)].fill_(1)\n                elif 'bias_hh' in name:\n                    p.data.fill_(0)\n    \n    def forward(self, numerical_array,\n                mask_array,\n                attention_mask):\n        \n        numerical_embedding = self.numerical_linear(numerical_array)\n        output,_ = self.rnn(numerical_embedding)\n        output = self.linear_out(output)\n        return output\n","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:07:49.961154Z","iopub.execute_input":"2023-07-04T16:07:49.961894Z","iopub.status.idle":"2023-07-04T16:07:49.985584Z","shell.execute_reply.started":"2023-07-04T16:07:49.961851Z","shell.execute_reply":"2023-07-04T16:07:49.984060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_pred(test_loader,model):\n    test_preds = []\n    with torch.no_grad():  # Do not calculate gradient since we are only predicting\n        # Predicting on validation set\n        for d in test_loader:\n            input_data_numerical_array = d['input_data_numerical_array'].to(device)\n            input_data_mask_array = d['input_data_mask_array'].to(device)\n            attention_mask = d['attention_mask'].to(device)\n            output = model(input_data_numerical_array, \n                       input_data_mask_array,\n                       attention_mask)\n            test_preds.append(output.sigmoid().cpu().numpy())\n    test_preds = np.concatenate(test_preds,axis=0)\n    return test_preds","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:07:49.987687Z","iopub.execute_input":"2023-07-04T16:07:49.988679Z","iopub.status.idle":"2023-07-04T16:07:49.998891Z","shell.execute_reply.started":"2023-07-04T16:07:49.988639Z","shell.execute_reply":"2023-07-04T16:07:49.997569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ===========================\n# tdcsfog model\n# ===========================\ntdcsfog_model_list = []\nfor i in tdcsfog_path:\n    model = TdcsfogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    tdcsfog_model_list.append(model)\n\n\n# ===========================\n# defog model\n# ===========================\ndefog_model_list = []\nfor i in defog_path:\n    model = DefogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    defog_model_list.append(model)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:07:50.004468Z","iopub.execute_input":"2023-07-04T16:07:50.005394Z","iopub.status.idle":"2023-07-04T16:07:54.295608Z","shell.execute_reply.started":"2023-07-04T16:07:50.005356Z","shell.execute_reply":"2023-07-04T16:07:54.294479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tdcsfog_model_all = []","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:07:54.296976Z","iopub.execute_input":"2023-07-04T16:07:54.297605Z","iopub.status.idle":"2023-07-04T16:07:54.304166Z","shell.execute_reply.started":"2023-07-04T16:07:54.297564Z","shell.execute_reply":"2023-07-04T16:07:54.303062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# tdcsfog model","metadata":{}},{"cell_type":"code","source":"# =========================\n# tdcsfog\n# =========================\nth_len = 5000\nw = 0.20\n\ncols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = ['AccV', 'AccML', 'AccAP', \n       'AccV_lag_diff', 'AccV_lead_diff', 'AccV_cumsum', 'AccML_lag_diff',\n       'AccML_lead_diff', 'AccML_cumsum', 'AccAP_lag_diff', 'AccAP_lead_diff',\n       'AccAP_cumsum']\nfor p in tqdm(data_list):\n    id_values = p.split(\"/\")[-1].split(\".\")[0]\n    df = pd.read_csv(p)\n    \n    if len(df) > th_len:\n        seq_len = 5000\n        shift = 2500\n        offset = 1250\n    else:\n        seq_len = 3000\n        shift = 1500\n        offset = 750\n        \n    batch = (len(df)-1) // shift\n    if batch == 0:\n        batch = 1\n    for c in cols:\n        df[f\"{c}_lag_diff\"] = df[c].diff()\n        df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n        df[f\"{c}_cumsum\"] = df[c].cumsum()\n    sc = RobustScaler()\n    df[num_cols] = sc.fit_transform(df[num_cols].values)\n    df[num_cols] = df[num_cols].fillna(0)\n    num = df[num_cols].values\n    time = df[\"Time\"].values\n    \n    num_array = np.zeros([batch,seq_len,12])\n    mask_array = np.zeros([batch,seq_len],dtype=int)\n    time_array = np.zeros([batch,seq_len],dtype=int)\n    pred_use_array = np.zeros([batch,seq_len],dtype=int)\n    \n    if len(df) <= seq_len:\n        b = 0\n        num_ = num.copy()\n        time_ = time.copy()\n        num_len = len(num_)\n\n        num_array[b,:num_len,:] = num_\n        time_array[b,:num_len] = time_\n        mask_array[b,:num_len] = 1\n        pred_use_array[b,:num_len] = 1\n    else:\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                time_ = time[b*shift : ]\n                num_len = len(num_)\n\n                num_array[b,:num_len,:] = num_\n                time_array[b,:num_len] = time_\n                mask_array[b,:num_len] = 1\n                pred_use_array[b,offset:num_len] = 1\n            elif b == 0:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,:shift+offset] = 1\n            else:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,offset:shift+offset] = 1\n            \n            \n\n    \n    \n    \n    test_ = FogDataset(num_array,\n                       mask_array,\n                       train=False)\n    test_loader = DataLoader(dataset=test_, \n                        batch_size=bs, \n                        shuffle = False)\n    for n,m in enumerate(tdcsfog_model_list):\n        if n == 0:\n            pred = make_pred(test_loader,m) / len(tdcsfog_model_list)\n        else:\n            pred += make_pred(test_loader,m) / len(tdcsfog_model_list)\n    pred_list = []\n    for i in range(batch):\n        mask_ = pred_use_array[i]\n        pred_ = pred[i,mask_ == 1,:]\n        time_ = time_array[i, mask_ == 1]\n        df_ = pd.DataFrame()\n        df_[\"StartHesitation\"] = pred_[:,0] * w\n        df_[\"Turn\"] = pred_[:,1] * w\n        df_[\"Walking\"] = pred_[:,2] * w\n        df_[\"Time\"] = time_\n        df_[\"Id\"] = id_values\n        df_[\"Id\"] = df_[\"Id\"].astype(str) + \"_\" + df_[\"Time\"].astype(str)\n        pred_list.append(df_)\n    pred = pd.concat(pred_list).reset_index(drop=True)\n    df_tdcsfog_model_all.append(pred)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:07:54.307301Z","iopub.execute_input":"2023-07-04T16:07:54.307969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tdcsfog_model_all = pd.concat(df_tdcsfog_model_all).reset_index(drop=True)\ndf_tdcsfog_model_all = df_tdcsfog_model_all.groupby(by=\"Id\")[['StartHesitation', 'Turn', 'Walking']].sum().reset_index()\ndf_tdcsfog_model_all[['Id', 'StartHesitation', 'Turn', 'Walking']].to_csv(\"tdcsfog_model_prediction_fold_0.csv\",index=False)\ndf_tdcsfog_model_all[['Id', 'StartHesitation', 'Turn', 'Walking']].head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# defog model","metadata":{}},{"cell_type":"code","source":"df_defog_model_all = []","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# =========================\n# defog\n# =========================\nth_len = 200000\nw = 0.25\n\ncols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = [\"AccV\",\"AccML\",\"AccAP\",'AccV_lag_diff',\n            'AccV_lead_diff', 'AccML_lag_diff', 'AccML_lead_diff',\n            'AccAP_lag_diff', 'AccAP_lead_diff']\nfor p in tqdm(data_list):\n    id_values = p.split(\"/\")[-1].split(\".\")[0]\n    df = pd.read_csv(p)\n    if len(df) > th_len:\n        seq_len = 30000\n        shift = 15000\n        offset = 7500\n    else:\n        seq_len = 15000\n        shift = 7500\n        offset = 3750\n    batch = (len(df)-1) // shift\n    if batch == 0:\n        batch = 1\n    for c in cols:\n        df[f\"{c}_lag_diff\"] = df[c].diff()\n        df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n    sc = StandardScaler()\n    df[num_cols] = sc.fit_transform(df[num_cols].values)\n    df[num_cols] = df[num_cols].fillna(0)\n    num = df[num_cols].values\n    time = df[\"Time\"].values\n    \n    num_array = np.zeros([batch,seq_len,9])\n    mask_array = np.zeros([batch,seq_len],dtype=int)\n    time_array = np.zeros([batch,seq_len],dtype=int)\n    pred_use_array = np.zeros([batch,seq_len],dtype=int)\n    \n    if len(df) <= seq_len:\n        b = 0\n        num_len = len(num)\n        num_array[b,:num_len,:] = num\n        time_array[b,:num_len] = time\n        mask_array[b,:num_len] = 1\n        pred_use_array[b,:num_len] = 1\n    else:\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                time_ = time[b*shift : ]\n                num_len = len(num_)\n\n                num_array[b,:num_len,:] = num_\n                time_array[b,:num_len] = time_\n                mask_array[b,:num_len] = 1\n                pred_use_array[b,offset:num_len] = 1\n            elif b == 0:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,:shift+offset] = 1\n            else:\n                num_ = num[b*shift:b*shift+seq_len]\n                time_ = time[b*shift:b*shift + seq_len]\n\n                num_array[b,:,:] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,offset:shift+offset] = 1  \n    \n    test_ = FogDataset(num_array,\n                       mask_array,\n                       train=False)\n    test_loader = DataLoader(dataset=test_, \n                        batch_size=bs, \n                        shuffle = False)\n    for n,m in enumerate(defog_model_list):\n        if n == 0:\n            pred = make_pred(test_loader,m) / len(defog_model_list)\n        else:\n            pred += make_pred(test_loader,m) / len(defog_model_list)\n    pred_list = []\n    for i in range(batch):\n        mask_ = pred_use_array[i]\n        pred_ = pred[i,mask_ == 1,:]\n        time_ = time_array[i, mask_ == 1]\n        df_ = pd.DataFrame()\n        df_[\"StartHesitation\"] = pred_[:,0] * w\n        df_[\"Turn\"] = pred_[:,1] * w\n        df_[\"Walking\"] = pred_[:,2] * w\n        df_[\"Time\"] = time_\n        df_[\"Id\"] = id_values\n        df_[\"Id\"] = df_[\"Id\"].astype(str) + \"_\" + df_[\"Time\"].astype(str)\n        pred_list.append(df_)\n    pred = pd.concat(pred_list).reset_index(drop=True)\n    df_defog_model_all.append(pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_defog_model_all = pd.concat(df_defog_model_all).reset_index(drop=True)\ndf_defog_model_all = df_defog_model_all.groupby(by=\"Id\")[['StartHesitation', 'Turn', 'Walking']].sum().reset_index()\ndf_defog_model_all[['Id', 'StartHesitation', 'Turn', 'Walking']].to_csv(\"defog_model_prediction_fold_0.csv\",index=False)\ndf_defog_model_all[['Id', 'StartHesitation', 'Turn', 'Walking']].head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}