{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<br>\n<br>\n\n# **Reference**:\n* `Kaggle Competition`: <a href=\"https://www.kaggle.com/competitions/tlvmc-parkinsons-freezing-gait-prediction/overview\" style=\"text-decoration:none\">Parkinson's Freezing of Gait Prediction</a>\n* `Disscusion`: <a href=\"https://www.kaggle.com/competitions/tlvmc-parkinsons-freezing-gait-prediction/discussion/416057\" style=\"text-decoration:none\">2nd place solution</a>\n* `Notebook`: <a href=\"https://www.kaggle.com/code/takoihiraokazu/cv-ensemble-sub-0607-1\" style=\"text-decoration:none\">Inference notebook</a>\n* `Github`: **TakoiHirokazu** <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction\" style=\"text-decoration:none\">Kaggle-Parkinsons-Freezing-of-Gait-Prediction</a> (Training)\n\nThanks <a href=\"https://www.kaggle.com/takoihiraokazu\" style=\"text-decoration:none\">Takoi</a> and his/her teammates for their sharing solution and code!\n\n<br>\n\n* My own works: <a href=\"https://www.kaggle.com/code/abrachan/exploratory-data-analysis#Some-Conclusions\" style=\"text-decoration:none\">Exploratory Data Analysis</a>\n\n\n<br>\n<br>\n\n**Note**: <br>\nThe checkpoints using in this notebook which lying in the `/kaggle/input/parkinson-fog-prediction` were obtained from the training stage using **CUDA**. So if you want to run the `【Inference】` by skipping the training stage in this notebook in the kaggle environment, please select the GPU accelerator. Otherwise it will throw exceptions.\n\n`RuntimeError: Attempting to deserialize object on a CUDA device but torch.cuda.is_available() is False. If you are running on a CPU-only machine, please use torch.load with map_location=torch.device('cpu') to map your storages to the CPU.`\n\n<br>\n\n------\n","metadata":{}},{"cell_type":"markdown","source":"<br>\n\n# **Import Libraries**","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nimport gc\nimport sys\nimport glob\nimport json\nimport pickle\nimport logging\nfrom tqdm import tqdm\nfrom tqdm import tqdm_notebook as tqdm_nb\nfrom contextlib import contextmanager\n\nimport time\nimport datetime\n\nimport random\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport PIL.Image as Image\n\nfrom IPython.display import Video\n\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler, RobustScaler\nfrom sklearn.metrics import roc_auc_score, accuracy_score, f1_score, average_precision_score\nfrom sklearn.model_selection import KFold, GroupKFold, StratifiedKFold, StratifiedGroupKFold\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.nn import LayerNorm\nfrom torch.nn import TransformerEncoder, TransformerDecoder\n\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom transformers import AdamW, get_linear_schedule_with_warmup     # pip install transformers\n\nfrom torch.cuda import amp\nfrom torch.utils.data import Dataset, DataLoader, TensorDataset","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:48:57.887726Z","iopub.execute_input":"2023-07-03T07:48:57.888502Z","iopub.status.idle":"2023-07-03T07:49:10.096368Z","shell.execute_reply.started":"2023-07-03T07:48:57.888457Z","shell.execute_reply":"2023-07-03T07:49:10.095404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sys.path.append(\"../src/\")\n# from logger import setup_logger, LOGGER    # Abrachan: see my revise below: `def init_logger()`\n# from util_tool import reduce_mem_usage\n\npd.set_option('display.max_columns', 300)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:10.098112Z","iopub.execute_input":"2023-07-03T07:49:10.098882Z","iopub.status.idle":"2023-07-03T07:49:10.103815Z","shell.execute_reply.started":"2023-07-03T07:49:10.098843Z","shell.execute_reply":"2023-07-03T07:49:10.102908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n<br>\n\n# **Config**","metadata":{}},{"cell_type":"code","source":"# When doing `inference`, set below as False so that we can run the whole notebook directly.\nTRAIN_FLAG   = False # True\nPREDICT_FLAG = False # True","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:10.105240Z","iopub.execute_input":"2023-07-03T07:49:10.105836Z","iopub.status.idle":"2023-07-03T07:49:10.112991Z","shell.execute_reply.started":"2023-07-03T07:49:10.105781Z","shell.execute_reply":"2023-07-03T07:49:10.112104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n<br>\n\n# **Load Data**","metadata":{}},{"cell_type":"code","source":"root_data = '/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/'\n\ntrain_defog = glob.glob(root_data + 'train/defog/**')\ntrain_tdcsfog = glob.glob(root_data + 'train/tdcsfog/**')\ntrain_notype = glob.glob(root_data + 'train/notype/**')\n\ntest_defog = glob.glob(root_data + 'test/defog/**')\ntest_tdcsfog = glob.glob(root_data + 'test/tdcsfog/**')\n\nsubjects = pd.read_csv(root_data + 'subjects.csv')\ntasks = pd.read_csv(root_data + 'tasks.csv')\nevents = pd.read_csv(root_data + 'events.csv')\n\ndaily_metadata=pd.read_csv(root_data + 'daily_metadata.csv')\ntdcsfog_metadata=pd.read_csv(root_data + 'tdcsfog_metadata.csv')\ndefog_metadata=pd.read_csv(root_data + 'defog_metadata.csv')\n\nsub = pd.read_csv(root_data + 'sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:10.116692Z","iopub.execute_input":"2023-07-03T07:49:10.117051Z","iopub.status.idle":"2023-07-03T07:49:10.962726Z","shell.execute_reply.started":"2023-07-03T07:49:10.117021Z","shell.execute_reply":"2023-07-03T07:49:10.960538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For some basic Exploratory Data Analysis (EDA), refers to <a href=\"https://www.kaggle.com/code/abrachan/exploratory-data-analysis\" style=\"text-decoration:none\">my own work</a> if you are interested.","metadata":{}},{"cell_type":"markdown","source":"<br>\n<br>\n<br>\n\n# **tdcsfog** 【Training】\n\n<br>\n\n<br>\n\n## **Feature Engineering**\n\n<br>\n\n### fe001_tdcsfog_base_feature\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/fe/fe001_tdcsfog_base_feature.ipynb\" style=\"text-decoration:none\">fe001_tdcsfog_base_feature.ipynb</a>","metadata":{}},{"cell_type":"code","source":"fe = \"001\"\nif not os.path.exists(f\"./output/fe/fe{fe}\"):\n    os.makedirs(f\"./output/fe/fe{fe}\", exist_ok=True)\n    os.makedirs(f\"./output/fe/fe{fe}/save\", exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:10.965105Z","iopub.execute_input":"2023-07-03T07:49:10.965485Z","iopub.status.idle":"2023-07-03T07:49:10.975526Z","shell.execute_reply.started":"2023-07-03T07:49:10.965452Z","shell.execute_reply":"2023-07-03T07:49:10.974604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TDCSFOG_META_PATH = \"../data/tdcsfog_metadata.csv\"\n# TDCSFOG_FOLDER = \"../data/train/tdcsfog/*.csv\"\n\n# meta = pd.read_csv(TDCSFOG_META_PATH)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:10.979133Z","iopub.execute_input":"2023-07-03T07:49:10.979471Z","iopub.status.idle":"2023-07-03T07:49:10.985175Z","shell.execute_reply.started":"2023-07-03T07:49:10.979444Z","shell.execute_reply":"2023-07-03T07:49:10.984254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_dict = {}\nfor n,i in enumerate(tdcsfog_metadata[\"Subject\"].unique()):\n    sub_dict[i] = n\nsub_dict","metadata":{"scrolled":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-03T07:49:10.987337Z","iopub.execute_input":"2023-07-03T07:49:10.988113Z","iopub.status.idle":"2023-07-03T07:49:11.004515Z","shell.execute_reply.started":"2023-07-03T07:49:10.988082Z","shell.execute_reply":"2023-07-03T07:49:11.003690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdcsfog_metadata[\"Sub_id\"] = tdcsfog_metadata[\"Subject\"].map(sub_dict)\n\n\n# pip install pyarrow\n# pip install fastparquet\ntdcsfog_metadata.to_parquet(\"./output/fe/fe001/fe001_tdcsfog_meta.parquet\")\n\nwith open(f'./output/fe/fe{fe}/fe{fe}_sub_id.pkl', 'wb') as p:\n    pickle.dump(sub_dict, p)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:11.007083Z","iopub.execute_input":"2023-07-03T07:49:11.007378Z","iopub.status.idle":"2023-07-03T07:49:11.417679Z","shell.execute_reply.started":"2023-07-03T07:49:11.007342Z","shell.execute_reply":"2023-07-03T07:49:11.416729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n\n### fe022_tdcsfog_1000\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/fe/fe022_tdcsfog_1000.ipynb\" style=\"text-decoration:none\">fe022_tdcsfog_1000.ipynb</a>","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder, StandardScaler, RobustScaler","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:11.419089Z","iopub.execute_input":"2023-07-03T07:49:11.419478Z","iopub.status.idle":"2023-07-03T07:49:11.425193Z","shell.execute_reply.started":"2023-07-03T07:49:11.419445Z","shell.execute_reply":"2023-07-03T07:49:11.424165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fe = \"022\"\nif not os.path.exists(f\"./output/fe/fe{fe}\"):\n    os.makedirs(f\"./output/fe/fe{fe}\")\n    os.makedirs(f\"./output/fe/fe{fe}/save\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:11.430371Z","iopub.execute_input":"2023-07-03T07:49:11.430776Z","iopub.status.idle":"2023-07-03T07:49:11.438177Z","shell.execute_reply.started":"2023-07-03T07:49:11.430752Z","shell.execute_reply":"2023-07-03T07:49:11.437235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TDCSFOG_META_PATH = \"../data/tdcsfog_metadata.csv\"\n# TDCSFOG_FOLDER = \"../data/train/tdcsfog/*.csv\"      \n\n# data_list = glob.glob(TDCSFOG_FOLDER)\n\n# data_list\n\n# Abrachan: The variable `data_list` is the same as `train_tdcdfog` above.","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:11.439779Z","iopub.execute_input":"2023-07-03T07:49:11.440454Z","iopub.status.idle":"2023-07-03T07:49:11.445874Z","shell.execute_reply.started":"2023-07-03T07:49:11.440424Z","shell.execute_reply":"2023-07-03T07:49:11.444879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdcsfog_metadata = pd.read_parquet(\"./output/fe/fe001/fe001_tdcsfog_meta.parquet\")\ntdcsfog_metadata.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:11.447191Z","iopub.execute_input":"2023-07-03T07:49:11.447695Z","iopub.status.idle":"2023-07-03T07:49:11.514961Z","shell.execute_reply.started":"2023-07-03T07:49:11.447665Z","shell.execute_reply":"2023-07-03T07:49:11.514083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols       = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols   = ['AccV', 'AccML', 'AccAP',  \n              'AccV_lag_diff',  'AccV_lead_diff',  'AccV_cumsum', \n              'AccML_lag_diff', 'AccML_lead_diff', 'AccML_cumsum', \n              'AccAP_lag_diff', 'AccAP_lead_diff', 'AccAP_cumsum']\ntarget_cols = [\"StartHesitation\",\"Turn\",\"Walking\"]\n\nseq_len = 1000\nshift = 500\noffset = 250\n\n# Abrachan:\n# https://www.kaggle.com/competitions/tlvmc-parkinsons-freezing-gait-prediction/discussion/416057\n#     Each Id was split into sequences of a specified length. During training, \n#     we used a shorter length (e.g., 1000 for tdcsfog, 5000 for defog)\n#     For tdcsfog, sequences were created by shifting 500 steps from the starting position.","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:11.516033Z","iopub.execute_input":"2023-07-03T07:49:11.516567Z","iopub.status.idle":"2023-07-03T07:49:11.523345Z","shell.execute_reply.started":"2023-07-03T07:49:11.516535Z","shell.execute_reply":"2023-07-03T07:49:11.522519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_array = []\ntarget_array = []\nsubject_list = []\nid_list = []\nmask_array = []\npred_use_array = []\ntime_array = []","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:11.525534Z","iopub.execute_input":"2023-07-03T07:49:11.525815Z","iopub.status.idle":"2023-07-03T07:49:11.534274Z","shell.execute_reply.started":"2023-07-03T07:49:11.525792Z","shell.execute_reply":"2023-07-03T07:49:11.533327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i,s in tqdm(zip(tdcsfog_metadata[\"Id\"].values, tdcsfog_metadata[\"Sub_id\"].values), desc=\"tdcsfog_metadata: \"):\n# for i,s in tqdm(zip(meta[\"Id\"].values, meta[\"sub_id\"].values)):\n    # path = f\"../data/train/tdcsfog/{i}.csv\"\n    path = root_data + f\"train/tdcsfog/{i}.csv\"\n    df = pd.read_csv(path)\n    \n    batch = (len(df)-1) // shift    # Abrachan: Why minus 1 here?\n    \n    for c in cols:\n        df[f\"{c}_lag_diff\"] = df[c].diff()\n        df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n        df[f\"{c}_cumsum\"] = df[c].cumsum()\n    \n    sc = RobustScaler()\n    df[num_cols] = sc.fit_transform(df[num_cols].values)\n    df[num_cols] = df[num_cols].fillna(0)\n    # for c in num_cols:\n    #     df[c] = (df[c] - mean_std_dict[c][0]) / mean_std_dict[c][1]\n    #     df[c] = df[c].fillna(0)\n    \n    num    = df[num_cols].values\n    target = df[target_cols].values\n    time_values   = df[\"Time\"].values\n    \n    num_array_      = np.zeros([batch, seq_len, 12])\n    target_array_   = np.zeros([batch, seq_len,  3])\n    time_array_     = np.zeros([batch, seq_len    ], dtype=int)\n    \n    mask_array_     = np.zeros([batch, seq_len    ], dtype=int)\n    pred_use_array_ = np.zeros([batch, seq_len    ], dtype=int)\n    \n    \n    for n,b in enumerate(range(batch)):\n        if b == (batch - 1):\n            num_ = num[b*shift : ]\n            num_array_[b, :len(num_), :] = num_\n            \n            target_ = target[b*shift : ]\n            target_array_[b, :len(target_), :] = target_\n            \n            mask_array_[b, :len(target_)] = 1\n            \n            pred_use_array_[b, offset:len(target_)] = 1\n            \n            time_ = time_values[b*shift : ]\n            time_array_[b, :len(time_)] = time_\n            \n        elif b == 0:\n            num_ = num[b*shift : b*shift+seq_len]          # [0:1000]\n            num_array_[b, :, :] = num_\n            \n            target_ = target[b*shift : b*shift+seq_len]    # [0:1000]\n            target_array_[b, :, :] = target_\n            \n            mask_array_[b, :] = 1                          # [b, :1000]\n            \n            pred_use_array_[b, :offset+shift] = 1          # [b, :750]    <===\n            \n            time_ = time_values[b*shift : b*shift+seq_len] # [0:1000]\n            time_array_[b, :] = time_\n        \n        else:\n            num_ = num[b*shift : b*shift+seq_len]          # for b=1: [500:1500]\n            num_array_[b, :, :] = num_\n            \n            target_ = target[b*shift : b*shift+seq_len]    # for b=1: [500:1500]\n            target_array_[b, :, :] = target_\n            \n            mask_array_[b, :] = 1                          # for b=1: [b, :1000]\n            \n            pred_use_array_[b, offset:offset+shift] = 1    # for b=1: [b, 250:750]   <===\n            \n            time_ = time_values[b*shift : b*shift+seq_len] # for b=1: [500:1500]\n            time_array_[b,:] = time_\n    \n    \n    num_array.append(num_array_)\n    target_array.append(target_array_)\n    mask_array.append(mask_array_)\n    pred_use_array.append(pred_use_array_)\n    time_array.append(time_array_)\n    \n    subject_list += [s for _ in range(batch)]     # s is `Sub_id`\n    id_list      += [i for _ in range(batch)]     # i is `Id`","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:11.537008Z","iopub.execute_input":"2023-07-03T07:49:11.537331Z","iopub.status.idle":"2023-07-03T07:49:47.033029Z","shell.execute_reply.started":"2023-07-03T07:49:11.537301Z","shell.execute_reply":"2023-07-03T07:49:47.031897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_array      = np.concatenate(num_array, axis=0)\ntarget_array   = np.concatenate(target_array, axis=0)\nmask_array     = np.concatenate(mask_array, axis=0)\npred_use_array = np.concatenate(pred_use_array, axis=0)\ntime_array     = np.concatenate(time_array, axis=0)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:47.034587Z","iopub.execute_input":"2023-07-03T07:49:47.034935Z","iopub.status.idle":"2023-07-03T07:49:47.636860Z","shell.execute_reply.started":"2023-07-03T07:49:47.034902Z","shell.execute_reply":"2023-07-03T07:49:47.635911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_id = pd.DataFrame()\ndf_id[\"Id\"] = id_list\ndf_id[\"subject\"] = subject_list\n\ndf_id","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:47.638424Z","iopub.execute_input":"2023-07-03T07:49:47.638775Z","iopub.status.idle":"2023-07-03T07:49:47.714506Z","shell.execute_reply.started":"2023-07-03T07:49:47.638742Z","shell.execute_reply":"2023-07-03T07:49:47.713461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save(f\"./output/fe/fe{fe}/fe{fe}_num_array.npy\", num_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_target_array.npy\", target_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_mask_array.npy\", mask_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_time_array.npy\",time_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_pred_use_array.npy\", pred_use_array)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:47.715919Z","iopub.execute_input":"2023-07-03T07:49:47.716429Z","iopub.status.idle":"2023-07-03T07:49:50.090662Z","shell.execute_reply.started":"2023-07-03T07:49:47.716398Z","shell.execute_reply":"2023-07-03T07:49:50.089300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_id.to_parquet(f\"./output/fe/fe{fe}/fe{fe}_id.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:50.096703Z","iopub.execute_input":"2023-07-03T07:49:50.099817Z","iopub.status.idle":"2023-07-03T07:49:50.151026Z","shell.execute_reply.started":"2023-07-03T07:49:50.099771Z","shell.execute_reply":"2023-07-03T07:49:50.149785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n## <font color=red>**Training**</font>","metadata":{}},{"cell_type":"markdown","source":"<br>\n\nCode below is common for the tdcsfog notebooks whose names start with \"ex\", for example: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/exp/ex143_tdcsfog_gru.ipynb\" style=\"text-decoration:none\">ex143_tdcsfog_gru.ipynb</a>\n\n<br>","metadata":{}},{"cell_type":"code","source":"TRAIN_FLAG_TDCSFOG = True","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:50.156127Z","iopub.execute_input":"2023-07-03T07:49:50.156934Z","iopub.status.idle":"2023-07-03T07:49:51.501321Z","shell.execute_reply.started":"2023-07-03T07:49:50.156893Z","shell.execute_reply":"2023-07-03T07:49:51.499814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### helpers\n\n<br>\n\n#### def `set_seed()`","metadata":{}},{"cell_type":"code","source":"def set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:51.502958Z","iopub.execute_input":"2023-07-03T07:49:51.503702Z","iopub.status.idle":"2023-07-03T07:49:52.728689Z","shell.execute_reply.started":"2023-07-03T07:49:51.503663Z","shell.execute_reply":"2023-07-03T07:49:52.727562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### def `init_logger()`","metadata":{}},{"cell_type":"code","source":"def init_logger(log_file):\n    \n    from logging import getLogger, INFO, FileHandler, Formatter, StreamHandler\n    \n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    \n    return logger\n\n\n# LOGGER = init_logger(log_file=logger_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:52.730311Z","iopub.execute_input":"2023-07-03T07:49:52.730689Z","iopub.status.idle":"2023-07-03T07:49:56.442262Z","shell.execute_reply.started":"2023-07-03T07:49:52.730656Z","shell.execute_reply":"2023-07-03T07:49:56.441259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### def `timer()`","metadata":{}},{"cell_type":"code","source":"@contextmanager\ndef timer(name):\n    t0 = time.time()\n    \n    yield \n    # LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s')\n    print(f'[{name}] done in {time.time() - t0:.0f} s')\n    print(\"=\"*66)\n    print(\"\\n\"*2)\n    \n# setup_logger(out_file=logger_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:56.443737Z","iopub.execute_input":"2023-07-03T07:49:56.444584Z","iopub.status.idle":"2023-07-03T07:49:56.456335Z","shell.execute_reply.started":"2023-07-03T07:49:56.444549Z","shell.execute_reply":"2023-07-03T07:49:56.455335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### def `preprocess()`","metadata":{}},{"cell_type":"code","source":"def preprocess(numerical_array, mask_array,):\n    \n    attention_mask = (mask_array == 0)\n\n    return {'input_data_numerical_array': numerical_array,\n            'input_data_mask_array': mask_array,\n            'attention_mask': attention_mask,\n           }","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:56.457653Z","iopub.execute_input":"2023-07-03T07:49:56.458060Z","iopub.status.idle":"2023-07-03T07:49:56.467031Z","shell.execute_reply.started":"2023-07-03T07:49:56.458030Z","shell.execute_reply":"2023-07-03T07:49:56.466197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### class `FogDataset()`","metadata":{}},{"cell_type":"code","source":"class FogDataset(Dataset):\n    def __init__(self, numerical_array, mask_array, train = True, y = None):\n        self.numerical_array = numerical_array\n        self.mask_array = mask_array\n        self.train = train\n        self.y = y\n    \n    def __len__(self):\n        return len(self.numerical_array)\n\n    def __getitem__(self, item):\n        data = preprocess(self.numerical_array[item], self.mask_array[item],)\n        \n        # Return the processed data where the lists are converted to `torch.tensor`s\n        if self.train : \n            return {\n              'input_data_numerical_array': torch.tensor(data['input_data_numerical_array'], dtype=torch.float32),\n              'input_data_mask_array'     : torch.tensor(data['input_data_mask_array'], dtype=torch.long),  \n              'attention_mask'            : torch.tensor(data[\"attention_mask\"], dtype=torch.bool),\n              \"y\"                         : torch.tensor(self.y[item], dtype=torch.float32)\n               }\n        else:\n            return {\n                'input_data_numerical_array': torch.tensor(data['input_data_numerical_array'], dtype=torch.float32),\n                'input_data_mask_array'     : torch.tensor(data['input_data_mask_array'], dtype=torch.long),  \n                'attention_mask'            : torch.tensor(data[\"attention_mask\"], dtype=torch.bool),\n               }","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:56.468427Z","iopub.execute_input":"2023-07-03T07:49:56.468861Z","iopub.status.idle":"2023-07-03T07:49:56.479722Z","shell.execute_reply.started":"2023-07-03T07:49:56.468831Z","shell.execute_reply":"2023-07-03T07:49:56.478849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### class `FogRnnModel()`","metadata":{}},{"cell_type":"code","source":"class FogRnnModel(nn.Module):\n    def __init__(self, \n                 dropout=0.2,\n                 input_numerical_size=12,        # len(num_cols) = 12\n                 numeraical_linear_size = 64,\n                 model_size = 128,\n                 linear_out = 128,\n                 out_size=3):\n        \n        super(FogRnnModel, self).__init__()\n        \n        # input shape : [batch, seq_len, len(num_cols)]            ->  [24, 1000, 12]\n        # output shape: [batch, seq_len, numeraical_linear_size]   ->  [24, 1000, 64]\n        self.numerical_linear  = nn.Sequential(nn.Linear(input_numerical_size, numeraical_linear_size),\n                                               nn.LayerNorm(numeraical_linear_size)\n                                              )\n        \n        # input shape : [batch, seq_len, numeraical_linear_size]   ->  [24, 1000, 64]\n        # h_0 shape   : [D∗num_layers, batch, hidden_size]         ->  [2*2, 24, 128]\n        # output shape: [batch, seq_len, D*hidden_size]            ->  [24,1000, 128*2]\n        self.rnn = nn.GRU(numeraical_linear_size,    # input_size – The number of expected features in the input x\n                          model_size,                # hidden_size – The number of features in the hidden state h\n                          num_layers = 2,            # num_layers – Number of recurrent layers. E.g., setting num_layers=2 would mean stacking two GRUs together to form a stacked GRU\n                          batch_first=True,          # If True, the input and output tensors are provided as (batch, seq, feature) instead of (seq, batch, feature).\n                          bidirectional=True)\n        \n        # input shape : [batch, seq_len, D*hidden_size]  ->  [24, 1000, 128*2]\n        # output shape: [batch, seq_len, linear_out]     ->  [24, 1000, 3]\n        self.linear_out  = nn.Sequential(nn.Linear(model_size*2, linear_out),    # [128*2, 3]\n                                         nn.LayerNorm(linear_out),\n                                         nn.ReLU(),\n                                         nn.Dropout(dropout),\n                                         nn.Linear(linear_out, out_size)\n                                        )\n        self._reinitialize()\n        \n        \n    def _reinitialize(self):\n        \"\"\"Tensorflow/Keras-like initialization\"\"\"\n\n        for name, p in self.named_parameters():\n            if 'rnn' in name:\n                if 'weight_ih' in name:\n                    nn.init.xavier_uniform_(p.data)\n                elif 'weight_hh' in name:\n                    nn.init.orthogonal_(p.data)\n                elif 'bias_ih' in name:\n                    p.data.fill_(0)\n                    # Set forget-gate bias to 1\n                    n = p.size(0)\n                    p.data[(n // 4):(n // 2)].fill_(1)\n                elif 'bias_hh' in name:\n                    p.data.fill_(0)\n    \n    \n    def forward(self, numerical_array, mask_array, attention_mask):        \n        numerical_embedding = self.numerical_linear(numerical_array)\n        output, _           = self.rnn(numerical_embedding)\n        output              = self.linear_out(output)\n        return output","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:56.481005Z","iopub.execute_input":"2023-07-03T07:49:56.481548Z","iopub.status.idle":"2023-07-03T07:49:56.495715Z","shell.execute_reply.started":"2023-07-03T07:49:56.481489Z","shell.execute_reply":"2023-07-03T07:49:56.494817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### load data & preprocessing","metadata":{}},{"cell_type":"code","source":"id_path        = f\"./output/fe/fe022/fe022_id.parquet\"\n\nnumerical_path = f\"./output/fe/fe022/fe022_num_array.npy\"\ntarget_path    = f\"./output/fe/fe022/fe022_target_array.npy\"\nmask_path      = f\"./output/fe/fe022/fe022_mask_array.npy\"\npred_use_path  = f\"./output/fe/fe022/fe022_pred_use_array.npy\"\n\n\ndf_id           = pd.read_parquet(id_path)\n\nnumerical_array = np.load(numerical_path)\ntarget_array    = np.load(target_path)\nmask_array      = np.load(mask_path)\npred_use_array  = np.load(pred_use_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:56.496909Z","iopub.execute_input":"2023-07-03T07:49:56.497660Z","iopub.status.idle":"2023-07-03T07:49:57.750877Z","shell.execute_reply.started":"2023-07-03T07:49:56.497629Z","shell.execute_reply":"2023-07-03T07:49:57.749859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target1 = []\ntarget2 = []\ntarget3 = []\nfor i in range(len(target_array)):\n    target1.append(np.sum(target_array[i, :, 0]))\n    target2.append(np.sum(target_array[i, :, 1]))\n    target3.append(np.sum(target_array[i, :, 2]))","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:57.760392Z","iopub.execute_input":"2023-07-03T07:49:57.760694Z","iopub.status.idle":"2023-07-03T07:49:58.104507Z","shell.execute_reply.started":"2023-07-03T07:49:57.760670Z","shell.execute_reply":"2023-07-03T07:49:58.103385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_id[\"target1\"] = target1\ndf_id[\"target2\"] = target2\ndf_id[\"target3\"] = target3\ndf_id[\"target1_1\"] = df_id[\"target1\"] > 0\ndf_id[\"target2_1\"] = df_id[\"target2\"] > 0\ndf_id[\"target3_1\"] = df_id[\"target3\"] > 0\ndf_id[\"target1_1\"] = df_id[\"target1_1\"].astype(np.int)\ndf_id[\"target2_1\"] = df_id[\"target2_1\"].astype(np.int)\ndf_id[\"target3_1\"] = df_id[\"target3_1\"].astype(np.int)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.106004Z","iopub.execute_input":"2023-07-03T07:49:58.106430Z","iopub.status.idle":"2023-07-03T07:49:58.132551Z","shell.execute_reply.started":"2023-07-03T07:49:58.106394Z","shell.execute_reply":"2023-07-03T07:49:58.131620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_id[\"group\"] = 0\ndf_id.loc[df_id[\"target2_1\"] > 0,\"group\"] = 1\ndf_id.loc[df_id[\"target1_1\"] > 0,\"group\"] = 2\ndf_id.loc[df_id[\"target3_1\"] > 0,\"group\"] = 3","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.135531Z","iopub.execute_input":"2023-07-03T07:49:58.136039Z","iopub.status.idle":"2023-07-03T07:49:58.146755Z","shell.execute_reply.started":"2023-07-03T07:49:58.136007Z","shell.execute_reply":"2023-07-03T07:49:58.145733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_id[\"group\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.148157Z","iopub.execute_input":"2023-07-03T07:49:58.148533Z","iopub.status.idle":"2023-07-03T07:49:58.162559Z","shell.execute_reply.started":"2023-07-03T07:49:58.148499Z","shell.execute_reply":"2023-07-03T07:49:58.161501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### config","metadata":{}},{"cell_type":"code","source":"# config\nseed = 0\nshuffle = True\nn_splits = 5\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# model config\nbatch_size = 24\nn_epochs = 10\nlr = 1e-3\nweight_decay = 0.05\nnum_warmup_steps = 10","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.164170Z","iopub.execute_input":"2023-07-03T07:49:58.164609Z","iopub.status.idle":"2023-07-03T07:49:58.242485Z","shell.execute_reply.started":"2023-07-03T07:49:58.164578Z","shell.execute_reply":"2023-07-03T07:49:58.241268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### <font color=maroon>**ex143_tdcsfog_gru.ipynb**</font>\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/exp/ex143_tdcsfog_gru.ipynb\" style=\"text-decoration:none\">ex143_tdcsfog_gru.ipynb</a>","metadata":{}},{"cell_type":"code","source":"debug = False\n\nex = \"143_tdcsfog\"\nif not os.path.exists(f\"./output/exp/ex{ex}\"):\n    os.makedirs(f\"./output/exp/ex{ex}\")\n    os.makedirs(f\"./output/exp/ex{ex}/ex{ex}_model\")\n    \nlogger_path = f\"./output/exp/ex{ex}/log_ex_{ex}.txt\"\nLOGGER = init_logger(log_file=logger_path)\n\n# model_path  = f\"./output/exp/ex{ex}/ex{ex}_model/ex{ex}.pth\"","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.246562Z","iopub.execute_input":"2023-07-03T07:49:58.246914Z","iopub.status.idle":"2023-07-03T07:49:58.255315Z","shell.execute_reply.started":"2023-07-03T07:49:58.246888Z","shell.execute_reply":"2023-07-03T07:49:58.254234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### main","metadata":{}},{"cell_type":"code","source":"%%time\n\n# with timer(\"gru\"):\nif TRAIN_FLAG:\n    set_seed(seed)\n    y_oof = np.empty([len(target_array), 1000, 3])      # Abrachan: out of fold\n    gkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state = seed)\n    for fold, (train_idx, valid_idx) in enumerate(gkf.split(numerical_array, \n                                                            y = df_id[\"group\"].values,\n                                                            groups=df_id[\"subject\"].values)):\n        # LOGGER.info(f\"start fold: {fold+1}\")\n        print(f\"start fold: {fold+1}\")\n        \n        with timer(f\"fold {fold+1}\"):\n            train_numerical_array = numerical_array[train_idx]\n            train_target_array = target_array[train_idx]\n            train_mask_array = mask_array[train_idx]\n\n            val_numerical_array = numerical_array[valid_idx]\n            val_target_array = target_array[valid_idx]\n            val_mask_array = mask_array[valid_idx]\n            val_pred_array = pred_use_array[valid_idx]\n            \n            train_ = FogDataset(train_numerical_array,\n                                train_mask_array,\n                                train=True,\n                                y=train_target_array)\n            val_ = FogDataset(val_numerical_array,\n                              val_mask_array,\n                              train=True,\n                              y=val_target_array)\n            \n            train_loader = DataLoader(dataset=train_, \n                                      batch_size=batch_size, \n                                      shuffle = True, \n                                      # num_workers=8,\n                                     )\n            val_loader = DataLoader(dataset=val_, \n                                    batch_size=batch_size, \n                                    shuffle = False , \n                                    # num_workers=8\n                                   )\n            \n            \n            model = FogRnnModel()\n            model = model.to(device)\n            \n            param_optimizer = list(model.named_parameters())\n            no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n            optimizer_grouped_parameters = [\n                {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], \n                 'weight_decay': weight_decay\n                },\n                {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], \n                 'weight_decay': 0.0\n                }]\n            optimizer = AdamW(optimizer_grouped_parameters,\n                              lr=lr,\n                              weight_decay=weight_decay,\n                              )\n            num_train_optimization_steps = int(len(train_loader) * n_epochs)\n            scheduler = get_linear_schedule_with_warmup(optimizer,\n                                                        num_warmup_steps=num_warmup_steps,\n                                                        num_training_steps=num_train_optimization_steps)\n            criterion = nn.BCEWithLogitsLoss()\n            best_val_score = 0\n            \n            for epoch in range(n_epochs):\n                model.train() \n                train_losses_batch = []\n                val_losses_batch = []\n                epoch_loss = 0\n                train_preds = np.ndarray((0,3))   # array([], shape=(0, 3), dtype=float64)\n                \n                tk0 = tqdm_nb(train_loader, total=len(train_loader), desc=\"train_loader: \")\n                for d in tk0:\n                    # ======================================================\n                    # data loader\n                    # ======================================================\n                    input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                    input_data_mask_array      = d['input_data_mask_array'].to(device)\n                    attention_mask             = d['attention_mask'].to(device)\n                    y                          = d[\"y\"].to(device)\n                    \n                    optimizer.zero_grad()\n                    output = model(input_data_numerical_array, \n                                   input_data_mask_array,\n                                   attention_mask)\n                    loss = criterion(output[input_data_mask_array == 1], y[input_data_mask_array == 1])\n                    loss.backward()\n                    optimizer.step()\n                    scheduler.step()\n                    train_losses_batch.append(loss.item())\n                train_loss = np.mean(train_losses_batch)\n                \n                \n                # ======================================================\n                # eval\n                # ======================================================\n                model.eval()       # switch model to the evaluation mode\n                \n                val_preds = np.ndarray((0, 1000, 3))\n                tk0 = tqdm_nb(val_loader, total=len(val_loader), desc=\"val_loader  : \")\n                with torch.no_grad():  # Do not calculate gradient since we are only predicting\n                    # Predicting on validation set\n                    for d in tk0:\n                        input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                        input_data_mask_array      = d['input_data_mask_array'].to(device)\n                        attention_mask             = d['attention_mask'].to(device)\n                        \n                        output = model(input_data_numerical_array, \n                                       input_data_mask_array,\n                                       attention_mask)\n                        val_preds = np.concatenate([val_preds, output.sigmoid().detach().cpu().numpy()], axis=0)\n                \n                pred_valid_index = (val_mask_array == 1) & (val_pred_array == 1)\n                StartHesitation = average_precision_score(val_target_array[pred_valid_index][:, 0],\n                                                          val_preds[pred_valid_index][:, 0])\n                Turn = average_precision_score(val_target_array[pred_valid_index][:, 1],\n                                               val_preds[pred_valid_index][:, 1])\n                Walking = average_precision_score(val_target_array[pred_valid_index][:, 2],\n                                                  val_preds[pred_valid_index][:, 2])\n                map_score = np.mean([StartHesitation, Turn, Walking])\n                \n                # LOGGER.info(f\"fold: {fold+1} epoch: {epoch+1},train loss {train_loss} map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                print(f\"fold {fold+1} epoch {epoch+1}\\ntrain loss {train_loss} map:{map_score}\\nstart_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                \n                if map_score >= best_val_score:\n                    print(\"save weight\")\n                    best_val_score = map_score\n                    best_val_preds = val_preds.copy()\n                    torch.save(model.state_dict(), f\"./output/exp/ex{ex}/ex{ex}_model/ex{ex}_fold{fold+1}_epoch{epoch+1}__map{best_val_score:.6f}.pth\")\n                print()\n            y_oof[valid_idx] = best_val_preds\n    \n    np.save(f\"./output/exp/ex{ex}/ex{ex}_oof.npy\", y_oof)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.257009Z","iopub.execute_input":"2023-07-03T07:49:58.257672Z","iopub.status.idle":"2023-07-03T07:49:58.294068Z","shell.execute_reply.started":"2023-07-03T07:49:58.257640Z","shell.execute_reply":"2023-07-03T07:49:58.293013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### oof score","metadata":{}},{"cell_type":"code","source":"if TRAIN_FLAG:\n    val_pred_index = (mask_array == 1) & (pred_use_array == 1)\n    StartHesitation = average_precision_score(target_array[val_pred_index][:,0], \n                                              y_oof[val_pred_index][:,0])\n    Turn            = average_precision_score(target_array[val_pred_index][:,1], \n                                              y_oof[val_pred_index][:,1])\n    Walking         = average_precision_score(target_array[val_pred_index][:,2],\n                                              y_oof[val_pred_index][:,2])\n\n    map_score = np.mean([StartHesitation, Turn, Walking])\n    # LOGGER.info(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n    print(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n\n\n\n    # kaggle_json = {\"title\": f\"fog-ex{ex}\",\n    #                \"id\": f\"takoihiraokazu/fog-ex{ex}\",\n    #                \"licenses\": [{\"name\": \"CC0-1.0\"}]}\n\n    # with open(f\"./output/exp/ex{ex}/ex{ex}_model/dataset-metadata.json\", 'w') as f:\n    #     json.dump(kaggle_json, f)\n\n\n    # del LOGGER\n    # gc.collect()\n    # logging.shutdown()","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.295541Z","iopub.execute_input":"2023-07-03T07:49:58.296121Z","iopub.status.idle":"2023-07-03T07:49:58.303925Z","shell.execute_reply.started":"2023-07-03T07:49:58.296089Z","shell.execute_reply":"2023-07-03T07:49:58.302714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### <font color=maroon>ex145_tdcsfog_gru_StartHesitation.ipynb</font>\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/exp/ex145_tdcsfog_gru_StartHesitation.ipynb\" style=\"text-decoration:none\">ex145_tdcsfog_gru_StartHesitation.ipynb</a>","metadata":{}},{"cell_type":"code","source":"debug = False\n\nex = \"145_tdcsfog\"\nif not os.path.exists(f\"./output/exp/ex{ex}\"):\n    os.makedirs(f\"./output/exp/ex{ex}\")\n    os.makedirs(f\"./output/exp/ex{ex}/ex{ex}_model\")\n    \n# logger_path = f\"./output/exp/ex{ex}/log_ex_{ex}.txt\"\n# LOGGER = init_logger(log_file=logger_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.305574Z","iopub.execute_input":"2023-07-03T07:49:58.306248Z","iopub.status.idle":"2023-07-03T07:49:58.317988Z","shell.execute_reply.started":"2023-07-03T07:49:58.306184Z","shell.execute_reply":"2023-07-03T07:49:58.316860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### main","metadata":{}},{"cell_type":"code","source":"%%time\n\n# with timer(\"gru\"):\nif TRAIN_FLAG:\n    set_seed(seed)\n    y_oof = np.empty([len(target_array), 1000, 3])      # Abrachan: out of fold\n    gkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state = seed)\n    for fold, (train_idx, valid_idx) in enumerate(gkf.split(numerical_array, \n                                                            y = df_id[\"group\"].values,\n                                                            groups=df_id[\"subject\"].values)):\n        # LOGGER.info(f\"start fold:{fold+1}\")\n        print(f\"start fold:{fold+1}\")\n        \n        with timer(f\"fold {fold+1}\"):\n            train_numerical_array = numerical_array[train_idx]\n            train_target_array = target_array[train_idx]\n            train_mask_array = mask_array[train_idx]\n\n            val_numerical_array = numerical_array[valid_idx]\n            val_target_array = target_array[valid_idx]\n            val_mask_array = mask_array[valid_idx]\n            val_pred_array = pred_use_array[valid_idx]\n            \n            train_ = FogDataset(train_numerical_array,\n                                train_mask_array,\n                                train=True,\n                                y=train_target_array)\n            val_ = FogDataset(val_numerical_array,\n                              val_mask_array,\n                              train=True,\n                              y=val_target_array)\n            \n            train_loader = DataLoader(dataset=train_, \n                                      batch_size=batch_size, \n                                      shuffle = True, \n                                      # num_workers=8,\n                                     )\n            val_loader = DataLoader(dataset=val_, \n                                    batch_size=batch_size, \n                                    shuffle = False , \n                                    # num_workers=8\n                                   )\n            \n            \n            model = FogRnnModel()\n            model = model.to(device)\n            \n            param_optimizer = list(model.named_parameters())\n            no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n            optimizer_grouped_parameters = [\n                {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], \n                 'weight_decay': weight_decay\n                },\n                {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], \n                 'weight_decay': 0.0\n                }]\n            optimizer = AdamW(optimizer_grouped_parameters,\n                              lr=lr,\n                              weight_decay=weight_decay,\n                              )\n            num_train_optimization_steps = int(len(train_loader) * n_epochs)\n            scheduler = get_linear_schedule_with_warmup(optimizer,\n                                                        num_warmup_steps=num_warmup_steps,\n                                                        num_training_steps=num_train_optimization_steps)\n            criterion = nn.BCEWithLogitsLoss()\n            best_val_score = 0\n            \n            for epoch in range(n_epochs):\n                model.train() \n                train_losses_batch = []\n                val_losses_batch = []\n                epoch_loss = 0\n                train_preds = np.ndarray((0,3))   # array([], shape=(0, 3), dtype=float64)\n                \n                tk0 = tqdm_nb(train_loader, total=len(train_loader), desc=\"train_loader: \")\n                for d in tk0:\n                    # ======================================================\n                    # data loader\n                    # ======================================================\n                    input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                    input_data_mask_array      = d['input_data_mask_array'].to(device)\n                    attention_mask             = d['attention_mask'].to(device)\n                    y                          = d[\"y\"].to(device)\n                    \n                    optimizer.zero_grad()\n                    output = model(input_data_numerical_array, \n                                   input_data_mask_array,\n                                   attention_mask)\n                    # loss = criterion(output[input_data_mask_array == 1], y[input_data_mask_array == 1])\n                    \n                    output1 = output[:, :, 0    ]\n                    output2 = output[:, :, [1,2]]\n                    y1 = y[:, :, 0    ]\n                    y2 = y[:, :, [1,2]]\n                    loss1 = criterion(output1[input_data_mask_array == 1], y1[input_data_mask_array == 1])\n                    loss2 = criterion(output2[input_data_mask_array == 1], y2[input_data_mask_array == 1])\n                    \n                    loss = loss1*0.6 + loss2*0.4\n                    loss.backward()\n                    optimizer.step()\n                    scheduler.step()\n                    train_losses_batch.append(loss.item())\n                train_loss = np.mean(train_losses_batch)\n                \n                \n                # ======================================================\n                # eval\n                # ======================================================\n                model.eval()       # switch model to the evaluation mode\n                \n                val_preds = np.ndarray((0, 1000, 3))\n                tk0 = tqdm_nb(val_loader, total=len(val_loader), desc=\"val_loader  : \")\n                with torch.no_grad():  # Do not calculate gradient since we are only predicting\n                    # Predicting on validation set\n                    for d in tk0:\n                        input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                        input_data_mask_array      = d['input_data_mask_array'].to(device)\n                        attention_mask             = d['attention_mask'].to(device)\n                        \n                        output = model(input_data_numerical_array, \n                                       input_data_mask_array,\n                                       attention_mask)\n                        val_preds = np.concatenate([val_preds, output.sigmoid().detach().cpu().numpy()], axis=0)\n                \n                pred_valid_index = (val_mask_array == 1) & (val_pred_array == 1)\n                StartHesitation = average_precision_score(val_target_array[pred_valid_index][:, 0],\n                                                          val_preds[pred_valid_index][:, 0])\n                Turn = average_precision_score(val_target_array[pred_valid_index][:, 1],\n                                               val_preds[pred_valid_index][:, 1])\n                Walking = average_precision_score(val_target_array[pred_valid_index][:, 2],\n                                                  val_preds[pred_valid_index][:, 2])\n                map_score = np.mean([StartHesitation, Turn, Walking])\n                \n                # LOGGER.info(f\"fold: {fold+1} epoch: {epoch+1},train loss {train_loss} map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                print(f\"fold {fold+1} epoch {epoch+1}\\ntrain loss {train_loss} map:{map_score}\\nstart_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                \n                if map_score >= best_val_score:\n                    print(\"save weight\")\n                    best_val_score = map_score\n                    best_val_preds = val_preds.copy()\n                    torch.save(model.state_dict(), f\"./output/exp/ex{ex}/ex{ex}_model/ex{ex}_fold{fold+1}_epoch{epoch+1}__map{best_val_score:.6f}.pth\")\n                print()\n            y_oof[valid_idx] = best_val_preds\n    \n    np.save(f\"./output/exp/ex{ex}/ex{ex}_oof.npy\", y_oof)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.319583Z","iopub.execute_input":"2023-07-03T07:49:58.320296Z","iopub.status.idle":"2023-07-03T07:49:58.358309Z","shell.execute_reply.started":"2023-07-03T07:49:58.320264Z","shell.execute_reply":"2023-07-03T07:49:58.357275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### oof score","metadata":{}},{"cell_type":"code","source":"if TRAIN_FLAG:\n    val_pred_index = (mask_array == 1) & (pred_use_array == 1)\n    StartHesitation = average_precision_score(target_array[val_pred_index][:,0], \n                                              y_oof[val_pred_index][:,0])\n    Turn            = average_precision_score(target_array[val_pred_index][:,1], \n                                              y_oof[val_pred_index][:,1])\n    Walking         = average_precision_score(target_array[val_pred_index][:,2],\n                                              y_oof[val_pred_index][:,2])\n\n    map_score = np.mean([StartHesitation, Turn, Walking])\n    # LOGGER.info(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n    print(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n\n\n\n    # kaggle_json = {\"title\": f\"fog-ex{ex}\",\n    #                \"id\": f\"takoihiraokazu/fog-ex{ex}\",\n    #                \"licenses\": [{\"name\": \"CC0-1.0\"}]}\n\n    # with open(f\"./output/exp/ex{ex}/ex{ex}_model/dataset-metadata.json\", 'w') as f:\n    #     json.dump(kaggle_json, f)\n\n\n    # del LOGGER\n    # gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.361510Z","iopub.execute_input":"2023-07-03T07:49:58.361934Z","iopub.status.idle":"2023-07-03T07:49:58.371785Z","shell.execute_reply.started":"2023-07-03T07:49:58.361896Z","shell.execute_reply":"2023-07-03T07:49:58.369790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### <font color=maroon>ex146_tdcsfog_gru_Turn.ipynb</font>\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/exp/ex146_tdcsfog_gru_Turn.ipynb\" style=\"text-decoration:none\">ex146_tdcsfog_gru_Turn.ipynb</a>","metadata":{}},{"cell_type":"code","source":"debug = False\nex = \"146_tdcsfog\"\nif not os.path.exists(f\"./output/exp/ex{ex}\"):\n    os.makedirs(f\"./output/exp/ex{ex}\")\n    os.makedirs(f\"./output/exp/ex{ex}/ex{ex}_model\")\n\n# logger_path = f\"./output/exp/ex{ex}/ex_{ex}.txt\"\n# LOGGER = init_logger(log_file=logger_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.373745Z","iopub.execute_input":"2023-07-03T07:49:58.374823Z","iopub.status.idle":"2023-07-03T07:49:58.384602Z","shell.execute_reply.started":"2023-07-03T07:49:58.374769Z","shell.execute_reply":"2023-07-03T07:49:58.383277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### main","metadata":{}},{"cell_type":"code","source":"%%time\n\n# with timer(\"gru\"):\nif TRAIN_FLAG:\n    set_seed(seed)\n    y_oof = np.empty([len(target_array), 1000, 3])      # Abrachan: out of fold\n    gkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state = seed)\n    for fold, (train_idx, valid_idx) in enumerate(gkf.split(numerical_array, \n                                                            y = df_id[\"group\"].values,\n                                                            groups=df_id[\"subject\"].values)):\n        # LOGGER.info(f\"start fold:{fold+1}\")\n        print(f\"start fold: {fold+1}\")\n        \n        with timer(f\"fold {fold+1}\"):\n            train_numerical_array = numerical_array[train_idx]\n            train_target_array = target_array[train_idx]\n            train_mask_array = mask_array[train_idx]\n\n            val_numerical_array = numerical_array[valid_idx]\n            val_target_array = target_array[valid_idx]\n            val_mask_array = mask_array[valid_idx]\n            val_pred_array = pred_use_array[valid_idx]\n            \n            train_ = FogDataset(train_numerical_array,\n                                train_mask_array,\n                                train=True,\n                                y=train_target_array)\n            val_ = FogDataset(val_numerical_array,\n                              val_mask_array,\n                              train=True,\n                              y=val_target_array)\n            \n            train_loader = DataLoader(dataset=train_, \n                                      batch_size=batch_size, \n                                      shuffle = True, \n                                      # num_workers=8,\n                                     )\n            val_loader = DataLoader(dataset=val_, \n                                    batch_size=batch_size, \n                                    shuffle = False , \n                                    # num_workers=8\n                                   )\n            \n            \n            model = FogRnnModel()\n            model = model.to(device)\n            \n            param_optimizer = list(model.named_parameters())\n            no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n            optimizer_grouped_parameters = [\n                {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], \n                 'weight_decay': weight_decay\n                },\n                {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], \n                 'weight_decay': 0.0\n                }]\n            optimizer = AdamW(optimizer_grouped_parameters,\n                              lr=lr,\n                              weight_decay=weight_decay,\n                              )\n            num_train_optimization_steps = int(len(train_loader) * n_epochs)\n            scheduler = get_linear_schedule_with_warmup(optimizer,\n                                                        num_warmup_steps=num_warmup_steps,\n                                                        num_training_steps=num_train_optimization_steps)\n            criterion = nn.BCEWithLogitsLoss()\n            best_val_score = 0\n            \n            for epoch in range(n_epochs):\n                model.train() \n                train_losses_batch = []\n                val_losses_batch = []\n                epoch_loss = 0\n                train_preds = np.ndarray((0,3))   # array([], shape=(0, 3), dtype=float64)\n                \n                tk0 = tqdm_nb(train_loader, total=len(train_loader), desc=\"train_loader: \")\n                for d in tk0:\n                    # ======================================================\n                    # data loader\n                    # ======================================================\n                    input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                    input_data_mask_array      = d['input_data_mask_array'].to(device)\n                    attention_mask             = d['attention_mask'].to(device)\n                    y                          = d[\"y\"].to(device)\n                    \n                    optimizer.zero_grad()\n                    output = model(input_data_numerical_array, \n                                   input_data_mask_array,\n                                   attention_mask)\n                    # loss = criterion(output[input_data_mask_array == 1], y[input_data_mask_array == 1])\n                    \n                    output1 = output[:, :, 1    ]\n                    output2 = output[:, :, [0,2]]\n                    y1 = y[:, :, 1    ]\n                    y2 = y[:, :, [0,2]]\n                    loss1 = criterion(output1[input_data_mask_array == 1], y1[input_data_mask_array == 1])\n                    loss2 = criterion(output2[input_data_mask_array == 1], y2[input_data_mask_array == 1])\n                    \n                    loss = loss1*0.6 + loss2*0.4\n                    loss.backward()\n                    optimizer.step()\n                    scheduler.step()\n                    train_losses_batch.append(loss.item())\n                train_loss = np.mean(train_losses_batch)\n                \n                \n                # ======================================================\n                # eval\n                # ======================================================\n                model.eval()       # switch model to the evaluation mode\n                \n                val_preds = np.ndarray((0, 1000, 3))\n                tk0 = tqdm_nb(val_loader, total=len(val_loader), desc=\"val_loader  : \")\n                with torch.no_grad():  # Do not calculate gradient since we are only predicting\n                    # Predicting on validation set\n                    for d in tk0:\n                        input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                        input_data_mask_array      = d['input_data_mask_array'].to(device)\n                        attention_mask             = d['attention_mask'].to(device)\n                        \n                        output = model(input_data_numerical_array, \n                                       input_data_mask_array,\n                                       attention_mask)\n                        val_preds = np.concatenate([val_preds, output.sigmoid().detach().cpu().numpy()], axis=0)\n                \n                pred_valid_index = (val_mask_array == 1) & (val_pred_array == 1)\n                StartHesitation = average_precision_score(val_target_array[pred_valid_index][:, 0],\n                                                          val_preds[pred_valid_index][:, 0])\n                Turn = average_precision_score(val_target_array[pred_valid_index][:, 1],\n                                               val_preds[pred_valid_index][:, 1])\n                Walking = average_precision_score(val_target_array[pred_valid_index][:, 2],\n                                                  val_preds[pred_valid_index][:, 2])\n                map_score = np.mean([StartHesitation, Turn, Walking])\n                \n                # LOGGER.info(f\"fold: {fold+1} epoch: {epoch+1},train loss {train_loss} map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                print(f\"fold {fold+1} epoch {epoch+1}\\ntrain loss {train_loss} map:{map_score}\\nstart_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                \n                if map_score >= best_val_score:\n                    print(\"save weight\")\n                    best_val_score = map_score\n                    best_val_preds = val_preds.copy()\n                    torch.save(model.state_dict(), f\"./output/exp/ex{ex}/ex{ex}_model/ex{ex}_fold{fold+1}_epoch{epoch+1}__map{best_val_score:.6f}.pth\")\n                print()\n            y_oof[valid_idx] = best_val_preds\n    \n    np.save(f\"./output/exp/ex{ex}/ex{ex}_oof.npy\", y_oof)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.387796Z","iopub.execute_input":"2023-07-03T07:49:58.389476Z","iopub.status.idle":"2023-07-03T07:49:58.424283Z","shell.execute_reply.started":"2023-07-03T07:49:58.389444Z","shell.execute_reply":"2023-07-03T07:49:58.423260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### oof score","metadata":{}},{"cell_type":"code","source":"if TRAIN_FLAG:\n    val_pred_index = (mask_array == 1) & (pred_use_array == 1)\n    StartHesitation = average_precision_score(target_array[val_pred_index][:,0], \n                                              y_oof[val_pred_index][:,0])\n    Turn            = average_precision_score(target_array[val_pred_index][:,1], \n                                              y_oof[val_pred_index][:,1])\n    Walking         = average_precision_score(target_array[val_pred_index][:,2],\n                                              y_oof[val_pred_index][:,2])\n\n    map_score = np.mean([StartHesitation, Turn, Walking])\n    # LOGGER.info(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n    print(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.425473Z","iopub.execute_input":"2023-07-03T07:49:58.425989Z","iopub.status.idle":"2023-07-03T07:49:58.433707Z","shell.execute_reply.started":"2023-07-03T07:49:58.425957Z","shell.execute_reply":"2023-07-03T07:49:58.432639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### <font color=maroon>ex147_tdcsfog_gru_Walking.ipynb</font>\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/exp/ex147_tdcsfog_gru_Walking.ipynb\" style=\"text-decoration:none\">ex147_tdcsfog_gru_Walking.ipynb</a>","metadata":{}},{"cell_type":"code","source":"debug = False\nex = \"147_tdcsfog\"\nif not os.path.exists(f\"./output/exp/ex{ex}\"):\n    os.makedirs(f\"./output/exp/ex{ex}\")\n    os.makedirs(f\"./output/exp/ex{ex}/ex{ex}_model\")\n\n# logger_path = f\"./output/exp/ex{ex}/ex_{ex}.txt\"\n# LOGGER = init_logger(log_file=logger_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.435029Z","iopub.execute_input":"2023-07-03T07:49:58.435950Z","iopub.status.idle":"2023-07-03T07:49:58.446960Z","shell.execute_reply.started":"2023-07-03T07:49:58.435892Z","shell.execute_reply":"2023-07-03T07:49:58.445885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### main","metadata":{}},{"cell_type":"code","source":"%%time\n\n# with timer(\"gru\"):\nif TRAIN_FLAG:\n    set_seed(seed)\n    y_oof = np.empty([len(target_array), 1000, 3])      # Abrachan: out of fold\n    gkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state = seed)\n    for fold, (train_idx, valid_idx) in enumerate(gkf.split(numerical_array, \n                                                            y = df_id[\"group\"].values,\n                                                            groups=df_id[\"subject\"].values)):\n        # LOGGER.info(f\"start fold:{fold+1}\")\n        print(f\"start fold: {fold+1}\")\n        \n        with timer(f\"fold {fold+1}\"):\n            train_numerical_array = numerical_array[train_idx]\n            train_target_array = target_array[train_idx]\n            train_mask_array = mask_array[train_idx]\n\n            val_numerical_array = numerical_array[valid_idx]\n            val_target_array = target_array[valid_idx]\n            val_mask_array = mask_array[valid_idx]\n            val_pred_array = pred_use_array[valid_idx]\n            \n            train_ = FogDataset(train_numerical_array,\n                                train_mask_array,\n                                train=True,\n                                y=train_target_array)\n            val_ = FogDataset(val_numerical_array,\n                              val_mask_array,\n                              train=True,\n                              y=val_target_array)\n            \n            train_loader = DataLoader(dataset=train_, \n                                      batch_size=batch_size, \n                                      shuffle = True, \n                                      # num_workers=8,\n                                     )\n            val_loader = DataLoader(dataset=val_, \n                                    batch_size=batch_size, \n                                    shuffle = False , \n                                    # num_workers=8\n                                   )\n            \n            \n            model = FogRnnModel()\n            model = model.to(device)\n            \n            param_optimizer = list(model.named_parameters())\n            no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n            optimizer_grouped_parameters = [\n                {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], \n                 'weight_decay': weight_decay\n                },\n                {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], \n                 'weight_decay': 0.0\n                }]\n            optimizer = AdamW(optimizer_grouped_parameters,\n                              lr=lr,\n                              weight_decay=weight_decay,\n                              )\n            num_train_optimization_steps = int(len(train_loader) * n_epochs)\n            scheduler = get_linear_schedule_with_warmup(optimizer,\n                                                        num_warmup_steps=num_warmup_steps,\n                                                        num_training_steps=num_train_optimization_steps)\n            criterion = nn.BCEWithLogitsLoss()\n            best_val_score = 0\n            \n            for epoch in range(n_epochs):\n                model.train() \n                train_losses_batch = []\n                val_losses_batch = []\n                epoch_loss = 0\n                train_preds = np.ndarray((0,3))   # array([], shape=(0, 3), dtype=float64)\n                \n                tk0 = tqdm_nb(train_loader, total=len(train_loader), desc=\"train_loader: \")\n                for d in tk0:\n                    # ======================================================\n                    # data loader\n                    # ======================================================\n                    input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                    input_data_mask_array      = d['input_data_mask_array'].to(device)\n                    attention_mask             = d['attention_mask'].to(device)\n                    y                          = d[\"y\"].to(device)\n                    \n                    optimizer.zero_grad()\n                    output = model(input_data_numerical_array, \n                                   input_data_mask_array,\n                                   attention_mask)\n                    # loss = criterion(output[input_data_mask_array == 1], y[input_data_mask_array == 1])\n                    \n                    output1 = output[:, :, 2    ]\n                    output2 = output[:, :, [0,1]]\n                    y1 = y[:, :, 2    ]\n                    y2 = y[:, :, [0,1]]\n                    loss1 = criterion(output1[input_data_mask_array == 1], y1[input_data_mask_array == 1])\n                    loss2 = criterion(output2[input_data_mask_array == 1], y2[input_data_mask_array == 1])\n                    \n                    loss = loss1*0.6 + loss2*0.4\n                    loss.backward()\n                    optimizer.step()\n                    scheduler.step()\n                    train_losses_batch.append(loss.item())\n                train_loss = np.mean(train_losses_batch)\n                \n                \n                # ======================================================\n                # eval\n                # ======================================================\n                model.eval()       # switch model to the evaluation mode\n                \n                val_preds = np.ndarray((0, 1000, 3))\n                tk0 = tqdm_nb(val_loader, total=len(val_loader), desc=\"val_loader  : \")\n                with torch.no_grad():  # Do not calculate gradient since we are only predicting\n                    # Predicting on validation set\n                    for d in tk0:\n                        input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                        input_data_mask_array      = d['input_data_mask_array'].to(device)\n                        attention_mask             = d['attention_mask'].to(device)\n                        \n                        output = model(input_data_numerical_array, \n                                       input_data_mask_array,\n                                       attention_mask)\n                        val_preds = np.concatenate([val_preds, output.sigmoid().detach().cpu().numpy()], axis=0)\n                \n                pred_valid_index = (val_mask_array == 1) & (val_pred_array == 1)\n                StartHesitation = average_precision_score(val_target_array[pred_valid_index][:, 0],\n                                                          val_preds[pred_valid_index][:, 0])\n                Turn = average_precision_score(val_target_array[pred_valid_index][:, 1],\n                                               val_preds[pred_valid_index][:, 1])\n                Walking = average_precision_score(val_target_array[pred_valid_index][:, 2],\n                                                  val_preds[pred_valid_index][:, 2])\n                map_score = np.mean([StartHesitation, Turn, Walking])\n                \n                # LOGGER.info(f\"fold: {fold+1} epoch: {epoch+1},train loss {train_loss} map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                print(f\"fold {fold+1} epoch {epoch+1}\\ntrain loss {train_loss} map:{map_score}\\nstart_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                \n                if map_score >= best_val_score:\n                    print(\"save weight\")\n                    best_val_score = map_score\n                    best_val_preds = val_preds.copy()\n                    torch.save(model.state_dict(), f\"./output/exp/ex{ex}/ex{ex}_model/ex{ex}_fold{fold+1}_epoch{epoch+1}__map{best_val_score:.6f}.pth\")\n                print()\n            y_oof[valid_idx] = best_val_preds\n    \n    np.save(f\"./output/exp/ex{ex}/ex{ex}_oof.npy\", y_oof)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.448597Z","iopub.execute_input":"2023-07-03T07:49:58.449795Z","iopub.status.idle":"2023-07-03T07:49:58.486506Z","shell.execute_reply.started":"2023-07-03T07:49:58.449763Z","shell.execute_reply":"2023-07-03T07:49:58.485551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### oof score","metadata":{}},{"cell_type":"code","source":"if TRAIN_FLAG:\n    val_pred_index = (mask_array == 1) & (pred_use_array == 1)\n    StartHesitation = average_precision_score(target_array[val_pred_index][:,0], \n                                              y_oof[val_pred_index][:,0])\n    Turn            = average_precision_score(target_array[val_pred_index][:,1], \n                                              y_oof[val_pred_index][:,1])\n    Walking         = average_precision_score(target_array[val_pred_index][:,2],\n                                              y_oof[val_pred_index][:,2])\n\n    map_score = np.mean([StartHesitation, Turn, Walking])\n    # LOGGER.info(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n    print(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.488135Z","iopub.execute_input":"2023-07-03T07:49:58.488809Z","iopub.status.idle":"2023-07-03T07:49:58.496180Z","shell.execute_reply.started":"2023-07-03T07:49:58.488775Z","shell.execute_reply":"2023-07-03T07:49:58.495249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### <font color=maroon>ex182_tdcsfog_gru_StartHesitation_Turn.ipynb</font>\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/exp/ex182_tdcsfog_gru_StartHesitation_Turn.ipynb\" style=\"text-decoration:none\">ex182_tdcsfog_gru_StartHesitation_Turn.ipynb</a>","metadata":{}},{"cell_type":"code","source":"debug = False\nex = \"182_tdcsfog\"\nif not os.path.exists(f\"./output/exp/ex{ex}\"):\n    os.makedirs(f\"./output/exp/ex{ex}\")\n    os.makedirs(f\"./output/exp/ex{ex}/ex{ex}_model\")\n\n# logger_path = f\"./output/exp/ex{ex}/ex_{ex}.txt\"\n# LOGGER = init_logger(log_file=logger_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.497835Z","iopub.execute_input":"2023-07-03T07:49:58.498527Z","iopub.status.idle":"2023-07-03T07:49:58.509984Z","shell.execute_reply.started":"2023-07-03T07:49:58.498494Z","shell.execute_reply":"2023-07-03T07:49:58.509022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### main","metadata":{}},{"cell_type":"code","source":"%%time\n\n# with timer(\"gru\"):\nif TRAIN_FLAG:\n    set_seed(seed)\n    y_oof = np.empty([len(target_array), 1000, 3])      # Abrachan: out of fold\n    gkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state = seed)\n    for fold, (train_idx, valid_idx) in enumerate(gkf.split(numerical_array, \n                                                            y = df_id[\"group\"].values,\n                                                            groups=df_id[\"subject\"].values)):\n        # LOGGER.info(f\"start fold:{fold+1}\")\n        print(f\"start fold: {fold+1}\")\n        \n        with timer(f\"fold {fold+1}\"):\n            train_numerical_array = numerical_array[train_idx]\n            train_target_array = target_array[train_idx]\n            train_mask_array = mask_array[train_idx]\n\n            val_numerical_array = numerical_array[valid_idx]\n            val_target_array = target_array[valid_idx]\n            val_mask_array = mask_array[valid_idx]\n            val_pred_array = pred_use_array[valid_idx]\n            \n            train_ = FogDataset(train_numerical_array,\n                                train_mask_array,\n                                train=True,\n                                y=train_target_array)\n            val_ = FogDataset(val_numerical_array,\n                              val_mask_array,\n                              train=True,\n                              y=val_target_array)\n            \n            train_loader = DataLoader(dataset=train_, \n                                      batch_size=batch_size, \n                                      shuffle = True, \n                                      # num_workers=8,\n                                     )\n            val_loader = DataLoader(dataset=val_, \n                                    batch_size=batch_size, \n                                    shuffle = False , \n                                    # num_workers=8\n                                   )\n            \n            \n            model = FogRnnModel()\n            model = model.to(device)\n            \n            param_optimizer = list(model.named_parameters())\n            no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n            optimizer_grouped_parameters = [\n                {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], \n                 'weight_decay': weight_decay\n                },\n                {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], \n                 'weight_decay': 0.0\n                }]\n            optimizer = AdamW(optimizer_grouped_parameters,\n                              lr=lr,\n                              weight_decay=weight_decay,\n                              )\n            num_train_optimization_steps = int(len(train_loader) * n_epochs)\n            scheduler = get_linear_schedule_with_warmup(optimizer,\n                                                        num_warmup_steps=num_warmup_steps,\n                                                        num_training_steps=num_train_optimization_steps)\n            criterion = nn.BCEWithLogitsLoss()\n            best_val_score = 0\n            \n            for epoch in range(n_epochs):\n                model.train() \n                train_losses_batch = []\n                val_losses_batch = []\n                epoch_loss = 0\n                train_preds = np.ndarray((0,3))   # array([], shape=(0, 3), dtype=float64)\n                \n                tk0 = tqdm_nb(train_loader, total=len(train_loader), desc=\"train_loader: \")\n                for d in tk0:\n                    # ======================================================\n                    # data loader\n                    # ======================================================\n                    input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                    input_data_mask_array      = d['input_data_mask_array'].to(device)\n                    attention_mask             = d['attention_mask'].to(device)\n                    y                          = d[\"y\"].to(device)\n                    \n                    optimizer.zero_grad()\n                    output = model(input_data_numerical_array, \n                                   input_data_mask_array,\n                                   attention_mask)\n                    # loss = criterion(output[input_data_mask_array == 1], y[input_data_mask_array == 1])\n                    \n                    output1 = output[:, :, [0,1]]\n                    output2 = output[:, :, 2]\n                    y1 = y[:, :, [0,1]]\n                    y2 = y[:, :, 2]\n                    loss1 = criterion(output1[input_data_mask_array == 1], y1[input_data_mask_array == 1])\n                    loss2 = criterion(output2[input_data_mask_array == 1], y2[input_data_mask_array == 1])\n                    \n                    loss = loss1*0.8 + loss2*0.2\n                    loss.backward()\n                    optimizer.step()\n                    scheduler.step()\n                    train_losses_batch.append(loss.item())\n                train_loss = np.mean(train_losses_batch)\n                \n                \n                # ======================================================\n                # eval\n                # ======================================================\n                model.eval()       # switch model to the evaluation mode\n                \n                val_preds = np.ndarray((0, 1000, 3))\n                tk0 = tqdm_nb(val_loader, total=len(val_loader), desc=\"val_loader  : \")\n                with torch.no_grad():  # Do not calculate gradient since we are only predicting\n                    # Predicting on validation set\n                    for d in tk0:\n                        input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                        input_data_mask_array      = d['input_data_mask_array'].to(device)\n                        attention_mask             = d['attention_mask'].to(device)\n                        \n                        output = model(input_data_numerical_array, \n                                       input_data_mask_array,\n                                       attention_mask)\n                        val_preds = np.concatenate([val_preds, output.sigmoid().detach().cpu().numpy()], axis=0)\n                \n                pred_valid_index = (val_mask_array == 1) & (val_pred_array == 1)\n                StartHesitation = average_precision_score(val_target_array[pred_valid_index][:, 0],\n                                                          val_preds[pred_valid_index][:, 0])\n                Turn = average_precision_score(val_target_array[pred_valid_index][:, 1],\n                                               val_preds[pred_valid_index][:, 1])\n                Walking = average_precision_score(val_target_array[pred_valid_index][:, 2],\n                                                  val_preds[pred_valid_index][:, 2])\n                map_score = np.mean([StartHesitation, Turn, Walking])\n                \n                # LOGGER.info(f\"fold: {fold+1} epoch: {epoch+1},train loss {train_loss} map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                print(f\"fold {fold+1} epoch {epoch+1}\\ntrain loss {train_loss} map:{map_score}\\nstart_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                \n                if map_score >= best_val_score:\n                    print(\"save weight\")\n                    best_val_score = map_score\n                    best_val_preds = val_preds.copy()\n                    torch.save(model.state_dict(), f\"./output/exp/ex{ex}/ex{ex}_model/ex{ex}_fold{fold+1}_epoch{epoch+1}__map{best_val_score:.6f}.pth\")\n                print()\n            y_oof[valid_idx] = best_val_preds\n    \n    np.save(f\"./output/exp/ex{ex}/ex{ex}_oof.npy\", y_oof)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.511682Z","iopub.execute_input":"2023-07-03T07:49:58.512432Z","iopub.status.idle":"2023-07-03T07:49:58.548677Z","shell.execute_reply.started":"2023-07-03T07:49:58.512402Z","shell.execute_reply":"2023-07-03T07:49:58.547736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### oof score","metadata":{}},{"cell_type":"code","source":"if TRAIN_FLAG:\n    val_pred_index = (mask_array == 1) & (pred_use_array == 1)\n    StartHesitation = average_precision_score(target_array[val_pred_index][:,0], \n                                              y_oof[val_pred_index][:,0])\n    Turn            = average_precision_score(target_array[val_pred_index][:,1], \n                                              y_oof[val_pred_index][:,1])\n    Walking         = average_precision_score(target_array[val_pred_index][:,2],\n                                              y_oof[val_pred_index][:,2])\n\n    map_score = np.mean([StartHesitation, Turn, Walking])\n    # LOGGER.info(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n    print(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.550140Z","iopub.execute_input":"2023-07-03T07:49:58.550808Z","iopub.status.idle":"2023-07-03T07:49:58.558419Z","shell.execute_reply.started":"2023-07-03T07:49:58.550775Z","shell.execute_reply":"2023-07-03T07:49:58.557443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### <font color=maroon>ex183_tdcsfog_gru_StartHesitation_Walking.ipynb</font>\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/exp/ex183_tdcsfog_gru_StartHesitation_Walking.ipynb\" style=\"text-decoration:none\">ex183_tdcsfog_gru_StartHesitation_Walking.ipynb</a>","metadata":{}},{"cell_type":"code","source":"debug = False\nex = \"183_tdcsfog\"\nif not os.path.exists(f\"./output/exp/ex{ex}\"):\n    os.makedirs(f\"./output/exp/ex{ex}\")\n    os.makedirs(f\"./output/exp/ex{ex}/ex{ex}_model\")\n\n# logger_path = f\"./output/exp/ex{ex}/ex_{ex}.txt\"\n# LOGGER = init_logger(log_file=logger_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.560140Z","iopub.execute_input":"2023-07-03T07:49:58.560844Z","iopub.status.idle":"2023-07-03T07:49:58.572060Z","shell.execute_reply.started":"2023-07-03T07:49:58.560813Z","shell.execute_reply":"2023-07-03T07:49:58.570968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### main","metadata":{}},{"cell_type":"code","source":"%%time\n\n# with timer(\"gru\"):\nif TRAIN_FLAG:\n    set_seed(seed)\n    y_oof = np.empty([len(target_array), 1000, 3])      # Abrachan: out of fold\n    gkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state = seed)\n    for fold, (train_idx, valid_idx) in enumerate(gkf.split(numerical_array, \n                                                            y = df_id[\"group\"].values,\n                                                            groups=df_id[\"subject\"].values)):\n        # LOGGER.info(f\"start fold:{fold+1}\")\n        print(f\"start fold: {fold+1}\")\n        \n        with timer(f\"fold {fold+1}\"):\n            train_numerical_array = numerical_array[train_idx]\n            train_target_array = target_array[train_idx]\n            train_mask_array = mask_array[train_idx]\n\n            val_numerical_array = numerical_array[valid_idx]\n            val_target_array = target_array[valid_idx]\n            val_mask_array = mask_array[valid_idx]\n            val_pred_array = pred_use_array[valid_idx]\n            \n            train_ = FogDataset(train_numerical_array,\n                                train_mask_array,\n                                train=True,\n                                y=train_target_array)\n            val_ = FogDataset(val_numerical_array,\n                              val_mask_array,\n                              train=True,\n                              y=val_target_array)\n            \n            train_loader = DataLoader(dataset=train_, \n                                      batch_size=batch_size, \n                                      shuffle = True, \n                                      # num_workers=8,\n                                     )\n            val_loader = DataLoader(dataset=val_, \n                                    batch_size=batch_size, \n                                    shuffle = False , \n                                    # num_workers=8\n                                   )\n            \n            \n            model = FogRnnModel()\n            model = model.to(device)\n            \n            param_optimizer = list(model.named_parameters())\n            no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n            optimizer_grouped_parameters = [\n                {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], \n                 'weight_decay': weight_decay\n                },\n                {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], \n                 'weight_decay': 0.0\n                }]\n            optimizer = AdamW(optimizer_grouped_parameters,\n                              lr=lr,\n                              weight_decay=weight_decay,\n                              )\n            num_train_optimization_steps = int(len(train_loader) * n_epochs)\n            scheduler = get_linear_schedule_with_warmup(optimizer,\n                                                        num_warmup_steps=num_warmup_steps,\n                                                        num_training_steps=num_train_optimization_steps)\n            criterion = nn.BCEWithLogitsLoss()\n            best_val_score = 0\n            \n            for epoch in range(n_epochs):\n                model.train() \n                train_losses_batch = []\n                val_losses_batch = []\n                epoch_loss = 0\n                train_preds = np.ndarray((0,3))   # array([], shape=(0, 3), dtype=float64)\n                \n                tk0 = tqdm_nb(train_loader, total=len(train_loader), desc=\"train_loader: \")\n                for d in tk0:\n                    # ======================================================\n                    # data loader\n                    # ======================================================\n                    input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                    input_data_mask_array      = d['input_data_mask_array'].to(device)\n                    attention_mask             = d['attention_mask'].to(device)\n                    y                          = d[\"y\"].to(device)\n                    \n                    optimizer.zero_grad()\n                    output = model(input_data_numerical_array, \n                                   input_data_mask_array,\n                                   attention_mask)\n                    # loss = criterion(output[input_data_mask_array == 1], y[input_data_mask_array == 1])\n                    \n                    output1 = output[:, :, [0,2]]\n                    output2 = output[:, :, 1]\n                    y1 = y[:, :, [0,2]]\n                    y2 = y[:, :, 1]\n                    loss1 = criterion(output1[input_data_mask_array == 1], y1[input_data_mask_array == 1])\n                    loss2 = criterion(output2[input_data_mask_array == 1], y2[input_data_mask_array == 1])\n                    \n                    loss = loss1*0.8 + loss2*0.2\n                    loss.backward()\n                    optimizer.step()\n                    scheduler.step()\n                    train_losses_batch.append(loss.item())\n                train_loss = np.mean(train_losses_batch)\n                \n                \n                # ======================================================\n                # eval\n                # ======================================================\n                model.eval()       # switch model to the evaluation mode\n                \n                val_preds = np.ndarray((0, 1000, 3))\n                tk0 = tqdm_nb(val_loader, total=len(val_loader), desc=\"val_loader  : \")\n                with torch.no_grad():  # Do not calculate gradient since we are only predicting\n                    # Predicting on validation set\n                    for d in tk0:\n                        input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                        input_data_mask_array      = d['input_data_mask_array'].to(device)\n                        attention_mask             = d['attention_mask'].to(device)\n                        \n                        output = model(input_data_numerical_array, \n                                       input_data_mask_array,\n                                       attention_mask)\n                        val_preds = np.concatenate([val_preds, output.sigmoid().detach().cpu().numpy()], axis=0)\n                \n                pred_valid_index = (val_mask_array == 1) & (val_pred_array == 1)\n                StartHesitation = average_precision_score(val_target_array[pred_valid_index][:, 0],\n                                                          val_preds[pred_valid_index][:, 0])\n                Turn = average_precision_score(val_target_array[pred_valid_index][:, 1],\n                                               val_preds[pred_valid_index][:, 1])\n                Walking = average_precision_score(val_target_array[pred_valid_index][:, 2],\n                                                  val_preds[pred_valid_index][:, 2])\n                map_score = np.mean([StartHesitation, Turn, Walking])\n                \n                # LOGGER.info(f\"fold: {fold+1} epoch: {epoch+1},train loss {train_loss} map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                print(f\"fold {fold+1} epoch {epoch+1}\\ntrain loss {train_loss} map:{map_score}\\nstart_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                \n                if map_score >= best_val_score:\n                    print(\"save weight\")\n                    best_val_score = map_score\n                    best_val_preds = val_preds.copy()\n                    torch.save(model.state_dict(), f\"./output/exp/ex{ex}/ex{ex}_model/ex{ex}_fold{fold+1}_epoch{epoch+1}__map{best_val_score:.6f}.pth\")\n                print()\n            y_oof[valid_idx] = best_val_preds\n    \n    np.save(f\"./output/exp/ex{ex}/ex{ex}_oof.npy\", y_oof)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.575865Z","iopub.execute_input":"2023-07-03T07:49:58.576220Z","iopub.status.idle":"2023-07-03T07:49:58.616777Z","shell.execute_reply.started":"2023-07-03T07:49:58.576170Z","shell.execute_reply":"2023-07-03T07:49:58.616040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### oof score","metadata":{}},{"cell_type":"code","source":"if TRAIN_FLAG:\n    val_pred_index = (mask_array == 1) & (pred_use_array == 1)\n    StartHesitation = average_precision_score(target_array[val_pred_index][:,0], \n                                              y_oof[val_pred_index][:,0])\n    Turn            = average_precision_score(target_array[val_pred_index][:,1], \n                                              y_oof[val_pred_index][:,1])\n    Walking         = average_precision_score(target_array[val_pred_index][:,2],\n                                              y_oof[val_pred_index][:,2])\n\n    map_score = np.mean([StartHesitation, Turn, Walking])\n    # LOGGER.info(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n    print(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.619113Z","iopub.execute_input":"2023-07-03T07:49:58.622387Z","iopub.status.idle":"2023-07-03T07:49:58.629725Z","shell.execute_reply.started":"2023-07-03T07:49:58.622361Z","shell.execute_reply":"2023-07-03T07:49:58.628827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### <font color=maroon>ex184_tdcsfog_gru_Turn_Walking.ipynb</font>\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/exp/ex184_tdcsfog_gru_Turn_Walking.ipynb\" style=\"text-decoration:none\">ex184_tdcsfog_gru_Turn_Walking.ipynb</a>","metadata":{}},{"cell_type":"code","source":"debug = False\nex = \"184_tdcsfog\"\nif not os.path.exists(f\"./output/exp/ex{ex}\"):\n    os.makedirs(f\"./output/exp/ex{ex}\")\n    os.makedirs(f\"./output/exp/ex{ex}/ex{ex}_model\")\n\n# logger_path = f\"./output/exp/ex{ex}/ex_{ex}.txt\"\n# LOGGER = init_logger(log_file=logger_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.631362Z","iopub.execute_input":"2023-07-03T07:49:58.632119Z","iopub.status.idle":"2023-07-03T07:49:58.642727Z","shell.execute_reply.started":"2023-07-03T07:49:58.632088Z","shell.execute_reply":"2023-07-03T07:49:58.641906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### main","metadata":{}},{"cell_type":"code","source":"%%time\n\n# with timer(\"gru\"):\nif TRAIN_FLAG:\n    set_seed(seed)\n    y_oof = np.empty([len(target_array), 1000, 3])      # Abrachan: out of fold\n    gkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state = seed)\n    for fold, (train_idx, valid_idx) in enumerate(gkf.split(numerical_array, \n                                                            y = df_id[\"group\"].values,\n                                                            groups=df_id[\"subject\"].values)):\n        # LOGGER.info(f\"start fold:{fold+1}\")\n        print(f\"start fold: {fold+1}\")\n        \n        with timer(f\"fold {fold+1}\"):\n            train_numerical_array = numerical_array[train_idx]\n            train_target_array = target_array[train_idx]\n            train_mask_array = mask_array[train_idx]\n\n            val_numerical_array = numerical_array[valid_idx]\n            val_target_array = target_array[valid_idx]\n            val_mask_array = mask_array[valid_idx]\n            val_pred_array = pred_use_array[valid_idx]\n            \n            train_ = FogDataset(train_numerical_array,\n                                train_mask_array,\n                                train=True,\n                                y=train_target_array)\n            val_ = FogDataset(val_numerical_array,\n                              val_mask_array,\n                              train=True,\n                              y=val_target_array)\n            \n            train_loader = DataLoader(dataset=train_, \n                                      batch_size=batch_size, \n                                      shuffle = True, \n                                      # num_workers=8,\n                                     )\n            val_loader = DataLoader(dataset=val_, \n                                    batch_size=batch_size, \n                                    shuffle = False , \n                                    # num_workers=8\n                                   )\n            \n            \n            model = FogRnnModel()\n            model = model.to(device)\n            \n            param_optimizer = list(model.named_parameters())\n            no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n            optimizer_grouped_parameters = [\n                {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], \n                 'weight_decay': weight_decay\n                },\n                {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], \n                 'weight_decay': 0.0\n                }]\n            optimizer = AdamW(optimizer_grouped_parameters,\n                              lr=lr,\n                              weight_decay=weight_decay,\n                              )\n            num_train_optimization_steps = int(len(train_loader) * n_epochs)\n            scheduler = get_linear_schedule_with_warmup(optimizer,\n                                                        num_warmup_steps=num_warmup_steps,\n                                                        num_training_steps=num_train_optimization_steps)\n            criterion = nn.BCEWithLogitsLoss()\n            best_val_score = 0\n            \n            for epoch in range(n_epochs):\n                model.train() \n                train_losses_batch = []\n                val_losses_batch = []\n                epoch_loss = 0\n                train_preds = np.ndarray((0,3))   # array([], shape=(0, 3), dtype=float64)\n                \n                tk0 = tqdm_nb(train_loader, total=len(train_loader), desc=\"train_loader: \")\n                for d in tk0:\n                    # ======================================================\n                    # data loader\n                    # ======================================================\n                    input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                    input_data_mask_array      = d['input_data_mask_array'].to(device)\n                    attention_mask             = d['attention_mask'].to(device)\n                    y                          = d[\"y\"].to(device)\n                    \n                    optimizer.zero_grad()\n                    output = model(input_data_numerical_array, \n                                   input_data_mask_array,\n                                   attention_mask)\n                    # loss = criterion(output[input_data_mask_array == 1], y[input_data_mask_array == 1])\n                    \n                    output1 = output[:, :, [1,2]]\n                    output2 = output[:, :, 0]\n                    y1 = y[:, :, [1,2]]\n                    y2 = y[:, :, 0]\n                    loss1 = criterion(output1[input_data_mask_array == 1], y1[input_data_mask_array == 1])\n                    loss2 = criterion(output2[input_data_mask_array == 1], y2[input_data_mask_array == 1])\n                    \n                    loss = loss1*0.8 + loss2*0.2\n                    loss.backward()\n                    optimizer.step()\n                    scheduler.step()\n                    train_losses_batch.append(loss.item())\n                train_loss = np.mean(train_losses_batch)\n                \n                \n                # ======================================================\n                # eval\n                # ======================================================\n                model.eval()       # switch model to the evaluation mode\n                \n                val_preds = np.ndarray((0, 1000, 3))\n                tk0 = tqdm_nb(val_loader, total=len(val_loader), desc=\"val_loader  : \")\n                with torch.no_grad():  # Do not calculate gradient since we are only predicting\n                    # Predicting on validation set\n                    for d in tk0:\n                        input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                        input_data_mask_array      = d['input_data_mask_array'].to(device)\n                        attention_mask             = d['attention_mask'].to(device)\n                        \n                        output = model(input_data_numerical_array, \n                                       input_data_mask_array,\n                                       attention_mask)\n                        val_preds = np.concatenate([val_preds, output.sigmoid().detach().cpu().numpy()], axis=0)\n                \n                pred_valid_index = (val_mask_array == 1) & (val_pred_array == 1)\n                StartHesitation = average_precision_score(val_target_array[pred_valid_index][:, 0],\n                                                          val_preds[pred_valid_index][:, 0])\n                Turn = average_precision_score(val_target_array[pred_valid_index][:, 1],\n                                               val_preds[pred_valid_index][:, 1])\n                Walking = average_precision_score(val_target_array[pred_valid_index][:, 2],\n                                                  val_preds[pred_valid_index][:, 2])\n                map_score = np.mean([StartHesitation, Turn, Walking])\n                \n                # LOGGER.info(f\"fold: {fold+1} epoch: {epoch+1},train loss {train_loss} map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                print(f\"fold {fold+1} epoch {epoch+1}\\ntrain loss {train_loss} map:{map_score}\\nstart_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                \n                if map_score >= best_val_score:\n                    print(\"save weight\")\n                    best_val_score = map_score\n                    best_val_preds = val_preds.copy()\n                    torch.save(model.state_dict(), f\"./output/exp/ex{ex}/ex{ex}_model/ex{ex}_fold{fold+1}_epoch{epoch+1}__map{best_val_score:.6f}.pth\")\n                print()\n            y_oof[valid_idx] = best_val_preds\n    \n    np.save(f\"./output/exp/ex{ex}/ex{ex}_oof.npy\", y_oof)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.644483Z","iopub.execute_input":"2023-07-03T07:49:58.645050Z","iopub.status.idle":"2023-07-03T07:49:58.686777Z","shell.execute_reply.started":"2023-07-03T07:49:58.645020Z","shell.execute_reply":"2023-07-03T07:49:58.685918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### oof score","metadata":{}},{"cell_type":"code","source":"if TRAIN_FLAG:\n    val_pred_index = (mask_array == 1) & (pred_use_array == 1)\n    StartHesitation = average_precision_score(target_array[val_pred_index][:,0], \n                                              y_oof[val_pred_index][:,0])\n    Turn            = average_precision_score(target_array[val_pred_index][:,1], \n                                              y_oof[val_pred_index][:,1])\n    Walking         = average_precision_score(target_array[val_pred_index][:,2],\n                                              y_oof[val_pred_index][:,2])\n\n    map_score = np.mean([StartHesitation, Turn, Walking])\n    # LOGGER.info(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n    print(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.690969Z","iopub.execute_input":"2023-07-03T07:49:58.693195Z","iopub.status.idle":"2023-07-03T07:49:58.701992Z","shell.execute_reply.started":"2023-07-03T07:49:58.693157Z","shell.execute_reply":"2023-07-03T07:49:58.701240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n## **models choosed**\n\n<br>\n\n<font color=maroon size=5>All models above were used for final submission.</font>","metadata":{}},{"cell_type":"markdown","source":"<br>\n<br>\n<br>\n\n# **defog** 【Training】\n\n<br>\n\n## **Feature Engineering**\n\n\n### fe039_defog_base_feature\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/fe/fe039_defog_base_feature.ipynb\" style=\"text-decoration:none\">fe039_defog_base_feature.ipynb</a>","metadata":{}},{"cell_type":"code","source":"fe = \"039\"\nif not os.path.exists(f\"./output/fe/fe{fe}\"):\n    os.makedirs(f\"./output/fe/fe{fe}\")\n    os.makedirs(f\"./output/fe/fe{fe}/save\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.706139Z","iopub.execute_input":"2023-07-03T07:49:58.708691Z","iopub.status.idle":"2023-07-03T07:49:58.715896Z","shell.execute_reply.started":"2023-07-03T07:49:58.708657Z","shell.execute_reply":"2023-07-03T07:49:58.715010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_dict = {}\nfor n,i in enumerate(defog_metadata[\"Subject\"].unique()):\n    sub_dict[i] = n","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.720415Z","iopub.execute_input":"2023-07-03T07:49:58.723075Z","iopub.status.idle":"2023-07-03T07:49:58.729902Z","shell.execute_reply.started":"2023-07-03T07:49:58.723037Z","shell.execute_reply":"2023-07-03T07:49:58.729016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog_metadata[\"sub_id\"] = defog_metadata[\"Subject\"].map(sub_dict)\ndefog_metadata","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.734575Z","iopub.execute_input":"2023-07-03T07:49:58.737040Z","iopub.status.idle":"2023-07-03T07:49:58.760904Z","shell.execute_reply.started":"2023-07-03T07:49:58.737008Z","shell.execute_reply":"2023-07-03T07:49:58.759986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(f'./output/fe/fe{fe}/fe{fe}_sub_id.pkl', 'wb') as p:\n    pickle.dump(sub_dict, p)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.765269Z","iopub.execute_input":"2023-07-03T07:49:58.767516Z","iopub.status.idle":"2023-07-03T07:49:58.774012Z","shell.execute_reply.started":"2023-07-03T07:49:58.767482Z","shell.execute_reply":"2023-07-03T07:49:58.773005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DEFOG_META_PATH = \"../data/defog_metadata.csv\"\n# DEFOG_FOLDER = \"../data/train/defog/*.csv\"\n\n# meta = pd.read_csv(DEFOG_META_PATH)\n# data_list = glob.glob(DEFOG_FOLDER)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.778457Z","iopub.execute_input":"2023-07-03T07:49:58.780810Z","iopub.status.idle":"2023-07-03T07:49:58.787661Z","shell.execute_reply.started":"2023-07-03T07:49:58.780775Z","shell.execute_reply":"2023-07-03T07:49:58.786284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all = []\nfor i in tqdm(train_defog):\n    df = pd.read_csv(i)\n    df_all.append(df)\ndf_all = pd.concat(df_all).reset_index(drop=True)\n\n# len(df_all)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:49:58.789456Z","iopub.execute_input":"2023-07-03T07:49:58.790280Z","iopub.status.idle":"2023-07-03T07:50:23.783006Z","shell.execute_reply.started":"2023-07-03T07:49:58.790247Z","shell.execute_reply":"2023-07-03T07:50:23.781981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:50:23.784646Z","iopub.execute_input":"2023-07-03T07:50:23.785358Z","iopub.status.idle":"2023-07-03T07:50:23.809100Z","shell.execute_reply.started":"2023-07-03T07:50:23.785318Z","shell.execute_reply":"2023-07-03T07:50:23.808225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_std_dict = {}\nfor c in [\"AccV\", \"AccML\", \"AccAP\"]:\n    mean = df_all[c].mean()\n    std = df_all[c].std()\n    mean_std_dict[c] = [mean,std]\n    print(c, mean, std)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:50:23.812826Z","iopub.execute_input":"2023-07-03T07:50:23.815134Z","iopub.status.idle":"2023-07-03T07:50:24.443639Z","shell.execute_reply.started":"2023-07-03T07:50:23.815101Z","shell.execute_reply":"2023-07-03T07:50:24.442702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(f'./output/fe/fe{fe}/save/fe{fe}_sc.pkl', 'wb') as p:\n    pickle.dump(mean_std_dict, p)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:50:24.448083Z","iopub.execute_input":"2023-07-03T07:50:24.450280Z","iopub.status.idle":"2023-07-03T07:50:24.457027Z","shell.execute_reply.started":"2023-07-03T07:50:24.450245Z","shell.execute_reply":"2023-07-03T07:50:24.455900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d_list = []\nnum_array = []\ntarget_array = []\nvalid_array = []\nsubject_list = []\nid_list = []\nmask_array = []\nnum_cols = [\"AccV\", \"AccML\", \"AccAP\"]\ntarget_cols = [\"StartHesitation\", \"Turn\", \"Walking\"]\nseq_len = 1000\n\nfor i,s in tqdm(zip(defog_metadata[\"Id\"].values, defog_metadata[\"sub_id\"].values), desc=\"defog_metadata: \"):\n    path = root_data + f\"train/defog/{i}.csv\"\n    if path in [x.replace(\"\\\\\", \"/\") for x in train_defog]:\n    # if path in data_list:\n        d_list.append(1)\n        \n        df = pd.read_csv(path)\n        df[\"valid\"] = df[\"Valid\"] & df[\"Task\"]\n        df[\"valid\"] = df[\"valid\"].astype(int)\n        \n        batch = (len(df) // seq_len) + 1\n        \n        for c in num_cols:\n            df[c] = (df[c] - mean_std_dict[c][0]) / mean_std_dict[c][1]\n        num = df[num_cols].values\n        target = df[target_cols].values\n        valid = df[\"valid\"].values\n        \n        num_array_    = np.zeros([batch, seq_len, 3])\n        target_array_ = np.zeros([batch, seq_len, 3])\n        mask_array_   = np.zeros([batch, seq_len])\n        valid_array_  = np.zeros([batch, seq_len])\n        \n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*seq_len : ]\n                num_array_[b, :len(num_), :] = num_\n                target_ = target[b*seq_len : ]\n                target_array_[b,:len(target_), :] = target_\n                valid_ = valid[b*seq_len : ]\n                valid_array_[b, :len(valid_)] = valid_\n                mask_array_[b, :len(target_)] = 1\n            else:\n                num_ = num[b*seq_len : (b+1)*seq_len]\n                num_array_[b, :, :] = num_\n                target_ = target[b*seq_len : (b+1)*seq_len]\n                target_array_[b, :, :] = target_\n                valid_ = valid[b*seq_len : (b+1)*seq_len]\n                valid_array_[b, :] = valid_\n                mask_array_[b,:] = 1\n        num_array.append(num_array_)\n        target_array.append(target_array_)\n        mask_array.append(mask_array_)\n        valid_array.append(valid_array_)\n        subject_list += [s for _ in range(batch)]\n        id_list      += [i for _ in range(batch)] \n    else:\n        d_list.append(0)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:50:24.461285Z","iopub.execute_input":"2023-07-03T07:50:24.463642Z","iopub.status.idle":"2023-07-03T07:50:39.157956Z","shell.execute_reply.started":"2023-07-03T07:50:24.463610Z","shell.execute_reply":"2023-07-03T07:50:39.156889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_array    = np.concatenate(num_array, axis=0)\ntarget_array = np.concatenate(target_array, axis=0)\nmask_array   =  np.concatenate(mask_array, axis=0)\nvalid_array = np.concatenate(valid_array, axis=0)\n\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_num_array.npy\", num_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_target_array.npy\", target_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_mask_array.npy\", mask_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_valid_array.npy\", valid_array)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:50:39.159427Z","iopub.execute_input":"2023-07-03T07:50:39.159751Z","iopub.status.idle":"2023-07-03T07:50:40.031552Z","shell.execute_reply.started":"2023-07-03T07:50:39.159720Z","shell.execute_reply":"2023-07-03T07:50:40.030589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog_metadata[\"data_is\"] = d_list\n\ndefog_metadata.to_parquet(f\"./output/fe/fe{fe}/fe{fe}_defog_meta.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:50:40.033007Z","iopub.execute_input":"2023-07-03T07:50:40.033368Z","iopub.status.idle":"2023-07-03T07:50:40.044046Z","shell.execute_reply.started":"2023-07-03T07:50:40.033335Z","shell.execute_reply":"2023-07-03T07:50:40.043070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_id = pd.DataFrame()\ndf_id[\"Id\"] = id_list\ndf_id[\"subject\"] = subject_list\n\ndf_id.to_parquet(f\"./output/fe/fe{fe}/fe{fe}_id.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:50:40.045363Z","iopub.execute_input":"2023-07-03T07:50:40.045806Z","iopub.status.idle":"2023-07-03T07:50:40.119002Z","shell.execute_reply.started":"2023-07-03T07:50:40.045774Z","shell.execute_reply":"2023-07-03T07:50:40.118130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### fe047_defog_5000\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/fe/fe047_defog_5000.ipynb\" style=\"text-decoration:none\">fe047_defog_5000.ipynb</a>","metadata":{}},{"cell_type":"code","source":"fe = \"047\"\nif not os.path.exists(f\"./output/fe/fe{fe}\"):\n    os.makedirs(f\"./output/fe/fe{fe}\")\n    os.makedirs(f\"./output/fe/fe{fe}/save\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:50:40.120264Z","iopub.execute_input":"2023-07-03T07:50:40.120579Z","iopub.status.idle":"2023-07-03T07:50:40.126451Z","shell.execute_reply.started":"2023-07-03T07:50:40.120548Z","shell.execute_reply":"2023-07-03T07:50:40.125455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = [\"AccV\", \"AccML\", \"AccAP\"]\nnum_cols = [\"AccV\", \"AccML\", \"AccAP\",\n            'AccV_lag_diff',  'AccV_lead_diff', \n            'AccML_lag_diff', 'AccML_lead_diff', \n            'AccAP_lag_diff', 'AccAP_lead_diff']\ntarget_cols = [\"StartHesitation\", \"Turn\", \"Walking\"]\n\nseq_len = 5000\nshift = 2500\noffset = 1250","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:50:40.127957Z","iopub.execute_input":"2023-07-03T07:50:40.129728Z","iopub.status.idle":"2023-07-03T07:50:40.136297Z","shell.execute_reply.started":"2023-07-03T07:50:40.129697Z","shell.execute_reply":"2023-07-03T07:50:40.135424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_array = []\ntarget_array = []\nvalid_array = []\nmask_array = []\npred_use_array = []\ntime_array = []\n\nsubject_list = []\nid_list = []\nd_list = []","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:50:40.138075Z","iopub.execute_input":"2023-07-03T07:50:40.138345Z","iopub.status.idle":"2023-07-03T07:50:40.149624Z","shell.execute_reply.started":"2023-07-03T07:50:40.138323Z","shell.execute_reply":"2023-07-03T07:50:40.148771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_list = glob.glob(DEFOG_FOLDER)\n\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler, RobustScaler\n\nfor i,s in tqdm(zip(defog_metadata[\"Id\"].values, defog_metadata[\"sub_id\"].values), desc=\"defog_metadata: \"):\n    path = root_data + f\"train/defog/{i}.csv\"\n    if path in [x.replace(\"\\\\\", \"/\") for x in train_defog]:\n    # if path in data_list:\n        d_list.append(1)\n        \n        df = pd.read_csv(path)\n        df[\"valid\"] = df[\"Valid\"] & df[\"Task\"]\n        df[\"valid\"] = df[\"valid\"].astype(int)\n        \n        batch = (len(df)-1) // shift\n        \n        for c in cols:\n            df[f\"{c}_lag_diff\"] = df[c].diff()\n            df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n        \n        sc = StandardScaler()\n        df[num_cols] = sc.fit_transform(df[num_cols].values)\n        df[num_cols] = df[num_cols].fillna(0)\n\n        num = df[num_cols].values\n        target = df[target_cols].values\n        valid = df[\"valid\"].values\n        time_values = df[\"Time\"].values\n        \n        num_array_ = np.zeros([batch,seq_len, 9])\n        target_array_ = np.zeros([batch, seq_len, 3])\n        valid_array_ = np.zeros([batch, seq_len], dtype=int)\n        time_array_ = np.zeros([batch, seq_len], dtype=int)\n        mask_array_ = np.zeros([batch, seq_len], dtype=int)\n        pred_use_array_ = np.zeros([batch, seq_len], dtype=int)\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                num_array_[b,:len(num_), :] = num_\n                target_ = target[b*shift : ]\n                target_array_[b, :len(target_), :] = target_\n                mask_array_[b, :len(target_)] = 1\n                pred_use_array_[b, offset:len(target_)] = 1\n                time_ = time_values[b*shift : ]\n                time_array_[b, :len(time_)] = time_\n                valid_ = valid[b*shift : ]\n                valid_array_[b, :len(valid_)] = valid_\n            elif b == 0:\n                num_ = num[b*shift : b*shift+seq_len]\n                num_array_[b, :, :] = num_\n                target_ = target[b*shift : b*shift + seq_len]\n                target_array_[b, :, :] = target_\n                mask_array_[b, :] = 1\n                pred_use_array_[b, :shift + offset] = 1\n                time_ = time_values[b*shift : b*shift + seq_len]\n                time_array_[b, :] = time_\n                valid_ = valid[b*shift : b*shift + seq_len]\n                valid_array_[b, :] = valid_\n            else:\n                num_ = num[b*shift : b*shift+seq_len]\n                num_array_[b, :, :] = num_\n                target_ = target[b*shift : b*shift + seq_len]\n                target_array_[b, :, :] = target_\n                mask_array_[b, :] = 1\n                pred_use_array_[b, offset:shift + offset] = 1\n                time_ = time_values[b*shift : b*shift + seq_len]\n                time_array_[b, :] = time_\n                valid_ = valid[b*shift : b*shift + seq_len]\n                valid_array_[b, :] = valid_\n\n        num_array.append(num_array_)\n        target_array.append(target_array_)\n        mask_array.append(mask_array_)\n        pred_use_array.append(pred_use_array_)\n        time_array.append(time_array_)\n        valid_array.append(valid_array_)\n        subject_list += [s for _ in range(batch)]\n        id_list      += [i for _ in range(batch)] \n    else:\n        d_list.append(0)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:50:40.150896Z","iopub.execute_input":"2023-07-03T07:50:40.151410Z","iopub.status.idle":"2023-07-03T07:51:03.194418Z","shell.execute_reply.started":"2023-07-03T07:50:40.151380Z","shell.execute_reply":"2023-07-03T07:51:03.193365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_array = np.concatenate(num_array, axis=0)\ntarget_array =np.concatenate(target_array, axis=0)\nmask_array =  np.concatenate(mask_array, axis=0)\npred_use_array = np.concatenate(pred_use_array, axis=0)\ntime_array = np.concatenate(time_array, axis=0)\nvalid_array = np.concatenate(valid_array, axis=0)\n\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_num_array.npy\", num_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_target_array.npy\", target_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_mask_array.npy\", mask_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_time_array.npy\", time_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_pred_use_array.npy\", pred_use_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_valid_array.npy\", valid_array)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:03.196119Z","iopub.execute_input":"2023-07-03T07:51:03.196517Z","iopub.status.idle":"2023-07-03T07:51:17.884566Z","shell.execute_reply.started":"2023-07-03T07:51:03.196483Z","shell.execute_reply":"2023-07-03T07:51:17.882849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_id = pd.DataFrame()\ndf_id[\"Id\"] = id_list\ndf_id[\"subject\"] = subject_list\n\ndf_id.to_parquet(f\"./output/fe/fe{fe}/fe{fe}_id.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:17.890499Z","iopub.execute_input":"2023-07-03T07:51:17.893590Z","iopub.status.idle":"2023-07-03T07:51:17.989689Z","shell.execute_reply.started":"2023-07-03T07:51:17.893546Z","shell.execute_reply":"2023-07-03T07:51:17.988644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n## <font color=red>**Training**</font>\n\n\n<br>\n\n### helpers","metadata":{}},{"cell_type":"code","source":"TRAIN_FLAG_DEFOG = True","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:17.991886Z","iopub.execute_input":"2023-07-03T07:51:17.992565Z","iopub.status.idle":"2023-07-03T07:51:17.998817Z","shell.execute_reply.started":"2023-07-03T07:51:17.992531Z","shell.execute_reply":"2023-07-03T07:51:17.997573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### def `set_seed()`","metadata":{}},{"cell_type":"code","source":"def set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:18.003546Z","iopub.execute_input":"2023-07-03T07:51:18.003905Z","iopub.status.idle":"2023-07-03T07:51:18.014603Z","shell.execute_reply.started":"2023-07-03T07:51:18.003873Z","shell.execute_reply":"2023-07-03T07:51:18.012251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### def `timer()`","metadata":{}},{"cell_type":"code","source":"@contextmanager\ndef timer(name):\n    t0 = time.time()\n    \n    yield \n    # LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s')\n    print(f'[{name}] done in {time.time() - t0:.0f} s')\n    print(\"=\"*66)\n    print(\"\\n\"*2)\n    \n# setup_logger(out_file=logger_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:18.018116Z","iopub.execute_input":"2023-07-03T07:51:18.020167Z","iopub.status.idle":"2023-07-03T07:51:18.032172Z","shell.execute_reply.started":"2023-07-03T07:51:18.020124Z","shell.execute_reply":"2023-07-03T07:51:18.030464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### def `preprocess()`","metadata":{}},{"cell_type":"code","source":"def preprocess(numerical_array, mask_array, valid_array,):\n    \n    attention_mask = mask_array == 0\n\n    return {'input_data_numerical_array': numerical_array,\n            'input_data_mask_array': mask_array,\n            'input_data_valid_array': valid_array,\n            'attention_mask': attention_mask,\n           }","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:18.034531Z","iopub.execute_input":"2023-07-03T07:51:18.036040Z","iopub.status.idle":"2023-07-03T07:51:18.047760Z","shell.execute_reply.started":"2023-07-03T07:51:18.036004Z","shell.execute_reply":"2023-07-03T07:51:18.046685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### class `FogDataset()`","metadata":{}},{"cell_type":"code","source":"class FogDataset(Dataset):\n    def __init__(self, numerical_array, mask_array, valid_array, train = True, y = None):\n        self.numerical_array = numerical_array\n        self.mask_array = mask_array\n        self.valid_array = valid_array\n        self.train = train\n        self.y = y\n    \n    \n    def __len__(self):\n        return len(self.numerical_array)\n    \n    \n    def __getitem__(self, item):\n        data = preprocess(self.numerical_array[item], self.mask_array[item], self.valid_array[item],)\n\n        # Return the processed data where the lists are converted to `torch.tensor`s\n        if self.train : \n            return {\n              'input_data_numerical_array': torch.tensor(data['input_data_numerical_array'], dtype=torch.float32),\n              'input_data_mask_array': torch.tensor(data['input_data_mask_array'],  dtype=torch.long),  \n              'input_data_valid_array': torch.tensor(data['input_data_valid_array'], dtype=torch.long),   \n              'attention_mask': torch.tensor(data[\"attention_mask\"], dtype=torch.bool),\n              \"y\": torch.tensor(self.y[item], dtype=torch.float32)\n            }\n        else:\n            return {\n              'input_data_numerical_array': torch.tensor(data['input_data_numerical_array'], dtype=torch.float32),\n              'input_data_mask_array': torch.tensor(data['input_data_mask_array'], dtype=torch.long),  \n              'input_data_valid_array': torch.tensor(data['input_data_valid_array'], dtype=torch.long),  \n              'attention_mask': torch.tensor(data[\"attention_mask\"], dtype=torch.bool),\n               }","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:18.048964Z","iopub.execute_input":"2023-07-03T07:51:18.049957Z","iopub.status.idle":"2023-07-03T07:51:18.066894Z","shell.execute_reply.started":"2023-07-03T07:51:18.049926Z","shell.execute_reply":"2023-07-03T07:51:18.064886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### class `FogRnnModel()`","metadata":{}},{"cell_type":"code","source":"class FogRnnModel(nn.Module):\n    def __init__(self, \n                 dropout=0.2,\n                 input_numerical_size=9,\n                 numeraical_linear_size = 64,\n                 model_size = 128,\n                 linear_out = 128,\n                 out_size=3):\n        \n        super(FogRnnModel, self).__init__()\n        \n        self.numerical_linear  = nn.Sequential(nn.Linear(input_numerical_size, numeraical_linear_size),\n                                               nn.LayerNorm(numeraical_linear_size))\n        \n        self.rnn = nn.GRU(numeraical_linear_size, \n                          model_size,\n                          num_layers = 2, \n                          batch_first=True,\n                          bidirectional=True)\n        \n        self.linear_out  = nn.Sequential(nn.Linear(model_size*2, linear_out),\n                                         nn.LayerNorm(linear_out),\n                                         nn.ReLU(),\n                                         nn.Dropout(dropout),\n                                         nn.Linear(linear_out, out_size))\n        self._reinitialize()\n        \n        \n        \n    def _reinitialize(self):\n        \"\"\"Tensorflow/Keras-like initialization\"\"\"\n        for name, p in self.named_parameters():\n            if 'rnn' in name:\n                if 'weight_ih' in name:\n                    nn.init.xavier_uniform_(p.data)\n                elif 'weight_hh' in name:\n                    nn.init.orthogonal_(p.data)\n                elif 'bias_ih' in name:\n                    p.data.fill_(0)\n                    # Set forget-gate bias to 1\n                    n = p.size(0)\n                    p.data[(n // 4):(n // 2)].fill_(1)\n                elif 'bias_hh' in name:\n                    p.data.fill_(0)\n        \n        \n        \n    def forward(self, numerical_array, mask_array, attention_mask):\n        numerical_embedding = self.numerical_linear(numerical_array)\n        output, _ = self.rnn(numerical_embedding)\n        output = self.linear_out(output)\n        return output","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:18.070115Z","iopub.execute_input":"2023-07-03T07:51:18.071565Z","iopub.status.idle":"2023-07-03T07:51:18.101731Z","shell.execute_reply.started":"2023-07-03T07:51:18.071512Z","shell.execute_reply":"2023-07-03T07:51:18.100090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### load data & preprocessing","metadata":{}},{"cell_type":"code","source":"id_path        = f\"./output/fe/fe047/fe047_id.parquet\"\nnumerical_path = f\"./output/fe/fe047/fe047_num_array.npy\"\ntarget_path    = f\"./output/fe/fe047/fe047_target_array.npy\"\nmask_path      = f\"./output/fe/fe047/fe047_mask_array.npy\"\nvalid_path     = f\"./output/fe/fe047/fe047_valid_array.npy\"\npred_use_path  = f\"./output/fe/fe047/fe047_pred_use_array.npy\"","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:18.106531Z","iopub.execute_input":"2023-07-03T07:51:18.109488Z","iopub.status.idle":"2023-07-03T07:51:18.118516Z","shell.execute_reply.started":"2023-07-03T07:51:18.109447Z","shell.execute_reply":"2023-07-03T07:51:18.116768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_id = pd.read_parquet(id_path)\nnumerical_array = np.load(numerical_path)\ntarget_array = np.load(target_path)\nmask_array = np.load(mask_path)\nvalid_array = np.load(valid_path)\npred_use_array = np.load(pred_use_path)\n\ntarget1 = []\ntarget2 = []\ntarget3 = []\nfor i in range(len(target_array)):\n    target1.append(np.sum(target_array[i,:,0]))\n    target2.append(np.sum(target_array[i,:,1]))\n    target3.append(np.sum(target_array[i,:,2]))\n\n\ndf_id[\"target1\"] = target1\ndf_id[\"target2\"] = target2\ndf_id[\"target3\"] = target3\ndf_id[\"target1_1\"] = df_id[\"target1\"] > 0\ndf_id[\"target2_1\"] = df_id[\"target2\"] > 0\ndf_id[\"target3_1\"] = df_id[\"target3\"] > 0\ndf_id[\"target1_1\"] = df_id[\"target1_1\"].astype(np.int)\ndf_id[\"target2_1\"] = df_id[\"target2_1\"].astype(np.int)\ndf_id[\"target3_1\"] = df_id[\"target3_1\"].astype(np.int)\n\ndf_id[\"group\"] = 0\ndf_id.loc[df_id[\"target1_1\"] > 0,\"group\"] = 1\ndf_id.loc[df_id[\"target2_1\"] > 0,\"group\"] = 2\ndf_id.loc[df_id[\"target3_1\"] > 0,\"group\"] = 3\n\n\ndf_id[\"group\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:18.120483Z","iopub.execute_input":"2023-07-03T07:51:18.121891Z","iopub.status.idle":"2023-07-03T07:51:34.331946Z","shell.execute_reply.started":"2023-07-03T07:51:18.121857Z","shell.execute_reply":"2023-07-03T07:51:34.331047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### config","metadata":{}},{"cell_type":"code","source":"# config\nseed = 0\nshuffle = True\nn_splits = 5\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# model config\nbatch_size = 24\nn_epochs = 15\nlr = 1e-3\nweight_decay = 0.05\nnum_warmup_steps = 10","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:34.333213Z","iopub.execute_input":"2023-07-03T07:51:34.333801Z","iopub.status.idle":"2023-07-03T07:51:34.339241Z","shell.execute_reply.started":"2023-07-03T07:51:34.333769Z","shell.execute_reply":"2023-07-03T07:51:34.338246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### <font color=maroon>ex153_defog_gru.ipynb</font>\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/exp/ex153_defog_gru.ipynb\" style=\"text-decoration:none\">ex153_defog_gru.ipynb</a>","metadata":{}},{"cell_type":"code","source":"debug = False\nex = \"153_defog\"\nif not os.path.exists(f\"./output/exp/ex{ex}\"):\n    os.makedirs(f\"./output/exp/ex{ex}\")\n    os.makedirs(f\"./output/exp/ex{ex}/ex{ex}_model\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:34.362520Z","iopub.execute_input":"2023-07-03T07:51:34.362843Z","iopub.status.idle":"2023-07-03T07:51:34.368262Z","shell.execute_reply.started":"2023-07-03T07:51:34.362816Z","shell.execute_reply":"2023-07-03T07:51:34.367385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### main","metadata":{}},{"cell_type":"code","source":"%%time\n\n# with timer(\"gru\"):\nif TRAIN_FLAG:\n    set_seed(seed)\n    y_oof = np.empty([len(target_array), 5000, 3])      # Abrachan: out of fold\n    gkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state = seed)\n    for fold, (train_idx, valid_idx) in enumerate(gkf.split(numerical_array, \n                                                            y = df_id[\"group\"].values,\n                                                            groups=df_id[\"subject\"].values)):\n        # LOGGER.info(f\"start fold:{fold+1}\")\n        print(f\"start fold: {fold+1}\")\n        \n        with timer(f\"fold {fold+1}\"):\n            train_numerical_array = numerical_array[train_idx]\n            train_target_array = target_array[train_idx]\n            train_mask_array = mask_array[train_idx]\n            train_valid_array = valid_array[train_idx]\n\n            val_numerical_array = numerical_array[valid_idx]\n            val_target_array = target_array[valid_idx]\n            val_mask_array = mask_array[valid_idx]\n            val_valid_array = valid_array[valid_idx]\n            val_pred_array = pred_use_array[valid_idx]\n            \n            train_ = FogDataset(train_numerical_array,\n                                train_mask_array,\n                                train_valid_array,\n                                train=True,\n                                y=train_target_array)\n            val_ = FogDataset(val_numerical_array,\n                              val_mask_array,\n                              val_valid_array,\n                              train=True,\n                              y=val_target_array)\n            \n            train_loader = DataLoader(dataset=train_, \n                                      batch_size=batch_size, \n                                      shuffle = True, \n                                      # num_workers=8,\n                                     )\n            val_loader = DataLoader(dataset=val_, \n                                    batch_size=batch_size, \n                                    shuffle = False , \n                                    # num_workers=8\n                                   )\n            \n            \n            model = FogRnnModel()\n            model = model.to(device)\n            \n            param_optimizer = list(model.named_parameters())\n            no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n            optimizer_grouped_parameters = [\n                {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], \n                 'weight_decay': weight_decay\n                },\n                {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], \n                 'weight_decay': 0.0\n                }]\n            optimizer = AdamW(optimizer_grouped_parameters,\n                              lr=lr,\n                              weight_decay=weight_decay,\n                              )\n            num_train_optimization_steps = int(len(train_loader) * n_epochs)\n            scheduler = get_linear_schedule_with_warmup(optimizer,\n                                                        num_warmup_steps=num_warmup_steps,\n                                                        num_training_steps=num_train_optimization_steps)\n            criterion = nn.BCEWithLogitsLoss()\n            best_val_score = 0\n            \n            for epoch in range(n_epochs):\n                model.train() \n                train_losses_batch = []\n                val_losses_batch = []\n                epoch_loss = 0\n                train_preds = np.ndarray((0,3))   # array([], shape=(0, 3), dtype=float64)\n                \n                tk0 = tqdm_nb(train_loader, total=len(train_loader), desc=\"train_loader: \")\n                for d in tk0:\n                    # ======================================================\n                    # data loader\n                    # ======================================================\n                    input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                    input_data_mask_array      = d['input_data_mask_array'].to(device)\n                    input_data_valid_array     = d['input_data_valid_array'].to(device)\n                    attention_mask             = d['attention_mask'].to(device)\n                    y                          = d[\"y\"].to(device)\n                    \n                    optimizer.zero_grad()\n                    output = model(input_data_numerical_array, \n                                   input_data_mask_array,\n                                   attention_mask)\n                    loss = criterion(output[input_data_mask_array == 1], y[input_data_mask_array == 1])\n                    \n                    # output1 = output[:, :, [1,2]]\n                    # output2 = output[:, :, 0]\n                    # y1 = y[:, :, [1,2]]\n                    # y2 = y[:, :, 0]\n                    # loss1 = criterion(output1[input_data_mask_array == 1], y1[input_data_mask_array == 1])\n                    # loss2 = criterion(output2[input_data_mask_array == 1], y2[input_data_mask_array == 1])\n                    # loss = loss1*0.8 + loss2*0.2\n                    \n                    loss.backward()\n                    optimizer.step()\n                    scheduler.step()\n                    train_losses_batch.append(loss.item())\n                train_loss = np.mean(train_losses_batch)\n                \n                \n                # ======================================================\n                # eval\n                # ======================================================\n                model.eval()       # switch model to the evaluation mode\n                \n                val_preds = np.ndarray((0, 5000, 3))\n                tk0 = tqdm_nb(val_loader, total=len(val_loader), desc=\"val_loader  : \")\n                with torch.no_grad():  # Do not calculate gradient since we are only predicting\n                    # Predicting on validation set\n                    for d in tk0:\n                        input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                        input_data_mask_array      = d['input_data_mask_array'].to(device)\n                        attention_mask             = d['attention_mask'].to(device)\n                        \n                        output = model(input_data_numerical_array, \n                                       input_data_mask_array,\n                                       attention_mask)\n                        val_preds = np.concatenate([val_preds, output.sigmoid().detach().cpu().numpy()], axis=0)\n                \n                pred_valid_index = (val_mask_array == 1) & (val_pred_array == 1) & (val_valid_array == 1)\n                StartHesitation = average_precision_score(val_target_array[pred_valid_index][:, 0],\n                                                          val_preds[pred_valid_index][:, 0])\n                Turn = average_precision_score(val_target_array[pred_valid_index][:, 1],\n                                               val_preds[pred_valid_index][:, 1])\n                Walking = average_precision_score(val_target_array[pred_valid_index][:, 2],\n                                                  val_preds[pred_valid_index][:, 2])\n                # map_score = np.mean([StartHesitation, Turn, Walking])\n                map_score = np.mean([Turn, Walking])\n                \n                # LOGGER.info(f\"fold: {fold+1} epoch: {epoch+1},train loss {train_loss} map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                print(f\"fold {fold+1} epoch {epoch+1}\\ntrain loss {train_loss} map:{map_score}\\nstart_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                \n                if map_score >= best_val_score:\n                    print(\"save weight\")\n                    best_val_score = map_score\n                    best_val_preds = val_preds.copy()\n                    torch.save(model.state_dict(), f\"./output/exp/ex{ex}/ex{ex}_model/ex{ex}_fold{fold+1}_epoch{epoch+1}__map{best_val_score:.6f}.pth\")\n                print()\n            y_oof[valid_idx] = best_val_preds\n    \n    np.save(f\"./output/exp/ex{ex}/ex{ex}_oof.npy\", y_oof)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:34.369862Z","iopub.execute_input":"2023-07-03T07:51:34.370443Z","iopub.status.idle":"2023-07-03T07:51:34.399178Z","shell.execute_reply.started":"2023-07-03T07:51:34.370408Z","shell.execute_reply":"2023-07-03T07:51:34.398277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### oof score","metadata":{}},{"cell_type":"code","source":"if TRAIN_FLAG:\n    val_pred_index = (mask_array == 1) & (pred_use_array == 1) & (valid_array == 1)\n    StartHesitation = average_precision_score(target_array[val_pred_index][:,0], \n                                              y_oof[val_pred_index][:,0])\n    Turn            = average_precision_score(target_array[val_pred_index][:,1], \n                                              y_oof[val_pred_index][:,1])\n    Walking         = average_precision_score(target_array[val_pred_index][:,2],\n                                              y_oof[val_pred_index][:,2])\n\n    map_score = np.mean([StartHesitation, Turn, Walking])\n    # LOGGER.info(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n    print(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:34.400592Z","iopub.execute_input":"2023-07-03T07:51:34.403760Z","iopub.status.idle":"2023-07-03T07:51:34.411864Z","shell.execute_reply.started":"2023-07-03T07:51:34.403728Z","shell.execute_reply":"2023-07-03T07:51:34.410930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### <font color=maroon>ex154_defog_gru.ipynb</font>\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/exp/ex154_defog_gru.ipynb\" style=\"text-decoration:none\">ex154_defog_gru.ipynb</a>","metadata":{}},{"cell_type":"code","source":"debug = False\nex = \"154_defog\"\nif not os.path.exists(f\"./output/exp/ex{ex}\"):\n    os.makedirs(f\"./output/exp/ex{ex}\")\n    os.makedirs(f\"./output/exp/ex{ex}/ex{ex}_model\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:34.413701Z","iopub.execute_input":"2023-07-03T07:51:34.414070Z","iopub.status.idle":"2023-07-03T07:51:34.426154Z","shell.execute_reply.started":"2023-07-03T07:51:34.414039Z","shell.execute_reply":"2023-07-03T07:51:34.425142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### class `FogRnnModel()`","metadata":{}},{"cell_type":"code","source":"class FogRnnModel(nn.Module):\n    def __init__(self, \n                 dropout=0.2,\n                 input_numerical_size=9,\n                 numeraical_linear_size = 96,\n                 model_size = 256,\n                 linear_out = 256,\n                 out_size=3):\n        \n        super(FogRnnModel, self).__init__()\n        \n        self.numerical_linear  = nn.Sequential(nn.Linear(input_numerical_size, numeraical_linear_size),\n                                               nn.LayerNorm(numeraical_linear_size))\n        \n        self.rnn = nn.GRU(numeraical_linear_size, \n                          model_size,\n                          num_layers = 2, \n                          batch_first=True,\n                          bidirectional=True)\n        \n        self.linear_out  = nn.Sequential(nn.Linear(model_size*2, linear_out),\n                                         nn.LayerNorm(linear_out),\n                                         nn.ReLU(),\n                                         nn.Dropout(dropout),\n                                         nn.Linear(linear_out, out_size))\n        self._reinitialize()\n        \n        \n        \n    def _reinitialize(self):\n        \"\"\"Tensorflow/Keras-like initialization\"\"\"\n        for name, p in self.named_parameters():\n            if 'rnn' in name:\n                if 'weight_ih' in name:\n                    nn.init.xavier_uniform_(p.data)\n                elif 'weight_hh' in name:\n                    nn.init.orthogonal_(p.data)\n                elif 'bias_ih' in name:\n                    p.data.fill_(0)\n                    # Set forget-gate bias to 1\n                    n = p.size(0)\n                    p.data[(n // 4):(n // 2)].fill_(1)\n                elif 'bias_hh' in name:\n                    p.data.fill_(0)\n        \n        \n        \n    def forward(self, numerical_array, mask_array, attention_mask):\n        numerical_embedding = self.numerical_linear(numerical_array)\n        output, _ = self.rnn(numerical_embedding)\n        output = self.linear_out(output)\n        return output","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:34.429381Z","iopub.execute_input":"2023-07-03T07:51:34.429667Z","iopub.status.idle":"2023-07-03T07:51:34.443152Z","shell.execute_reply.started":"2023-07-03T07:51:34.429644Z","shell.execute_reply":"2023-07-03T07:51:34.441962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### main","metadata":{}},{"cell_type":"code","source":"%%time\n\n# with timer(\"gru\"):\nif TRAIN_FLAG:\n    set_seed(seed)\n    y_oof = np.empty([len(target_array), 5000, 3])      # Abrachan: out of fold\n    gkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state = seed)\n    for fold, (train_idx, valid_idx) in enumerate(gkf.split(numerical_array, \n                                                            y = df_id[\"group\"].values,\n                                                            groups=df_id[\"subject\"].values)):\n        # LOGGER.info(f\"start fold:{fold+1}\")\n        print(f\"start fold: {fold+1}\")\n        \n        with timer(f\"fold {fold+1}\"):\n            train_numerical_array = numerical_array[train_idx]\n            train_target_array = target_array[train_idx]\n            train_mask_array = mask_array[train_idx]\n            train_valid_array = valid_array[train_idx]\n\n            val_numerical_array = numerical_array[valid_idx]\n            val_target_array = target_array[valid_idx]\n            val_mask_array = mask_array[valid_idx]\n            val_valid_array = valid_array[valid_idx]\n            val_pred_array = pred_use_array[valid_idx]\n            \n            train_ = FogDataset(train_numerical_array,\n                                train_mask_array,\n                                train_valid_array,\n                                train=True,\n                                y=train_target_array)\n            val_ = FogDataset(val_numerical_array,\n                              val_mask_array,\n                              val_valid_array,\n                              train=True,\n                              y=val_target_array)\n            \n            train_loader = DataLoader(dataset=train_, \n                                      batch_size=batch_size, \n                                      shuffle = True, \n                                      # num_workers=8,\n                                     )\n            val_loader = DataLoader(dataset=val_, \n                                    batch_size=batch_size, \n                                    shuffle = False , \n                                    # num_workers=8\n                                   )\n            \n            \n            model = FogRnnModel()\n            model = model.to(device)\n            \n            param_optimizer = list(model.named_parameters())\n            no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n            optimizer_grouped_parameters = [\n                {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], \n                 'weight_decay': weight_decay\n                },\n                {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], \n                 'weight_decay': 0.0\n                }]\n            optimizer = AdamW(optimizer_grouped_parameters,\n                              lr=lr,\n                              weight_decay=weight_decay,\n                              )\n            num_train_optimization_steps = int(len(train_loader) * n_epochs)\n            scheduler = get_linear_schedule_with_warmup(optimizer,\n                                                        num_warmup_steps=num_warmup_steps,\n                                                        num_training_steps=num_train_optimization_steps)\n            criterion = nn.BCEWithLogitsLoss()\n            best_val_score = 0\n            \n            for epoch in range(n_epochs):\n                model.train() \n                train_losses_batch = []\n                val_losses_batch = []\n                epoch_loss = 0\n                train_preds = np.ndarray((0,3))   # array([], shape=(0, 3), dtype=float64)\n                \n                tk0 = tqdm_nb(train_loader, total=len(train_loader), desc=\"train_loader: \")\n                for d in tk0:\n                    # ======================================================\n                    # data loader\n                    # ======================================================\n                    input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                    input_data_mask_array      = d['input_data_mask_array'].to(device)\n                    input_data_valid_array     = d['input_data_valid_array'].to(device)\n                    attention_mask             = d['attention_mask'].to(device)\n                    y                          = d[\"y\"].to(device)\n                    \n                    optimizer.zero_grad()\n                    output = model(input_data_numerical_array, \n                                   input_data_mask_array,\n                                   attention_mask)\n                    loss = criterion(output[input_data_mask_array == 1], y[input_data_mask_array == 1])\n                    \n                    # output1 = output[:, :, [1,2]]\n                    # output2 = output[:, :, 0]\n                    # y1 = y[:, :, [1,2]]\n                    # y2 = y[:, :, 0]\n                    # loss1 = criterion(output1[input_data_mask_array == 1], y1[input_data_mask_array == 1])\n                    # loss2 = criterion(output2[input_data_mask_array == 1], y2[input_data_mask_array == 1])\n                    # loss = loss1*0.8 + loss2*0.2\n                    \n                    loss.backward()\n                    optimizer.step()\n                    scheduler.step()\n                    train_losses_batch.append(loss.item())\n                train_loss = np.mean(train_losses_batch)\n                \n                \n                # ======================================================\n                # eval\n                # ======================================================\n                model.eval()       # switch model to the evaluation mode\n                \n                val_preds = np.ndarray((0, 5000, 3))\n                tk0 = tqdm_nb(val_loader, total=len(val_loader), desc=\"val_loader  : \")\n                with torch.no_grad():  # Do not calculate gradient since we are only predicting\n                    # Predicting on validation set\n                    for d in tk0:\n                        input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                        input_data_mask_array      = d['input_data_mask_array'].to(device)\n                        attention_mask             = d['attention_mask'].to(device)\n                        \n                        output = model(input_data_numerical_array, \n                                       input_data_mask_array,\n                                       attention_mask)\n                        val_preds = np.concatenate([val_preds, output.sigmoid().detach().cpu().numpy()], axis=0)\n                \n                pred_valid_index = (val_mask_array == 1) & (val_pred_array == 1) & (val_valid_array == 1)\n                StartHesitation = average_precision_score(val_target_array[pred_valid_index][:, 0],\n                                                          val_preds[pred_valid_index][:, 0])\n                Turn = average_precision_score(val_target_array[pred_valid_index][:, 1],\n                                               val_preds[pred_valid_index][:, 1])\n                Walking = average_precision_score(val_target_array[pred_valid_index][:, 2],\n                                                  val_preds[pred_valid_index][:, 2])\n                # map_score = np.mean([StartHesitation, Turn, Walking])\n                map_score = np.mean([Turn, Walking])\n                \n                # LOGGER.info(f\"fold: {fold+1} epoch: {epoch+1},train loss {train_loss} map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                print(f\"fold {fold+1} epoch {epoch+1}\\ntrain loss {train_loss} map:{map_score}\\nstart_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                \n                if map_score >= best_val_score:\n                    print(\"save weight\")\n                    best_val_score = map_score\n                    best_val_preds = val_preds.copy()\n                    torch.save(model.state_dict(), f\"./output/exp/ex{ex}/ex{ex}_model/ex{ex}_fold{fold+1}_epoch{epoch+1}__map{best_val_score:.6f}.pth\")\n                print()\n            y_oof[valid_idx] = best_val_preds\n    \n    np.save(f\"./output/exp/ex{ex}/ex{ex}_oof.npy\", y_oof)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:34.444744Z","iopub.execute_input":"2023-07-03T07:51:34.445128Z","iopub.status.idle":"2023-07-03T07:51:34.474315Z","shell.execute_reply.started":"2023-07-03T07:51:34.445090Z","shell.execute_reply":"2023-07-03T07:51:34.473140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### oof score","metadata":{}},{"cell_type":"code","source":"if TRAIN_FLAG:\n    val_pred_index = (mask_array == 1) & (pred_use_array == 1) & (valid_array == 1)\n    StartHesitation = average_precision_score(target_array[val_pred_index][:,0], \n                                              y_oof[val_pred_index][:,0])\n    Turn            = average_precision_score(target_array[val_pred_index][:,1], \n                                              y_oof[val_pred_index][:,1])\n    Walking         = average_precision_score(target_array[val_pred_index][:,2],\n                                              y_oof[val_pred_index][:,2])\n\n    map_score = np.mean([StartHesitation, Turn, Walking])\n    # LOGGER.info(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n    print(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:34.476338Z","iopub.execute_input":"2023-07-03T07:51:34.476704Z","iopub.status.idle":"2023-07-03T07:51:34.486092Z","shell.execute_reply.started":"2023-07-03T07:51:34.476672Z","shell.execute_reply":"2023-07-03T07:51:34.485247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n<br>\n<br>\n\n\n# **notype** 【Training】\n\n\n<br>\n\n## **Feature Engineering**\n\n\n<br>\n\n### fe061_notype_5000\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/fe/fe061_notype_5000.ipynb\" style=\"text-decoration:none\">fe061_notype_5000.ipynb</a>","metadata":{}},{"cell_type":"code","source":"fe = \"061_notype\"\nif not os.path.exists(f\"./output/fe/fe{fe}\"):\n    os.makedirs(f\"./output/fe/fe{fe}\")\n    os.makedirs(f\"./output/fe/fe{fe}/save\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:34.487702Z","iopub.execute_input":"2023-07-03T07:51:34.488024Z","iopub.status.idle":"2023-07-03T07:51:34.499442Z","shell.execute_reply.started":"2023-07-03T07:51:34.487995Z","shell.execute_reply":"2023-07-03T07:51:34.498517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DEFOG_META_PATH = \"../data/defog_metadata.csv\"\n# DEFOG_FOLDER = \"../data/train/notype/*.csv\"\n\ndefog_meta = pd.read_parquet(\"./output/fe/fe039/fe039_defog_meta.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:34.502433Z","iopub.execute_input":"2023-07-03T07:51:34.502764Z","iopub.status.idle":"2023-07-03T07:51:34.518197Z","shell.execute_reply.started":"2023-07-03T07:51:34.502738Z","shell.execute_reply":"2023-07-03T07:51:34.517249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = [\"AccV\", \"AccML\", \"AccAP\"]\nnum_cols = [\"AccV\", \"AccML\",  \"AccAP\",\n            'AccV_lag_diff',  'AccV_lead_diff',  \n            'AccML_lag_diff', 'AccML_lead_diff', \n            'AccAP_lag_diff', 'AccAP_lead_diff']\ntarget_cols = [\"Event\"]\nseq_len = 5000\nshift = 2500\noffset = 1250","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:34.521456Z","iopub.execute_input":"2023-07-03T07:51:34.522287Z","iopub.status.idle":"2023-07-03T07:51:34.527965Z","shell.execute_reply.started":"2023-07-03T07:51:34.522248Z","shell.execute_reply":"2023-07-03T07:51:34.526790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_array = []\ntarget_array = []\nvalid_array = []\nmask_array = []\npred_use_array = []\ntime_array = []\n\nsubject_list = []\nid_list = []\nd_list = []","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:34.529787Z","iopub.execute_input":"2023-07-03T07:51:34.530107Z","iopub.status.idle":"2023-07-03T07:51:34.581935Z","shell.execute_reply.started":"2023-07-03T07:51:34.530079Z","shell.execute_reply":"2023-07-03T07:51:34.580911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_list = glob.glob(DEFOG_FOLDER)\n\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler, RobustScaler\n\nfor i,s in tqdm(zip(defog_meta[\"Id\"].values, defog_meta[\"sub_id\"].values), desc=\"defog_meta: \"):\n    path = root_data + f\"train/notype/{i}.csv\"\n    if path in [x.replace(\"\\\\\", \"/\") for x in train_notype]:\n    # if path in data_list:\n        d_list.append(1)\n        \n        df = pd.read_csv(path)\n        df[\"valid\"] = df[\"Valid\"] & df[\"Task\"]\n        df[\"valid\"] = df[\"valid\"].astype(int)\n        \n        batch = (len(df)-1) // shift\n        \n        for c in cols:\n            df[f\"{c}_lag_diff\"] = df[c].diff()\n            df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n        \n        sc = StandardScaler()\n        df[num_cols] = sc.fit_transform(df[num_cols].values)\n        df[num_cols] = df[num_cols].fillna(0)\n\n        num = df[num_cols].values\n        target = df[target_cols].values\n        valid = df[\"valid\"].values\n        time_values = df[\"Time\"].values\n        \n        num_array_ = np.zeros([batch,seq_len, 9])\n        target_array_ = np.zeros([batch, seq_len, 1])\n        valid_array_ = np.zeros([batch, seq_len], dtype=int)\n        time_array_ = np.zeros([batch, seq_len], dtype=int)\n        mask_array_ = np.zeros([batch, seq_len], dtype=int)\n        pred_use_array_ = np.zeros([batch, seq_len], dtype=int)\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                num_array_[b,:len(num_), :] = num_\n                target_ = target[b*shift : ]\n                target_array_[b, :len(target_), :] = target_\n                mask_array_[b, :len(target_)] = 1\n                pred_use_array_[b, offset:len(target_)] = 1\n                time_ = time_values[b*shift : ]\n                time_array_[b, :len(time_)] = time_\n                valid_ = valid[b*shift : ]\n                valid_array_[b, :len(valid_)] = valid_\n            elif b == 0:\n                num_ = num[b*shift : b*shift+seq_len]\n                num_array_[b, :, :] = num_\n                target_ = target[b*shift : b*shift + seq_len]\n                target_array_[b, :, :] = target_\n                mask_array_[b, :] = 1\n                pred_use_array_[b, :shift + offset] = 1\n                time_ = time_values[b*shift : b*shift + seq_len]\n                time_array_[b, :] = time_\n                valid_ = valid[b*shift : b*shift + seq_len]\n                valid_array_[b, :] = valid_\n            else:\n                num_ = num[b*shift : b*shift+seq_len]\n                num_array_[b, :, :] = num_\n                target_ = target[b*shift : b*shift + seq_len]\n                target_array_[b, :, :] = target_\n                mask_array_[b, :] = 1\n                pred_use_array_[b, offset:shift + offset] = 1\n                time_ = time_values[b*shift : b*shift + seq_len]\n                time_array_[b, :] = time_\n                valid_ = valid[b*shift : b*shift + seq_len]\n                valid_array_[b, :] = valid_\n\n        num_array.append(num_array_)\n        target_array.append(target_array_)\n        mask_array.append(mask_array_)\n        pred_use_array.append(pred_use_array_)\n        time_array.append(time_array_)\n        valid_array.append(valid_array_)\n        subject_list += [s for _ in range(batch)]\n        id_list      += [i for _ in range(batch)] \n    else:\n        d_list.append(0)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:34.585829Z","iopub.execute_input":"2023-07-03T07:51:34.586185Z","iopub.status.idle":"2023-07-03T07:51:58.856848Z","shell.execute_reply.started":"2023-07-03T07:51:34.586143Z","shell.execute_reply":"2023-07-03T07:51:58.855881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_array = np.concatenate(num_array, axis=0)\ntarget_array =np.concatenate(target_array, axis=0)\nmask_array =  np.concatenate(mask_array, axis=0)\npred_use_array = np.concatenate(pred_use_array, axis=0)\ntime_array = np.concatenate(time_array, axis=0)\nvalid_array = np.concatenate(valid_array, axis=0)\n\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_num_array.npy\", num_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_target_array.npy\", target_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_mask_array.npy\", mask_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_time_array.npy\", time_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_pred_use_array.npy\", pred_use_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_valid_array.npy\", valid_array)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:51:58.858183Z","iopub.execute_input":"2023-07-03T07:51:58.858656Z","iopub.status.idle":"2023-07-03T07:52:05.097639Z","shell.execute_reply.started":"2023-07-03T07:51:58.858622Z","shell.execute_reply":"2023-07-03T07:52:05.096624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_id = pd.DataFrame()\ndf_id[\"Id\"] = id_list\ndf_id[\"subject\"] = subject_list\n\ndf_id[\"Id\"].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:52:05.099034Z","iopub.execute_input":"2023-07-03T07:52:05.099666Z","iopub.status.idle":"2023-07-03T07:52:05.136676Z","shell.execute_reply.started":"2023-07-03T07:52:05.099631Z","shell.execute_reply":"2023-07-03T07:52:05.135742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_id.to_parquet(f\"./output/fe/fe{fe}/fe{fe}_id.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:52:05.137999Z","iopub.execute_input":"2023-07-03T07:52:05.138380Z","iopub.status.idle":"2023-07-03T07:52:09.594800Z","shell.execute_reply.started":"2023-07-03T07:52:05.138346Z","shell.execute_reply":"2023-07-03T07:52:09.593669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### fe064_notype_10000\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/fe/fe064_notype_10000.ipynb\" style=\"text-decoration:none\">fe064_notype_10000.ipynb</a>","metadata":{}},{"cell_type":"code","source":"fe = \"064_notype\"\nif not os.path.exists(f\"./output/fe/fe{fe}\"):\n    os.makedirs(f\"./output/fe/fe{fe}\")\n    os.makedirs(f\"./output/fe/fe{fe}/save\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:52:09.596275Z","iopub.execute_input":"2023-07-03T07:52:09.597286Z","iopub.status.idle":"2023-07-03T07:52:09.604990Z","shell.execute_reply.started":"2023-07-03T07:52:09.597251Z","shell.execute_reply":"2023-07-03T07:52:09.603825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DEFOG_META_PATH = \"../data/defog_metadata.csv\"\n# DEFOG_FOLDER = \"../data/train/notype/*.csv\"\n\ndefog_meta = pd.read_parquet(\"./output/fe/fe039/fe039_defog_meta.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:52:09.607264Z","iopub.execute_input":"2023-07-03T07:52:09.607541Z","iopub.status.idle":"2023-07-03T07:52:09.617928Z","shell.execute_reply.started":"2023-07-03T07:52:09.607518Z","shell.execute_reply":"2023-07-03T07:52:09.617049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = [\"AccV\", \"AccML\", \"AccAP\",\n            'AccV_lag_diff', 'AccV_lead_diff', \n            'AccML_lag_diff', 'AccML_lead_diff',\n            'AccAP_lag_diff', 'AccAP_lead_diff']\ntarget_cols = [\"Event\"]\nseq_len = 10000\nshift = 5000\noffset = 2500","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:52:09.619624Z","iopub.execute_input":"2023-07-03T07:52:09.620017Z","iopub.status.idle":"2023-07-03T07:52:09.626142Z","shell.execute_reply.started":"2023-07-03T07:52:09.619987Z","shell.execute_reply":"2023-07-03T07:52:09.624222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_array = []\ntarget_array = []\nvalid_array = []\nmask_array = []\npred_use_array = []\ntime_array = []\n\nsubject_list = []\nid_list = []\nd_list = []","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:52:09.627722Z","iopub.execute_input":"2023-07-03T07:52:09.628188Z","iopub.status.idle":"2023-07-03T07:52:09.653960Z","shell.execute_reply.started":"2023-07-03T07:52:09.628155Z","shell.execute_reply":"2023-07-03T07:52:09.653030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_list = glob.glob(DEFOG_FOLDER)\n\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler, RobustScaler\n\nfor i,s in tqdm(zip(defog_meta[\"Id\"].values, defog_meta[\"sub_id\"].values), desc=\"defog_meta: \"):\n    path = root_data + f\"train/notype/{i}.csv\"\n    if path in [x.replace(\"\\\\\", \"/\") for x in train_notype]:\n    # if path in data_list:\n        d_list.append(1)\n        \n        df = pd.read_csv(path)\n        df[\"valid\"] = df[\"Valid\"] & df[\"Task\"]\n        df[\"valid\"] = df[\"valid\"].astype(int)\n        \n        batch = (len(df)-1) // shift\n        \n        for c in cols:\n            df[f\"{c}_lag_diff\"] = df[c].diff()\n            df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n        \n        sc = StandardScaler()\n        df[num_cols] = sc.fit_transform(df[num_cols].values)\n        df[num_cols] = df[num_cols].fillna(0)\n\n        num = df[num_cols].values\n        target = df[target_cols].values\n        valid = df[\"valid\"].values\n        time_values = df[\"Time\"].values\n        \n        num_array_ = np.zeros([batch,seq_len, 9])\n        target_array_ = np.zeros([batch, seq_len, 1])\n        valid_array_ = np.zeros([batch, seq_len], dtype=int)\n        time_array_ = np.zeros([batch, seq_len], dtype=int)\n        mask_array_ = np.zeros([batch, seq_len], dtype=int)\n        pred_use_array_ = np.zeros([batch, seq_len], dtype=int)\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                num_array_[b,:len(num_), :] = num_\n                target_ = target[b*shift : ]\n                target_array_[b, :len(target_), :] = target_\n                mask_array_[b, :len(target_)] = 1\n                pred_use_array_[b, offset:len(target_)] = 1\n                time_ = time_values[b*shift : ]\n                time_array_[b, :len(time_)] = time_\n                valid_ = valid[b*shift : ]\n                valid_array_[b, :len(valid_)] = valid_\n            elif b == 0:\n                num_ = num[b*shift : b*shift+seq_len]\n                num_array_[b, :, :] = num_\n                target_ = target[b*shift : b*shift + seq_len]\n                target_array_[b, :, :] = target_\n                mask_array_[b, :] = 1\n                pred_use_array_[b, :shift + offset] = 1\n                time_ = time_values[b*shift : b*shift + seq_len]\n                time_array_[b, :] = time_\n                valid_ = valid[b*shift : b*shift + seq_len]\n                valid_array_[b, :] = valid_\n            else:\n                num_ = num[b*shift : b*shift+seq_len]\n                num_array_[b, :, :] = num_\n                target_ = target[b*shift : b*shift + seq_len]\n                target_array_[b, :, :] = target_\n                mask_array_[b, :] = 1\n                pred_use_array_[b, offset:shift + offset] = 1\n                time_ = time_values[b*shift : b*shift + seq_len]\n                time_array_[b, :] = time_\n                valid_ = valid[b*shift : b*shift + seq_len]\n                valid_array_[b, :] = valid_\n\n        num_array.append(num_array_)\n        target_array.append(target_array_)\n        mask_array.append(mask_array_)\n        pred_use_array.append(pred_use_array_)\n        time_array.append(time_array_)\n        valid_array.append(valid_array_)\n        subject_list += [s for _ in range(batch)]\n        id_list      += [i for _ in range(batch)] \n    else:\n        d_list.append(0)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:52:09.657463Z","iopub.execute_input":"2023-07-03T07:52:09.657729Z","iopub.status.idle":"2023-07-03T07:52:32.269520Z","shell.execute_reply.started":"2023-07-03T07:52:09.657706Z","shell.execute_reply":"2023-07-03T07:52:32.268611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_array = np.concatenate(num_array, axis=0)\ntarget_array =np.concatenate(target_array, axis=0)\nmask_array =  np.concatenate(mask_array, axis=0)\npred_use_array = np.concatenate(pred_use_array, axis=0)\ntime_array = np.concatenate(time_array, axis=0)\nvalid_array = np.concatenate(valid_array, axis=0)\n\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_num_array.npy\", num_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_target_array.npy\", target_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_mask_array.npy\", mask_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_time_array.npy\", time_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_pred_use_array.npy\", pred_use_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_valid_array.npy\", valid_array)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:52:32.270976Z","iopub.execute_input":"2023-07-03T07:52:32.271587Z","iopub.status.idle":"2023-07-03T07:52:38.866025Z","shell.execute_reply.started":"2023-07-03T07:52:32.271551Z","shell.execute_reply":"2023-07-03T07:52:38.864173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_id = pd.DataFrame()\ndf_id[\"Id\"] = id_list\ndf_id[\"subject\"] = subject_list\n\ndf_id[\"Id\"].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:52:38.867450Z","iopub.execute_input":"2023-07-03T07:52:38.867810Z","iopub.status.idle":"2023-07-03T07:52:38.889449Z","shell.execute_reply.started":"2023-07-03T07:52:38.867774Z","shell.execute_reply":"2023-07-03T07:52:38.888234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_id.to_parquet(f\"./output/fe/fe{fe}/fe{fe}_id.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:52:38.890736Z","iopub.execute_input":"2023-07-03T07:52:38.891161Z","iopub.status.idle":"2023-07-03T07:52:43.787913Z","shell.execute_reply.started":"2023-07-03T07:52:38.891127Z","shell.execute_reply":"2023-07-03T07:52:43.786843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### fe074_notype_15000\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/fe/fe074_notype_15000.ipynb\" style=\"text-decoration:none\">fe074_notype_15000.ipynb</a>","metadata":{}},{"cell_type":"code","source":"fe = \"074_notype\"\nif not os.path.exists(f\"./output/fe/fe{fe}\"):\n    os.makedirs(f\"./output/fe/fe{fe}\")\n    os.makedirs(f\"./output/fe/fe{fe}/save\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:52:43.789987Z","iopub.execute_input":"2023-07-03T07:52:43.790410Z","iopub.status.idle":"2023-07-03T07:52:43.799711Z","shell.execute_reply.started":"2023-07-03T07:52:43.790373Z","shell.execute_reply":"2023-07-03T07:52:43.798641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DEFOG_META_PATH = \"../data/defog_metadata.csv\"\n# DEFOG_FOLDER = \"../data/train/notype/*.csv\"\n\ndefog_meta = pd.read_parquet(\"./output/fe/fe039/fe039_defog_meta.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:52:43.803003Z","iopub.execute_input":"2023-07-03T07:52:43.803446Z","iopub.status.idle":"2023-07-03T07:52:43.816286Z","shell.execute_reply.started":"2023-07-03T07:52:43.803421Z","shell.execute_reply":"2023-07-03T07:52:43.815421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = [\"AccV\", \"AccML\", \"AccAP\",\n            'AccV_lag_diff', 'AccV_lead_diff', \n            'AccML_lag_diff', 'AccML_lead_diff',\n            'AccAP_lag_diff', 'AccAP_lead_diff']\ntarget_cols = [\"Event\"]\nseq_len = 15000\nshift = 7500\noffset = 3750","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:52:43.818117Z","iopub.execute_input":"2023-07-03T07:52:43.819304Z","iopub.status.idle":"2023-07-03T07:52:43.824812Z","shell.execute_reply.started":"2023-07-03T07:52:43.819263Z","shell.execute_reply":"2023-07-03T07:52:43.823739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_array = []\ntarget_array = []\nvalid_array = []\nmask_array = []\npred_use_array = []\ntime_array = []\n\nsubject_list = []\nid_list = []\nd_list = []","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:52:43.826419Z","iopub.execute_input":"2023-07-03T07:52:43.827720Z","iopub.status.idle":"2023-07-03T07:52:43.849172Z","shell.execute_reply.started":"2023-07-03T07:52:43.827662Z","shell.execute_reply":"2023-07-03T07:52:43.847961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_list = glob.glob(DEFOG_FOLDER)\n\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler, RobustScaler\n\nfor i,s in tqdm(zip(defog_meta[\"Id\"].values, defog_meta[\"sub_id\"].values), desc=\"defog_meta: \"):\n    path = root_data + f\"train/notype/{i}.csv\"\n    if path in [x.replace(\"\\\\\", \"/\") for x in train_notype]:\n    # if path in data_list:\n        d_list.append(1)\n        \n        df = pd.read_csv(path)\n        df[\"valid\"] = df[\"Valid\"] & df[\"Task\"]\n        df[\"valid\"] = df[\"valid\"].astype(int)\n        \n        batch = (len(df)-1) // shift\n        \n        for c in cols:\n            df[f\"{c}_lag_diff\"] = df[c].diff()\n            df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n        \n        sc = StandardScaler()\n        df[num_cols] = sc.fit_transform(df[num_cols].values)\n        df[num_cols] = df[num_cols].fillna(0)\n\n        num = df[num_cols].values\n        target = df[target_cols].values\n        valid = df[\"valid\"].values\n        time_values = df[\"Time\"].values\n        \n        num_array_ = np.zeros([batch,seq_len, 9])\n        target_array_ = np.zeros([batch, seq_len, 1])\n        valid_array_ = np.zeros([batch, seq_len], dtype=int)\n        time_array_ = np.zeros([batch, seq_len], dtype=int)\n        mask_array_ = np.zeros([batch, seq_len], dtype=int)\n        pred_use_array_ = np.zeros([batch, seq_len], dtype=int)\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                num_array_[b,:len(num_), :] = num_\n                target_ = target[b*shift : ]\n                target_array_[b, :len(target_), :] = target_\n                mask_array_[b, :len(target_)] = 1\n                pred_use_array_[b, offset:len(target_)] = 1\n                time_ = time_values[b*shift : ]\n                time_array_[b, :len(time_)] = time_\n                valid_ = valid[b*shift : ]\n                valid_array_[b, :len(valid_)] = valid_\n            elif b == 0:\n                num_ = num[b*shift : b*shift+seq_len]\n                num_array_[b, :, :] = num_\n                target_ = target[b*shift : b*shift + seq_len]\n                target_array_[b, :, :] = target_\n                mask_array_[b, :] = 1\n                pred_use_array_[b, :shift + offset] = 1\n                time_ = time_values[b*shift : b*shift + seq_len]\n                time_array_[b, :] = time_\n                valid_ = valid[b*shift : b*shift + seq_len]\n                valid_array_[b, :] = valid_\n            else:\n                num_ = num[b*shift : b*shift+seq_len]\n                num_array_[b, :, :] = num_\n                target_ = target[b*shift : b*shift + seq_len]\n                target_array_[b, :, :] = target_\n                mask_array_[b, :] = 1\n                pred_use_array_[b, offset:shift + offset] = 1\n                time_ = time_values[b*shift : b*shift + seq_len]\n                time_array_[b, :] = time_\n                valid_ = valid[b*shift : b*shift + seq_len]\n                valid_array_[b, :] = valid_\n\n        num_array.append(num_array_)\n        target_array.append(target_array_)\n        mask_array.append(mask_array_)\n        pred_use_array.append(pred_use_array_)\n        time_array.append(time_array_)\n        valid_array.append(valid_array_)\n        subject_list += [s for _ in range(batch)]\n        id_list      += [i for _ in range(batch)] \n    else:\n        d_list.append(0)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:52:43.853636Z","iopub.execute_input":"2023-07-03T07:52:43.853942Z","iopub.status.idle":"2023-07-03T07:53:00.749591Z","shell.execute_reply.started":"2023-07-03T07:52:43.853917Z","shell.execute_reply":"2023-07-03T07:53:00.748678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_array = np.concatenate(num_array, axis=0)\ntarget_array =np.concatenate(target_array, axis=0)\nmask_array =  np.concatenate(mask_array, axis=0)\npred_use_array = np.concatenate(pred_use_array, axis=0)\ntime_array = np.concatenate(time_array, axis=0)\nvalid_array = np.concatenate(valid_array, axis=0)\n\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_num_array.npy\", num_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_target_array.npy\", target_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_mask_array.npy\", mask_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_time_array.npy\", time_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_pred_use_array.npy\", pred_use_array)\nnp.save(f\"./output/fe/fe{fe}/fe{fe}_valid_array.npy\", valid_array)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:00.750998Z","iopub.execute_input":"2023-07-03T07:53:00.752463Z","iopub.status.idle":"2023-07-03T07:53:07.387329Z","shell.execute_reply.started":"2023-07-03T07:53:00.752427Z","shell.execute_reply":"2023-07-03T07:53:07.386383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_id = pd.DataFrame()\ndf_id[\"Id\"] = id_list\ndf_id[\"subject\"] = subject_list\n\ndf_id[\"Id\"].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:07.388782Z","iopub.execute_input":"2023-07-03T07:53:07.389132Z","iopub.status.idle":"2023-07-03T07:53:07.408976Z","shell.execute_reply.started":"2023-07-03T07:53:07.389087Z","shell.execute_reply":"2023-07-03T07:53:07.407789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_id.to_parquet(f\"./output/fe/fe{fe}/fe{fe}_id.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:07.410549Z","iopub.execute_input":"2023-07-03T07:53:07.411331Z","iopub.status.idle":"2023-07-03T07:53:11.605856Z","shell.execute_reply.started":"2023-07-03T07:53:07.411294Z","shell.execute_reply":"2023-07-03T07:53:11.604776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n## **Helpers**\n\n<br>\n\n### def `set_seed()`","metadata":{}},{"cell_type":"code","source":"def set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:11.607124Z","iopub.execute_input":"2023-07-03T07:53:11.607489Z","iopub.status.idle":"2023-07-03T07:53:11.619946Z","shell.execute_reply.started":"2023-07-03T07:53:11.607459Z","shell.execute_reply":"2023-07-03T07:53:11.618959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### def `timer()`","metadata":{}},{"cell_type":"code","source":"@contextmanager\ndef timer(name):\n    t0 = time.time()\n    \n    yield \n    # LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s')\n    print(f'[{name}] done in {time.time() - t0:.0f} s')\n    print(\"=\"*66)\n    \n# setup_logger(out_file=logger_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:11.621537Z","iopub.execute_input":"2023-07-03T07:53:11.622068Z","iopub.status.idle":"2023-07-03T07:53:11.630951Z","shell.execute_reply.started":"2023-07-03T07:53:11.622027Z","shell.execute_reply":"2023-07-03T07:53:11.629836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### def `preprocess()`","metadata":{}},{"cell_type":"code","source":"def preprocess(numerical_array, mask_array, valid_array,):\n    \n    attention_mask = mask_array == 0\n\n    return {'input_data_numerical_array': numerical_array,\n            'input_data_mask_array': mask_array,\n            'input_data_valid_array': valid_array,\n            'attention_mask': attention_mask,\n           }","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:11.634090Z","iopub.execute_input":"2023-07-03T07:53:11.635445Z","iopub.status.idle":"2023-07-03T07:53:11.642415Z","shell.execute_reply.started":"2023-07-03T07:53:11.635410Z","shell.execute_reply":"2023-07-03T07:53:11.641278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### class `FogDataset()`","metadata":{}},{"cell_type":"code","source":"class FogDataset(Dataset):\n    def __init__(self, numerical_array, mask_array, valid_array, train = True, y = None):\n        self.numerical_array = numerical_array\n        self.mask_array = mask_array\n        self.valid_array = valid_array\n        self.train = train\n        self.y = y\n    \n    \n    def __len__(self):\n        return len(self.numerical_array)\n    \n    \n    def __getitem__(self, item):\n        data = preprocess(self.numerical_array[item], self.mask_array[item], self.valid_array[item],)\n\n        # Return the processed data where the lists are converted to `torch.tensor`s\n        if self.train : \n            return {\n              'input_data_numerical_array': torch.tensor(data['input_data_numerical_array'], dtype=torch.float32),\n              'input_data_mask_array': torch.tensor(data['input_data_mask_array'],  dtype=torch.long),  \n              'input_data_valid_array': torch.tensor(data['input_data_valid_array'], dtype=torch.long),   \n              'attention_mask': torch.tensor(data[\"attention_mask\"], dtype=torch.bool),\n              \"y\": torch.tensor(self.y[item], dtype=torch.float32)\n            }\n        else:\n            return {\n              'input_data_numerical_array': torch.tensor(data['input_data_numerical_array'], dtype=torch.float32),\n              'input_data_mask_array': torch.tensor(data['input_data_mask_array'], dtype=torch.long),  \n              'input_data_valid_array': torch.tensor(data['input_data_valid_array'], dtype=torch.long),  \n              'attention_mask': torch.tensor(data[\"attention_mask\"], dtype=torch.bool),\n               }","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:11.643994Z","iopub.execute_input":"2023-07-03T07:53:11.644377Z","iopub.status.idle":"2023-07-03T07:53:11.656420Z","shell.execute_reply.started":"2023-07-03T07:53:11.644346Z","shell.execute_reply":"2023-07-03T07:53:11.655381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### class `FogRnnModel()`","metadata":{}},{"cell_type":"code","source":"class FogRnnModel(nn.Module):\n    def __init__(self, \n                 dropout=0.2,\n                 input_numerical_size=9,\n                 numeraical_linear_size = 64,\n                 model_size = 128,\n                 linear_out = 128,\n                 out_size=3):\n        \n        super(FogRnnModel, self).__init__()\n        \n        self.numerical_linear  = nn.Sequential(nn.Linear(input_numerical_size, numeraical_linear_size),\n                                               nn.LayerNorm(numeraical_linear_size))\n        \n        self.rnn = nn.GRU(numeraical_linear_size, \n                          model_size,\n                          num_layers = 2, \n                          batch_first=True,\n                          bidirectional=True)\n        \n        self.linear_out  = nn.Sequential(nn.Linear(model_size*2, linear_out),\n                                         nn.LayerNorm(linear_out),\n                                         nn.ReLU(),\n                                         nn.Dropout(dropout),\n                                         nn.Linear(linear_out, out_size))\n        self._reinitialize()\n        \n        \n        \n    def _reinitialize(self):\n        \"\"\"Tensorflow/Keras-like initialization\"\"\"\n        for name, p in self.named_parameters():\n            if 'rnn' in name:\n                if 'weight_ih' in name:\n                    nn.init.xavier_uniform_(p.data)\n                elif 'weight_hh' in name:\n                    nn.init.orthogonal_(p.data)\n                elif 'bias_ih' in name:\n                    p.data.fill_(0)\n                    # Set forget-gate bias to 1\n                    n = p.size(0)\n                    p.data[(n // 4):(n // 2)].fill_(1)\n                elif 'bias_hh' in name:\n                    p.data.fill_(0)\n        \n        \n        \n    def forward(self, numerical_array, mask_array, attention_mask):\n        numerical_embedding = self.numerical_linear(numerical_array)\n        output, _ = self.rnn(numerical_embedding)\n        output = self.linear_out(output)\n        return output","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:11.657984Z","iopub.execute_input":"2023-07-03T07:53:11.658440Z","iopub.status.idle":"2023-07-03T07:53:11.673048Z","shell.execute_reply.started":"2023-07-03T07:53:11.658373Z","shell.execute_reply":"2023-07-03T07:53:11.672137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n<br>\n\n## <font color=blue>**ROUND 1**</font>\n\n<br>\n<br>\n\n\n## <font color=red>**Predicting**</font>: **Making** <font color=red>**pseudo**</font> **label** (round1: use models of ex153 & ex154)\n\n<br>\n\n### config","metadata":{}},{"cell_type":"code","source":"# config\nseed = 0\nshuffle = True\nn_splits = 5\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# model config\nbatch_size = 24\nn_epochs = 15\nlr = 1e-3\nweight_decay = 0.05\nnum_warmup_steps = 10","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:11.674696Z","iopub.execute_input":"2023-07-03T07:53:11.675047Z","iopub.status.idle":"2023-07-03T07:53:11.686337Z","shell.execute_reply.started":"2023-07-03T07:53:11.675016Z","shell.execute_reply":"2023-07-03T07:53:11.685192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### <font color=maroon>ex153_defog_gru_inference_notype_10000.ipynb</font>\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/exp/ex153_defog_gru_inference_notype_10000.ipynb\" style=\"text-decoration:none\">ex153_defog_gru_inference_notype_10000.ipynb</a>","metadata":{}},{"cell_type":"code","source":"debug = False\nex = \"153_defog\"\nif not os.path.exists(f\"./output/exp/ex{ex}\"):\n    os.makedirs(f\"./output/exp/ex{ex}\")\n    os.makedirs(f\"./output/exp/ex{ex}/ex{ex}_model\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:11.687941Z","iopub.execute_input":"2023-07-03T07:53:11.688328Z","iopub.status.idle":"2023-07-03T07:53:11.699751Z","shell.execute_reply.started":"2023-07-03T07:53:11.688297Z","shell.execute_reply":"2023-07-03T07:53:11.698771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### load data","metadata":{}},{"cell_type":"code","source":"seq_len = 10000\n\nid_path        = f\"./output/fe/fe064_notype/fe064_notype_id.parquet\"\nnumerical_path = f\"./output/fe/fe064_notype/fe064_notype_num_array.npy\"\ntarget_path    = f\"./output/fe/fe064_notype/fe064_notype_target_array.npy\"\nmask_path      = f\"./output/fe/fe064_notype/fe064_notype_mask_array.npy\"\nvalid_path     = f\"./output/fe/fe064_notype/fe064_notype_valid_array.npy\"\npred_use_path  = f\"./output/fe/fe064_notype/fe064_notype_pred_use_array.npy\"\ntime_path      = f\"./output/fe/fe064_notype/fe064_notype_time_array.npy\"\n\ndf_id           = pd.read_parquet(id_path)\nnumerical_array = np.load(numerical_path)\ntarget_array    = np.load(target_path)\nmask_array      = np.load(mask_path)\nvalid_array     = np.load(valid_path)\npred_use_array  = np.load(pred_use_path)\ntime_array      = np.load(time_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:11.701069Z","iopub.execute_input":"2023-07-03T07:53:11.701544Z","iopub.status.idle":"2023-07-03T07:53:23.387653Z","shell.execute_reply.started":"2023-07-03T07:53:11.701507Z","shell.execute_reply":"2023-07-03T07:53:23.386474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### main (predict)","metadata":{}},{"cell_type":"code","source":"models_list = os.listdir(f\"./output/exp/ex{ex}/ex{ex}_model\")\nmodels_list","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:23.388888Z","iopub.execute_input":"2023-07-03T07:53:23.389261Z","iopub.status.idle":"2023-07-03T07:53:23.398500Z","shell.execute_reply.started":"2023-07-03T07:53:23.389229Z","shell.execute_reply":"2023-07-03T07:53:23.397371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if PREDICT_FLAG:\n# with timer(\"gru\"):\n    set_seed(seed)\n    for fold in range(1, 6):\n        with timer(f\"fold {fold}\"):\n            val_numerical_array = numerical_array.copy()\n            val_target_array = target_array.copy()\n            val_mask_array = mask_array.copy()\n            val_valid_array = valid_array.copy()\n            val_pred_array = pred_use_array.copy()\n\n            val_ = FogDataset(val_numerical_array, \n                              val_mask_array,\n                              val_valid_array, \n                              train=True,\n                              y=val_target_array)\n\n            val_loader = DataLoader(dataset=val_, \n                                    batch_size=batch_size, \n                                    shuffle = False,\n                                    # num_workers=8\n                                   )\n\n            model = FogRnnModel()\n            # model.load_state_dict(torch.load(f\"./output/exp/ex{ex}/ex{ex}_model/ex{ex}_{fold}.pth\"))\n            model.load_state_dict(torch.load(f\"./output/exp/ex{ex}/ex{ex}_model/\" + models_list[fold-1]))\n            model = model.to(device)\n            model.eval()  # switch model to the evaluation mode\n            \n            val_preds = np.ndarray((0, 10000, 3))\n            tk0 = tqdm(val_loader, total=len(val_loader))\n            with torch.no_grad():  # Do not calculate gradient since we are only predicting\n                # Predicting on validation set\n                for d in tk0:\n                    input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                    input_data_mask_array      = d['input_data_mask_array'].to(device)\n                    attention_mask             = d['attention_mask'].to(device)\n                    output = model(input_data_numerical_array, \n                                   input_data_mask_array,\n                                   attention_mask)\n                    val_preds = np.concatenate([val_preds, output.sigmoid().detach().cpu().numpy()], axis=0)\n            np.save(f\"./output/exp/ex{ex}/ex{ex}_notype_{fold}_oof_{seq_len}.npy\",val_preds)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:23.400259Z","iopub.execute_input":"2023-07-03T07:53:23.400665Z","iopub.status.idle":"2023-07-03T07:53:23.413312Z","shell.execute_reply.started":"2023-07-03T07:53:23.400633Z","shell.execute_reply":"2023-07-03T07:53:23.412006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if PREDICT_FLAG:\n    id_array = df_id[\"Id\"].values\n    for i in range(1, 6):\n        pred_all = []\n        pred = np.load(f\"./output/exp/ex{ex}/ex{ex}_notype_{i}_oof_{seq_len}.npy\")\n        for v in tqdm(range(len(pred_use_array))):\n            use_ = pred_use_array[v, :] == 1\n            pred_ = pred[v, use_ == 1, :]\n            time_ = time_array[v, use_ == 1]\n            Id = id_array[v]\n            pred_df = pd.DataFrame()\n            pred_df[\"Time\"] = time_\n            pred_df[\"Id\"] = Id\n            pred_df[\"StartHesitation\"] = pred_[:, 0]\n            pred_df[\"Turn\"] = pred_[:, 1]\n            pred_df[\"Walking\"] = pred_[:, 2]\n            pred_all.append(pred_df)\n        pred_all = pd.concat(pred_all).reset_index(drop=True)\n        pred_all.to_parquet(f\"./output/exp/ex{ex}/ex{ex}_notype_{i}_pred_{seq_len}.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:23.414828Z","iopub.execute_input":"2023-07-03T07:53:23.415174Z","iopub.status.idle":"2023-07-03T07:53:23.426997Z","shell.execute_reply.started":"2023-07-03T07:53:23.415144Z","shell.execute_reply":"2023-07-03T07:53:23.425932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### <font color=maroon>ex153_defog_gru_inference_notype_15000.ipynb</font>\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/exp/ex153_defog_gru_inference_notype_15000.ipynb\" style=\"text-decoration:none\">ex153_defog_gru_inference_notype_15000.ipynb</a>","metadata":{}},{"cell_type":"code","source":"debug = False\nex = \"153_defog\"\nif not os.path.exists(f\"./output/exp/ex{ex}\"):\n    os.makedirs(f\"./output/exp/ex{ex}\")\n    os.makedirs(f\"./output/exp/ex{ex}/ex{ex}_model\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:23.428408Z","iopub.execute_input":"2023-07-03T07:53:23.428810Z","iopub.status.idle":"2023-07-03T07:53:23.442000Z","shell.execute_reply.started":"2023-07-03T07:53:23.428777Z","shell.execute_reply":"2023-07-03T07:53:23.441001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### load data","metadata":{}},{"cell_type":"code","source":"seq_len = 15000\n\nid_path        = f\"./output/fe/fe074_notype/fe074_notype_id.parquet\"\nnumerical_path = f\"./output/fe/fe074_notype/fe074_notype_num_array.npy\"\ntarget_path    = f\"./output/fe/fe074_notype/fe074_notype_target_array.npy\"\nmask_path      = f\"./output/fe/fe074_notype/fe074_notype_mask_array.npy\"\nvalid_path     = f\"./output/fe/fe074_notype/fe074_notype_valid_array.npy\"\npred_use_path  = f\"./output/fe/fe074_notype/fe074_notype_pred_use_array.npy\"\ntime_path      = f\"./output/fe/fe074_notype/fe074_notype_time_array.npy\"\n\ndf_id           = pd.read_parquet(id_path)\nnumerical_array = np.load(numerical_path)\ntarget_array    = np.load(target_path)\nmask_array      = np.load(mask_path)\nvalid_array     = np.load(valid_path)\npred_use_array  = np.load(pred_use_path)\ntime_array      = np.load(time_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:23.443593Z","iopub.execute_input":"2023-07-03T07:53:23.444033Z","iopub.status.idle":"2023-07-03T07:53:35.533259Z","shell.execute_reply.started":"2023-07-03T07:53:23.444002Z","shell.execute_reply":"2023-07-03T07:53:35.532178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### main (predict)","metadata":{}},{"cell_type":"code","source":"models_list = os.listdir(f\"./output/exp/ex{ex}/ex{ex}_model\")\nmodels_list","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:35.535886Z","iopub.execute_input":"2023-07-03T07:53:35.536274Z","iopub.status.idle":"2023-07-03T07:53:35.545791Z","shell.execute_reply.started":"2023-07-03T07:53:35.536236Z","shell.execute_reply":"2023-07-03T07:53:35.544447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if PREDICT_FLAG:\n# with timer(\"gru\"):\n    set_seed(seed)\n    for fold in range(1, 6):\n        with timer(f\"fold {fold}\"):\n            val_numerical_array = numerical_array.copy()\n            val_target_array = target_array.copy()\n            val_mask_array = mask_array.copy()\n            val_valid_array = valid_array.copy()\n            val_pred_array = pred_use_array.copy()\n\n            val_ = FogDataset(val_numerical_array, \n                              val_mask_array,\n                              val_valid_array, \n                              train=True,\n                              y=val_target_array)\n\n            val_loader = DataLoader(dataset=val_, \n                                    batch_size=batch_size, \n                                    shuffle = False,\n                                    # num_workers=8\n                                   )\n\n            model = FogRnnModel()\n            # model.load_state_dict(torch.load(f\"./output/exp/ex{ex}/ex{ex}_model/ex{ex}_{fold}.pth\"))\n            model.load_state_dict(torch.load(f\"./output/exp/ex{ex}/ex{ex}_model/\" + models_list[fold-1]))\n            model = model.to(device)\n            model.eval()  # switch model to the evaluation mode\n            \n            val_preds = np.ndarray((0, seq_len, 3))\n            tk0 = tqdm(val_loader, total=len(val_loader))\n            with torch.no_grad():  # Do not calculate gradient since we are only predicting\n                # Predicting on validation set\n                for d in tk0:\n                    input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                    input_data_mask_array      = d['input_data_mask_array'].to(device)\n                    attention_mask             = d['attention_mask'].to(device)\n                    output = model(input_data_numerical_array, \n                                   input_data_mask_array,\n                                   attention_mask)\n                    val_preds = np.concatenate([val_preds, output.sigmoid().detach().cpu().numpy()], axis=0)\n            np.save(f\"./output/exp/ex{ex}/ex{ex}_notype_{fold}_oof_{seq_len}.npy\",val_preds)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:35.547801Z","iopub.execute_input":"2023-07-03T07:53:35.548177Z","iopub.status.idle":"2023-07-03T07:53:35.560653Z","shell.execute_reply.started":"2023-07-03T07:53:35.548146Z","shell.execute_reply":"2023-07-03T07:53:35.559759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if PREDICT_FLAG:\n    id_array = df_id[\"Id\"].values\n    for i in range(1, 6):\n        pred_all = []\n        pred = np.load(f\"./output/exp/ex{ex}/ex{ex}_notype_{i}_oof_{seq_len}.npy\")\n        for v in tqdm(range(len(pred_use_array))):\n            use_ = pred_use_array[v, :] == 1\n            pred_ = pred[v, use_ == 1, :]\n            time_ = time_array[v, use_ == 1]\n            Id = id_array[v]\n            pred_df = pd.DataFrame()\n            pred_df[\"Time\"] = time_\n            pred_df[\"Id\"] = Id\n            pred_df[\"StartHesitation\"] = pred_[:, 0]\n            pred_df[\"Turn\"] = pred_[:, 1]\n            pred_df[\"Walking\"] = pred_[:, 2]\n            pred_all.append(pred_df)\n        pred_all = pd.concat(pred_all).reset_index(drop=True)\n        pred_all.to_parquet(f\"./output/exp/ex{ex}/ex{ex}_notype_{i}_pred_{seq_len}.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:35.562040Z","iopub.execute_input":"2023-07-03T07:53:35.562500Z","iopub.status.idle":"2023-07-03T07:53:35.577171Z","shell.execute_reply.started":"2023-07-03T07:53:35.562465Z","shell.execute_reply":"2023-07-03T07:53:35.576249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### <font color=maroon>ex154_defog_gru_inference_notype_15000.ipynb</font>\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/exp/ex154_defog_gru_inference_notype_15000.ipynb\" style=\"text-decoration:none\">ex154_defog_gru_inference_notype_15000.ipynb</a>","metadata":{}},{"cell_type":"code","source":"debug = False\nex = \"154_defog\"\nif not os.path.exists(f\"./output/exp/ex{ex}\"):\n    os.makedirs(f\"./output/exp/ex{ex}\")\n    os.makedirs(f\"./output/exp/ex{ex}/ex{ex}_model\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:35.578655Z","iopub.execute_input":"2023-07-03T07:53:35.579191Z","iopub.status.idle":"2023-07-03T07:53:35.587531Z","shell.execute_reply.started":"2023-07-03T07:53:35.579159Z","shell.execute_reply":"2023-07-03T07:53:35.586570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### load data","metadata":{}},{"cell_type":"code","source":"seq_len = 15000\n\nid_path        = f\"./output/fe/fe074_notype/fe074_notype_id.parquet\"\nnumerical_path = f\"./output/fe/fe074_notype/fe074_notype_num_array.npy\"\ntarget_path    = f\"./output/fe/fe074_notype/fe074_notype_target_array.npy\"\nmask_path      = f\"./output/fe/fe074_notype/fe074_notype_mask_array.npy\"\nvalid_path     = f\"./output/fe/fe074_notype/fe074_notype_valid_array.npy\"\npred_use_path  = f\"./output/fe/fe074_notype/fe074_notype_pred_use_array.npy\"\ntime_path      = f\"./output/fe/fe074_notype/fe074_notype_time_array.npy\"\n\ndf_id           = pd.read_parquet(id_path)\nnumerical_array = np.load(numerical_path)\ntarget_array    = np.load(target_path)\nmask_array      = np.load(mask_path)\nvalid_array     = np.load(valid_path)\npred_use_array  = np.load(pred_use_path)\ntime_array      = np.load(time_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:35.590641Z","iopub.execute_input":"2023-07-03T07:53:35.590933Z","iopub.status.idle":"2023-07-03T07:53:45.391788Z","shell.execute_reply.started":"2023-07-03T07:53:35.590900Z","shell.execute_reply":"2023-07-03T07:53:45.390708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### main (predict)","metadata":{}},{"cell_type":"code","source":"models_list = os.listdir(f\"./output/exp/ex{ex}/ex{ex}_model\")\nmodels_list","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.393084Z","iopub.execute_input":"2023-07-03T07:53:45.393456Z","iopub.status.idle":"2023-07-03T07:53:45.403420Z","shell.execute_reply.started":"2023-07-03T07:53:45.393421Z","shell.execute_reply":"2023-07-03T07:53:45.402264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if PREDICT_FLAG:\n# with timer(\"gru\"):\n    set_seed(seed)\n    for fold in range(1, 6):\n        with timer(f\"fold {fold}\"):\n            val_numerical_array = numerical_array.copy()\n            val_target_array = target_array.copy()\n            val_mask_array = mask_array.copy()\n            val_valid_array = valid_array.copy()\n            val_pred_array = pred_use_array.copy()\n\n            val_ = FogDataset(val_numerical_array, \n                              val_mask_array,\n                              val_valid_array, \n                              train=True,\n                              y=val_target_array)\n\n            val_loader = DataLoader(dataset=val_, \n                                    batch_size=batch_size, \n                                    shuffle = False,\n                                    # num_workers=8\n                                   )\n\n            # model = FogRnnModel()\n            model = FogRnnModel(numeraical_linear_size = 96,\n                                model_size = 256,\n                                linear_out = 256,)\n            # model.load_state_dict(torch.load(f\"./output/exp/ex{ex}/ex{ex}_model/ex{ex}_{fold}.pth\"))\n            model.load_state_dict(torch.load(f\"./output/exp/ex{ex}/ex{ex}_model/\" + models_list[fold-1]))\n            model = model.to(device)\n            model.eval()  # switch model to the evaluation mode\n            \n            val_preds = np.ndarray((0, seq_len, 3))\n            tk0 = tqdm(val_loader, total=len(val_loader))\n            with torch.no_grad():  # Do not calculate gradient since we are only predicting\n                # Predicting on validation set\n                for d in tk0:\n                    input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                    input_data_mask_array      = d['input_data_mask_array'].to(device)\n                    attention_mask             = d['attention_mask'].to(device)\n                    output = model(input_data_numerical_array, \n                                   input_data_mask_array,\n                                   attention_mask)\n                    val_preds = np.concatenate([val_preds, output.sigmoid().detach().cpu().numpy()], axis=0)\n            np.save(f\"./output/exp/ex{ex}/ex{ex}_notype_{fold}_oof_{seq_len}.npy\",val_preds)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.404851Z","iopub.execute_input":"2023-07-03T07:53:45.405696Z","iopub.status.idle":"2023-07-03T07:53:45.418389Z","shell.execute_reply.started":"2023-07-03T07:53:45.405662Z","shell.execute_reply":"2023-07-03T07:53:45.417496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if PREDICT_FLAG:\n    id_array = df_id[\"Id\"].values\n    for i in range(1, 6):\n        pred_all = []\n        pred = np.load(f\"./output/exp/ex{ex}/ex{ex}_notype_{i}_oof_{seq_len}.npy\")\n        for v in tqdm(range(len(pred_use_array))):\n            use_ = pred_use_array[v, :] == 1\n            pred_ = pred[v, use_ == 1, :]\n            time_ = time_array[v, use_ == 1]\n            Id = id_array[v]\n            pred_df = pd.DataFrame()\n            pred_df[\"Time\"] = time_\n            pred_df[\"Id\"] = Id\n            pred_df[\"StartHesitation\"] = pred_[:, 0]\n            pred_df[\"Turn\"] = pred_[:, 1]\n            pred_df[\"Walking\"] = pred_[:, 2]\n            pred_all.append(pred_df)\n        pred_all = pd.concat(pred_all).reset_index(drop=True)\n        pred_all.to_parquet(f\"./output/exp/ex{ex}/ex{ex}_notype_{i}_pred_{seq_len}.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.419942Z","iopub.execute_input":"2023-07-03T07:53:45.420316Z","iopub.status.idle":"2023-07-03T07:53:45.433876Z","shell.execute_reply.started":"2023-07-03T07:53:45.420284Z","shell.execute_reply":"2023-07-03T07:53:45.432822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n<br>\n\n## **Feature Engineering with** <font color=red>**pseudo**</font>  **label**\n\n<br>\n\n### fe073_notype_pseudo_ex153_10000.ipynb\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/fe/fe073_notype_pseudo_ex153_10000.ipynb\" style=\"text-decoration:none\">fe073_notype_pseudo_ex153_10000.ipynb</a>","metadata":{}},{"cell_type":"code","source":"fe = \"073_notype_pseudo\"\nex = \"153_defog\"\nif not os.path.exists(f\"./output/fe/fe{fe}\"):\n    os.makedirs(f\"./output/fe/fe{fe}\")\n    os.makedirs(f\"./output/fe/fe{fe}/save\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.437158Z","iopub.execute_input":"2023-07-03T07:53:45.437981Z","iopub.status.idle":"2023-07-03T07:53:45.444960Z","shell.execute_reply.started":"2023-07-03T07:53:45.437955Z","shell.execute_reply":"2023-07-03T07:53:45.443793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DEFOG_META_PATH = \"../data/defog_metadata.csv\"\n# DEFOG_FOLDER = \"../data/train/notype/*.csv\"\n\ndefog_meta = pd.read_parquet(\"./output/fe/fe039/fe039_defog_meta.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.446739Z","iopub.execute_input":"2023-07-03T07:53:45.447185Z","iopub.status.idle":"2023-07-03T07:53:45.467772Z","shell.execute_reply.started":"2023-07-03T07:53:45.447149Z","shell.execute_reply":"2023-07-03T07:53:45.466595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = [\"AccV\", \"AccML\", \"AccAP\"]\nnum_cols = [\"AccV\", \"AccML\", \"AccAP\",\n            'AccV_lag_diff', 'AccV_lead_diff', \n            'AccML_lag_diff', 'AccML_lead_diff',\n            'AccAP_lag_diff', 'AccAP_lead_diff']\n\ntarget_use_cols = [\"Event\"]\ntarget_cols = [\"StartHesitation\", \"Turn\", \"Walking\"]\nseq_len = 5000\nshift = 2500\noffset = 1250","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.469170Z","iopub.execute_input":"2023-07-03T07:53:45.470512Z","iopub.status.idle":"2023-07-03T07:53:45.476590Z","shell.execute_reply.started":"2023-07-03T07:53:45.470479Z","shell.execute_reply":"2023-07-03T07:53:45.475745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_list = glob.glob(DEFOG_FOLDER)\n\nif TRAIN_FLAG:\n    for fold in range(1, 6):\n        print(fold)\n        pred = pd.read_parquet(f\"./output/exp/ex{ex}/ex{ex}_notype_{fold}_pred_10000.parquet\")\n        target_array = []\n        for i,s in tqdm(zip(defog_meta[\"Id\"].values, defog_meta[\"sub_id\"].values)):\n            path = root_data + f\"train/notype/{i}.csv\"\n            if path in [x.replace(\"\\\\\", \"/\") for x in train_notype]:\n            # if path in data_list:\n                df = pd.read_csv(path)\n                df_ = pred[pred[\"Id\"] == i].reset_index(drop=True)\n                df = df.merge(df_, how=\"left\", on=\"Time\")\n                df[\"target_max\"] = np.argmax(df[[\"StartHesitation\", \"Turn\", \"Walking\"]].values, axis=1)\n\n                df.loc[df[\"target_max\"] == 0, \"StartHesitation\"] = 1\n                df.loc[df[\"target_max\"] == 0, [\"Turn\",\"Walking\"]] = 0\n\n                df.loc[df[\"target_max\"] == 1, \"Turn\"] = 1\n                df.loc[df[\"target_max\"] == 1, [\"StartHesitation\",\"Walking\"]] = 0\n\n                df.loc[df[\"target_max\"] == 2, \"Walking\"] = 1\n                df.loc[df[\"target_max\"] == 2, [\"StartHesitation\",\"Turn\"]] = 0\n\n                df.loc[df[\"Event\"] == 0, [\"StartHesitation\", \"Turn\", \"Walking\"]] = 0\n\n                df[\"valid\"] = df[\"Valid\"] & df[\"Task\"]\n                df[\"valid\"] = df[\"valid\"].astype(int)\n                batch = (len(df)-1) // shift\n                target = df[target_cols].values\n                target_array_ = np.zeros([batch, seq_len, 3])\n                for n,b in enumerate(range(batch)):\n                    if b == (batch - 1):\n                        target_ = target[b*shift : ]\n                        target_array_[b, :len(target_), :] = target_\n                    elif b == 0:\n                        target_ = target[b*shift : b*shift + seq_len]\n                        target_array_[b, :, :] = target_\n                    else:\n                        target_ = target[b*shift : b*shift + seq_len]\n                        target_array_[b, :, :] = target_\n\n                target_array.append(target_array_)\n        target_array = np.concatenate(target_array, axis=0)\n        np.save(f\"./output/fe/fe{fe}/fe{fe}_target_array_{fold}.npy\", target_array)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.478248Z","iopub.execute_input":"2023-07-03T07:53:45.478926Z","iopub.status.idle":"2023-07-03T07:53:45.494692Z","shell.execute_reply.started":"2023-07-03T07:53:45.478894Z","shell.execute_reply":"2023-07-03T07:53:45.493665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.497117Z","iopub.execute_input":"2023-07-03T07:53:45.497811Z","iopub.status.idle":"2023-07-03T07:53:45.508955Z","shell.execute_reply.started":"2023-07-03T07:53:45.497779Z","shell.execute_reply":"2023-07-03T07:53:45.508056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.510356Z","iopub.execute_input":"2023-07-03T07:53:45.511043Z","iopub.status.idle":"2023-07-03T07:53:45.518620Z","shell.execute_reply.started":"2023-07-03T07:53:45.511009Z","shell.execute_reply":"2023-07-03T07:53:45.517715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### fe075_notype_pseudo_ex153_15000.ipynb\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/fe/fe075_notype_pseudo_ex153_15000.ipynb\" style=\"text-decoration:none\">fe075_notype_pseudo_ex153_15000.ipynb</a>","metadata":{}},{"cell_type":"code","source":"fe = \"075_notype_pseudo\"\nex = \"153_defog\"\nif not os.path.exists(f\"./output/fe/fe{fe}\"):\n    os.makedirs(f\"./output/fe/fe{fe}\")\n    os.makedirs(f\"./output/fe/fe{fe}/save\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.521600Z","iopub.execute_input":"2023-07-03T07:53:45.522586Z","iopub.status.idle":"2023-07-03T07:53:45.529088Z","shell.execute_reply.started":"2023-07-03T07:53:45.522554Z","shell.execute_reply":"2023-07-03T07:53:45.528253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DEFOG_META_PATH = \"../data/defog_metadata.csv\"\n# DEFOG_FOLDER = \"../data/train/notype/*.csv\"\n\ndefog_meta = pd.read_parquet(\"./output/fe/fe039/fe039_defog_meta.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.530609Z","iopub.execute_input":"2023-07-03T07:53:45.531461Z","iopub.status.idle":"2023-07-03T07:53:45.544286Z","shell.execute_reply.started":"2023-07-03T07:53:45.531426Z","shell.execute_reply":"2023-07-03T07:53:45.543200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = [\"AccV\", \"AccML\", \"AccAP\"]\nnum_cols = [\"AccV\", \"AccML\", \"AccAP\",\n            'AccV_lag_diff', 'AccV_lead_diff', \n            'AccML_lag_diff', 'AccML_lead_diff',\n            'AccAP_lag_diff', 'AccAP_lead_diff']\n\ntarget_use_cols = [\"Event\"]\ntarget_cols = [\"StartHesitation\", \"Turn\", \"Walking\"]\nseq_len = 5000\nshift = 2500\noffset = 1250","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.545901Z","iopub.execute_input":"2023-07-03T07:53:45.546472Z","iopub.status.idle":"2023-07-03T07:53:45.552849Z","shell.execute_reply.started":"2023-07-03T07:53:45.546436Z","shell.execute_reply":"2023-07-03T07:53:45.552183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_list = glob.glob(DEFOG_FOLDER)\n\nif TRAIN_FLAG:\n    for fold in range(1, 6):\n        print(fold)\n        pred = pd.read_parquet(f\"./output/exp/ex{ex}/ex{ex}_notype_{fold}_pred_15000.parquet\")\n        target_array = []\n        for i,s in tqdm(zip(defog_meta[\"Id\"].values, defog_meta[\"sub_id\"].values)):\n            path = root_data + f\"train/notype/{i}.csv\"\n            if path in [x.replace(\"\\\\\", \"/\") for x in train_notype]:\n            # if path in data_list:\n                df = pd.read_csv(path)\n                df_ = pred[pred[\"Id\"] == i].reset_index(drop=True)\n                df = df.merge(df_, how=\"left\", on=\"Time\")\n                df[\"target_max\"] = np.argmax(df[[\"StartHesitation\", \"Turn\", \"Walking\"]].values, axis=1)\n\n                df.loc[df[\"target_max\"] == 0, \"StartHesitation\"] = 1\n                df.loc[df[\"target_max\"] == 0, [\"Turn\",\"Walking\"]] = 0\n\n                df.loc[df[\"target_max\"] == 1, \"Turn\"] = 1\n                df.loc[df[\"target_max\"] == 1, [\"StartHesitation\",\"Walking\"]] = 0\n\n                df.loc[df[\"target_max\"] == 2, \"Walking\"] = 1\n                df.loc[df[\"target_max\"] == 2, [\"StartHesitation\",\"Turn\"]] = 0\n\n                df.loc[df[\"Event\"] == 0, [\"StartHesitation\", \"Turn\", \"Walking\"]] = 0\n\n                df[\"valid\"] = df[\"Valid\"] & df[\"Task\"]\n                df[\"valid\"] = df[\"valid\"].astype(int)\n                batch = (len(df)-1) // shift\n                target = df[target_cols].values\n                target_array_ = np.zeros([batch, seq_len, 3])\n                for n,b in enumerate(range(batch)):\n                    if b == (batch - 1):\n                        target_ = target[b*shift : ]\n                        target_array_[b, :len(target_), :] = target_\n                    elif b == 0:\n                        target_ = target[b*shift : b*shift + seq_len]\n                        target_array_[b, :, :] = target_\n                    else:\n                        target_ = target[b*shift : b*shift + seq_len]\n                        target_array_[b, :, :] = target_\n\n                target_array.append(target_array_)\n        target_array = np.concatenate(target_array, axis=0)\n        np.save(f\"./output/fe/fe{fe}/fe{fe}_target_array_{fold}.npy\", target_array)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.554554Z","iopub.execute_input":"2023-07-03T07:53:45.555307Z","iopub.status.idle":"2023-07-03T07:53:45.570451Z","shell.execute_reply.started":"2023-07-03T07:53:45.555270Z","shell.execute_reply":"2023-07-03T07:53:45.569603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.572932Z","iopub.execute_input":"2023-07-03T07:53:45.573251Z","iopub.status.idle":"2023-07-03T07:53:45.583005Z","shell.execute_reply.started":"2023-07-03T07:53:45.573215Z","shell.execute_reply":"2023-07-03T07:53:45.582164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.584702Z","iopub.execute_input":"2023-07-03T07:53:45.585145Z","iopub.status.idle":"2023-07-03T07:53:45.592187Z","shell.execute_reply.started":"2023-07-03T07:53:45.585110Z","shell.execute_reply":"2023-07-03T07:53:45.591251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### fe086_notype_pseudo_ex154_15000.ipynb\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/fe/fe086_notype_pseudo_ex154_15000.ipynb\" style=\"text-decoration:none\">fe086_notype_pseudo_ex154_15000.ipynb</a>","metadata":{}},{"cell_type":"code","source":"fe = \"086_notype_pseudo\"\nex = \"154_defog\"\nif not os.path.exists(f\"./output/fe/fe{fe}\"):\n    os.makedirs(f\"./output/fe/fe{fe}\")\n    os.makedirs(f\"./output/fe/fe{fe}/save\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.593734Z","iopub.execute_input":"2023-07-03T07:53:45.593978Z","iopub.status.idle":"2023-07-03T07:53:45.603909Z","shell.execute_reply.started":"2023-07-03T07:53:45.593956Z","shell.execute_reply":"2023-07-03T07:53:45.602956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DEFOG_META_PATH = \"../data/defog_metadata.csv\"\n# DEFOG_FOLDER = \"../data/train/notype/*.csv\"\n\ndefog_meta = pd.read_parquet(\"./output/fe/fe039/fe039_defog_meta.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.607625Z","iopub.execute_input":"2023-07-03T07:53:45.607888Z","iopub.status.idle":"2023-07-03T07:53:45.621460Z","shell.execute_reply.started":"2023-07-03T07:53:45.607866Z","shell.execute_reply":"2023-07-03T07:53:45.620498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = [\"AccV\", \"AccML\", \"AccAP\"]\nnum_cols = [\"AccV\", \"AccML\", \"AccAP\",\n            'AccV_lag_diff', 'AccV_lead_diff', \n            'AccML_lag_diff', 'AccML_lead_diff',\n            'AccAP_lag_diff', 'AccAP_lead_diff']\n\ntarget_use_cols = [\"Event\"]\ntarget_cols = [\"StartHesitation\", \"Turn\", \"Walking\"]\nseq_len = 5000\nshift = 2500\noffset = 1250","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.625354Z","iopub.execute_input":"2023-07-03T07:53:45.625948Z","iopub.status.idle":"2023-07-03T07:53:45.631502Z","shell.execute_reply.started":"2023-07-03T07:53:45.625923Z","shell.execute_reply":"2023-07-03T07:53:45.630627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_list = glob.glob(DEFOG_FOLDER)\nif TRAIN_FLAG:\n    for fold in range(1, 6):\n        print(fold)\n        pred = pd.read_parquet(f\"./output/exp/ex{ex}/ex{ex}_notype_{fold}_pred_15000.parquet\")\n        target_array = []\n        for i,s in tqdm(zip(defog_meta[\"Id\"].values, defog_meta[\"sub_id\"].values)):\n            path = root_data + f\"train/notype/{i}.csv\"\n            if path in [x.replace(\"\\\\\", \"/\") for x in train_notype]:\n            # if path in data_list:\n                df = pd.read_csv(path)\n                df_ = pred[pred[\"Id\"] == i].reset_index(drop=True)\n                df = df.merge(df_, how=\"left\", on=\"Time\")\n                df[\"target_max\"] = np.argmax(df[[\"StartHesitation\", \"Turn\", \"Walking\"]].values, axis=1)\n\n                df.loc[df[\"target_max\"] == 0, \"StartHesitation\"] = 1\n                df.loc[df[\"target_max\"] == 0, [\"Turn\",\"Walking\"]] = 0\n\n                df.loc[df[\"target_max\"] == 1, \"Turn\"] = 1\n                df.loc[df[\"target_max\"] == 1, [\"StartHesitation\",\"Walking\"]] = 0\n\n                df.loc[df[\"target_max\"] == 2, \"Walking\"] = 1\n                df.loc[df[\"target_max\"] == 2, [\"StartHesitation\",\"Turn\"]] = 0\n\n                df.loc[df[\"Event\"] == 0, [\"StartHesitation\", \"Turn\", \"Walking\"]] = 0\n\n                df[\"valid\"] = df[\"Valid\"] & df[\"Task\"]\n                df[\"valid\"] = df[\"valid\"].astype(int)\n                batch = (len(df)-1) // shift\n                target = df[target_cols].values\n                target_array_ = np.zeros([batch, seq_len, 3])\n                for n,b in enumerate(range(batch)):\n                    if b == (batch - 1):\n                        target_ = target[b*shift : ]\n                        target_array_[b, :len(target_), :] = target_\n                    elif b == 0:\n                        target_ = target[b*shift : b*shift + seq_len]\n                        target_array_[b, :, :] = target_\n                    else:\n                        target_ = target[b*shift : b*shift + seq_len]\n                        target_array_[b, :, :] = target_\n\n                target_array.append(target_array_)\n        target_array = np.concatenate(target_array, axis=0)\n        np.save(f\"./output/fe/fe{fe}/fe{fe}_target_array_{fold}.npy\", target_array)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.633045Z","iopub.execute_input":"2023-07-03T07:53:45.633718Z","iopub.status.idle":"2023-07-03T07:53:45.649404Z","shell.execute_reply.started":"2023-07-03T07:53:45.633682Z","shell.execute_reply":"2023-07-03T07:53:45.648470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.650778Z","iopub.execute_input":"2023-07-03T07:53:45.651383Z","iopub.status.idle":"2023-07-03T07:53:45.662243Z","shell.execute_reply.started":"2023-07-03T07:53:45.651353Z","shell.execute_reply":"2023-07-03T07:53:45.660956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.663856Z","iopub.execute_input":"2023-07-03T07:53:45.664633Z","iopub.status.idle":"2023-07-03T07:53:45.672157Z","shell.execute_reply.started":"2023-07-03T07:53:45.664584Z","shell.execute_reply":"2023-07-03T07:53:45.671084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n<br>\n\n##  <font color=red>**Training**</font> **with** <font color=red>**pseudo**</font> **label** (round1)\n\n\n\n<br>\n\n### load data & preprocessing","metadata":{}},{"cell_type":"code","source":"id_path        = f\"./output/fe/fe047/fe047_id.parquet\"\nnumerical_path = f\"./output/fe/fe047/fe047_num_array.npy\"\ntarget_path    = f\"./output/fe/fe047/fe047_target_array.npy\"\nmask_path      = f\"./output/fe/fe047/fe047_mask_array.npy\"\nvalid_path     = f\"./output/fe/fe047/fe047_valid_array.npy\"\npred_use_path  = f\"./output/fe/fe047/fe047_pred_use_array.npy\"\n\ndf_id           = pd.read_parquet(id_path)\nnumerical_array = np.load(numerical_path)\ntarget_array    = np.load(target_path)\nmask_array      = np.load(mask_path)\nvalid_array     = np.load(valid_path)\npred_use_array  = np.load(pred_use_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:53:45.673774Z","iopub.execute_input":"2023-07-03T07:53:45.674580Z","iopub.status.idle":"2023-07-03T07:54:00.524395Z","shell.execute_reply.started":"2023-07-03T07:53:45.674433Z","shell.execute_reply":"2023-07-03T07:54:00.523271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target1 = []\ntarget2 = []\ntarget3 = []\nfor i in range(len(target_array)):\n    target1.append(np.sum(target_array[i,:,0]))\n    target2.append(np.sum(target_array[i,:,1]))\n    target3.append(np.sum(target_array[i,:,2]))\n\ndf_id[\"target1\"] = target1\ndf_id[\"target2\"] = target2\ndf_id[\"target3\"] = target3\ndf_id[\"target1_1\"] = df_id[\"target1\"] > 0\ndf_id[\"target2_1\"] = df_id[\"target2\"] > 0\ndf_id[\"target3_1\"] = df_id[\"target3\"] > 0\ndf_id[\"target1_1\"] = df_id[\"target1_1\"].astype(np.int)\ndf_id[\"target2_1\"] = df_id[\"target2_1\"].astype(np.int)\ndf_id[\"target3_1\"] = df_id[\"target3_1\"].astype(np.int)\n\ndf_id[\"group\"] = 0\ndf_id.loc[df_id[\"target1_1\"] > 0,\"group\"] = 1\ndf_id.loc[df_id[\"target2_1\"] > 0,\"group\"] = 2\ndf_id.loc[df_id[\"target3_1\"] > 0,\"group\"] = 3\n\ndf_id[\"group\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:00.526658Z","iopub.execute_input":"2023-07-03T07:54:00.527498Z","iopub.status.idle":"2023-07-03T07:54:00.846597Z","shell.execute_reply.started":"2023-07-03T07:54:00.527394Z","shell.execute_reply":"2023-07-03T07:54:00.844754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pseudo_id_path        = f\"./output/fe/fe061_notype/fe061_notype_id.parquet\"\npseudo_numerical_path = f\"./output/fe/fe061_notype/fe061_notype_num_array.npy\"\npseudo_mask_path      = f\"./output/fe/fe061_notype/fe061_notype_mask_array.npy\"\npseudo_valid_path     = f\"./output/fe/fe061_notype/fe061_notype_valid_array.npy\"\npseudo_pred_use_path  = f\"./output/fe/fe061_notype/fe061_notype_pred_use_array.npy\"\n\npseudo_numerical_array = np.load(pseudo_numerical_path)\npseudo_mask_array      = np.load(pseudo_mask_path)\npseudo_valid_array     = np.load(pseudo_valid_path)\npseudo_pred_use_array  = np.load(pseudo_pred_use_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:00.847980Z","iopub.execute_input":"2023-07-03T07:54:00.848513Z","iopub.status.idle":"2023-07-03T07:54:10.335707Z","shell.execute_reply.started":"2023-07-03T07:54:00.848478Z","shell.execute_reply":"2023-07-03T07:54:10.334489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### config","metadata":{}},{"cell_type":"code","source":"# config\nseed = 0\nshuffle = True\nn_splits = 5\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# model config\nbatch_size = 24\nn_epochs = 10\nlr = 1e-3\nweight_decay = 0.05\nnum_warmup_steps = 10","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:10.337519Z","iopub.execute_input":"2023-07-03T07:54:10.337939Z","iopub.status.idle":"2023-07-03T07:54:10.351884Z","shell.execute_reply.started":"2023-07-03T07:54:10.337903Z","shell.execute_reply":"2023-07-03T07:54:10.350898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### <font color=maroon>ex175_defog_gru_pseudo.ipynb</font>\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/exp/ex175_defog_gru_pseudo.ipynb\" style=\"text-decoration:none\">ex175_defog_gru_pseudo.ipynb</a>","metadata":{}},{"cell_type":"code","source":"debug = False\nex = \"175_notype_pseudo_target\"\nif not os.path.exists(f\"./output/exp/ex{ex}\"):\n    os.makedirs(f\"./output/exp/ex{ex}\")\n    os.makedirs(f\"./output/exp/ex{ex}/ex{ex}_model\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:10.353284Z","iopub.execute_input":"2023-07-03T07:54:10.353802Z","iopub.status.idle":"2023-07-03T07:54:10.364613Z","shell.execute_reply.started":"2023-07-03T07:54:10.353770Z","shell.execute_reply":"2023-07-03T07:54:10.363724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pseudo_target = \"073_notype_pseudo\"","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:10.366741Z","iopub.execute_input":"2023-07-03T07:54:10.367531Z","iopub.status.idle":"2023-07-03T07:54:10.375239Z","shell.execute_reply.started":"2023-07-03T07:54:10.367458Z","shell.execute_reply":"2023-07-03T07:54:10.374268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### main","metadata":{}},{"cell_type":"code","source":"%%time\n\n# with timer(\"gru\"):\nif TRAIN_FLAG:\n    set_seed(seed)\n    y_oof = np.empty([len(target_array), 5000, 3])      # Abrachan: out of fold\n    gkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state = seed)\n    for fold, (train_idx, valid_idx) in enumerate(gkf.split(numerical_array, \n                                                            y = df_id[\"group\"].values,\n                                                            groups=df_id[\"subject\"].values)):\n        # LOGGER.info(f\"start fold:{fold+1}\")\n        print(f\"start fold: {fold+1}\")\n        \n        with timer(f\"fold {fold+1}\"):\n            train_numerical_array = numerical_array[train_idx]\n            train_target_array = target_array[train_idx]\n            train_mask_array = mask_array[train_idx]\n            train_valid_array = valid_array[train_idx]\n            \n            pseudo_target_array = np.load(f\"./output/fe/fe{pseudo_target}/fe{pseudo_target}_target_array_{fold+1}.npy\")\n            train_numerical_array = np.concatenate([train_numerical_array, pseudo_numerical_array], axis=0)\n            train_target_array = np.concatenate([train_target_array, pseudo_target_array], axis=0)\n            train_mask_array = np.concatenate([train_mask_array, pseudo_mask_array], axis=0)\n            train_valid_array = np.concatenate([train_valid_array, pseudo_valid_array], axis=0)\n\n            val_numerical_array = numerical_array[valid_idx]\n            val_target_array = target_array[valid_idx]\n            val_mask_array = mask_array[valid_idx]\n            val_valid_array = valid_array[valid_idx]\n            val_pred_array = pred_use_array[valid_idx]\n            \n            train_ = FogDataset(train_numerical_array,\n                                train_mask_array,\n                                train_valid_array,\n                                train=True,\n                                y=train_target_array)\n            val_ = FogDataset(val_numerical_array,\n                              val_mask_array,\n                              val_valid_array,\n                              train=True,\n                              y=val_target_array)\n            \n            train_loader = DataLoader(dataset=train_, \n                                      batch_size=batch_size, \n                                      shuffle = True, \n                                      # num_workers=8,\n                                     )\n            val_loader = DataLoader(dataset=val_, \n                                    batch_size=batch_size, \n                                    shuffle = False , \n                                    # num_workers=8\n                                   )\n            \n            \n            model = FogRnnModel()\n            model = model.to(device)\n            \n            param_optimizer = list(model.named_parameters())\n            no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n            optimizer_grouped_parameters = [\n                {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], \n                 'weight_decay': weight_decay\n                },\n                {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], \n                 'weight_decay': 0.0\n                }]\n            optimizer = AdamW(optimizer_grouped_parameters,\n                              lr=lr,\n                              weight_decay=weight_decay,\n                              )\n            num_train_optimization_steps = int(len(train_loader) * n_epochs)\n            scheduler = get_linear_schedule_with_warmup(optimizer,\n                                                        num_warmup_steps=num_warmup_steps,\n                                                        num_training_steps=num_train_optimization_steps)\n            criterion = nn.BCEWithLogitsLoss()\n            best_val_score = 0\n            \n            for epoch in range(n_epochs):\n                model.train() \n                train_losses_batch = []\n                val_losses_batch = []\n                epoch_loss = 0\n                train_preds = np.ndarray((0,3))   # array([], shape=(0, 3), dtype=float64)\n                \n                tk0 = tqdm_nb(train_loader, total=len(train_loader), desc=\"train_loader: \")\n                for d in tk0:\n                    # ======================================================\n                    # data loader\n                    # ======================================================\n                    input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                    input_data_mask_array      = d['input_data_mask_array'].to(device)\n                    input_data_valid_array     = d['input_data_valid_array'].to(device)\n                    attention_mask             = d['attention_mask'].to(device)\n                    y                          = d[\"y\"].to(device)\n                    \n                    optimizer.zero_grad()\n                    output = model(input_data_numerical_array, \n                                   input_data_mask_array,\n                                   attention_mask)\n                    loss = criterion(output[input_data_mask_array == 1], y[input_data_mask_array == 1])\n                    \n                    # output1 = output[:, :, [1,2]]\n                    # output2 = output[:, :, 0]\n                    # y1 = y[:, :, [1,2]]\n                    # y2 = y[:, :, 0]\n                    # loss1 = criterion(output1[input_data_mask_array == 1], y1[input_data_mask_array == 1])\n                    # loss2 = criterion(output2[input_data_mask_array == 1], y2[input_data_mask_array == 1])\n                    # loss = loss1*0.8 + loss2*0.2\n                    \n                    loss.backward()\n                    optimizer.step()\n                    scheduler.step()\n                    train_losses_batch.append(loss.item())\n                train_loss = np.mean(train_losses_batch)\n                \n                \n                # ======================================================\n                # eval\n                # ======================================================\n                model.eval()       # switch model to the evaluation mode\n                \n                val_preds = np.ndarray((0, 5000, 3))\n                tk0 = tqdm_nb(val_loader, total=len(val_loader), desc=\"val_loader  : \")\n                with torch.no_grad():  # Do not calculate gradient since we are only predicting\n                    # Predicting on validation set\n                    for d in tk0:\n                        input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                        input_data_mask_array      = d['input_data_mask_array'].to(device)\n                        attention_mask             = d['attention_mask'].to(device)\n                        \n                        output = model(input_data_numerical_array, \n                                       input_data_mask_array,\n                                       attention_mask)\n                        val_preds = np.concatenate([val_preds, output.sigmoid().detach().cpu().numpy()], axis=0)\n                \n                pred_valid_index = (val_mask_array == 1) & (val_pred_array == 1) & (val_valid_array == 1)\n                StartHesitation = average_precision_score(val_target_array[pred_valid_index][:, 0],\n                                                          val_preds[pred_valid_index][:, 0])\n                Turn = average_precision_score(val_target_array[pred_valid_index][:, 1],\n                                               val_preds[pred_valid_index][:, 1])\n                Walking = average_precision_score(val_target_array[pred_valid_index][:, 2],\n                                                  val_preds[pred_valid_index][:, 2])\n                # map_score = np.mean([StartHesitation, Turn, Walking])\n                map_score = np.mean([Turn, Walking])\n                \n                # LOGGER.info(f\"fold: {fold+1} epoch: {epoch+1},train loss {train_loss} map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                print(f\"fold {fold+1} epoch {epoch+1}\\ntrain loss {train_loss} map:{map_score}\\nstart_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                \n                if map_score >= best_val_score:\n                    print(\"save weight\")\n                    best_val_score = map_score\n                    best_val_preds = val_preds.copy()\n                    torch.save(model.state_dict(), f\"./output/exp/ex{ex}/ex{ex}_model/ex{ex}_fold{fold+1}_epoch{epoch+1}__map{best_val_score:.6f}.pth\")\n                print()\n            y_oof[valid_idx] = best_val_preds\n    \n    np.save(f\"./output/exp/ex{ex}/ex{ex}_oof.npy\", y_oof)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:10.376934Z","iopub.execute_input":"2023-07-03T07:54:10.377384Z","iopub.status.idle":"2023-07-03T07:54:10.415720Z","shell.execute_reply.started":"2023-07-03T07:54:10.377349Z","shell.execute_reply":"2023-07-03T07:54:10.414655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### oof score","metadata":{}},{"cell_type":"code","source":"if TRAIN_FLAG:\n    val_pred_index = (mask_array == 1) & (pred_use_array == 1) & (valid_array == 1)\n    StartHesitation = average_precision_score(target_array[val_pred_index][:,0], \n                                              y_oof[val_pred_index][:,0])\n    Turn            = average_precision_score(target_array[val_pred_index][:,1], \n                                              y_oof[val_pred_index][:,1])\n    Walking         = average_precision_score(target_array[val_pred_index][:,2],\n                                              y_oof[val_pred_index][:,2])\n\n    map_score = np.mean([StartHesitation, Turn, Walking])\n    # LOGGER.info(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n    print(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:10.417246Z","iopub.execute_input":"2023-07-03T07:54:10.417602Z","iopub.status.idle":"2023-07-03T07:54:10.424249Z","shell.execute_reply.started":"2023-07-03T07:54:10.417557Z","shell.execute_reply":"2023-07-03T07:54:10.423172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### <font color=maroon>ex179_defog_gru_pseudo.ipynb</font>\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/exp/ex179_defog_gru_pseudo.ipynb\" style=\"text-decoration:none\">ex179_defog_gru_pseudo.ipynb</a>","metadata":{}},{"cell_type":"code","source":"debug = False\nex = \"179_notype_pseudo_target\"\nif not os.path.exists(f\"./output/exp/ex{ex}\"):\n    os.makedirs(f\"./output/exp/ex{ex}\")\n    os.makedirs(f\"./output/exp/ex{ex}/ex{ex}_model\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:10.425874Z","iopub.execute_input":"2023-07-03T07:54:10.426347Z","iopub.status.idle":"2023-07-03T07:54:10.435051Z","shell.execute_reply.started":"2023-07-03T07:54:10.426315Z","shell.execute_reply":"2023-07-03T07:54:10.434097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seq_len = 5000\n\npseudo_target = \"075_notype_pseudo\"","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:10.436641Z","iopub.execute_input":"2023-07-03T07:54:10.437014Z","iopub.status.idle":"2023-07-03T07:54:10.444255Z","shell.execute_reply.started":"2023-07-03T07:54:10.436979Z","shell.execute_reply":"2023-07-03T07:54:10.443398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### main","metadata":{}},{"cell_type":"code","source":"%%time\n\n# with timer(\"gru\"):\nif TRAIN_FLAG:\n    set_seed(seed)\n    y_oof = np.empty([len(target_array), seq_len, 3])      # Abrachan: out of fold\n    gkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state = seed)\n    for fold, (train_idx, valid_idx) in enumerate(gkf.split(numerical_array, \n                                                            y = df_id[\"group\"].values,\n                                                            groups=df_id[\"subject\"].values)):\n        # LOGGER.info(f\"start fold:{fold+1}\")\n        print(f\"start fold: {fold+1}\")\n        \n        with timer(f\"fold {fold+1}\"):\n            train_numerical_array = numerical_array[train_idx]\n            train_target_array = target_array[train_idx]\n            train_mask_array = mask_array[train_idx]\n            train_valid_array = valid_array[train_idx]\n            \n            pseudo_target_array = np.load(f\"./output/fe/fe{pseudo_target}/fe{pseudo_target}_target_array_{fold+1}.npy\")\n            train_numerical_array = np.concatenate([train_numerical_array, pseudo_numerical_array], axis=0)\n            train_target_array = np.concatenate([train_target_array, pseudo_target_array], axis=0)\n            train_mask_array = np.concatenate([train_mask_array, pseudo_mask_array], axis=0)\n            train_valid_array = np.concatenate([train_valid_array, pseudo_valid_array], axis=0)\n\n            val_numerical_array = numerical_array[valid_idx]\n            val_target_array = target_array[valid_idx]\n            val_mask_array = mask_array[valid_idx]\n            val_valid_array = valid_array[valid_idx]\n            val_pred_array = pred_use_array[valid_idx]\n            \n            train_ = FogDataset(train_numerical_array,\n                                train_mask_array,\n                                train_valid_array,\n                                train=True,\n                                y=train_target_array)\n            val_ = FogDataset(val_numerical_array,\n                              val_mask_array,\n                              val_valid_array,\n                              train=True,\n                              y=val_target_array)\n            \n            train_loader = DataLoader(dataset=train_, \n                                      batch_size=batch_size, \n                                      shuffle = True, \n                                      # num_workers=8,\n                                     )\n            val_loader = DataLoader(dataset=val_, \n                                    batch_size=batch_size, \n                                    shuffle = False , \n                                    # num_workers=8\n                                   )\n            \n            \n            model = FogRnnModel()\n            model = model.to(device)\n            \n            param_optimizer = list(model.named_parameters())\n            no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n            optimizer_grouped_parameters = [\n                {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], \n                 'weight_decay': weight_decay\n                },\n                {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], \n                 'weight_decay': 0.0\n                }]\n            optimizer = AdamW(optimizer_grouped_parameters,\n                              lr=lr,\n                              weight_decay=weight_decay,\n                              )\n            num_train_optimization_steps = int(len(train_loader) * n_epochs)\n            scheduler = get_linear_schedule_with_warmup(optimizer,\n                                                        num_warmup_steps=num_warmup_steps,\n                                                        num_training_steps=num_train_optimization_steps)\n            criterion = nn.BCEWithLogitsLoss()\n            best_val_score = 0\n            \n            for epoch in range(n_epochs):\n                model.train() \n                train_losses_batch = []\n                val_losses_batch = []\n                epoch_loss = 0\n                train_preds = np.ndarray((0,3))   # array([], shape=(0, 3), dtype=float64)\n                \n                tk0 = tqdm_nb(train_loader, total=len(train_loader), desc=\"train_loader: \")\n                for d in tk0:\n                    # ======================================================\n                    # data loader\n                    # ======================================================\n                    input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                    input_data_mask_array      = d['input_data_mask_array'].to(device)\n                    input_data_valid_array     = d['input_data_valid_array'].to(device)\n                    attention_mask             = d['attention_mask'].to(device)\n                    y                          = d[\"y\"].to(device)\n                    \n                    optimizer.zero_grad()\n                    output = model(input_data_numerical_array, \n                                   input_data_mask_array,\n                                   attention_mask)\n                    loss = criterion(output[input_data_mask_array == 1], y[input_data_mask_array == 1])\n                    \n                    # output1 = output[:, :, [1,2]]\n                    # output2 = output[:, :, 0]\n                    # y1 = y[:, :, [1,2]]\n                    # y2 = y[:, :, 0]\n                    # loss1 = criterion(output1[input_data_mask_array == 1], y1[input_data_mask_array == 1])\n                    # loss2 = criterion(output2[input_data_mask_array == 1], y2[input_data_mask_array == 1])\n                    # loss = loss1*0.8 + loss2*0.2\n                    \n                    loss.backward()\n                    optimizer.step()\n                    scheduler.step()\n                    train_losses_batch.append(loss.item())\n                train_loss = np.mean(train_losses_batch)\n                \n                \n                # ======================================================\n                # eval\n                # ======================================================\n                model.eval()       # switch model to the evaluation mode\n                \n                val_preds = np.ndarray((0, seq_len, 3))\n                tk0 = tqdm_nb(val_loader, total=len(val_loader), desc=\"val_loader  : \")\n                with torch.no_grad():  # Do not calculate gradient since we are only predicting\n                    # Predicting on validation set\n                    for d in tk0:\n                        input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                        input_data_mask_array      = d['input_data_mask_array'].to(device)\n                        attention_mask             = d['attention_mask'].to(device)\n                        \n                        output = model(input_data_numerical_array, \n                                       input_data_mask_array,\n                                       attention_mask)\n                        val_preds = np.concatenate([val_preds, output.sigmoid().detach().cpu().numpy()], axis=0)\n                \n                pred_valid_index = (val_mask_array == 1) & (val_pred_array == 1) & (val_valid_array == 1)\n                StartHesitation = average_precision_score(val_target_array[pred_valid_index][:, 0],\n                                                          val_preds[pred_valid_index][:, 0])\n                Turn = average_precision_score(val_target_array[pred_valid_index][:, 1],\n                                               val_preds[pred_valid_index][:, 1])\n                Walking = average_precision_score(val_target_array[pred_valid_index][:, 2],\n                                                  val_preds[pred_valid_index][:, 2])\n                # map_score = np.mean([StartHesitation, Turn, Walking])\n                map_score = np.mean([Turn, Walking])\n                \n                # LOGGER.info(f\"fold: {fold+1} epoch: {epoch+1},train loss {train_loss} map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                print(f\"fold {fold+1} epoch {epoch+1}\\ntrain loss {train_loss} map:{map_score}\\nstart_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                \n                if map_score >= best_val_score:\n                    print(\"save weight\")\n                    best_val_score = map_score\n                    best_val_preds = val_preds.copy()\n                    torch.save(model.state_dict(), f\"./output/exp/ex{ex}/ex{ex}_model/ex{ex}_fold{fold+1}_epoch{epoch+1}__map{best_val_score:.6f}.pth\")\n                print()\n            y_oof[valid_idx] = best_val_preds\n    \n    np.save(f\"./output/exp/ex{ex}/ex{ex}_oof.npy\", y_oof)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:10.445785Z","iopub.execute_input":"2023-07-03T07:54:10.446131Z","iopub.status.idle":"2023-07-03T07:54:10.475552Z","shell.execute_reply.started":"2023-07-03T07:54:10.446079Z","shell.execute_reply":"2023-07-03T07:54:10.474514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### oof score","metadata":{}},{"cell_type":"code","source":"if TRAIN_FLAG:\n    val_pred_index = (mask_array == 1) & (pred_use_array == 1) & (valid_array == 1)\n    StartHesitation = average_precision_score(target_array[val_pred_index][:,0], \n                                              y_oof[val_pred_index][:,0])\n    Turn            = average_precision_score(target_array[val_pred_index][:,1], \n                                              y_oof[val_pred_index][:,1])\n    Walking         = average_precision_score(target_array[val_pred_index][:,2],\n                                              y_oof[val_pred_index][:,2])\n\n    map_score = np.mean([StartHesitation, Turn, Walking])\n    # LOGGER.info(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n    print(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:10.476918Z","iopub.execute_input":"2023-07-03T07:54:10.478103Z","iopub.status.idle":"2023-07-03T07:54:10.487775Z","shell.execute_reply.started":"2023-07-03T07:54:10.478071Z","shell.execute_reply":"2023-07-03T07:54:10.486952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### <font color=maroon>ex204_defog_gru_pseudo.ipynb</font>\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/exp/ex204_defog_gru_pseudo.ipynb\" style=\"text-decoration:none\">ex204_defog_gru_pseudo.ipynb</a>","metadata":{}},{"cell_type":"code","source":"debug = False\nex = \"204_notype_pseudo_target\"\nif not os.path.exists(f\"./output/exp/ex{ex}\"):\n    os.makedirs(f\"./output/exp/ex{ex}\")\n    os.makedirs(f\"./output/exp/ex{ex}/ex{ex}_model\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:10.488926Z","iopub.execute_input":"2023-07-03T07:54:10.489867Z","iopub.status.idle":"2023-07-03T07:54:10.498518Z","shell.execute_reply.started":"2023-07-03T07:54:10.489830Z","shell.execute_reply":"2023-07-03T07:54:10.497589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pseudo_target = \"086_notype_pseudo\"","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:10.501740Z","iopub.execute_input":"2023-07-03T07:54:10.502057Z","iopub.status.idle":"2023-07-03T07:54:10.508337Z","shell.execute_reply.started":"2023-07-03T07:54:10.502034Z","shell.execute_reply":"2023-07-03T07:54:10.507344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### main","metadata":{}},{"cell_type":"code","source":"%%time\n\n# with timer(\"gru\"):\nif TRAIN_FLAG:\n    set_seed(seed)\n    y_oof = np.empty([len(target_array), 5000, 3])      # Abrachan: out of fold\n    gkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state = seed)\n    for fold, (train_idx, valid_idx) in enumerate(gkf.split(numerical_array, \n                                                            y = df_id[\"group\"].values,\n                                                            groups=df_id[\"subject\"].values)):\n        # LOGGER.info(f\"start fold:{fold+1}\")\n        print(f\"start fold: {fold+1}\")\n        \n        with timer(f\"fold {fold+1}\"):\n            train_numerical_array = numerical_array[train_idx]\n            train_target_array = target_array[train_idx]\n            train_mask_array = mask_array[train_idx]\n            train_valid_array = valid_array[train_idx]\n            \n            pseudo_target_array = np.load(f\"./output/fe/fe{pseudo_target}/fe{pseudo_target}_target_array_{fold+1}.npy\")\n            train_numerical_array = np.concatenate([train_numerical_array, pseudo_numerical_array], axis=0)\n            train_target_array = np.concatenate([train_target_array, pseudo_target_array], axis=0)\n            train_mask_array = np.concatenate([train_mask_array, pseudo_mask_array], axis=0)\n            train_valid_array = np.concatenate([train_valid_array, pseudo_valid_array], axis=0)\n\n            val_numerical_array = numerical_array[valid_idx]\n            val_target_array = target_array[valid_idx]\n            val_mask_array = mask_array[valid_idx]\n            val_valid_array = valid_array[valid_idx]\n            val_pred_array = pred_use_array[valid_idx]\n            \n            train_ = FogDataset(train_numerical_array,\n                                train_mask_array,\n                                train_valid_array,\n                                train=True,\n                                y=train_target_array)\n            val_ = FogDataset(val_numerical_array,\n                              val_mask_array,\n                              val_valid_array,\n                              train=True,\n                              y=val_target_array)\n            \n            train_loader = DataLoader(dataset=train_, \n                                      batch_size=batch_size, \n                                      shuffle = True, \n                                      # num_workers=8,\n                                     )\n            val_loader = DataLoader(dataset=val_, \n                                    batch_size=batch_size, \n                                    shuffle = False , \n                                    # num_workers=8\n                                   )\n            \n            \n            # model = FogRnnModel()\n            model = FogRnnModel(numeraical_linear_size = 96,\n                                model_size = 256,\n                                linear_out = 256,)            \n            model = model.to(device)\n            \n            param_optimizer = list(model.named_parameters())\n            no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n            optimizer_grouped_parameters = [\n                {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], \n                 'weight_decay': weight_decay\n                },\n                {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], \n                 'weight_decay': 0.0\n                }]\n            optimizer = AdamW(optimizer_grouped_parameters,\n                              lr=lr,\n                              weight_decay=weight_decay,\n                              )\n            num_train_optimization_steps = int(len(train_loader) * n_epochs)\n            scheduler = get_linear_schedule_with_warmup(optimizer,\n                                                        num_warmup_steps=num_warmup_steps,\n                                                        num_training_steps=num_train_optimization_steps)\n            criterion = nn.BCEWithLogitsLoss()\n            best_val_score = 0\n            \n            for epoch in range(n_epochs):\n                model.train() \n                train_losses_batch = []\n                val_losses_batch = []\n                epoch_loss = 0\n                train_preds = np.ndarray((0,3))   # array([], shape=(0, 3), dtype=float64)\n                \n                tk0 = tqdm_nb(train_loader, total=len(train_loader), desc=\"train_loader: \")\n                for d in tk0:\n                    # ======================================================\n                    # data loader\n                    # ======================================================\n                    input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                    input_data_mask_array      = d['input_data_mask_array'].to(device)\n                    input_data_valid_array     = d['input_data_valid_array'].to(device)\n                    attention_mask             = d['attention_mask'].to(device)\n                    y                          = d[\"y\"].to(device)\n                    \n                    optimizer.zero_grad()\n                    output = model(input_data_numerical_array, \n                                   input_data_mask_array,\n                                   attention_mask)\n                    loss = criterion(output[input_data_mask_array == 1], y[input_data_mask_array == 1])\n                    \n                    # output1 = output[:, :, [1,2]]\n                    # output2 = output[:, :, 0]\n                    # y1 = y[:, :, [1,2]]\n                    # y2 = y[:, :, 0]\n                    # loss1 = criterion(output1[input_data_mask_array == 1], y1[input_data_mask_array == 1])\n                    # loss2 = criterion(output2[input_data_mask_array == 1], y2[input_data_mask_array == 1])\n                    # loss = loss1*0.8 + loss2*0.2\n                    \n                    loss.backward()\n                    optimizer.step()\n                    scheduler.step()\n                    train_losses_batch.append(loss.item())\n                train_loss = np.mean(train_losses_batch)\n                \n                \n                # ======================================================\n                # eval\n                # ======================================================\n                model.eval()       # switch model to the evaluation mode\n                \n                val_preds = np.ndarray((0, 5000, 3))\n                tk0 = tqdm_nb(val_loader, total=len(val_loader), desc=\"val_loader  : \")\n                with torch.no_grad():  # Do not calculate gradient since we are only predicting\n                    # Predicting on validation set\n                    for d in tk0:\n                        input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                        input_data_mask_array      = d['input_data_mask_array'].to(device)\n                        attention_mask             = d['attention_mask'].to(device)\n                        \n                        output = model(input_data_numerical_array, \n                                       input_data_mask_array,\n                                       attention_mask)\n                        val_preds = np.concatenate([val_preds, output.sigmoid().detach().cpu().numpy()], axis=0)\n                \n                pred_valid_index = (val_mask_array == 1) & (val_pred_array == 1) & (val_valid_array == 1)\n                StartHesitation = average_precision_score(val_target_array[pred_valid_index][:, 0],\n                                                          val_preds[pred_valid_index][:, 0])\n                Turn = average_precision_score(val_target_array[pred_valid_index][:, 1],\n                                               val_preds[pred_valid_index][:, 1])\n                Walking = average_precision_score(val_target_array[pred_valid_index][:, 2],\n                                                  val_preds[pred_valid_index][:, 2])\n                # map_score = np.mean([StartHesitation, Turn, Walking])\n                map_score = np.mean([Turn, Walking])\n                \n                # LOGGER.info(f\"fold: {fold+1} epoch: {epoch+1},train loss {train_loss} map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                print(f\"fold {fold+1} epoch {epoch+1}\\ntrain loss {train_loss} map:{map_score}\\nstart_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                \n                if map_score >= best_val_score:\n                    print(\"save weight\")\n                    best_val_score = map_score\n                    best_val_preds = val_preds.copy()\n                    torch.save(model.state_dict(), f\"./output/exp/ex{ex}/ex{ex}_model/ex{ex}_fold{fold+1}_epoch{epoch+1}__map{best_val_score:.6f}.pth\")\n                print()\n            y_oof[valid_idx] = best_val_preds\n    \n    np.save(f\"./output/exp/ex{ex}/ex{ex}_oof.npy\", y_oof)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:10.510010Z","iopub.execute_input":"2023-07-03T07:54:10.510591Z","iopub.status.idle":"2023-07-03T07:54:10.539746Z","shell.execute_reply.started":"2023-07-03T07:54:10.510560Z","shell.execute_reply":"2023-07-03T07:54:10.538871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### oof score","metadata":{}},{"cell_type":"code","source":"if TRAIN_FLAG:\n    val_pred_index = (mask_array == 1) & (pred_use_array == 1) & (valid_array == 1)\n    StartHesitation = average_precision_score(target_array[val_pred_index][:,0], \n                                              y_oof[val_pred_index][:,0])\n    Turn            = average_precision_score(target_array[val_pred_index][:,1], \n                                              y_oof[val_pred_index][:,1])\n    Walking         = average_precision_score(target_array[val_pred_index][:,2],\n                                              y_oof[val_pred_index][:,2])\n\n    map_score = np.mean([StartHesitation, Turn, Walking])\n    # LOGGER.info(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n    print(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:10.540901Z","iopub.execute_input":"2023-07-03T07:54:10.541266Z","iopub.status.idle":"2023-07-03T07:54:10.551806Z","shell.execute_reply.started":"2023-07-03T07:54:10.541233Z","shell.execute_reply":"2023-07-03T07:54:10.550960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n<br>\n\n## <font color=blue>**ROUND 2**</font>\n\n<br>\n<br>\n\n## <font color=red>**Predicting**</font>: **Making** <font color=red>**pseudo**</font> **label** (round2: use models of ex175 in round 1)\n\n\n<br>\n\n### <font color=maroon>ex175_defog_gru_inference_notype_15000.ipynbb</font>\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/exp/ex175_defog_gru_inference_notype_15000.ipynb\" style=\"text-decoration:none\">ex175_defog_gru_inference_notype_15000.ipynb</a>","metadata":{}},{"cell_type":"code","source":"debug = False\nex = \"175_notype_pseudo_target\"\nif not os.path.exists(f\"./output/exp/ex{ex}\"):\n    os.makedirs(f\"./output/exp/ex{ex}\")\n    os.makedirs(f\"./output/exp/ex{ex}/ex{ex}_model\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:10.553402Z","iopub.execute_input":"2023-07-03T07:54:10.553780Z","iopub.status.idle":"2023-07-03T07:54:10.561858Z","shell.execute_reply.started":"2023-07-03T07:54:10.553749Z","shell.execute_reply":"2023-07-03T07:54:10.560867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### load data & preprocessing","metadata":{}},{"cell_type":"code","source":"seq_len = 15000\n\nid_path        = f\"./output/fe/fe074_notype/fe074_notype_id.parquet\"\nnumerical_path = f\"./output/fe/fe074_notype/fe074_notype_num_array.npy\"\ntarget_path    = f\"./output/fe/fe074_notype/fe074_notype_target_array.npy\"\nmask_path      = f\"./output/fe/fe074_notype/fe074_notype_mask_array.npy\"\nvalid_path     = f\"./output/fe/fe074_notype/fe074_notype_valid_array.npy\"\npred_use_path  = f\"./output/fe/fe074_notype/fe074_notype_pred_use_array.npy\"\ntime_path      = f\"./output/fe/fe074_notype/fe074_notype_time_array.npy\"\n\ndf_id           = pd.read_parquet(id_path)\nnumerical_array = np.load(numerical_path)\ntarget_array    = np.load(target_path)\nmask_array      = np.load(mask_path)\nvalid_array     = np.load(valid_path)\npred_use_array  = np.load(pred_use_path)\ntime_array      = np.load(time_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:10.563265Z","iopub.execute_input":"2023-07-03T07:54:10.563738Z","iopub.status.idle":"2023-07-03T07:54:23.075578Z","shell.execute_reply.started":"2023-07-03T07:54:10.563693Z","shell.execute_reply":"2023-07-03T07:54:23.074530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### config","metadata":{}},{"cell_type":"code","source":"# config\nseed = 0\nshuffle = True\nn_splits = 5\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# model config\nbatch_size = 24\nn_epochs = 15\nlr = 1e-3\nweight_decay = 0.05\nnum_warmup_steps = 10","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:23.079360Z","iopub.execute_input":"2023-07-03T07:54:23.079775Z","iopub.status.idle":"2023-07-03T07:54:23.091081Z","shell.execute_reply.started":"2023-07-03T07:54:23.079734Z","shell.execute_reply":"2023-07-03T07:54:23.090052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### main (predict)","metadata":{}},{"cell_type":"code","source":"models_list = os.listdir(f\"./output/exp/ex{ex}/ex{ex}_model\")\nmodels_list","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:23.093354Z","iopub.execute_input":"2023-07-03T07:54:23.093641Z","iopub.status.idle":"2023-07-03T07:54:23.107961Z","shell.execute_reply.started":"2023-07-03T07:54:23.093617Z","shell.execute_reply":"2023-07-03T07:54:23.106968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if PREDICT_FLAG:\n# with timer(\"gru\"):\n    set_seed(seed)\n    for fold in range(1, 6):\n        with timer(f\"fold {fold}\"):\n            val_numerical_array = numerical_array.copy()\n            val_target_array = target_array.copy()\n            val_mask_array = mask_array.copy()\n            val_valid_array = valid_array.copy()\n            val_pred_array = pred_use_array.copy()\n\n            val_ = FogDataset(val_numerical_array, \n                              val_mask_array,\n                              val_valid_array, \n                              train=True,\n                              y=val_target_array)\n\n            val_loader = DataLoader(dataset=val_, \n                                    batch_size=batch_size, \n                                    shuffle = False,\n                                    # num_workers=8\n                                   )\n\n            model = FogRnnModel()\n            # model.load_state_dict(torch.load(f\"./output/exp/ex{ex}/ex{ex}_model/ex{ex}_{fold}.pth\"))\n            model.load_state_dict(torch.load(f\"./output/exp/ex{ex}/ex{ex}_model/\" + models_list[fold-1]))\n            model = model.to(device)\n            model.eval()  # switch model to the evaluation mode\n            \n            val_preds = np.ndarray((0, seq_len, 3))\n            tk0 = tqdm(val_loader, total=len(val_loader))\n            with torch.no_grad():  # Do not calculate gradient since we are only predicting\n                # Predicting on validation set\n                for d in tk0:\n                    input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                    input_data_mask_array      = d['input_data_mask_array'].to(device)\n                    attention_mask             = d['attention_mask'].to(device)\n                    output = model(input_data_numerical_array, \n                                   input_data_mask_array,\n                                   attention_mask)\n                    val_preds = np.concatenate([val_preds, output.sigmoid().detach().cpu().numpy()], axis=0)\n            # np.save(f\"./output/exp/ex{ex}/ex{ex}_notype_{fold}_oof_{seq_len}.npy\",val_preds)\n            np.save(f\"./output/exp/ex{ex}/ex{ex}_R2_{fold}_oof_{seq_len}.npy\",val_preds)            ","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:23.109584Z","iopub.execute_input":"2023-07-03T07:54:23.109900Z","iopub.status.idle":"2023-07-03T07:54:23.121928Z","shell.execute_reply.started":"2023-07-03T07:54:23.109870Z","shell.execute_reply":"2023-07-03T07:54:23.120961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if PREDICT_FLAG:\n    id_array = df_id[\"Id\"].values\n    for i in range(1, 6):\n        pred_all = []\n        # pred = np.load(f\"./output/exp/ex{ex}/ex{ex}_notype_{i}_oof_{seq_len}.npy\")\n        pred = np.load(f\"./output/exp/ex{ex}/ex{ex}_R2_{i}_oof_{seq_len}.npy\")\n        for v in tqdm(range(len(pred_use_array))):\n            use_ = pred_use_array[v, :] == 1\n            pred_ = pred[v, use_ == 1, :]\n            time_ = time_array[v, use_ == 1]\n            Id = id_array[v]\n            pred_df = pd.DataFrame()\n            pred_df[\"Time\"] = time_\n            pred_df[\"Id\"] = Id\n            pred_df[\"StartHesitation\"] = pred_[:, 0]\n            pred_df[\"Turn\"] = pred_[:, 1]\n            pred_df[\"Walking\"] = pred_[:, 2]\n            pred_all.append(pred_df)\n        pred_all = pd.concat(pred_all).reset_index(drop=True)\n        # pred_all.to_parquet(f\"./output/exp/ex{ex}/ex{ex}_notype_{i}_pred_{seq_len}.parquet\")\n        pred_all.to_parquet(f\"./output/exp/ex{ex}/ex{ex}_R2_{i}_pred_{seq_len}.parquet\")    ","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:23.123534Z","iopub.execute_input":"2023-07-03T07:54:23.123854Z","iopub.status.idle":"2023-07-03T07:54:23.135315Z","shell.execute_reply.started":"2023-07-03T07:54:23.123824Z","shell.execute_reply":"2023-07-03T07:54:23.134384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n<br>\n\n## **Feature Engineering with** <font color=red>**pseudo**</font>  **label** \n\n<br>\n\n\n### <font color=maroon>fe078_notype_pseudo_ex175_15000.ipynb</font>\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/fe/fe078_notype_pseudo_ex175_15000.ipynb\" style=\"text-decoration:none\">fe078_notype_pseudo_ex175_15000.ipynb</a>","metadata":{}},{"cell_type":"code","source":"fe = \"078_notype_pseudo\"\nex = \"175_notype_pseudo_target\"\nif not os.path.exists(f\"./output/fe/fe{fe}\"):\n    os.makedirs(f\"./output/fe/fe{fe}\")\n    os.makedirs(f\"./output/fe/fe{fe}/save\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:23.136773Z","iopub.execute_input":"2023-07-03T07:54:23.137155Z","iopub.status.idle":"2023-07-03T07:54:23.149334Z","shell.execute_reply.started":"2023-07-03T07:54:23.137123Z","shell.execute_reply":"2023-07-03T07:54:23.148267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DEFOG_META_PATH = \"../data/defog_metadata.csv\"\n# DEFOG_FOLDER = \"../data/train/notype/*.csv\"\n\ndefog_meta = pd.read_parquet(\"./output/fe/fe039/fe039_defog_meta.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:23.150913Z","iopub.execute_input":"2023-07-03T07:54:23.151267Z","iopub.status.idle":"2023-07-03T07:54:23.179293Z","shell.execute_reply.started":"2023-07-03T07:54:23.151235Z","shell.execute_reply":"2023-07-03T07:54:23.178264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = [\"AccV\", \"AccML\", \"AccAP\"]\nnum_cols = [\"AccV\", \"AccML\", \"AccAP\",\n            'AccV_lag_diff', 'AccV_lead_diff', \n            'AccML_lag_diff', 'AccML_lead_diff',\n            'AccAP_lag_diff', 'AccAP_lead_diff']\n\ntarget_use_cols = [\"Event\"]\ntarget_cols = [\"StartHesitation\", \"Turn\", \"Walking\"]\nseq_len = 5000\nshift = 2500\noffset = 1250","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:23.180590Z","iopub.execute_input":"2023-07-03T07:54:23.180999Z","iopub.status.idle":"2023-07-03T07:54:23.186735Z","shell.execute_reply.started":"2023-07-03T07:54:23.180967Z","shell.execute_reply":"2023-07-03T07:54:23.185787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_list = glob.glob(DEFOG_FOLDER)\nif TRAIN_FLAG:\n    for fold in range(1, 6):\n        print(fold)\n        # pred = pd.read_parquet(f\"./output/exp/ex{ex}/ex{ex}_notype_{fold}_pred_15000.parquet\")\n        pred = pd.read_parquet(f\"./output/exp/ex{ex}/ex{ex}_R2_{fold}_pred_15000.parquet\")\n        target_array = []\n        for i,s in tqdm(zip(defog_meta[\"Id\"].values, defog_meta[\"sub_id\"].values)):\n            path = root_data + f\"train/notype/{i}.csv\"\n            if path in [x.replace(\"\\\\\", \"/\") for x in train_notype]:\n            # if path in data_list:\n                df = pd.read_csv(path)\n                df_ = pred[pred[\"Id\"] == i].reset_index(drop=True)\n                df = df.merge(df_, how=\"left\", on=\"Time\")\n                df[\"target_max\"] = np.argmax(df[[\"StartHesitation\", \"Turn\", \"Walking\"]].values, axis=1)\n\n                df.loc[df[\"target_max\"] == 0, \"StartHesitation\"] = 1\n                df.loc[df[\"target_max\"] == 0, [\"Turn\",\"Walking\"]] = 0\n\n                df.loc[df[\"target_max\"] == 1, \"Turn\"] = 1\n                df.loc[df[\"target_max\"] == 1, [\"StartHesitation\",\"Walking\"]] = 0\n\n                df.loc[df[\"target_max\"] == 2, \"Walking\"] = 1\n                df.loc[df[\"target_max\"] == 2, [\"StartHesitation\",\"Turn\"]] = 0\n\n                df.loc[df[\"Event\"] == 0, [\"StartHesitation\", \"Turn\", \"Walking\"]] = 0\n\n                df[\"valid\"] = df[\"Valid\"] & df[\"Task\"]\n                df[\"valid\"] = df[\"valid\"].astype(int)\n                batch = (len(df)-1) // shift\n                target = df[target_cols].values\n                target_array_ = np.zeros([batch, seq_len, 3])\n                for n,b in enumerate(range(batch)):\n                    if b == (batch - 1):\n                        target_ = target[b*shift : ]\n                        target_array_[b, :len(target_), :] = target_\n                    elif b == 0:\n                        target_ = target[b*shift : b*shift + seq_len]\n                        target_array_[b, :, :] = target_\n                    else:\n                        target_ = target[b*shift : b*shift + seq_len]\n                        target_array_[b, :, :] = target_\n\n                target_array.append(target_array_)\n        target_array = np.concatenate(target_array, axis=0)\n        np.save(f\"./output/fe/fe{fe}/fe{fe}_target_array_{fold}.npy\", target_array)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:23.188565Z","iopub.execute_input":"2023-07-03T07:54:23.189297Z","iopub.status.idle":"2023-07-03T07:54:23.206183Z","shell.execute_reply.started":"2023-07-03T07:54:23.189265Z","shell.execute_reply":"2023-07-03T07:54:23.205235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n<br>\n\n##  <font color=red>**Training**</font> **with** <font color=red>**pseudo**</font> **label** (round2)\n\n<br>\n\n### load data & preprocessing","metadata":{}},{"cell_type":"code","source":"id_path        = f\"./output/fe/fe047/fe047_id.parquet\"\nnumerical_path = f\"./output/fe/fe047/fe047_num_array.npy\"\ntarget_path    = f\"./output/fe/fe047/fe047_target_array.npy\"\nmask_path      = f\"./output/fe/fe047/fe047_mask_array.npy\"\nvalid_path     = f\"./output/fe/fe047/fe047_valid_array.npy\"\npred_use_path  = f\"./output/fe/fe047/fe047_pred_use_array.npy\"\n\ndf_id           = pd.read_parquet(id_path)\nnumerical_array = np.load(numerical_path)\ntarget_array    = np.load(target_path)\nmask_array      = np.load(mask_path)\nvalid_array     = np.load(valid_path)\npred_use_array  = np.load(pred_use_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:23.208909Z","iopub.execute_input":"2023-07-03T07:54:23.209173Z","iopub.status.idle":"2023-07-03T07:54:38.394101Z","shell.execute_reply.started":"2023-07-03T07:54:23.209151Z","shell.execute_reply":"2023-07-03T07:54:38.393059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target1 = []\ntarget2 = []\ntarget3 = []\nfor i in range(len(target_array)):\n    target1.append(np.sum(target_array[i,:,0]))\n    target2.append(np.sum(target_array[i,:,1]))\n    target3.append(np.sum(target_array[i,:,2]))\n\ndf_id[\"target1\"] = target1\ndf_id[\"target2\"] = target2\ndf_id[\"target3\"] = target3\ndf_id[\"target1_1\"] = df_id[\"target1\"] > 0\ndf_id[\"target2_1\"] = df_id[\"target2\"] > 0\ndf_id[\"target3_1\"] = df_id[\"target3\"] > 0\ndf_id[\"target1_1\"] = df_id[\"target1_1\"].astype(np.int)\ndf_id[\"target2_1\"] = df_id[\"target2_1\"].astype(np.int)\ndf_id[\"target3_1\"] = df_id[\"target3_1\"].astype(np.int)\n\ndf_id[\"group\"] = 0\ndf_id.loc[df_id[\"target1_1\"] > 0,\"group\"] = 1\ndf_id.loc[df_id[\"target2_1\"] > 0,\"group\"] = 2\ndf_id.loc[df_id[\"target3_1\"] > 0,\"group\"] = 3\n\ndf_id[\"group\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:38.396656Z","iopub.execute_input":"2023-07-03T07:54:38.397473Z","iopub.status.idle":"2023-07-03T07:54:38.631245Z","shell.execute_reply.started":"2023-07-03T07:54:38.397444Z","shell.execute_reply":"2023-07-03T07:54:38.630183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pseudo_id_path        = f\"./output/fe/fe061_notype/fe061_notype_id.parquet\"\npseudo_numerical_path = f\"./output/fe/fe061_notype/fe061_notype_num_array.npy\"\npseudo_mask_path      = f\"./output/fe/fe061_notype/fe061_notype_mask_array.npy\"\npseudo_valid_path     = f\"./output/fe/fe061_notype/fe061_notype_valid_array.npy\"\npseudo_pred_use_path  = f\"./output/fe/fe061_notype/fe061_notype_pred_use_array.npy\"\n\npseudo_numerical_array = np.load(pseudo_numerical_path)\npseudo_mask_array      = np.load(pseudo_mask_path)\npseudo_valid_array     = np.load(pseudo_valid_path)\npseudo_pred_use_array  = np.load(pseudo_pred_use_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:38.632509Z","iopub.execute_input":"2023-07-03T07:54:38.633113Z","iopub.status.idle":"2023-07-03T07:54:47.935226Z","shell.execute_reply.started":"2023-07-03T07:54:38.633078Z","shell.execute_reply":"2023-07-03T07:54:47.934213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### config","metadata":{}},{"cell_type":"code","source":"seq_len = 5000\n\n# config\nseed = 0\nshuffle = True\nn_splits = 5\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# model config\nbatch_size = 24\nn_epochs = 10\nlr = 1e-3\nweight_decay = 0.05\nnum_warmup_steps = 10","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:47.938334Z","iopub.execute_input":"2023-07-03T07:54:47.938942Z","iopub.status.idle":"2023-07-03T07:54:47.948429Z","shell.execute_reply.started":"2023-07-03T07:54:47.938906Z","shell.execute_reply":"2023-07-03T07:54:47.947542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### <font color=maroon>ex185_defog_gru_pseudo2.ipynb</font>\n\n<br>\n\nCode below originates from: \n* <a href=\"https://github.com/TakoiHirokazu/Kaggle-Parkinsons-Freezing-of-Gait-Prediction/blob/main/takoi/exp/ex185_defog_gru_pseudo2.ipynb\" style=\"text-decoration:none\">ex185_defog_gru_pseudo2.ipynb</a>","metadata":{}},{"cell_type":"code","source":"debug = False\nex = \"185_notype_pseudo_target\"\nif not os.path.exists(f\"./output/exp/ex{ex}\"):\n    os.makedirs(f\"./output/exp/ex{ex}\")\n    os.makedirs(f\"./output/exp/ex{ex}/ex{ex}_model\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:47.950587Z","iopub.execute_input":"2023-07-03T07:54:47.950999Z","iopub.status.idle":"2023-07-03T07:54:47.967121Z","shell.execute_reply.started":"2023-07-03T07:54:47.950962Z","shell.execute_reply":"2023-07-03T07:54:47.966189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pseudo_target = \"078_notype_pseudo\"","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:47.968446Z","iopub.execute_input":"2023-07-03T07:54:47.968935Z","iopub.status.idle":"2023-07-03T07:54:47.976444Z","shell.execute_reply.started":"2023-07-03T07:54:47.968865Z","shell.execute_reply":"2023-07-03T07:54:47.975601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### main","metadata":{}},{"cell_type":"code","source":"%%time\n\n# with timer(\"gru\"):\nif TRAIN_FLAG:\n    set_seed(seed)\n    y_oof = np.empty([len(target_array), seq_len, 3])      # Abrachan: out of fold\n    gkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state = seed)\n    for fold, (train_idx, valid_idx) in enumerate(gkf.split(numerical_array, \n                                                            y = df_id[\"group\"].values,\n                                                            groups=df_id[\"subject\"].values)):\n        # LOGGER.info(f\"start fold:{fold+1}\")\n        print(f\"start fold: {fold+1}\")\n        \n        with timer(f\"fold {fold+1}\"):\n            train_numerical_array = numerical_array[train_idx]\n            train_target_array = target_array[train_idx]\n            train_mask_array = mask_array[train_idx]\n            train_valid_array = valid_array[train_idx]\n            \n            pseudo_target_array = np.load(f\"./output/fe/fe{pseudo_target}/fe{pseudo_target}_target_array_{fold+1}.npy\")\n            train_numerical_array = np.concatenate([train_numerical_array, pseudo_numerical_array], axis=0)\n            train_target_array = np.concatenate([train_target_array, pseudo_target_array], axis=0)\n            train_mask_array = np.concatenate([train_mask_array, pseudo_mask_array], axis=0)\n            train_valid_array = np.concatenate([train_valid_array, pseudo_valid_array], axis=0)\n\n            val_numerical_array = numerical_array[valid_idx]\n            val_target_array = target_array[valid_idx]\n            val_mask_array = mask_array[valid_idx]\n            val_valid_array = valid_array[valid_idx]\n            val_pred_array = pred_use_array[valid_idx]\n            \n            train_ = FogDataset(train_numerical_array,\n                                train_mask_array,\n                                train_valid_array,\n                                train=True,\n                                y=train_target_array)\n            val_ = FogDataset(val_numerical_array,\n                              val_mask_array,\n                              val_valid_array,\n                              train=True,\n                              y=val_target_array)\n            \n            train_loader = DataLoader(dataset=train_, \n                                      batch_size=batch_size, \n                                      shuffle = True, \n                                      # num_workers=8,\n                                     )\n            val_loader = DataLoader(dataset=val_, \n                                    batch_size=batch_size, \n                                    shuffle = False , \n                                    # num_workers=8\n                                   )\n            \n            \n            model = FogRnnModel()\n            model = model.to(device)\n            \n            param_optimizer = list(model.named_parameters())\n            no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n            optimizer_grouped_parameters = [\n                {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], \n                 'weight_decay': weight_decay\n                },\n                {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], \n                 'weight_decay': 0.0\n                }]\n            optimizer = AdamW(optimizer_grouped_parameters,\n                              lr=lr,\n                              weight_decay=weight_decay,\n                              )\n            num_train_optimization_steps = int(len(train_loader) * n_epochs)\n            scheduler = get_linear_schedule_with_warmup(optimizer,\n                                                        num_warmup_steps=num_warmup_steps,\n                                                        num_training_steps=num_train_optimization_steps)\n            criterion = nn.BCEWithLogitsLoss()\n            best_val_score = 0\n            \n            for epoch in range(n_epochs):\n                model.train() \n                train_losses_batch = []\n                val_losses_batch = []\n                epoch_loss = 0\n                train_preds = np.ndarray((0,3))   # array([], shape=(0, 3), dtype=float64)\n                \n                tk0 = tqdm_nb(train_loader, total=len(train_loader), desc=\"train_loader: \")\n                for d in tk0:\n                    # ======================================================\n                    # data loader\n                    # ======================================================\n                    input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                    input_data_mask_array      = d['input_data_mask_array'].to(device)\n                    input_data_valid_array     = d['input_data_valid_array'].to(device)\n                    attention_mask             = d['attention_mask'].to(device)\n                    y                          = d[\"y\"].to(device)\n                    \n                    optimizer.zero_grad()\n                    output = model(input_data_numerical_array, \n                                   input_data_mask_array,\n                                   attention_mask)\n                    loss = criterion(output[input_data_mask_array == 1], y[input_data_mask_array == 1])\n                    \n                    # output1 = output[:, :, [1,2]]\n                    # output2 = output[:, :, 0]\n                    # y1 = y[:, :, [1,2]]\n                    # y2 = y[:, :, 0]\n                    # loss1 = criterion(output1[input_data_mask_array == 1], y1[input_data_mask_array == 1])\n                    # loss2 = criterion(output2[input_data_mask_array == 1], y2[input_data_mask_array == 1])\n                    # loss = loss1*0.8 + loss2*0.2\n                    \n                    loss.backward()\n                    optimizer.step()\n                    scheduler.step()\n                    train_losses_batch.append(loss.item())\n                train_loss = np.mean(train_losses_batch)\n                \n                \n                # ======================================================\n                # eval\n                # ======================================================\n                model.eval()       # switch model to the evaluation mode\n                \n                val_preds = np.ndarray((0, seq_len, 3))\n                tk0 = tqdm_nb(val_loader, total=len(val_loader), desc=\"val_loader  : \")\n                with torch.no_grad():  # Do not calculate gradient since we are only predicting\n                    # Predicting on validation set\n                    for d in tk0:\n                        input_data_numerical_array = d['input_data_numerical_array'].to(device)\n                        input_data_mask_array      = d['input_data_mask_array'].to(device)\n                        attention_mask             = d['attention_mask'].to(device)\n                        \n                        output = model(input_data_numerical_array, \n                                       input_data_mask_array,\n                                       attention_mask)\n                        val_preds = np.concatenate([val_preds, output.sigmoid().detach().cpu().numpy()], axis=0)\n                \n                pred_valid_index = (val_mask_array == 1) & (val_pred_array == 1) & (val_valid_array == 1)\n                StartHesitation = average_precision_score(val_target_array[pred_valid_index][:, 0],\n                                                          val_preds[pred_valid_index][:, 0])\n                Turn = average_precision_score(val_target_array[pred_valid_index][:, 1],\n                                               val_preds[pred_valid_index][:, 1])\n                Walking = average_precision_score(val_target_array[pred_valid_index][:, 2],\n                                                  val_preds[pred_valid_index][:, 2])\n                # map_score = np.mean([StartHesitation, Turn, Walking])\n                map_score = np.mean([Turn, Walking])\n                \n                # LOGGER.info(f\"fold: {fold+1} epoch: {epoch+1},train loss {train_loss} map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                print(f\"fold {fold+1} epoch {epoch+1}\\ntrain loss {train_loss} map:{map_score}\\nstart_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n                \n                if map_score >= best_val_score:\n                    print(\"save weight\")\n                    best_val_score = map_score\n                    best_val_preds = val_preds.copy()\n                    torch.save(model.state_dict(), f\"./output/exp/ex{ex}/ex{ex}_model/ex{ex}_fold{fold+1}_epoch{epoch+1}__map{best_val_score:.6f}.pth\")\n                print()\n            y_oof[valid_idx] = best_val_preds\n    \n    np.save(f\"./output/exp/ex{ex}/ex{ex}_oof.npy\", y_oof)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:47.978174Z","iopub.execute_input":"2023-07-03T07:54:47.978553Z","iopub.status.idle":"2023-07-03T07:54:48.011384Z","shell.execute_reply.started":"2023-07-03T07:54:47.978520Z","shell.execute_reply":"2023-07-03T07:54:48.010252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### oof score","metadata":{}},{"cell_type":"code","source":"if TRAIN_FLAG:\n    val_pred_index = (mask_array == 1) & (pred_use_array == 1) & (valid_array == 1)\n    StartHesitation = average_precision_score(target_array[val_pred_index][:,0], \n                                              y_oof[val_pred_index][:,0])\n    Turn            = average_precision_score(target_array[val_pred_index][:,1], \n                                              y_oof[val_pred_index][:,1])\n    Walking         = average_precision_score(target_array[val_pred_index][:,2],\n                                              y_oof[val_pred_index][:,2])\n\n    map_score = np.mean([StartHesitation, Turn, Walking])\n    # LOGGER.info(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")\n    print(f\"cv map:{map_score} start_hesi:{StartHesitation} turn:{Turn} walking :{Walking}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:48.012506Z","iopub.execute_input":"2023-07-03T07:54:48.013442Z","iopub.status.idle":"2023-07-03T07:54:48.020867Z","shell.execute_reply.started":"2023-07-03T07:54:48.013410Z","shell.execute_reply":"2023-07-03T07:54:48.019704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n## **models choosed**\n\n<br>\n\n<font color=maroon size=4>The following numbered models were used for the final submission:</font>\n* <font color=maroon size=4>ex153</font>\n* <font color=maroon size=4>ex179</font>\n* <font color=maroon size=4>ex185</font>\n* <font color=maroon size=4>ex204</font>","metadata":{}},{"cell_type":"markdown","source":"<br>\n<br>\n<br>\n\n\n# 【**Inference**】\n\n<br>\n\nCode below originates from: \n* <a href=\"https://www.kaggle.com/code/takoihiraokazu/cv-ensemble-sub-0607-1\" style=\"text-decoration:none\">[cv]ensemble_sub_0607_1</a>","metadata":{}},{"cell_type":"markdown","source":"<br>\n\n## Config","metadata":{}},{"cell_type":"code","source":"# SUB_PATH          = \"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/sample_submission.csv\"\n# DEFOG_DATA_PATH   = \"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/test/defog/*.csv\"\n# TDCSFOG_DATA_PATH = \"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/test/tdcsfog/*.csv\"","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:48.022576Z","iopub.execute_input":"2023-07-03T07:54:48.022989Z","iopub.status.idle":"2023-07-03T07:54:48.034430Z","shell.execute_reply.started":"2023-07-03T07:54:48.022958Z","shell.execute_reply":"2023-07-03T07:54:48.033509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nbs = 32","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:48.036328Z","iopub.execute_input":"2023-07-03T07:54:48.036623Z","iopub.status.idle":"2023-07-03T07:54:48.044432Z","shell.execute_reply.started":"2023-07-03T07:54:48.036593Z","shell.execute_reply":"2023-07-03T07:54:48.043485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sub = pd.read_csv(SUB_PATH)\n\n# # sub = pd.read_csv(root_data + 'sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:48.045719Z","iopub.execute_input":"2023-07-03T07:54:48.046752Z","iopub.status.idle":"2023-07-03T07:54:48.053665Z","shell.execute_reply.started":"2023-07-03T07:54:48.046721Z","shell.execute_reply":"2023-07-03T07:54:48.052693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all = []","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:48.054921Z","iopub.execute_input":"2023-07-03T07:54:48.055936Z","iopub.status.idle":"2023-07-03T07:54:48.062581Z","shell.execute_reply.started":"2023-07-03T07:54:48.055905Z","shell.execute_reply":"2023-07-03T07:54:48.061690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n## Helpers\n\n<br>\n\n### def `preprocess()`","metadata":{}},{"cell_type":"code","source":"def preprocess(numerical_array, mask_array):\n    \n    attention_mask = mask_array == 0\n\n    return {'input_data_numerical_array': numerical_array,\n            'input_data_mask_array': mask_array,\n            'attention_mask': attention_mask,\n           }","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:48.063824Z","iopub.execute_input":"2023-07-03T07:54:48.064334Z","iopub.status.idle":"2023-07-03T07:54:48.072056Z","shell.execute_reply.started":"2023-07-03T07:54:48.064302Z","shell.execute_reply":"2023-07-03T07:54:48.071112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### class `FogDataset()`","metadata":{}},{"cell_type":"code","source":"class FogDataset(Dataset):\n    def __init__(self, numerical_array, mask_array, train = True, y = None):\n        self.numerical_array = numerical_array\n        self.mask_array = mask_array\n        self.train = train\n        self.y = y\n    \n    \n    def __len__(self):\n        return len(self.numerical_array)\n    \n    \n    def __getitem__(self, item):\n        data = preprocess(self.numerical_array[item], self.mask_array[item])\n\n        # Return the processed data where the lists are converted to `torch.tensor`s\n        if self.train : \n            return {\n              'input_data_numerical_array': torch.tensor(data['input_data_numerical_array'], dtype=torch.float32),\n              'input_data_mask_array': torch.tensor(data['input_data_mask_array'],  dtype=torch.long),              \n              'attention_mask': torch.tensor(data[\"attention_mask\"], dtype=torch.bool),\n              \"y\": torch.tensor(self.y[item], dtype=torch.float32)\n            }\n        else:\n            return {\n              'input_data_numerical_array': torch.tensor(data['input_data_numerical_array'], dtype=torch.float32),\n              'input_data_mask_array': torch.tensor(data['input_data_mask_array'], dtype=torch.long),\n              'attention_mask': torch.tensor(data[\"attention_mask\"], dtype=torch.bool),\n               }","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:48.073346Z","iopub.execute_input":"2023-07-03T07:54:48.074295Z","iopub.status.idle":"2023-07-03T07:54:48.084645Z","shell.execute_reply.started":"2023-07-03T07:54:48.074262Z","shell.execute_reply":"2023-07-03T07:54:48.084017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### class `TdcsfogRnnModel()`","metadata":{}},{"cell_type":"code","source":"class TdcsfogRnnModel(nn.Module):\n    def __init__(self, \n                 dropout=0.2,\n                 input_numerical_size=12,\n                 numeraical_linear_size = 64,\n                 model_size = 128,\n                 linear_out = 128,\n                 out_size=3):\n        \n        super(TdcsfogRnnModel, self).__init__()\n        \n        self.numerical_linear  = nn.Sequential(nn.Linear(input_numerical_size, numeraical_linear_size),\n                                               nn.LayerNorm(numeraical_linear_size))\n        \n        self.rnn = nn.GRU(numeraical_linear_size, \n                          model_size,\n                          num_layers = 2, \n                          batch_first=True,\n                          bidirectional=True)\n        \n        self.linear_out  = nn.Sequential(nn.Linear(model_size*2, linear_out),\n                                         nn.LayerNorm(linear_out),\n                                         nn.ReLU(),\n                                         nn.Dropout(dropout),\n                                         nn.Linear(linear_out, out_size))\n        self._reinitialize()\n        \n        \n        \n    def _reinitialize(self):\n        \"\"\"Tensorflow/Keras-like initialization\"\"\"\n        for name, p in self.named_parameters():\n            if 'rnn' in name:\n                if 'weight_ih' in name:\n                    nn.init.xavier_uniform_(p.data)\n                elif 'weight_hh' in name:\n                    nn.init.orthogonal_(p.data)\n                elif 'bias_ih' in name:\n                    p.data.fill_(0)\n                    # Set forget-gate bias to 1\n                    n = p.size(0)\n                    p.data[(n // 4):(n // 2)].fill_(1)\n                elif 'bias_hh' in name:\n                    p.data.fill_(0)\n        \n        \n        \n    def forward(self, numerical_array, mask_array, attention_mask):\n        numerical_embedding = self.numerical_linear(numerical_array)\n        output, _ = self.rnn(numerical_embedding)\n        output = self.linear_out(output)\n        return output","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:48.085853Z","iopub.execute_input":"2023-07-03T07:54:48.086916Z","iopub.status.idle":"2023-07-03T07:54:48.099012Z","shell.execute_reply.started":"2023-07-03T07:54:48.086883Z","shell.execute_reply":"2023-07-03T07:54:48.098258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### class `TdcsfogRnnModel2()`","metadata":{}},{"cell_type":"code","source":"class TdcsfogRnnModel2(nn.Module):\n    def __init__(self, \n                 dropout=0.2,\n                 input_numerical_size=12,\n                 numeraical_linear_size = 64,\n                 model_size = 128,\n                 linear_out = 128,\n                 out_size=3):\n        \n        super(TdcsfogRnnModel2, self).__init__()\n        \n        self.numerical_linear  = nn.Sequential(nn.Linear(input_numerical_size, numeraical_linear_size),\n                                               nn.LayerNorm(numeraical_linear_size))\n        \n        self.rnn = nn.GRU(numeraical_linear_size, \n                          model_size,\n                          num_layers = 2, \n                          batch_first=True,\n                          bidirectional=True)\n        \n        self.linear_out  = nn.Sequential(nn.Linear(model_size*2, linear_out),\n                                         nn.LayerNorm(linear_out),\n                                         nn.ReLU(),\n                                         nn.Dropout(dropout),\n                                         # nn.Linear(linear_out, out_size)\n                                         )\n        self.out1 = nn.Linear(linear_out, out_size)\n        self.out2 = nn.Linear(linear_out, out_size)\n        \n        \n        self._reinitialize()\n        \n        \n        \n    def _reinitialize(self):\n        \"\"\"Tensorflow/Keras-like initialization\"\"\"\n        for name, p in self.named_parameters():\n            if 'rnn' in name:\n                if 'weight_ih' in name:\n                    nn.init.xavier_uniform_(p.data)\n                elif 'weight_hh' in name:\n                    nn.init.orthogonal_(p.data)\n                elif 'bias_ih' in name:\n                    p.data.fill_(0)\n                    # Set forget-gate bias to 1\n                    n = p.size(0)\n                    p.data[(n // 4):(n // 2)].fill_(1)\n                elif 'bias_hh' in name:\n                    p.data.fill_(0)\n        \n        \n        \n    def forward(self, numerical_array, mask_array, attention_mask):\n        numerical_embedding = self.numerical_linear(numerical_array)\n        output, _ = self.rnn(numerical_embedding)\n        output = self.linear_out(output)\n        # return output\n        \n        output1 = self.out1(output)\n        output2 = self.out2(output)\n        return output1, output2","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:48.100728Z","iopub.execute_input":"2023-07-03T07:54:48.101476Z","iopub.status.idle":"2023-07-03T07:54:48.114723Z","shell.execute_reply.started":"2023-07-03T07:54:48.101445Z","shell.execute_reply":"2023-07-03T07:54:48.114113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### class `DefogRnnModel()`","metadata":{}},{"cell_type":"code","source":"class DefogRnnModel(nn.Module):\n    def __init__(self, \n                 dropout=0.2,\n                 input_numerical_size=9,\n                 numeraical_linear_size = 64,\n                 model_size = 128,\n                 linear_out = 128,\n                 out_size=3):\n        \n        super(DefogRnnModel, self).__init__()\n        \n        self.numerical_linear  = nn.Sequential(nn.Linear(input_numerical_size, numeraical_linear_size),\n                                               nn.LayerNorm(numeraical_linear_size))\n        \n        self.rnn = nn.GRU(numeraical_linear_size, \n                          model_size,\n                          num_layers = 2, \n                          batch_first=True,\n                          bidirectional=True)\n        \n        self.linear_out  = nn.Sequential(nn.Linear(model_size*2, linear_out),\n                                         nn.LayerNorm(linear_out),\n                                         nn.ReLU(),\n                                         nn.Dropout(dropout),\n                                         nn.Linear(linear_out, out_size))\n        self._reinitialize()\n        \n        \n        \n    def _reinitialize(self):\n        \"\"\"Tensorflow/Keras-like initialization\"\"\"\n        for name, p in self.named_parameters():\n            if 'rnn' in name:\n                if 'weight_ih' in name:\n                    nn.init.xavier_uniform_(p.data)\n                elif 'weight_hh' in name:\n                    nn.init.orthogonal_(p.data)\n                elif 'bias_ih' in name:\n                    p.data.fill_(0)\n                    # Set forget-gate bias to 1\n                    n = p.size(0)\n                    p.data[(n // 4):(n // 2)].fill_(1)\n                elif 'bias_hh' in name:\n                    p.data.fill_(0)\n        \n        \n        \n    def forward(self, numerical_array, mask_array, attention_mask):\n        numerical_embedding = self.numerical_linear(numerical_array)\n        output, _ = self.rnn(numerical_embedding)\n        output = self.linear_out(output)\n        return output","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:48.116191Z","iopub.execute_input":"2023-07-03T07:54:48.116949Z","iopub.status.idle":"2023-07-03T07:54:48.129181Z","shell.execute_reply.started":"2023-07-03T07:54:48.116918Z","shell.execute_reply":"2023-07-03T07:54:48.128357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### class `DefogRnnModel2()`","metadata":{}},{"cell_type":"code","source":"class DefogRnnModel2(nn.Module):\n    def __init__(self, \n                 dropout=0.2,\n                 input_numerical_size=9,\n                 numeraical_linear_size = 96,\n                 model_size = 256,\n                 linear_out = 256,\n                 out_size=3):\n        \n        super(DefogRnnModel2, self).__init__()\n        \n        self.numerical_linear  = nn.Sequential(nn.Linear(input_numerical_size, numeraical_linear_size),\n                                               nn.LayerNorm(numeraical_linear_size))\n        \n        self.rnn = nn.GRU(numeraical_linear_size, \n                          model_size,\n                          num_layers = 2, \n                          batch_first=True,\n                          bidirectional=True)\n        \n        self.linear_out  = nn.Sequential(nn.Linear(model_size*2, linear_out),\n                                         nn.LayerNorm(linear_out),\n                                         nn.ReLU(),\n                                         nn.Dropout(dropout),\n                                         nn.Linear(linear_out, out_size))\n        self._reinitialize()\n        \n        \n        \n    def _reinitialize(self):\n        \"\"\"Tensorflow/Keras-like initialization\"\"\"\n        for name, p in self.named_parameters():\n            if 'rnn' in name:\n                if 'weight_ih' in name:\n                    nn.init.xavier_uniform_(p.data)\n                elif 'weight_hh' in name:\n                    nn.init.orthogonal_(p.data)\n                elif 'bias_ih' in name:\n                    p.data.fill_(0)\n                    # Set forget-gate bias to 1\n                    n = p.size(0)\n                    p.data[(n // 4):(n // 2)].fill_(1)\n                elif 'bias_hh' in name:\n                    p.data.fill_(0)\n        \n        \n        \n    def forward(self, numerical_array, mask_array, attention_mask):\n        numerical_embedding = self.numerical_linear(numerical_array)\n        output, _ = self.rnn(numerical_embedding)\n        output = self.linear_out(output)\n        return output","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:48.130685Z","iopub.execute_input":"2023-07-03T07:54:48.131536Z","iopub.status.idle":"2023-07-03T07:54:48.143795Z","shell.execute_reply.started":"2023-07-03T07:54:48.131484Z","shell.execute_reply":"2023-07-03T07:54:48.142984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### class `Defog3Model()`","metadata":{}},{"cell_type":"code","source":"class Defog3Model(nn.Module):\n    def __init__(self, \n                 dropout=0.2,\n                 input_numerical_size=9,\n                 numeraical_linear_size = 64,\n                 model_size = 128,\n                 linear_out = 128,\n                 out_size=3):\n        \n        super(Defog3Model, self).__init__()\n        \n        self.numerical_linear  = nn.Sequential(nn.Linear(input_numerical_size, numeraical_linear_size),\n                                               nn.LayerNorm(numeraical_linear_size))\n        \n        self.lstm = nn.GRU(numeraical_linear_size, \n                          model_size,\n                          num_layers = 2, \n                          batch_first=True,\n                          bidirectional=True)\n        \n        self.linear_out  = nn.Sequential(nn.Linear(model_size*2, linear_out),\n                                         nn.LayerNorm(linear_out),\n                                         nn.ReLU(),\n                                         nn.Dropout(dropout),\n                                         nn.Linear(linear_out, out_size))\n        self._reinitialize()\n        \n        \n        \n    def _reinitialize(self):\n        \"\"\"Tensorflow/Keras-like initialization\"\"\"\n        for name, p in self.named_parameters():\n            if 'lstm' in name:\n                if 'weight_ih' in name:\n                    nn.init.xavier_uniform_(p.data)\n                elif 'weight_hh' in name:\n                    nn.init.orthogonal_(p.data)\n                elif 'bias_ih' in name:\n                    p.data.fill_(0)\n                    # Set forget-gate bias to 1\n                    n = p.size(0)\n                    p.data[(n // 4):(n // 2)].fill_(1)\n                elif 'bias_hh' in name:\n                    p.data.fill_(0)\n        \n        \n        \n    def forward(self, numerical_array, mask_array, attention_mask):\n        numerical_embedding = self.numerical_linear(numerical_array)\n        output, _ = self.rnn(numerical_embedding)\n        output = self.linear_out(output)\n        return output","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:48.145085Z","iopub.execute_input":"2023-07-03T07:54:48.145801Z","iopub.status.idle":"2023-07-03T07:54:48.158436Z","shell.execute_reply.started":"2023-07-03T07:54:48.145675Z","shell.execute_reply":"2023-07-03T07:54:48.157563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### def `make_pred()`","metadata":{}},{"cell_type":"code","source":"def make_pred(test_loader, model):\n    test_preds = []\n    # tk0 = tqdm(test_loader, total=len(test_loader), desc=\"test_loader: \")\n    with torch.no_grad():  # Do not calculate gradient since we are only predicting\n        # Predicting on validation set\n        # for d in tk0:\n        for d in test_loader:\n            input_data_numerical_array = d['input_data_numerical_array'].to(device)\n            input_data_mask_array      = d['input_data_mask_array'].to(device)\n            attention_mask             = d['attention_mask'].to(device)\n            output = model(input_data_numerical_array, \n                           input_data_mask_array,\n                           attention_mask)\n            test_preds.append(output.sigmoid().cpu().numpy())\n    test_preds = np.concatenate(test_preds, axis=0)\n    return test_preds","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:48.159474Z","iopub.execute_input":"2023-07-03T07:54:48.160016Z","iopub.status.idle":"2023-07-03T07:54:48.171797Z","shell.execute_reply.started":"2023-07-03T07:54:48.159966Z","shell.execute_reply":"2023-07-03T07:54:48.171178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### def `make_pred2()`","metadata":{}},{"cell_type":"code","source":"def make_pred2(test_loader, model):\n    test_preds = []\n    # tk0 = tqdm(test_loader, total=len(test_loader), desc=\"test_loader: \")\n    with torch.no_grad():  # Do not calculate gradient since we are only predicting\n        # Predicting on validation set\n        # for d in tk0:\n        for d in test_loader:\n            input_data_numerical_array = d['input_data_numerical_array'].to(device)\n            input_data_mask_array      = d['input_data_mask_array'].to(device)\n            attention_mask             = d['attention_mask'].to(device)\n            \n            output, _ = model(input_data_numerical_array, \n                             input_data_mask_array,\n                             attention_mask)\n            test_preds.append(output.sigmoid().cpu().numpy())\n    test_preds = np.concatenate(test_preds, axis=0)\n    return test_preds","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:48.172868Z","iopub.execute_input":"2023-07-03T07:54:48.173407Z","iopub.status.idle":"2023-07-03T07:54:48.182351Z","shell.execute_reply.started":"2023-07-03T07:54:48.173374Z","shell.execute_reply":"2023-07-03T07:54:48.181550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n## <font color=red>tdcsfog models</font>\n\n### checkpoint paths","metadata":{}},{"cell_type":"code","source":"# tdcsfog_path1   = [f\"/kaggle/input/fog-ex143/ex143_{i}.pth\" for i in range(5)] # len 3000 cv TdcsfogRnnModel\n# tdcsfog_path3_1 = [f\"/kaggle/input/fog-ex145/ex145_{i}.pth\" for i in range(5)] # len 3000 TdcsfogRnnModel  \n# tdcsfog_path3_2 = [f\"/kaggle/input/fog-ex146/ex146_{i}.pth\" for i in range(5)] # len 3000 TdcsfogRnnModel \n# tdcsfog_path3_3 = [f\"/kaggle/input/fog-ex147/ex147_{i}.pth\" for i in range(5)] # len 3000 TdcsfogRnnModel\n# tdcsfog_path4_1 = [f\"/kaggle/input/fog-ex182/ex182_{i}.pth\" for i in range(5)] # len 3000 TdcsfogRnnModel  \n# tdcsfog_path4_2 = [f\"/kaggle/input/fog-ex183/ex183_{i}.pth\" for i in range(5)] # len 3000 TdcsfogRnnModel \n# tdcsfog_path4_3 = [f\"/kaggle/input/fog-ex184/ex184_{i}.pth\" for i in range(5)] # len 3000 TdcsfogRnnModel","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:48.183387Z","iopub.execute_input":"2023-07-03T07:54:48.184357Z","iopub.status.idle":"2023-07-03T07:54:48.195697Z","shell.execute_reply.started":"2023-07-03T07:54:48.184324Z","shell.execute_reply":"2023-07-03T07:54:48.195073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdcsfog_path1     = glob.glob(\"/kaggle/input/parkinson-fog-prediction/ex143*\")\ntdcsfog_path3_1   = glob.glob(\"/kaggle/input/parkinson-fog-prediction/ex145*\")\ntdcsfog_path3_2   = glob.glob(\"/kaggle/input/parkinson-fog-prediction/ex146*\")\ntdcsfog_path3_3   = glob.glob(\"/kaggle/input/parkinson-fog-prediction/ex147*\")\ntdcsfog_path4_1   = glob.glob(\"/kaggle/input/parkinson-fog-prediction/ex182*\")\ntdcsfog_path4_2   = glob.glob(\"/kaggle/input/parkinson-fog-prediction/ex183*\")\ntdcsfog_path4_3   = glob.glob(\"/kaggle/input/parkinson-fog-prediction/ex184*\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:48.196736Z","iopub.execute_input":"2023-07-03T07:54:48.197724Z","iopub.status.idle":"2023-07-03T07:54:48.221799Z","shell.execute_reply.started":"2023-07-03T07:54:48.197692Z","shell.execute_reply":"2023-07-03T07:54:48.220890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### **ex143** (tdcsfog1: 0.2)\n\n\n#### load checkpoint","metadata":{}},{"cell_type":"code","source":"# TdcsfogRnnModel()\n\ntdcsfog_model_list1 = []\nfor i in tdcsfog_path1:  # ex143\n    model = TdcsfogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    tdcsfog_model_list1.append(model)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:48.222927Z","iopub.execute_input":"2023-07-03T07:54:48.223423Z","iopub.status.idle":"2023-07-03T07:54:54.835111Z","shell.execute_reply.started":"2023-07-03T07:54:48.223390Z","shell.execute_reply":"2023-07-03T07:54:54.834063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### predict","metadata":{}},{"cell_type":"code","source":"%%time\n\n# =========================\n# tdcsfog1\n# =========================\nth_len = 5000\nw = 0.20\n\ncols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = ['AccV', 'AccML', 'AccAP', \n            'AccV_lag_diff',  'AccV_lead_diff', 'AccV_cumsum', \n            'AccML_lag_diff', 'AccML_lead_diff', 'AccML_cumsum', \n            'AccAP_lag_diff', 'AccAP_lead_diff', 'AccAP_cumsum']\n\n# tdcsfog_list = glob.glob(TDCSFOG_DATA_PATH)\n# for p in tqdm(tdcsfog_list):\nfor p in tqdm(test_tdcsfog, desc=\"test_tdcsfog: \"):\n    # id_values = p.split(\"/\")[-1].split(\".\")[0]\n    id_values = p.split(\"/\")[-1].split(\".\")[0].split(\"\\\\\")[-1]\n    df = pd.read_csv(p)\n    \n    if len(df) > th_len:\n        seq_len = 5000\n        shift = 2500\n        offset = 1250\n    else:\n        seq_len = 3000\n        shift = 1500\n        offset = 750\n        \n    batch = (len(df)-1) // shift\n    if batch == 0:\n        batch = 1\n    for c in cols:\n        df[f\"{c}_lag_diff\"] = df[c].diff()\n        df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n        df[f\"{c}_cumsum\"] = df[c].cumsum()\n    sc = RobustScaler()\n    df[num_cols] = sc.fit_transform(df[num_cols].values)\n    df[num_cols] = df[num_cols].fillna(0)\n    num = df[num_cols].values\n    time = df[\"Time\"].values\n    \n    num_array = np.zeros([batch,seq_len, 12])\n    mask_array = np.zeros([batch,seq_len], dtype=int)\n    time_array = np.zeros([batch,seq_len], dtype=int)\n    pred_use_array = np.zeros([batch,seq_len], dtype=int)\n    \n    if len(df) <= seq_len:\n        b = 0\n        num_ = num.copy()\n        time_ = time.copy()\n        num_len = len(num_)\n\n        num_array[b,:num_len,:] = num_\n        time_array[b,:num_len] = time_\n        mask_array[b,:num_len] = 1\n        pred_use_array[b,:num_len] = 1\n    else:\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                time_ = time[b*shift : ]\n                num_len = len(num_)\n\n                num_array[b, :num_len, :] = num_\n                time_array[b,:num_len] = time_\n                mask_array[b,:num_len] = 1\n                pred_use_array[b, offset:num_len] = 1\n            elif b == 0:\n                num_ = num[b*shift : b*shift+seq_len]\n                time_ = time[b*shift : b*shift + seq_len]\n\n                num_array[b, :, :] = num_\n                time_array[b, :] = time_\n                mask_array[b, :] = 1\n                pred_use_array[b, :shift+offset] = 1\n            else:\n                num_ = num[b*shift : b*shift+seq_len]\n                time_ = time[b*shift : b*shift + seq_len]\n\n                num_array[b, :, :] = num_\n                time_array[b, :] = time_\n                mask_array[b, :] = 1\n                pred_use_array[b,offset : shift+offset] = 1\n            \n            \n\n    test_ = FogDataset(num_array, mask_array, train=False)\n    test_loader = DataLoader(dataset=test_, batch_size=bs, shuffle = False)\n    for n,m in enumerate(tdcsfog_model_list1):\n        if n == 0:\n            pred = make_pred(test_loader,m) / len(tdcsfog_model_list1)\n        else:\n            pred += make_pred(test_loader,m) / len(tdcsfog_model_list1)\n    pred_list = []\n    for i in range(batch):\n        mask_ = pred_use_array[i]\n        pred_ = pred[i, mask_ == 1, :]\n        time_ = time_array[i, mask_ == 1]\n        \n        df_ = pd.DataFrame()\n        df_[\"StartHesitation\"] = pred_[:, 0] * w\n        df_[\"Turn\"] = pred_[:, 1] * w\n        df_[\"Walking\"] = pred_[:, 2] * w\n        df_[\"Time\"] = time_\n        df_[\"Id\"] = id_values\n        df_[\"Id\"] = df_[\"Id\"].astype(str) + \"_\" + df_[\"Time\"].astype(str)\n        \n        pred_list.append(df_)\n    pred = pd.concat(pred_list).reset_index(drop=True)\n    df_all.append(pred)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-03T07:54:54.838998Z","iopub.execute_input":"2023-07-03T07:54:54.839330Z","iopub.status.idle":"2023-07-03T07:54:55.316198Z","shell.execute_reply.started":"2023-07-03T07:54:54.839303Z","shell.execute_reply":"2023-07-03T07:54:55.314964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### **ex145, ex146, ex147** (tdcsfog3: 0.4)\n\n#### load checkpoint","metadata":{}},{"cell_type":"code","source":"# TdcsfogRnnModel()\n\ntdcsfog_model_list3_1 = []\nfor i in tdcsfog_path3_1:  # ex145\n    model = TdcsfogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    tdcsfog_model_list3_1.append(model)\n\n\ntdcsfog_model_list3_2 = []\nfor i in tdcsfog_path3_2:  # ex146\n    model = TdcsfogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    tdcsfog_model_list3_2.append(model)\n\n\ntdcsfog_model_list3_3 = []\nfor i in tdcsfog_path3_3:  # ex147\n    model = TdcsfogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    tdcsfog_model_list3_3.append(model)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:55.317899Z","iopub.execute_input":"2023-07-03T07:54:55.318186Z","iopub.status.idle":"2023-07-03T07:54:55.926657Z","shell.execute_reply.started":"2023-07-03T07:54:55.318159Z","shell.execute_reply":"2023-07-03T07:54:55.925660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### predict","metadata":{}},{"cell_type":"code","source":"%%time\n\n# =========================\n# tdcsfog3\n# =========================\nth_len = 5000\nw = 0.40\n\ncols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = ['AccV', 'AccML', 'AccAP', \n            'AccV_lag_diff',  'AccV_lead_diff', 'AccV_cumsum', \n            'AccML_lag_diff', 'AccML_lead_diff', 'AccML_cumsum', \n            'AccAP_lag_diff', 'AccAP_lead_diff', 'AccAP_cumsum']\n\n# tdcsfog_list = glob.glob(TDCSFOG_DATA_PATH)\n# for p in tqdm(tdcsfog_list):\nfor p in tqdm(test_tdcsfog, desc=\"test_tdcsfog: \"):\n    # id_values = p.split(\"/\")[-1].split(\".\")[0]\n    id_values = p.split(\"/\")[-1].split(\".\")[0].split(\"\\\\\")[-1]\n    df = pd.read_csv(p)\n    \n    if len(df) > th_len:\n        seq_len = 5000\n        shift = 2500\n        offset = 1250\n    else:\n        seq_len = 3000\n        shift = 1500\n        offset = 750\n        \n    batch = (len(df)-1) // shift\n    if batch == 0:\n        batch = 1\n    for c in cols:\n        df[f\"{c}_lag_diff\"] = df[c].diff()\n        df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n        df[f\"{c}_cumsum\"] = df[c].cumsum()\n    sc = RobustScaler()\n    df[num_cols] = sc.fit_transform(df[num_cols].values)\n    df[num_cols] = df[num_cols].fillna(0)\n    num = df[num_cols].values\n    time = df[\"Time\"].values\n    \n    num_array = np.zeros([batch,seq_len, 12])\n    mask_array = np.zeros([batch,seq_len], dtype=int)\n    time_array = np.zeros([batch,seq_len], dtype=int)\n    pred_use_array = np.zeros([batch,seq_len], dtype=int)\n    \n    if len(df) <= seq_len:\n        b = 0\n        num_ = num.copy()\n        time_ = time.copy()\n        num_len = len(num_)\n\n        num_array[b,:num_len,:] = num_\n        time_array[b,:num_len] = time_\n        mask_array[b,:num_len] = 1\n        pred_use_array[b,:num_len] = 1\n    else:\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                time_ = time[b*shift : ]\n                num_len = len(num_)\n\n                num_array[b, :num_len, :] = num_\n                time_array[b,:num_len] = time_\n                mask_array[b,:num_len] = 1\n                pred_use_array[b, offset:num_len] = 1\n            elif b == 0:\n                num_ = num[b*shift : b*shift+seq_len]\n                time_ = time[b*shift : b*shift + seq_len]\n\n                num_array[b, :, :] = num_\n                time_array[b, :] = time_\n                mask_array[b, :] = 1\n                pred_use_array[b, :shift+offset] = 1\n            else:\n                num_ = num[b*shift : b*shift+seq_len]\n                time_ = time[b*shift : b*shift + seq_len]\n\n                num_array[b, :, :] = num_\n                time_array[b, :] = time_\n                mask_array[b, :] = 1\n                pred_use_array[b,offset : shift+offset] = 1\n            \n            \n\n    test_ = FogDataset(num_array, mask_array, train=False)\n    test_loader = DataLoader(dataset=test_, batch_size=bs, shuffle = False)\n    for n,m in enumerate(tdcsfog_model_list3_1):\n        if n == 0:\n            pred1 = make_pred(test_loader,m) / len(tdcsfog_model_list3_1)\n        else:\n            pred1 += make_pred(test_loader,m) / len(tdcsfog_model_list3_1)\n    for n,m in enumerate(tdcsfog_model_list3_2):\n        if n == 0:\n            pred2 = make_pred(test_loader,m) / len(tdcsfog_model_list3_2)\n        else:\n            pred2 += make_pred(test_loader,m) / len(tdcsfog_model_list3_2)\n    for n,m in enumerate(tdcsfog_model_list3_3):\n        if n == 0:\n            pred3 = make_pred(test_loader,m) / len(tdcsfog_model_list3_3)\n        else:\n            pred3 += make_pred(test_loader,m) / len(tdcsfog_model_list3_3)\n    pred = pred1.copy()\n    pred[:,:,1] = pred2[:,:,1]\n    pred[:,:,2] = pred3[:,:,2]\n    \n    \n    pred_list = []\n    for i in range(batch):\n        mask_ = pred_use_array[i]\n        pred_ = pred[i, mask_ == 1, :]\n        time_ = time_array[i, mask_ == 1]\n        \n        df_ = pd.DataFrame()\n        df_[\"StartHesitation\"] = pred_[:, 0] * w\n        df_[\"Turn\"] = pred_[:, 1] * w\n        df_[\"Walking\"] = pred_[:, 2] * w\n        df_[\"Time\"] = time_\n        df_[\"Id\"] = id_values\n        df_[\"Id\"] = df_[\"Id\"].astype(str) + \"_\" + df_[\"Time\"].astype(str)\n        \n        pred_list.append(df_)\n    pred = pd.concat(pred_list).reset_index(drop=True)\n    df_all.append(pred)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-03T07:54:55.930189Z","iopub.execute_input":"2023-07-03T07:54:55.930560Z","iopub.status.idle":"2023-07-03T07:54:56.333886Z","shell.execute_reply.started":"2023-07-03T07:54:55.930534Z","shell.execute_reply":"2023-07-03T07:54:56.332905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### **ex182, ex183, ex184** (tdcsfog4: 0.4)\n\n#### load checkpoint","metadata":{}},{"cell_type":"code","source":"# TdcsfogRnnModel()\n\ntdcsfog_model_list4_1 = []\nfor i in tdcsfog_path4_1:  # ex182\n    model = TdcsfogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    tdcsfog_model_list4_1.append(model)\n\n\ntdcsfog_model_list4_2 = []\nfor i in tdcsfog_path4_2:  # ex183\n    model = TdcsfogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    tdcsfog_model_list4_2.append(model)\n\n\ntdcsfog_model_list4_3 = []\nfor i in tdcsfog_path4_3:  # ex184\n    model = TdcsfogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    tdcsfog_model_list4_3.append(model)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:56.335289Z","iopub.execute_input":"2023-07-03T07:54:56.336339Z","iopub.status.idle":"2023-07-03T07:54:57.005583Z","shell.execute_reply.started":"2023-07-03T07:54:56.336303Z","shell.execute_reply":"2023-07-03T07:54:57.004510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### predict","metadata":{}},{"cell_type":"code","source":"%%time\n\n# =========================\n# tdcsfog4\n# =========================\nth_len = 5000\nw = 0.40\n\ncols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = ['AccV', 'AccML', 'AccAP', \n            'AccV_lag_diff',  'AccV_lead_diff', 'AccV_cumsum', \n            'AccML_lag_diff', 'AccML_lead_diff', 'AccML_cumsum', \n            'AccAP_lag_diff', 'AccAP_lead_diff', 'AccAP_cumsum']\n\n# tdcsfog_list = glob.glob(TDCSFOG_DATA_PATH)\n# for p in tqdm(tdcsfog_list):\nfor p in tqdm(test_tdcsfog, desc=\"test_tdcsfog: \"):\n    # id_values = p.split(\"/\")[-1].split(\".\")[0]\n    id_values = p.split(\"/\")[-1].split(\".\")[0].split(\"\\\\\")[-1]\n    df = pd.read_csv(p)\n    \n    if len(df) > th_len:\n        seq_len = 5000\n        shift = 2500\n        offset = 1250\n    else:\n        seq_len = 3000\n        shift = 1500\n        offset = 750\n        \n    batch = (len(df)-1) // shift\n    if batch == 0:\n        batch = 1\n    for c in cols:\n        df[f\"{c}_lag_diff\"] = df[c].diff()\n        df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n        df[f\"{c}_cumsum\"] = df[c].cumsum()\n    sc = RobustScaler()\n    df[num_cols] = sc.fit_transform(df[num_cols].values)\n    df[num_cols] = df[num_cols].fillna(0)\n    num = df[num_cols].values\n    time = df[\"Time\"].values\n    \n    num_array = np.zeros([batch,seq_len, 12])\n    mask_array = np.zeros([batch,seq_len], dtype=int)\n    time_array = np.zeros([batch,seq_len], dtype=int)\n    pred_use_array = np.zeros([batch,seq_len], dtype=int)\n    \n    if len(df) <= seq_len:\n        b = 0\n        num_ = num.copy()\n        time_ = time.copy()\n        num_len = len(num_)\n\n        num_array[b,:num_len,:] = num_\n        time_array[b,:num_len] = time_\n        mask_array[b,:num_len] = 1\n        pred_use_array[b,:num_len] = 1\n    else:\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                time_ = time[b*shift : ]\n                num_len = len(num_)\n\n                num_array[b, :num_len, :] = num_\n                time_array[b,:num_len] = time_\n                mask_array[b,:num_len] = 1\n                pred_use_array[b, offset:num_len] = 1\n            elif b == 0:\n                num_ = num[b*shift : b*shift+seq_len]\n                time_ = time[b*shift : b*shift + seq_len]\n\n                num_array[b, :, :] = num_\n                time_array[b, :] = time_\n                mask_array[b, :] = 1\n                pred_use_array[b, :shift+offset] = 1\n            else:\n                num_ = num[b*shift : b*shift+seq_len]\n                time_ = time[b*shift : b*shift + seq_len]\n\n                num_array[b, :, :] = num_\n                time_array[b, :] = time_\n                mask_array[b, :] = 1\n                pred_use_array[b,offset : shift+offset] = 1\n            \n            \n\n    test_ = FogDataset(num_array, mask_array, train=False)\n    test_loader = DataLoader(dataset=test_, batch_size=bs, shuffle = False)\n    for n,m in enumerate(tdcsfog_model_list4_1):\n        if n == 0:\n            pred1 = make_pred(test_loader,m) / len(tdcsfog_model_list4_1)\n        else:\n            pred1 += make_pred(test_loader,m) / len(tdcsfog_model_list4_1)\n    for n,m in enumerate(tdcsfog_model_list4_2):\n        if n == 0:\n            pred2 = make_pred(test_loader,m) / len(tdcsfog_model_list4_2)\n        else:\n            pred2 += make_pred(test_loader,m) / len(tdcsfog_model_list4_2)\n    for n,m in enumerate(tdcsfog_model_list4_3):\n        if n == 0:\n            pred3 = make_pred(test_loader,m) / len(tdcsfog_model_list4_3)\n        else:\n            pred3 += make_pred(test_loader,m) / len(tdcsfog_model_list4_3)\n    pred = pred1.copy()\n    pred[:,:,0] = pred1[:,:,0]*0.5 + pred2[:,:,0]*0.5\n    pred[:,:,1] = pred1[:,:,1]*0.5 + pred3[:,:,1]*0.5\n    pred[:,:,2] = pred2[:,:,2]*0.5 + pred3[:,:,2]*0.5\n    \n    \n    pred_list = []\n    for i in range(batch):\n        mask_ = pred_use_array[i]\n        pred_ = pred[i, mask_ == 1, :]\n        time_ = time_array[i, mask_ == 1]\n        \n        df_ = pd.DataFrame()\n        df_[\"StartHesitation\"] = pred_[:, 0] * w\n        df_[\"Turn\"] = pred_[:, 1] * w\n        df_[\"Walking\"] = pred_[:, 2] * w\n        df_[\"Time\"] = time_\n        df_[\"Id\"] = id_values\n        df_[\"Id\"] = df_[\"Id\"].astype(str) + \"_\" + df_[\"Time\"].astype(str)\n        \n        pred_list.append(df_)\n    pred = pd.concat(pred_list).reset_index(drop=True)\n    df_all.append(pred)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-03T07:54:57.007971Z","iopub.execute_input":"2023-07-03T07:54:57.008765Z","iopub.status.idle":"2023-07-03T07:54:57.416223Z","shell.execute_reply.started":"2023-07-03T07:54:57.008727Z","shell.execute_reply":"2023-07-03T07:54:57.415278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n## <font color=red>defog models</font>\n\n### checkpoint paths","metadata":{}},{"cell_type":"code","source":"# defog_path2 = [f\"/kaggle/input/fog-ex153/ex153_{i}.pth\" for i in range(5)] # len 30000 defog1 \n# defog_path4 = [f\"/kaggle/input/fog-ex179/ex179_{i}.pth\" for i in range(5)] # len 30000 defog1\n# defog_path5 = [f\"/kaggle/input/fog-ex185/ex185_{i}.pth\" for i in range(5)] # len 30000 defog1\n# defog_path6 = [f\"/kaggle/input/fog-ex204/ex204_{i}.pth\" for i in range(5)] # len 30000 defog2\n\n# defog_path7 = [f\"/kaggle/input/pd-exp238/fold{i}_best.pth\" for i in [0, 1, 2, 3, 4]]  # len 30000 Defog3Model","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:57.417581Z","iopub.execute_input":"2023-07-03T07:54:57.418674Z","iopub.status.idle":"2023-07-03T07:54:57.423740Z","shell.execute_reply.started":"2023-07-03T07:54:57.418633Z","shell.execute_reply":"2023-07-03T07:54:57.422507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog_path2 = glob.glob(\"/kaggle/input/parkinson-fog-prediction/ex153*\")\ndefog_path4 = glob.glob(\"/kaggle/input/parkinson-fog-prediction/ex179*\")\ndefog_path5 = glob.glob(\"/kaggle/input/parkinson-fog-prediction/ex185*\")\ndefog_path6 = glob.glob(\"/kaggle/input/parkinson-fog-prediction/ex204*\")\n\n# defog_path7 = glob.glob(\".\\\\output\\\\exp\\\\ex238_\\\\ex238_\\\\*\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:57.425459Z","iopub.execute_input":"2023-07-03T07:54:57.426222Z","iopub.status.idle":"2023-07-03T07:54:57.441550Z","shell.execute_reply.started":"2023-07-03T07:54:57.426169Z","shell.execute_reply":"2023-07-03T07:54:57.440330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### **ex153** (defog2: 0.35)\n\n#### load checkpoint","metadata":{}},{"cell_type":"code","source":"# DefogRnnModel()\n\ndefog_model_list2 = []\nfor i in defog_path2:         # ex153\n    model = DefogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    defog_model_list2.append(model)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:57.443115Z","iopub.execute_input":"2023-07-03T07:54:57.443703Z","iopub.status.idle":"2023-07-03T07:54:57.650183Z","shell.execute_reply.started":"2023-07-03T07:54:57.443667Z","shell.execute_reply":"2023-07-03T07:54:57.649259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### predict","metadata":{}},{"cell_type":"code","source":"# for p in train_defog:\n#     id_values = p.split(\"/\")[-1].split(\".\")[0].split(\"\\\\\")[-1]\n#     print(id_values)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:57.651631Z","iopub.execute_input":"2023-07-03T07:54:57.652431Z","iopub.status.idle":"2023-07-03T07:54:57.657047Z","shell.execute_reply.started":"2023-07-03T07:54:57.652395Z","shell.execute_reply":"2023-07-03T07:54:57.656083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# =========================\n# defog2\n# =========================\nth_len = 200000\nw = 0.35\n\ncols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = [\"AccV\", \"AccML\", \"AccAP\",\n            'AccV_lag_diff',  'AccV_lead_diff', \n            'AccML_lag_diff', 'AccML_lead_diff',\n            'AccAP_lag_diff', 'AccAP_lead_diff']\n\n# defog_list = glob.glob(DEFOG_DATA_PATH)\n# for p in tqdm(defog_list):\nfor p in tqdm(test_defog, desc=\"test_defog: \"):\n    # id_values = p.split(\"/\")[-1].split(\".\")[0]\n    id_values = p.split(\"/\")[-1].split(\".\")[0].split(\"\\\\\")[-1]\n    \n    df = pd.read_csv(p)\n    if len(df) > th_len:\n        seq_len = 30000\n        shift = 15000\n        offset = 7500\n    else:\n        seq_len = 15000\n        shift = 7500\n        offset = 3750\n    batch = (len(df)-1) // shift\n    if batch == 0:\n        batch = 1\n    for c in cols:\n        df[f\"{c}_lag_diff\"] = df[c].diff()\n        df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n    sc = StandardScaler()\n    df[num_cols] = sc.fit_transform(df[num_cols].values)\n    df[num_cols] = df[num_cols].fillna(0)\n    num = df[num_cols].values\n    time = df[\"Time\"].values\n    \n    num_array = np.zeros([batch,seq_len, 9])\n    mask_array = np.zeros([batch,seq_len], dtype=int)\n    time_array = np.zeros([batch,seq_len], dtype=int)\n    pred_use_array = np.zeros([batch,seq_len], dtype=int)\n    \n    if len(df) <= seq_len:\n        b = 0\n        num_len = len(num)\n        num_array[b,  :num_len, :] = num\n        time_array[b, :num_len] = time\n        mask_array[b, :num_len] = 1\n        pred_use_array[b, :num_len] = 1\n    else:\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                time_ = time[b*shift : ]\n                num_len = len(num_)\n\n                num_array[b,  :num_len, :] = num_\n                time_array[b, :num_len] = time_\n                mask_array[b, :num_len] = 1\n                pred_use_array[b, offset:num_len] = 1\n            elif b == 0:\n                num_ = num[b*shift : b*shift+seq_len]\n                time_ = time[b*shift : b*shift + seq_len]\n\n                num_array[b,  :, :] = num_\n                time_array[b, :] = time_\n                mask_array[b, :] = 1\n                pred_use_array[b,:shift+offset] = 1\n            else:\n                num_ = num[b*shift : b*shift+seq_len]\n                time_ = time[b*shift : b*shift + seq_len]\n\n                num_array[b, :, :] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,offset:shift+offset] = 1  \n    \n    test_ = FogDataset(num_array, mask_array, train=False)\n    test_loader = DataLoader(dataset=test_, batch_size=bs, shuffle = False)\n    for n,m in enumerate(defog_model_list2):\n        if n == 0:\n            pred = make_pred(test_loader,m) / len(defog_model_list2)\n        else:\n            pred += make_pred(test_loader,m) / len(defog_model_list2)\n    \n    pred_list = []\n    for i in range(batch):\n        mask_ = pred_use_array[i]\n        pred_ = pred[i,mask_ == 1,:]\n        time_ = time_array[i, mask_ == 1]\n        \n        df_ = pd.DataFrame()\n        df_[\"StartHesitation\"] = pred_[:,0] * w\n        df_[\"Turn\"] = pred_[:,1] * w\n        df_[\"Walking\"] = pred_[:,2] * w\n        df_[\"Time\"] = time_\n        df_[\"Id\"] = id_values\n        df_[\"Id\"] = df_[\"Id\"].astype(str) + \"_\" + df_[\"Time\"].astype(str)\n        \n        pred_list.append(df_)\n    pred = pd.concat(pred_list).reset_index(drop=True)\n    df_all.append(pred)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-03T07:54:57.658892Z","iopub.execute_input":"2023-07-03T07:54:57.659627Z","iopub.status.idle":"2023-07-03T07:54:59.944009Z","shell.execute_reply.started":"2023-07-03T07:54:57.659594Z","shell.execute_reply":"2023-07-03T07:54:59.943079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### **ex179** (defog4: 0.25)\n\n#### load checkpoint","metadata":{}},{"cell_type":"code","source":"defog_model_list4 = []\nfor i in defog_path4:       # ex179\n    model = DefogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    defog_model_list4.append(model)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:54:59.945660Z","iopub.execute_input":"2023-07-03T07:54:59.946368Z","iopub.status.idle":"2023-07-03T07:55:00.136898Z","shell.execute_reply.started":"2023-07-03T07:54:59.946330Z","shell.execute_reply":"2023-07-03T07:55:00.135840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### predict","metadata":{}},{"cell_type":"code","source":"%%time\n\n# =========================\n# defog4\n# =========================\nth_len = 200000\nw = 0.25\n\ncols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = [\"AccV\", \"AccML\", \"AccAP\",\n            'AccV_lag_diff',  'AccV_lead_diff', \n            'AccML_lag_diff', 'AccML_lead_diff',\n            'AccAP_lag_diff', 'AccAP_lead_diff']\n\n# defog_list = glob.glob(DEFOG_DATA_PATH)\n# for p in tqdm(defog_list):\nfor p in tqdm(test_defog, desc=\"test_defog: \"):\n    # id_values = p.split(\"/\")[-1].split(\".\")[0]\n    id_values = p.split(\"/\")[-1].split(\".\")[0].split(\"\\\\\")[-1]\n    \n    df = pd.read_csv(p)\n    if len(df) > th_len:\n        seq_len = 30000\n        shift = 15000\n        offset = 7500\n    else:\n        seq_len = 15000\n        shift = 7500\n        offset = 3750\n    batch = (len(df)-1) // shift\n    if batch == 0:\n        batch = 1\n    for c in cols:\n        df[f\"{c}_lag_diff\"] = df[c].diff()\n        df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n    sc = StandardScaler()\n    df[num_cols] = sc.fit_transform(df[num_cols].values)\n    df[num_cols] = df[num_cols].fillna(0)\n    num = df[num_cols].values\n    time = df[\"Time\"].values\n    \n    num_array = np.zeros([batch,seq_len, 9])\n    mask_array = np.zeros([batch,seq_len], dtype=int)\n    time_array = np.zeros([batch,seq_len], dtype=int)\n    pred_use_array = np.zeros([batch,seq_len], dtype=int)\n    \n    if len(df) <= seq_len:\n        b = 0\n        num_len = len(num)\n        num_array[b,  :num_len, :] = num\n        time_array[b, :num_len] = time\n        mask_array[b, :num_len] = 1\n        pred_use_array[b, :num_len] = 1\n    else:\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                time_ = time[b*shift : ]\n                num_len = len(num_)\n\n                num_array[b,  :num_len, :] = num_\n                time_array[b, :num_len] = time_\n                mask_array[b, :num_len] = 1\n                pred_use_array[b, offset:num_len] = 1\n            elif b == 0:\n                num_ = num[b*shift : b*shift+seq_len]\n                time_ = time[b*shift : b*shift + seq_len]\n\n                num_array[b,  :, :] = num_\n                time_array[b, :] = time_\n                mask_array[b, :] = 1\n                pred_use_array[b,:shift+offset] = 1\n            else:\n                num_ = num[b*shift : b*shift+seq_len]\n                time_ = time[b*shift : b*shift + seq_len]\n\n                num_array[b, :, :] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,offset:shift+offset] = 1  \n    \n    test_ = FogDataset(num_array, mask_array, train=False)\n    test_loader = DataLoader(dataset=test_, batch_size=bs, shuffle = False)\n    for n,m in enumerate(defog_model_list4):\n        if n == 0:\n            pred = make_pred(test_loader,m) / len(defog_model_list4)\n        else:\n            pred += make_pred(test_loader,m) / len(defog_model_list4)\n    \n    pred_list = []\n    for i in range(batch):\n        mask_ = pred_use_array[i]\n        pred_ = pred[i,mask_ == 1,:]\n        time_ = time_array[i, mask_ == 1]\n        \n        df_ = pd.DataFrame()\n        df_[\"StartHesitation\"] = pred_[:,0] * w\n        df_[\"Turn\"] = pred_[:,1] * w\n        df_[\"Walking\"] = pred_[:,2] * w\n        df_[\"Time\"] = time_\n        df_[\"Id\"] = id_values\n        df_[\"Id\"] = df_[\"Id\"].astype(str) + \"_\" + df_[\"Time\"].astype(str)\n        \n        pred_list.append(df_)\n    pred = pd.concat(pred_list).reset_index(drop=True)\n    df_all.append(pred)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-03T07:55:00.140812Z","iopub.execute_input":"2023-07-03T07:55:00.141408Z","iopub.status.idle":"2023-07-03T07:55:02.258227Z","shell.execute_reply.started":"2023-07-03T07:55:00.141379Z","shell.execute_reply":"2023-07-03T07:55:02.257373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### **ex185** (defog5: 0.25)\n\n#### load checkpoint","metadata":{}},{"cell_type":"code","source":"defog_model_list5 = []\nfor i in defog_path5:        # ex185\n    model = DefogRnnModel()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    defog_model_list5.append(model)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:55:02.259467Z","iopub.execute_input":"2023-07-03T07:55:02.260412Z","iopub.status.idle":"2023-07-03T07:55:02.494395Z","shell.execute_reply.started":"2023-07-03T07:55:02.260376Z","shell.execute_reply":"2023-07-03T07:55:02.493406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### predict","metadata":{}},{"cell_type":"code","source":"%%time\n\n# =========================\n# defog5\n# =========================\nth_len = 200000\nw = 0.25\n\ncols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = [\"AccV\", \"AccML\", \"AccAP\",\n            'AccV_lag_diff',  'AccV_lead_diff', \n            'AccML_lag_diff', 'AccML_lead_diff',\n            'AccAP_lag_diff', 'AccAP_lead_diff']\n\n# defog_list = glob.glob(DEFOG_DATA_PATH)\n# for p in tqdm(defog_list):\nfor p in tqdm(test_defog, desc=\"test_defog: \"):\n    # id_values = p.split(\"/\")[-1].split(\".\")[0]\n    id_values = p.split(\"/\")[-1].split(\".\")[0].split(\"\\\\\")[-1]\n    \n    df = pd.read_csv(p)\n    if len(df) > th_len:\n        seq_len = 30000\n        shift = 15000\n        offset = 7500\n    else:\n        seq_len = 15000\n        shift = 7500\n        offset = 3750\n    batch = (len(df)-1) // shift\n    if batch == 0:\n        batch = 1\n    for c in cols:\n        df[f\"{c}_lag_diff\"] = df[c].diff()\n        df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n    sc = StandardScaler()\n    df[num_cols] = sc.fit_transform(df[num_cols].values)\n    df[num_cols] = df[num_cols].fillna(0)\n    num = df[num_cols].values\n    time = df[\"Time\"].values\n    \n    num_array = np.zeros([batch,seq_len, 9])\n    mask_array = np.zeros([batch,seq_len], dtype=int)\n    time_array = np.zeros([batch,seq_len], dtype=int)\n    pred_use_array = np.zeros([batch,seq_len], dtype=int)\n    \n    if len(df) <= seq_len:\n        b = 0\n        num_len = len(num)\n        num_array[b,  :num_len, :] = num\n        time_array[b, :num_len] = time\n        mask_array[b, :num_len] = 1\n        pred_use_array[b, :num_len] = 1\n    else:\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                time_ = time[b*shift : ]\n                num_len = len(num_)\n\n                num_array[b,  :num_len, :] = num_\n                time_array[b, :num_len] = time_\n                mask_array[b, :num_len] = 1\n                pred_use_array[b, offset:num_len] = 1\n            elif b == 0:\n                num_ = num[b*shift : b*shift+seq_len]\n                time_ = time[b*shift : b*shift + seq_len]\n\n                num_array[b,  :, :] = num_\n                time_array[b, :] = time_\n                mask_array[b, :] = 1\n                pred_use_array[b,:shift+offset] = 1\n            else:\n                num_ = num[b*shift : b*shift+seq_len]\n                time_ = time[b*shift : b*shift + seq_len]\n\n                num_array[b, :, :] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,offset:shift+offset] = 1  \n    \n    test_ = FogDataset(num_array, mask_array, train=False)\n    test_loader = DataLoader(dataset=test_, batch_size=bs, shuffle = False)\n    for n,m in enumerate(defog_model_list5):\n        if n == 0:\n            pred = make_pred(test_loader,m) / len(defog_model_list5)\n        else:\n            pred += make_pred(test_loader,m) / len(defog_model_list5)\n    \n    pred_list = []\n    for i in range(batch):\n        mask_ = pred_use_array[i]\n        pred_ = pred[i,mask_ == 1,:]\n        time_ = time_array[i, mask_ == 1]\n        \n        df_ = pd.DataFrame()\n        df_[\"StartHesitation\"] = pred_[:,0] * w\n        df_[\"Turn\"] = pred_[:,1] * w\n        df_[\"Walking\"] = pred_[:,2] * w\n        df_[\"Time\"] = time_\n        df_[\"Id\"] = id_values\n        df_[\"Id\"] = df_[\"Id\"].astype(str) + \"_\" + df_[\"Time\"].astype(str)\n        \n        pred_list.append(df_)\n    pred = pd.concat(pred_list).reset_index(drop=True)\n    df_all.append(pred)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-03T07:55:02.495720Z","iopub.execute_input":"2023-07-03T07:55:02.496063Z","iopub.status.idle":"2023-07-03T07:55:04.780363Z","shell.execute_reply.started":"2023-07-03T07:55:02.496030Z","shell.execute_reply":"2023-07-03T07:55:04.779430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### **ex204** (defog6: 0.10)\n\n#### load checkpoint","metadata":{}},{"cell_type":"code","source":"# DefogRnnModel2()\n\ndefog_model_list6 = []\nfor i in defog_path6:           # ex204\n    model = DefogRnnModel2()\n    model.load_state_dict(torch.load(i))\n    model = model.to(device)\n    model.eval()\n    defog_model_list6.append(model)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:55:04.782013Z","iopub.execute_input":"2023-07-03T07:55:04.786679Z","iopub.status.idle":"2023-07-03T07:55:05.569461Z","shell.execute_reply.started":"2023-07-03T07:55:04.786638Z","shell.execute_reply":"2023-07-03T07:55:05.568498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### predict","metadata":{}},{"cell_type":"code","source":"%%time\n\n\n# =========================\n# defog6\n# =========================\nth_len = 200000\n# w = 0.10\nw = 0.15\n\ncols = [\"AccV\",\"AccML\",\"AccAP\"]\nnum_cols = [\"AccV\", \"AccML\", \"AccAP\",\n            'AccV_lag_diff',  'AccV_lead_diff', \n            'AccML_lag_diff', 'AccML_lead_diff',\n            'AccAP_lag_diff', 'AccAP_lead_diff']\n\n# defog_list = glob.glob(DEFOG_DATA_PATH)\n# for p in tqdm(defog_list):\nfor p in tqdm(test_defog, desc=\"test_defog: \"):\n    # id_values = p.split(\"/\")[-1].split(\".\")[0]\n    id_values = p.split(\"/\")[-1].split(\".\")[0].split(\"\\\\\")[-1]\n    \n    df = pd.read_csv(p)\n    if len(df) > th_len:\n        seq_len = 30000\n        shift = 15000\n        offset = 7500\n    else:\n        seq_len = 15000\n        shift = 7500\n        offset = 3750\n    batch = (len(df)-1) // shift\n    if batch == 0:\n        batch = 1\n    for c in cols:\n        df[f\"{c}_lag_diff\"] = df[c].diff()\n        df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n    sc = StandardScaler()\n    df[num_cols] = sc.fit_transform(df[num_cols].values)\n    df[num_cols] = df[num_cols].fillna(0)\n    num = df[num_cols].values\n    time = df[\"Time\"].values\n    \n    num_array = np.zeros([batch,seq_len, 9])\n    mask_array = np.zeros([batch,seq_len], dtype=int)\n    time_array = np.zeros([batch,seq_len], dtype=int)\n    pred_use_array = np.zeros([batch,seq_len], dtype=int)\n    \n    if len(df) <= seq_len:\n        b = 0\n        num_len = len(num)\n        num_array[b,  :num_len, :] = num\n        time_array[b, :num_len] = time\n        mask_array[b, :num_len] = 1\n        pred_use_array[b, :num_len] = 1\n    else:\n        for n,b in enumerate(range(batch)):\n            if b == (batch - 1):\n                num_ = num[b*shift : ]\n                time_ = time[b*shift : ]\n                num_len = len(num_)\n\n                num_array[b,  :num_len, :] = num_\n                time_array[b, :num_len] = time_\n                mask_array[b, :num_len] = 1\n                pred_use_array[b, offset:num_len] = 1\n            elif b == 0:\n                num_ = num[b*shift : b*shift+seq_len]\n                time_ = time[b*shift : b*shift + seq_len]\n\n                num_array[b,  :, :] = num_\n                time_array[b, :] = time_\n                mask_array[b, :] = 1\n                pred_use_array[b,:shift+offset] = 1\n            else:\n                num_ = num[b*shift : b*shift+seq_len]\n                time_ = time[b*shift : b*shift + seq_len]\n\n                num_array[b, :, :] = num_\n                time_array[b,:] = time_\n                mask_array[b,:] = 1\n                pred_use_array[b,offset:shift+offset] = 1  \n    \n    test_ = FogDataset(num_array, mask_array, train=False)\n    test_loader = DataLoader(dataset=test_, batch_size=bs, shuffle = False)\n    for n,m in enumerate(defog_model_list6):\n        if n == 0:\n            pred = make_pred(test_loader,m) / len(defog_model_list6)\n        else:\n            pred += make_pred(test_loader,m) / len(defog_model_list6)\n    \n    pred_list = []\n    for i in range(batch):\n        mask_ = pred_use_array[i]\n        pred_ = pred[i,mask_ == 1,:]\n        time_ = time_array[i, mask_ == 1]\n        \n        df_ = pd.DataFrame()\n        df_[\"StartHesitation\"] = pred_[:,0] * w\n        df_[\"Turn\"] = pred_[:,1] * w\n        df_[\"Walking\"] = pred_[:,2] * w\n        df_[\"Time\"] = time_\n        df_[\"Id\"] = id_values\n        df_[\"Id\"] = df_[\"Id\"].astype(str) + \"_\" + df_[\"Time\"].astype(str)\n        \n        pred_list.append(df_)\n    pred = pd.concat(pred_list).reset_index(drop=True)\n    df_all.append(pred)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-03T07:55:05.570745Z","iopub.execute_input":"2023-07-03T07:55:05.571091Z","iopub.status.idle":"2023-07-03T07:55:16.681485Z","shell.execute_reply.started":"2023-07-03T07:55:05.571058Z","shell.execute_reply":"2023-07-03T07:55:16.680349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n### **ex238** (defog7: 0.05)\n\n#### load checkpoint","metadata":{}},{"cell_type":"code","source":"# # Defog3Model()\n\n# defog_model_list7 = []\n# for path in defog_path7:    # ex238\n#     model = Defog3Model()\n#     state = torch.load(path, map_location=torch.device(\"cpu\"))\n#     model.load_state_dict(state[\"model\"])\n#     model = model.to(device)\n#     model.eval()\n#     defog_model_list7.append(model)\n#     print(f\"load weights from {path}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:55:16.683318Z","iopub.execute_input":"2023-07-03T07:55:16.684097Z","iopub.status.idle":"2023-07-03T07:55:16.689182Z","shell.execute_reply.started":"2023-07-03T07:55:16.684061Z","shell.execute_reply":"2023-07-03T07:55:16.688310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n\n#### predict","metadata":{}},{"cell_type":"code","source":"# %%time\n\n# # =========================\n# # defog7\n# # =========================\n# th_len = 200000\n# w = 0.05\n\n# cols = [\"AccV\",\"AccML\",\"AccAP\"]\n# num_cols = [\"AccV\", \"AccML\", \"AccAP\",\n#             'AccV_lag_diff',  'AccV_lead_diff', \n#             'AccML_lag_diff', 'AccML_lead_diff',\n#             'AccAP_lag_diff', 'AccAP_lead_diff']\n\n# # defog_list = glob.glob(DEFOG_DATA_PATH)\n# # for p in tqdm(defog_list):\n# for p in tqdm(test_defog, desc=\"test_defog: \"):\n#     # id_values = p.split(\"/\")[-1].split(\".\")[0]\n#     id_values = p.split(\"/\")[-1].split(\".\")[0].split(\"\\\\\")[-1]\n    \n#     df = pd.read_csv(p)\n#     if len(df) > th_len:\n#         seq_len = 30000\n#         shift = 15000\n#         offset = 7500\n#     else:\n#         seq_len = 15000\n#         shift = 7500\n#         offset = 3750\n#     batch = (len(df)-1) // shift\n#     if batch == 0:\n#         batch = 1\n#     for c in cols:\n#         df[f\"{c}_lag_diff\"] = df[c].diff()\n#         df[f\"{c}_lead_diff\"] = df[c].diff(-1)\n#     sc = StandardScaler()\n#     df[num_cols] = sc.fit_transform(df[num_cols].values)\n#     df[num_cols] = df[num_cols].fillna(0)\n#     num = df[num_cols].values\n#     time = df[\"Time\"].values\n    \n#     num_array = np.zeros([batch,seq_len, 9])\n#     mask_array = np.zeros([batch,seq_len], dtype=int)\n#     time_array = np.zeros([batch,seq_len], dtype=int)\n#     pred_use_array = np.zeros([batch,seq_len], dtype=int)\n    \n#     if len(df) <= seq_len:\n#         b = 0\n#         num_len = len(num)\n#         num_array[b,  :num_len, :] = num\n#         time_array[b, :num_len] = time\n#         mask_array[b, :num_len] = 1\n#         pred_use_array[b, :num_len] = 1\n#     else:\n#         for n,b in enumerate(range(batch)):\n#             if b == (batch - 1):\n#                 num_ = num[b*shift : ]\n#                 time_ = time[b*shift : ]\n#                 num_len = len(num_)\n\n#                 num_array[b,  :num_len, :] = num_\n#                 time_array[b, :num_len] = time_\n#                 mask_array[b, :num_len] = 1\n#                 pred_use_array[b, offset:num_len] = 1\n#             elif b == 0:\n#                 num_ = num[b*shift : b*shift+seq_len]\n#                 time_ = time[b*shift : b*shift + seq_len]\n\n#                 num_array[b,  :, :] = num_\n#                 time_array[b, :] = time_\n#                 mask_array[b, :] = 1\n#                 pred_use_array[b,:shift+offset] = 1\n#             else:\n#                 num_ = num[b*shift : b*shift+seq_len]\n#                 time_ = time[b*shift : b*shift + seq_len]\n\n#                 num_array[b, :, :] = num_\n#                 time_array[b,:] = time_\n#                 mask_array[b,:] = 1\n#                 pred_use_array[b,offset:shift+offset] = 1  \n    \n#     test_ = FogDataset(num_array, mask_array, train=False)\n#     test_loader = DataLoader(dataset=test_, batch_size=bs, shuffle = False)\n#     for n,m in enumerate(defog_model_list7):\n#         if n == 0:\n#             pred = make_pred(test_loader,m) / len(defog_model_list7)\n#         else:\n#             pred += make_pred(test_loader,m) / len(defog_model_list7)\n    \n#     pred_list = []\n#     for i in range(batch):\n#         mask_ = pred_use_array[i]\n#         pred_ = pred[i,mask_ == 1,:]\n#         time_ = time_array[i, mask_ == 1]\n        \n#         df_ = pd.DataFrame()\n#         df_[\"StartHesitation\"] = pred_[:,0] * w\n#         df_[\"Turn\"] = pred_[:,1] * w\n#         df_[\"Walking\"] = pred_[:,2] * w\n#         df_[\"Time\"] = time_\n#         df_[\"Id\"] = id_values\n#         df_[\"Id\"] = df_[\"Id\"].astype(str) + \"_\" + df_[\"Time\"].astype(str)\n        \n#         pred_list.append(df_)\n#     pred = pd.concat(pred_list).reset_index(drop=True)\n#     df_all.append(pred)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:55:16.692493Z","iopub.execute_input":"2023-07-03T07:55:16.692821Z","iopub.status.idle":"2023-07-03T07:55:16.706060Z","shell.execute_reply.started":"2023-07-03T07:55:16.692792Z","shell.execute_reply":"2023-07-03T07:55:16.705191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n<br>\n\n\n# **Submission** ","metadata":{}},{"cell_type":"code","source":"df_all = pd.concat(df_all).reset_index(drop=True)\ndf_all = df_all.groupby(by=\"Id\")[['StartHesitation', 'Turn', 'Walking']].sum().reset_index()\ndf_all[['Id', 'StartHesitation', 'Turn', 'Walking']].to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:55:16.707429Z","iopub.execute_input":"2023-07-03T07:55:16.707829Z","iopub.status.idle":"2023-07-03T07:55:19.167240Z","shell.execute_reply.started":"2023-07-03T07:55:16.707795Z","shell.execute_reply":"2023-07-03T07:55:19.166122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all","metadata":{"execution":{"iopub.status.busy":"2023-07-03T07:55:19.168768Z","iopub.execute_input":"2023-07-03T07:55:19.169130Z","iopub.status.idle":"2023-07-03T07:55:19.185558Z","shell.execute_reply.started":"2023-07-03T07:55:19.169095Z","shell.execute_reply":"2023-07-03T07:55:19.184521Z"},"trusted":true},"execution_count":null,"outputs":[]}]}