{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# EDA of Task & Subjects data (metadata) 🌈\n## [Objective]\n### - Load task data and cultivate a better understanding　using pandas, poilars and plotly, matplotlib\n    - 1. task\n    - 2. subjects\n    - 3. labeling task","metadata":{}},{"cell_type":"code","source":"# =========================\n# Import libraries\n# =========================\n# default\nimport gc, os, glob, random\nfrom os import path\nfrom pathlib import Path\n# make data\nimport polars as pl\nimport pandas as pd\npd.set_option('display.max_columns', None); # pd.set_option('display.max_rows', None)\nimport numpy as np\nfrom tqdm.auto import tqdm\nimport ydata_profiling as pdp","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-23T13:38:37.272792Z","iopub.execute_input":"2023-05-23T13:38:37.273182Z","iopub.status.idle":"2023-05-23T13:38:43.290131Z","shell.execute_reply.started":"2023-05-23T13:38:37.273152Z","shell.execute_reply":"2023-05-23T13:38:43.288287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_df(df, num=3, tail=True):\n    print(df.shape)\n    display(df.head(num))\n    if tail:\n        display(df.tail(num))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T13:38:43.292307Z","iopub.execute_input":"2023-05-23T13:38:43.292695Z","iopub.status.idle":"2023-05-23T13:38:43.301238Z","shell.execute_reply.started":"2023-05-23T13:38:43.292660Z","shell.execute_reply":"2023-05-23T13:38:43.299939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog_path = glob.glob(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/train/defog/*.csv\")\ntdcsfog_path = glob.glob(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/train/tdcsfog/*.csv\")\nnotype_path = glob.glob(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/train/notype/*.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-05-23T13:38:43.303142Z","iopub.execute_input":"2023-05-23T13:38:43.303796Z","iopub.status.idle":"2023-05-23T13:38:43.542837Z","shell.execute_reply.started":"2023-05-23T13:38:43.303749Z","shell.execute_reply":"2023-05-23T13:38:43.541907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Tasks","metadata":{}},{"cell_type":"code","source":"# ==============================================================================\n# Tasks - Task metadata for series in the defog dataset.(not tdcsfog & daily)-\n# ==============================================================================\ntasks    = pd.read_csv(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/tasks.csv\")\ntasks['Duration'] = tasks['End'] - tasks['Begin']\nprint(\"-\"*80);print(\"Tasks - Task metadata for series in the defog dataset.(not tdcsfog & daily)-\");print(\"-\"*80);show_df(tasks)\n","metadata":{"execution":{"iopub.status.busy":"2023-05-23T13:38:43.545536Z","iopub.execute_input":"2023-05-23T13:38:43.545911Z","iopub.status.idle":"2023-05-23T13:38:43.620105Z","shell.execute_reply.started":"2023-05-23T13:38:43.545880Z","shell.execute_reply":"2023-05-23T13:38:43.619260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tasks_pivot = pd.pivot_table(\n    tasks, \n    values=['Duration'], \n    index=['Id'], \n    columns=['Task'], \n    aggfunc='sum', \n    fill_value=0\n)\ntasks_pivot","metadata":{"execution":{"iopub.status.busy":"2023-05-23T13:38:43.621466Z","iopub.execute_input":"2023-05-23T13:38:43.621815Z","iopub.status.idle":"2023-05-23T13:38:43.735280Z","shell.execute_reply.started":"2023-05-23T13:38:43.621784Z","shell.execute_reply":"2023-05-23T13:38:43.733988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_defog_list = [os.path.basename(path).split(\".cs\")[0] for path in defog_path]\ntask_list = list(tasks.Id.unique())\n\nprint(f\"lentgh of train_defog_list: {len(train_defog_list)}\")\nprint(f\"lentgh of Task            : {len(task_list)}\")","metadata":{"execution":{"iopub.status.busy":"2023-05-23T13:38:43.736970Z","iopub.execute_input":"2023-05-23T13:38:43.737738Z","iopub.status.idle":"2023-05-23T13:38:43.747424Z","shell.execute_reply.started":"2023-05-23T13:38:43.737693Z","shell.execute_reply":"2023-05-23T13:38:43.746012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_defog_list = [path for path in task_list if path not in train_defog_list]\nprint(*test_defog_list)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T13:38:57.600622Z","iopub.execute_input":"2023-05-23T13:38:57.601080Z","iopub.status.idle":"2023-05-23T13:38:57.608960Z","shell.execute_reply.started":"2023-05-23T13:38:57.601043Z","shell.execute_reply":"2023-05-23T13:38:57.606916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_defog_table = tasks_pivot[tasks_pivot.index.isin(train_defog_list)]\ntest_defog_table = tasks_pivot[tasks_pivot.index.isin(test_defog_list)]","metadata":{"execution":{"iopub.status.busy":"2023-05-23T13:48:14.153833Z","iopub.execute_input":"2023-05-23T13:48:14.154212Z","iopub.status.idle":"2023-05-23T13:48:14.161503Z","shell.execute_reply.started":"2023-05-23T13:48:14.154176Z","shell.execute_reply":"2023-05-23T13:48:14.160217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def color_background_lightgreen(val):\n    color = 'lightgreen' if val > 1 else '' #1より大なら薄緑、その他は白\n    return 'background-color: %s' % color\n\n#表示\nprint(\"-\"*80);print(\"Tasks - train_defog_table-\");print(\"-\"*80);train_defog_table.style.applymap(color_background_lightgreen)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T13:48:38.676353Z","iopub.execute_input":"2023-05-23T13:48:38.676751Z","iopub.status.idle":"2023-05-23T13:48:38.843036Z","shell.execute_reply.started":"2023-05-23T13:48:38.676718Z","shell.execute_reply":"2023-05-23T13:48:38.841904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"-\"*80);print(\"Tasks - test_defog_table\");print(\"-\"*80);test_defog_table.style.applymap(color_background_lightgreen)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T13:41:09.928993Z","iopub.execute_input":"2023-05-23T13:41:09.929387Z","iopub.status.idle":"2023-05-23T13:41:10.021914Z","shell.execute_reply.started":"2023-05-23T13:41:09.929357Z","shell.execute_reply":"2023-05-23T13:41:10.020770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"-\"*80);print(\"Tasks - train_defog_table\");print(\"-\"*80);display(train_defog_table.describe())\nprint(\"-\"*80);print(\"Tasks - test_defog_table\");print(\"-\"*80);display(test_defog_table.describe())","metadata":{"execution":{"iopub.status.busy":"2023-05-23T13:49:56.562168Z","iopub.execute_input":"2023-05-23T13:49:56.562586Z","iopub.status.idle":"2023-05-23T13:49:56.863137Z","shell.execute_reply.started":"2023-05-23T13:49:56.562552Z","shell.execute_reply":"2023-05-23T13:49:56.861957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import pyplot as plt\n\n# 平均値の計算\nmean_values_train = train_defog_table.mean()\nmean_values_test = test_defog_table.mean()\n# サブプロットの作成\nfig, axes = plt.subplots(1, 2, figsize=(10, 5))\n\n# 左側のサブプロットに横棒グラフを描画\naxes[0].barh(mean_values_train[\"Duration\"].index, mean_values_train)\naxes[0].set_title('Duration -Mean Values Train-')\naxes[0].set_xlabel('Mean')\n\n# 右側のサブプロットに横棒グラフを描画\naxes[1].barh(mean_values_test[\"Duration\"].index, mean_values_test)\naxes[1].set_title('Duration -Mean Values Test-')\naxes[1].set_xlabel('Mean')\n\nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T14:02:50.873869Z","iopub.execute_input":"2023-05-23T14:02:50.874260Z","iopub.status.idle":"2023-05-23T14:02:51.965786Z","shell.execute_reply.started":"2023-05-23T14:02:50.874230Z","shell.execute_reply":"2023-05-23T14:02:51.964617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Subjects","metadata":{}},{"cell_type":"code","source":"# =========================================\n# Subjects -infomation about patients-\n# =========================================\nsubjects = pd.read_csv(\"../input/tlvmc-parkinsons-freezing-gait-prediction/subjects.csv\")\nsubjects.loc[subjects['Subject'] == 'fe5d84', 'Sex'] = 'F'\nprint(\"-\"*80);print(\"Subjects -infomation about patients-\");print(\"-\"*80);show_df(subjects)\nprint(\"YearsSinceDx: Years since Parkinson's diagnosis.\")\nprint(\"UPDRSIII_on/off: Unified Parkinson's Disease Rating Scale score during on/off medication respectively..\")\nprint(\"NFOGQ: Self-report FoG questionnaire score. See: https://pubmed.ncbi.nlm.nih.gov/19660949/.\")\nprint()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T14:03:53.017581Z","iopub.execute_input":"2023-05-23T14:03:53.018032Z","iopub.status.idle":"2023-05-23T14:03:53.063305Z","shell.execute_reply.started":"2023-05-23T14:03:53.017999Z","shell.execute_reply":"2023-05-23T14:03:53.062040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ===============================\n# defog_metadata (test at home)\n# ===============================\ndefog_metadata=pd.read_csv('../input/tlvmc-parkinsons-freezing-gait-prediction/defog_metadata.csv')\nprint(\"-\"*80);print(\"defog_metadata -test at home-\");print(\"-\"*80);show_df(defog_metadata)\nprint(f\"Visit: {defog_metadata.Visit.unique()}\")\nprint(f\"Medication:{defog_metadata.Medication.unique()}\")\nprint()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T14:04:51.566921Z","iopub.execute_input":"2023-05-23T14:04:51.567346Z","iopub.status.idle":"2023-05-23T14:04:51.600934Z","shell.execute_reply.started":"2023-05-23T14:04:51.567310Z","shell.execute_reply":"2023-05-23T14:04:51.599682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog_metadata_train = defog_metadata[defog_metadata[\"Id\"].isin(train_defog_list)]\ndefog_metadata_test  = defog_metadata[defog_metadata[\"Id\"].isin(test_defog_list)]\nsubjects_train = list(defog_metadata_train[\"Subject\"].unique())\nsubjects_test  = list(defog_metadata_test[\"Subject\"].unique())","metadata":{"execution":{"iopub.status.busy":"2023-05-23T14:14:48.113179Z","iopub.execute_input":"2023-05-23T14:14:48.113646Z","iopub.status.idle":"2023-05-23T14:14:48.124605Z","shell.execute_reply.started":"2023-05-23T14:14:48.113610Z","shell.execute_reply":"2023-05-23T14:14:48.122915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subjects_train = subjects[subjects[\"Subject\"].isin(subjects_train)].fillna(-1)\nsubjects_test = subjects[subjects[\"Subject\"].isin(subjects_test)].fillna(-1)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T14:14:50.127427Z","iopub.execute_input":"2023-05-23T14:14:50.127862Z","iopub.status.idle":"2023-05-23T14:14:50.136292Z","shell.execute_reply.started":"2023-05-23T14:14:50.127803Z","shell.execute_reply":"2023-05-23T14:14:50.135284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = [\"Visit\",\"Age\",\"Sex\",\"YearsSinceDx\",\"UPDRSIII_On\",\"UPDRSIII_Off\",\"NFOGQ\"]\n\n# サブプロットの作成\nfig, axes = plt.subplots(2, 4, figsize=(20, 6))\n\n# カラムごとにヒストグラムを描画\nfor i, col in enumerate(cols):\n    ax = axes[i // 4, i % 4]  # サブプロットの位置を指定\n    ax.hist(subjects_train[col], alpha=0.5, label='train', bins=10)\n    ax.hist(subjects_test[col], alpha=0.5, label='test', bins=10)\n    ax.set_title(col)  # カラム名をタイトルとして設定\n    ax.legend()  # 凡例を表示\n\n# サブプロット間のスペースを調整\nplt.tight_layout()\n\n# グラフの表示\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T14:15:37.908326Z","iopub.execute_input":"2023-05-23T14:15:37.908703Z","iopub.status.idle":"2023-05-23T14:15:39.963009Z","shell.execute_reply.started":"2023-05-23T14:15:37.908673Z","shell.execute_reply":"2023-05-23T14:15:39.962083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Labeling task","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(defog_path[1])\nprint(f\"Path: {defog_path[1]}\")\nshow_df(df)","metadata":{"execution":{"iopub.status.busy":"2023-05-17T01:32:40.687573Z","iopub.execute_input":"2023-05-17T01:32:40.688139Z","iopub.status.idle":"2023-05-17T01:32:40.853581Z","shell.execute_reply.started":"2023-05-17T01:32:40.688096Z","shell.execute_reply":"2023-05-17T01:32:40.851699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tasks[tasks['Id']=='4c3aa8ea6e']","metadata":{"execution":{"iopub.status.busy":"2023-05-16T10:48:27.574687Z","iopub.execute_input":"2023-05-16T10:48:27.574971Z","iopub.status.idle":"2023-05-16T10:48:27.587227Z","shell.execute_reply.started":"2023-05-16T10:48:27.574951Z","shell.execute_reply":"2023-05-16T10:48:27.586193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- tasks.csv Task metadata for series in the defog dataset. (Not relevant for the series in the tdcsfog or daily datasets.)\n\n    - Id The data series where the task was measured.\n    - Begin Time (s) the task began.\n    - End Time (s) the task ended.\n    - Task One of seven tasks types in the DeFOG protocol, described on this page.","metadata":{}},{"cell_type":"code","source":"from collections import defaultdict\ntask_dict = defaultdict(int) \ntask_list = list(tasks.Task.unique())\n\nfor i, _task in enumerate(task_list):\n    task_dict[_task] = i\n\nprint(task_dict)","metadata":{"execution":{"iopub.status.busy":"2023-05-17T01:28:11.720045Z","iopub.execute_input":"2023-05-17T01:28:11.720466Z","iopub.status.idle":"2023-05-17T01:28:11.728971Z","shell.execute_reply.started":"2023-05-17T01:28:11.720437Z","shell.execute_reply":"2023-05-17T01:28:11.727732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp_df = pd.read_csv(defog_path[1])\ntmp_task = tasks[tasks['Id']=='4c3aa8ea6e']\n\ntmp_df[\"Time\"] = tmp_df.index / 100 # -> sec\ntmp_df[\"Task_No\"] = -1 # initialization\n\nfor idx, row in tmp_task.iterrows():\n    _st, _ed = row[\"Begin\"], row[\"End\"]\n    _task = row[\"Task\"]\n    \n    # Add Task_No according to _st & _ed\n    tmp_df.loc[(tmp_df.Time >= _st) & (tmp_df.Time <= _ed), 'Task_No'] = task_dict[_task]","metadata":{"execution":{"iopub.status.busy":"2023-05-17T01:49:25.634777Z","iopub.execute_input":"2023-05-17T01:49:25.636060Z","iopub.status.idle":"2023-05-17T01:49:25.759605Z","shell.execute_reply.started":"2023-05-17T01:49:25.636021Z","shell.execute_reply":"2023-05-17T01:49:25.758391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(tmp_df.Task.min())\nprint(tmp_df.Task.max())\nshow_df(tmp_df)","metadata":{"execution":{"iopub.status.busy":"2023-05-17T01:49:26.753673Z","iopub.execute_input":"2023-05-17T01:49:26.754078Z","iopub.status.idle":"2023-05-17T01:49:26.784394Z","shell.execute_reply.started":"2023-05-17T01:49:26.754047Z","shell.execute_reply":"2023-05-17T01:49:26.783185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ==============================\n# 可視化して確認 (Plotly)\n# ==============================\n# plotly \nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nimport plotly.express as px\n\ndef show_defog_plotly(df, xcol, ycol, flags, st, length):\n    tmp_df = df.copy()\n    if (st !=None) | (length != None):\n        tmp_df = tmp_df[st: st + length].copy()\n\n    row_length = len(ycol)\n    flag_length = len(flags)\n\n    fig = make_subplots(\n        rows=row_length+1, cols=1,\n        shared_xaxes=True\n        )\n\n    for i in range(row_length):\n        fig.add_trace(go.Scatter(\n            x=tmp_df[xcol], y=tmp_df[ycol[i]], \n            name=ycol[i], \n            mode=\"lines\",\n            ), row=i+1, col=1)\n        \n    for j in range(flag_length):\n        fig.add_trace(go.Scatter(\n            x=tmp_df[xcol], y=tmp_df[flags[j]], \n            name=flags[j], \n            mode=\"lines\",\n            ), row=i+2, col=1)    \n            \n    # Update xaxis properties\n    fig.update_xaxes(title_text= xcol, row=row_length+1, col=1)\n\n    # Update yaxis properties\n    for i in range(row_length):\n        fig.update_yaxes(title_text=ycol[i], row=i+1, col=1)\n    fig.update_yaxes(title_text=\"flags\", row=i+2, col=1)\n    \n    fig.update_xaxes(rangeslider={\"visible\":True}, row=row_length+1, col=1) # X軸に range slider を表示（下図参照\n    fig.update_layout(title=\"Time Series Analysis\") # グラフタイトルを設定\n    fig.update_layout(font={\"family\":\"Meiryo\", \"size\":12}) # フォントファミリとフォントサイズを指定\n    fig.update_layout(showlegend=True) # 凡例を強制的に表示\n    fig.update_layout(xaxis_type=\"linear\", yaxis_type=\"linear\") # lenear / log\n    fig.update_layout(xaxis_tickformat=',g')\n    fig.update_layout(width=1000, height=600)  # 図の高さを幅を指定\n    fig.update_layout(template=\"plotly_white\") # 白背景のテーマに変更\n\n    fig.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-17T01:49:28.929214Z","iopub.execute_input":"2023-05-17T01:49:28.930447Z","iopub.status.idle":"2023-05-17T01:49:28.950891Z","shell.execute_reply.started":"2023-05-17T01:49:28.930392Z","shell.execute_reply":"2023-05-17T01:49:28.949684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp_df[\"StartHesitation\"] *= 0.8\ntmp_df[\"Turn\"] *= 1.0\ntmp_df[\"Walking\"] *= 1.2\ntmp_df[\"Task\"] *= -0.8\ntmp_df[\"Valid\"] *= -1.0\n\nxcol = \"Time\"\nycol = [\"AccV\", \"AccML\", \"AccAP\", \"Task_No\"]\nflags = [\"StartHesitation\", \"Turn\", \"Walking\", \"Task\", \"Valid\"]\nshow_defog_plotly(tmp_df, xcol, ycol, flags, st=0, length=len(tmp_df))","metadata":{"execution":{"iopub.status.busy":"2023-05-17T01:49:30.506093Z","iopub.execute_input":"2023-05-17T01:49:30.506501Z","iopub.status.idle":"2023-05-17T01:49:30.806914Z","shell.execute_reply.started":"2023-05-17T01:49:30.506472Z","shell.execute_reply":"2023-05-17T01:49:30.805098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}