{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The first answer that we should know: can we Concatenate TdcsFog and DeFOG. Let's try to find it!","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport os\nimport glob\nfrom scipy.stats import mannwhitneyu","metadata":{"execution":{"iopub.status.busy":"2023-05-29T07:31:41.345918Z","iopub.execute_input":"2023-05-29T07:31:41.346321Z","iopub.status.idle":"2023-05-29T07:31:41.352084Z","shell.execute_reply.started":"2023-05-29T07:31:41.346290Z","shell.execute_reply":"2023-05-29T07:31:41.351192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# root directory\nroot = '/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction'","metadata":{"execution":{"iopub.status.busy":"2023-05-29T07:12:18.648005Z","iopub.execute_input":"2023-05-29T07:12:18.648774Z","iopub.status.idle":"2023-05-29T07:12:18.654487Z","shell.execute_reply.started":"2023-05-29T07:12:18.648678Z","shell.execute_reply":"2023-05-29T07:12:18.653113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import data","metadata":{}},{"cell_type":"code","source":"df_tdcs_meta = pd.read_csv(os.path.join(root, 'tdcsfog_metadata.csv'))\ndf_tdcs_meta.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T07:12:18.658082Z","iopub.execute_input":"2023-05-29T07:12:18.658579Z","iopub.status.idle":"2023-05-29T07:12:18.721648Z","shell.execute_reply.started":"2023-05-29T07:12:18.658537Z","shell.execute_reply":"2023-05-29T07:12:18.720535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_defog_meta = pd.read_csv(os.path.join(root, 'defog_metadata.csv'))\ndf_defog_meta.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T07:12:18.725231Z","iopub.execute_input":"2023-05-29T07:12:18.725736Z","iopub.status.idle":"2023-05-29T07:12:18.746326Z","shell.execute_reply.started":"2023-05-29T07:12:18.725697Z","shell.execute_reply":"2023-05-29T07:12:18.745260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subjects = pd.read_csv(os.path.join(root, 'subjects.csv'))\ndf_subjects.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T07:12:18.747640Z","iopub.execute_input":"2023-05-29T07:12:18.747972Z","iopub.status.idle":"2023-05-29T07:12:18.773827Z","shell.execute_reply.started":"2023-05-29T07:12:18.747945Z","shell.execute_reply":"2023-05-29T07:12:18.772999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load tdcsfog data","metadata":{}},{"cell_type":"code","source":"# list of all tdcsfog csv file path\ntdcs_file_path = glob.glob(os.path.join(root, 'train', 'tdcsfog', '*.csv'), recursive=True)\n\n# In this notebook, we limit the number of files to be read in order to reduce the time required for model training.\ntdcs_file_path = tdcs_file_path\n\nprint(f'the number of files to be read: {len(tdcs_file_path)}')","metadata":{"execution":{"iopub.status.busy":"2023-05-29T07:12:18.775234Z","iopub.execute_input":"2023-05-29T07:12:18.775770Z","iopub.status.idle":"2023-05-29T07:12:18.788650Z","shell.execute_reply.started":"2023-05-29T07:12:18.775740Z","shell.execute_reply":"2023-05-29T07:12:18.786427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize a DataFrame to combine data from multiple CSV files.\ndf_tdcs = pd.DataFrame()\n\n# load tdcsfog time series in combination with metadata.\nfor fp in tdcs_file_path:    \n    \n    # load data into a variable 'tmp'.\n    tmp = pd.read_csv(fp)\n    \n    # get file Id from csv file name.\n    file_id = os.path.basename(fp).replace(\".csv\", \"\")\n    \n    # get subject Id.\n    subject = df_tdcs_meta.loc[df_tdcs_meta['Id'] == file_id, 'Subject'].iloc[0]\n    \n    # add metadata.\n    tmp['Medication'] = df_tdcs_meta.loc[df_tdcs_meta['Id'] == file_id, 'Medication'].iloc[0]\n    tmp['Age'] = df_subjects.loc[df_subjects['Subject'] == subject, 'Age'].iloc[0]\n    tmp['Sex'] = df_subjects.loc[df_subjects['Subject'] == subject, 'Sex'].iloc[0]\n    tmp['YearsSinceDx'] = df_subjects.loc[df_subjects['Subject'] == subject, 'YearsSinceDx'].iloc[0]\n    tmp['NFOGQ'] =df_subjects.loc[df_subjects['Subject'] == subject, 'NFOGQ'].iloc[0]\n    \n    # concat the data\n    df_tdcs = pd.concat([df_tdcs, tmp]).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-05-29T07:12:18.790337Z","iopub.execute_input":"2023-05-29T07:12:18.790840Z","iopub.status.idle":"2023-05-29T07:28:42.025118Z","shell.execute_reply.started":"2023-05-29T07:12:18.790800Z","shell.execute_reply":"2023-05-29T07:28:42.023977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load defog data","metadata":{}},{"cell_type":"code","source":"# list of all tdcsfog csv file path\ndefog_file_path = glob.glob(os.path.join(root, 'train', 'defog', '*.csv'), recursive=True)\n\n# In this notebook, we limit the number of files to be read in order to reduce the time required for model training.\ndefog_file_path = defog_file_path\n\nprint(f'the number of files to be read: {len(defog_file_path)}')","metadata":{"execution":{"iopub.status.busy":"2023-05-29T07:28:42.026654Z","iopub.execute_input":"2023-05-29T07:28:42.027857Z","iopub.status.idle":"2023-05-29T07:28:42.037104Z","shell.execute_reply.started":"2023-05-29T07:28:42.027806Z","shell.execute_reply":"2023-05-29T07:28:42.035970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize a DataFrame to combine data from multiple CSV files.\ndf_defog = pd.DataFrame()\n\nfor fp in defog_file_path:\n    # load data into a variable 'tmp'.\n    tmp = pd.read_csv(fp)\n    \n    # get file Id from csv file name.\n    file_id = os.path.basename(fp).replace(\".csv\", \"\")\n    \n    # get subject Id.\n    subject = df_defog_meta.loc[df_defog_meta['Id'] == file_id, 'Subject'].iloc[0]\n    \n    # add metadata.\n    tmp['Medication'] = df_defog_meta.loc[df_defog_meta['Id'] == file_id, 'Medication'].iloc[0]\n    tmp['Age'] = df_subjects.loc[df_subjects['Subject'] == subject, 'Age'].iloc[0]\n    tmp['Sex'] = df_subjects.loc[df_subjects['Subject'] == subject, 'Sex'].iloc[0]\n    tmp['YearsSinceDx'] = df_subjects.loc[df_subjects['Subject'] == subject, 'YearsSinceDx'].iloc[0]\n    tmp['NFOGQ'] =df_subjects.loc[df_subjects['Subject'] == subject, 'NFOGQ'].iloc[0]\n    \n    # extract data from the time period where Valid and Task are both True.\n    tmp = tmp[(tmp['Valid'] == True) & (tmp['Task']==True)]\n    tmp = tmp.drop(['Valid', 'Task'], axis=1)\n    \n    # concat the data\n    df_defog = pd.concat([df_defog, tmp]).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-05-29T07:28:42.040873Z","iopub.execute_input":"2023-05-29T07:28:42.041595Z","iopub.status.idle":"2023-05-29T07:30:04.682460Z","shell.execute_reply.started":"2023-05-29T07:28:42.041558Z","shell.execute_reply":"2023-05-29T07:30:04.680876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the contents of the df_defog\ndf_defog.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T07:30:04.683807Z","iopub.execute_input":"2023-05-29T07:30:04.684158Z","iopub.status.idle":"2023-05-29T07:30:04.703002Z","shell.execute_reply.started":"2023-05-29T07:30:04.684127Z","shell.execute_reply":"2023-05-29T07:30:04.701537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create function for Mann-Whitney U-test to test contribution.","metadata":{}},{"cell_type":"code","source":"def m_w_test(data_1, data_2):\n    stat, pvalue = mannwhitneyu (data_1, data_2, alternative='two-sided')\n    alpha = 0.05\n    if pvalue < alpha:\n        return print(f'''\n        H0: two populations are equal\n        H1: two populations are not equal\n        Alpha: {alpha}\n        P-value: {pvalue}, Same distribution, ''')\n    else:\n        return print(f'''\n        H0: two populations are equal\n        H1: two populations are not equal\n        Alpha: {alpha}\n        P-value: {pvalue}, Different distribution''')\n    ","metadata":{"execution":{"iopub.status.busy":"2023-05-29T07:30:04.704633Z","iopub.execute_input":"2023-05-29T07:30:04.705500Z","iopub.status.idle":"2023-05-29T07:30:04.716543Z","shell.execute_reply.started":"2023-05-29T07:30:04.705440Z","shell.execute_reply":"2023-05-29T07:30:04.715277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cheking distribution for 'AccV', 'AccML', 'AccAP'","metadata":{}},{"cell_type":"code","source":"dict_col = ['AccV', 'AccML', 'AccAP']","metadata":{"execution":{"iopub.status.busy":"2023-05-29T07:30:04.718207Z","iopub.execute_input":"2023-05-29T07:30:04.718648Z","iopub.status.idle":"2023-05-29T07:30:04.733971Z","shell.execute_reply.started":"2023-05-29T07:30:04.718615Z","shell.execute_reply":"2023-05-29T07:30:04.732438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in dict_col:\n    df_tdcs[col].hist(bins=200, alpha=0.5, density=True, \n                      label=f'Mean for {col}: {df_tdcs[col].mean()}')\n    df_defog[col].hist(bins=200, alpha=0.5, density=True, \n                       label=f'Mean for {col}: {df_defog[col].mean()}')\n    plt.title(f'Dispresion for {col}')\n    plt.xlabel(f'{col}')\n    plt.ylabel('Values')\n    plt.legend()\n    plt.show()\n    print(f'M_W test for column {col}:')\n    m_w_test(df_tdcs[col], df_defog[col])","metadata":{"execution":{"iopub.status.busy":"2023-05-29T07:31:46.839304Z","iopub.execute_input":"2023-05-29T07:31:46.840302Z","iopub.status.idle":"2023-05-29T07:33:19.071118Z","shell.execute_reply.started":"2023-05-29T07:31:46.840263Z","shell.execute_reply":"2023-05-29T07:33:19.069909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare data","metadata":{}},{"cell_type":"markdown","source":"From describe for data we know, that in DeFog and TdcsFog different data dimensions.  ","metadata":{}},{"cell_type":"markdown","source":"$$\n  1 * \\dfrac{m}{s^2} = g * 0.10197\n$$","metadata":{}},{"cell_type":"code","source":"CONST = 0.10197","metadata":{"execution":{"iopub.status.busy":"2023-05-29T07:33:19.073346Z","iopub.execute_input":"2023-05-29T07:33:19.075297Z","iopub.status.idle":"2023-05-29T07:33:19.080999Z","shell.execute_reply.started":"2023-05-29T07:33:19.075256Z","shell.execute_reply":"2023-05-29T07:33:19.079314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in dict_col:\n    df_tdcs[col] = df_tdcs[col] * CONST","metadata":{"execution":{"iopub.status.busy":"2023-05-29T07:33:19.083335Z","iopub.execute_input":"2023-05-29T07:33:19.083866Z","iopub.status.idle":"2023-05-29T07:33:19.324222Z","shell.execute_reply.started":"2023-05-29T07:33:19.083824Z","shell.execute_reply":"2023-05-29T07:33:19.322497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's cehck dispersion and m_w test after prepare","metadata":{}},{"cell_type":"code","source":"for col in dict_col:\n    df_tdcs[col].hist(bins=200, alpha=0.5, density=True, \n                      label=f'Mean for {col}: {df_tdcs[col].mean()}')\n    df_defog[col].hist(bins=200, alpha=0.5, density=True, \n                       label=f'Mean for {col}: {df_defog[col].mean()}')\n    plt.title(f'Dispresion for {col}')\n    plt.xlabel(f'{col}')\n    plt.ylabel('Values')\n    plt.legend()\n    plt.show()\n    print(f'M_W test for column {col}:')\n    m_w_test(df_tdcs[col], df_defog[col])","metadata":{"execution":{"iopub.status.busy":"2023-05-29T07:33:19.326927Z","iopub.execute_input":"2023-05-29T07:33:19.327319Z","iopub.status.idle":"2023-05-29T07:34:51.970433Z","shell.execute_reply.started":"2023-05-29T07:33:19.327277Z","shell.execute_reply":"2023-05-29T07:34:51.969168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After bringing all the values ​​to a single dimension, we can safely combine the data","metadata":{}},{"cell_type":"markdown","source":"# Concatenate datas","metadata":{}},{"cell_type":"code","source":"df_train = pd.concat([df_tdcs, df_defog]).reset_index(drop=True)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T07:34:51.983743Z","iopub.status.idle":"2023-05-29T07:34:51.984141Z","shell.execute_reply.started":"2023-05-29T07:34:51.983953Z","shell.execute_reply":"2023-05-29T07:34:51.983970Z"},"trusted":true},"execution_count":null,"outputs":[]}]}