{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport tqdm\nimport glob\nimport numpy as np\nimport pandas as pd","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# parent directory\npdir = '/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tdcs_meta = pd.read_csv(os.path.join(pdir, 'tdcsfog_metadata.csv'))\ndf_defog_meta = pd.read_csv(os.path.join(pdir, 'defog_metadata.csv'))\n\ndf_subjects = pd.read_csv(os.path.join(pdir, 'subjects.csv'))\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load tdcsfog data\n# list of all tdcsfog csv file path\ntdcs_file_path = glob.glob(os.path.join(pdir, 'train', 'tdcsfog', '*.csv'), recursive=True)\n\n# In this notebook, we limit the number of files to be read in order to reduce the time required for model training.\ntdcs_file_path = tdcs_file_path[::100]\n\nprint(f'the number of files to be read: {len(tdcs_file_path)}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize a DataFrame to combine data from multiple CSV files.\ndf_tdcs = pd.DataFrame()\n\n# load tdcsfog time series in combination with metadata.\nfor fp in tqdm.tqdm(tdcs_file_path):    \n    \n    # load data into a variable 'tmp'.\n    tmp = pd.read_csv(fp)\n    \n    # get file Id from csv file name.\n    file_id = os.path.basename(fp).replace(\".csv\", \"\")\n    \n    # get subject Id.\n    subject = df_tdcs_meta.loc[df_tdcs_meta['Id'] == file_id, 'Subject'].iloc[0]\n    \n    # add metadata.\n    tmp['Medication'] = df_tdcs_meta.loc[df_tdcs_meta['Id'] == file_id, 'Medication'].iloc[0]\n    tmp['Age'] = df_subjects.loc[df_subjects['Subject'] == subject, 'Age'].iloc[0]\n    tmp['Sex'] = df_subjects.loc[df_subjects['Subject'] == subject, 'Sex'].iloc[0]\n    tmp['YearsSinceDx'] = df_subjects.loc[df_subjects['Subject'] == subject, 'YearsSinceDx'].iloc[0]\n    tmp['NFOGQ'] =df_subjects.loc[df_subjects['Subject'] == subject, 'NFOGQ'].iloc[0]\n    \n    # concat the data\n    df_tdcs = pd.concat([df_tdcs, tmp]).reset_index(drop=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load defog data\n# list of all tdcsfog csv file path\ndefog_file_path = glob.glob(os.path.join(pdir, 'train', 'defog', '*.csv'), recursive=True)\n\n# In this notebook, we limit the number of files to be read in order to reduce the time required for model training.\ndefog_file_path = defog_file_path[::50]\n\nprint(f'the number of files to be read: {len(defog_file_path)}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize a DataFrame to combine data from multiple CSV files.\ndf_defog = pd.DataFrame()\n\nfor fp in tqdm.tqdm(defog_file_path):\n    # load data into a variable 'tmp'.\n    tmp = pd.read_csv(fp)\n    \n    # get file Id from csv file name.\n    file_id = os.path.basename(fp).replace(\".csv\", \"\")\n    \n    # get subject Id.\n    subject = df_defog_meta.loc[df_defog_meta['Id'] == file_id, 'Subject'].iloc[0]\n    \n    # add metadata.\n    tmp['Medication'] = df_defog_meta.loc[df_defog_meta['Id'] == file_id, 'Medication'].iloc[0]\n    tmp['Age'] = df_subjects.loc[df_subjects['Subject'] == subject, 'Age'].iloc[0]\n    tmp['Sex'] = df_subjects.loc[df_subjects['Subject'] == subject, 'Sex'].iloc[0]\n    tmp['YearsSinceDx'] = df_subjects.loc[df_subjects['Subject'] == subject, 'YearsSinceDx'].iloc[0]\n    tmp['NFOGQ'] =df_subjects.loc[df_subjects['Subject'] == subject, 'NFOGQ'].iloc[0]\n    \n    # extract data from the time period where Valid and Task are both True.\n    tmp = tmp[(tmp['Valid'] == True) & (tmp['Task']==True)]\n    tmp = tmp.drop(['Valid', 'Task'], axis=1)\n    \n    # concat the data\n    df_defog = pd.concat([df_defog, tmp]).reset_index(drop=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prepare the training data\n# concat tdcs and defog data.\ndf_train = pd.concat([df_tdcs, df_defog]).reset_index(drop=True)\ndf_train.head()\n\n# encode string columns into 0/1 format\ndf_train['Medication'] = np.where(df_train['Medication']=='on', 1, 0)\ndf_train['Sex'] = np.where(df_train['Sex']=='M', 1, 0)\ndf_train.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split data into features and target.\ny = df_train[['StartHesitation', 'Turn', 'Walking']]                       # target\nX = df_train.drop(['StartHesitation', 'Turn', 'Walking', 'Time'], axis=1)  # feature","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import XGBClassifier\n\nxgb = XGBClassifier(n_estimators=100)\nxgb.fit(X,y)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# list of all tdcsfog csv file path\ntdcs_test_file_path = glob.glob(os.path.join(pdir, 'test', 'tdcsfog', '*.csv'), recursive=True)\nprint(f'the number of files to be read: {len(tdcs_test_file_path)}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize a DataFrame to combine data from multiple CSV files.\ndf_tdcs_test = pd.DataFrame()\n\nfor fp in tqdm.tqdm(tdcs_test_file_path):\n    \n    # load data into a variable 'tmp'.\n    tmp = pd.read_csv(fp)\n    \n    # get file Id from csv file name.\n    file_id = os.path.basename(fp).replace(\".csv\", \"\")\n    \n    # get subject Id.\n    subject = df_tdcs_meta.loc[df_tdcs_meta['Id'] == file_id, 'Subject'].iloc[0]\n    \n    # add metadata.\n    tmp['Medication'] = df_tdcs_meta.loc[df_tdcs_meta['Id'] == file_id, 'Medication'].iloc[0]\n    tmp['Age'] = df_subjects.loc[df_subjects['Subject'] == subject, 'Age'].iloc[0]\n    tmp['Sex'] = df_subjects.loc[df_subjects['Subject'] == subject, 'Sex'].iloc[0]\n    tmp['YearsSinceDx'] = df_subjects.loc[df_subjects['Subject'] == subject, 'YearsSinceDx'].iloc[0]\n    tmp['NFOGQ'] =df_subjects.loc[df_subjects['Subject'] == subject, 'NFOGQ'].iloc[0]\n    \n    # add Id data to submit.\n    tmp['Id'] = file_id + '_' + tmp['Time'].astype(str)\n    \n    # concat the data\n    df_tdcs_test = pd.concat([df_tdcs_test, tmp]).reset_index(drop=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# list of all tdcsfog csv file path\ndefog_test_file_path = glob.glob(os.path.join(pdir, 'test', 'defog', '*.csv'), recursive=True)\nprint(f'the number of files to be read: {len(defog_test_file_path)}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize a DataFrame to combine data from multiple CSV files.\ndf_defog_test = pd.DataFrame()\n\nfor fp in tqdm.tqdm(defog_test_file_path):\n    # load data into a variable 'tmp'.\n    tmp = pd.read_csv(fp)\n    \n    # get file Id from csv file name.\n    file_id = os.path.basename(fp).replace(\".csv\", \"\")\n    \n    # get subject Id.\n    subject = df_defog_meta.loc[df_defog_meta['Id'] == file_id, 'Subject'].iloc[0]\n    \n    # add metadata.\n    tmp['Medication'] = df_defog_meta.loc[df_defog_meta['Id'] == file_id, 'Medication'].iloc[0]\n    tmp['Age'] = df_subjects.loc[df_subjects['Subject'] == subject, 'Age'].iloc[0]\n    tmp['Sex'] = df_subjects.loc[df_subjects['Subject'] == subject, 'Sex'].iloc[0]\n    tmp['YearsSinceDx'] = df_subjects.loc[df_subjects['Subject'] == subject, 'YearsSinceDx'].iloc[0]\n    tmp['NFOGQ'] =df_subjects.loc[df_subjects['Subject'] == subject, 'NFOGQ'].iloc[0]\n    \n    # add Id data to submit.\n    tmp['Id'] = file_id + '_' + tmp['Time'].astype(str)\n    \n    # concat the data\n    df_defog_test = pd.concat([df_defog_test, tmp]).reset_index(drop=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# concat tdcs and defog data.\ndf_test = pd.concat([df_tdcs_test, df_defog_test]).reset_index(drop=True)\n\n# encode string columns into 0/1 format\ndf_test['Medication'] = np.where(df_test['Medication']=='on', 1, 0)\ndf_test['Sex'] = np.where(df_test['Sex']=='M', 1, 0)\ndisplay(df_test)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split data into submission Id and feature.\nId = df_test['Id']                             # Id for submission data\nX_test = df_test.drop(['Time', 'Id'], axis=1)  # feature of test data\nX_test.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calculate prediction using trained RandomForestClassifier model.\nprediction = xgb.predict(X_test)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare submit data\nsubmit = pd.DataFrame(Id, columns=['Id'])\nsubmit['StartHesitation'] = prediction[:, 0]\nsubmit['Turn'] = prediction[:, 1]\nsubmit['Walking'] = prediction[:, 2]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(submit)\n# Save the created submission data.\nsubmit.to_csv('submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}]}