{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Support Vector Machine\nKieran Carigg, Julia Janssen, Elise Lems and Lars Zandbergen\n\nThis notebook is our attempt at creating a Support Vector Machine as a baseline for the Parkinsons FOG challenge. It uses many of the functions created in: https://www.kaggle.com/code/dataanalojisan/super-simple-baseline. We replaced the model with a support vector machine hoping to get better results. Instead we ran into memory errors and submission errors\n\n## Import Modules","metadata":{}},{"cell_type":"code","source":"import os\nfrom pathlib import Path\nimport tqdm\nimport glob\nimport numpy as np\nimport pandas as pd\nfrom sklearn import *\nimport lightgbm as lgb\n\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.svm import LinearSVC\nfrom sklearn.model_selection import GroupShuffleSplit\nfrom sklearn.preprocessing import scale","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2023-06-09T10:02:18.323818Z","iopub.execute_input":"2023-06-09T10:02:18.324745Z","iopub.status.idle":"2023-06-09T10:02:22.531062Z","shell.execute_reply.started":"2023-06-09T10:02:18.324699Z","shell.execute_reply":"2023-06-09T10:02:22.529643Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"# Function to reduce the memory usage of the dataset\ndef reduce_memory_usage(df):\n \n start_mem = df.memory_usage().sum() / 1024**2\n print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n \n for col in df.columns:\n col_type = df[col].dtype.name\n if ((col_type != 'datetime64[ns]') & (col_type != 'category')):\n if (col_type != 'object'):\n c_min = df[col].min()\n c_max = df[col].max()\n\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64)\n\n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n pass\n else:\n df[col] = df[col].astype('category')\n mem_usg = df.memory_usage().sum() / 1024**2 \n print(\"Memory usage became: \",mem_usg,\" MB\")\n \n return df","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:02:22.533429Z","iopub.execute_input":"2023-06-09T10:02:22.533901Z","iopub.status.idle":"2023-06-09T10:02:22.550576Z","shell.execute_reply.started":"2023-06-09T10:02:22.533858Z","shell.execute_reply":"2023-06-09T10:02:22.549079Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"# parent directory\npdir = '/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction'","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:02:22.553069Z","iopub.execute_input":"2023-06-09T10:02:22.553456Z","iopub.status.idle":"2023-06-09T10:02:22.573042Z","shell.execute_reply.started":"2023-06-09T10:02:22.553422Z","shell.execute_reply":"2023-06-09T10:02:22.571592Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"markdown","source":"# load meta data","metadata":{}},{"cell_type":"code","source":"subjects = pd.read_csv(os.path.join(pdir, 'subjects.csv'))\ntasks = pd.read_csv(os.path.join(pdir, 'tasks.csv'))\nevents = pd.read_csv(os.path.join(pdir, 'events.csv'))\n\ndf_tdcs_meta = pd.read_csv(os.path.join(pdir, 'tdcsfog_metadata.csv'))\ndf_defog_meta = pd.read_csv(os.path.join(pdir, 'defog_metadata.csv'))\ndf_tdcs_meta['Module'] = 'tdcsfog'\ndf_defog_meta['Module'] = 'defog'\n\nfull_metadata = pd.concat([df_tdcs_meta, df_defog_meta])\n\nsubjects.loc[subjects['Subject'] == 'fe5d84', 'Sex'] = 'F'\nsubjects['Sex'] = subjects['Sex'].factorize()[0]\nsubjects = subjects.fillna(0).groupby('Subject').median()\nsubjects['s_group'] = cluster.KMeans(n_clusters = 8, random_state = 42).fit_predict(subjects[subjects.columns[1:]])","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-06-09T10:02:22.575796Z","iopub.execute_input":"2023-06-09T10:02:22.576311Z","iopub.status.idle":"2023-06-09T10:02:23.461082Z","shell.execute_reply.started":"2023-06-09T10:02:22.576278Z","shell.execute_reply":"2023-06-09T10:02:23.457317Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/sklearn/cluster/_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n warnings.warn(\n","output_type":"stream"}]},{"cell_type":"code","source":"tasks['Duration'] = tasks['End'] - tasks['Begin']\ntasks['Task'] = tasks['Task'].factorize()[0]\ntasks['t_group'] = cluster.KMeans(n_clusters = 8, random_state = 42).fit_predict(tasks[tasks.columns[1:]])\n\nmetadata_w_subjects = full_metadata.merge(subjects, how='left', on='Subject').copy()\nmetadata_w_subjects['Medication'] = metadata_w_subjects['Medication'].factorize()[0]","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:02:23.465734Z","iopub.execute_input":"2023-06-09T10:02:23.466152Z","iopub.status.idle":"2023-06-09T10:02:23.662947Z","shell.execute_reply.started":"2023-06-09T10:02:23.46612Z","shell.execute_reply":"2023-06-09T10:02:23.661723Z"},"trusted":true},"execution_count":5,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/sklearn/cluster/_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n warnings.warn(\n","output_type":"stream"}]},{"cell_type":"code","source":"reduce_memory_usage(tasks)\nreduce_memory_usage(metadata_w_subjects)","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:02:23.664464Z","iopub.execute_input":"2023-06-09T10:02:23.66517Z","iopub.status.idle":"2023-06-09T10:02:23.780569Z","shell.execute_reply.started":"2023-06-09T10:02:23.665129Z","shell.execute_reply":"2023-06-09T10:02:23.779378Z"},"trusted":true},"execution_count":6,"outputs":[{"name":"stdout","text":"Memory usage of dataframe is 0.12 MB\nMemory usage became: 0.03200721740722656 MB\nMemory usage of dataframe is 0.11 MB\nMemory usage became: 0.07251930236816406 MB\n","output_type":"stream"},{"execution_count":6,"output_type":"execute_result","data":{"text/plain":" Id Subject Visit_x Test Medication Module Visit_y Age \\\n0 003f117e14 4dc2f8 3 2.0 0 tdcsfog 0.0 68.0 \n1 009ee11563 f62eec 4 2.0 0 tdcsfog 0.0 71.0 \n2 011322847a 231c3b 2 2.0 0 tdcsfog 0.0 67.0 \n3 01d0fe7266 231c3b 2 1.0 1 tdcsfog 0.0 67.0 \n4 024418ba39 fa8764 19 3.0 0 tdcsfog 0.0 60.0 \n.. ... ... ... ... ... ... ... ... \n965 f3a921edee 1a778d 1 NaN 1 defog 1.5 65.0 \n966 f40e8c6ebe 575c60 1 NaN 1 defog 1.0 28.0 \n967 f8ddbdd98d 107712 1 NaN 0 defog 1.0 82.0 \n968 f9efef91fb 5d9cae 2 NaN 1 defog 1.5 72.0 \n969 f9fc61ce85 040587 1 NaN 0 defog 1.5 75.0 \n\n Sex YearsSinceDx UPDRSIII_On UPDRSIII_Off NFOGQ s_group \n0 1.0 9.0 17.0 15.0 15.0 3 \n1 0.0 10.0 42.0 0.0 24.0 0 \n2 0.0 12.0 27.0 28.0 19.0 3 \n3 0.0 12.0 27.0 28.0 19.0 3 \n4 1.0 7.0 30.0 0.0 19.0 2 \n.. ... ... ... ... ... ... \n965 0.0 7.0 50.0 59.5 24.5 1 \n966 0.0 4.0 54.0 50.0 25.0 1 \n967 1.0 11.0 38.0 42.0 21.0 6 \n968 0.5 14.0 22.5 39.0 16.0 7 \n969 0.0 26.0 49.5 72.0 22.5 5 \n\n[970 rows x 14 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
IdSubjectVisit_xTestMedicationModuleVisit_yAgeSexYearsSinceDxUPDRSIII_OnUPDRSIII_OffNFOGQs_group
0003f117e144dc2f832.00tdcsfog0.068.01.09.017.015.015.03
1009ee11563f62eec42.00tdcsfog0.071.00.010.042.00.024.00
2011322847a231c3b22.00tdcsfog0.067.00.012.027.028.019.03
301d0fe7266231c3b21.01tdcsfog0.067.00.012.027.028.019.03
4024418ba39fa8764193.00tdcsfog0.060.01.07.030.00.019.02
.............................................
965f3a921edee1a778d1NaN1defog1.565.00.07.050.059.524.51
966f40e8c6ebe575c601NaN1defog1.028.00.04.054.050.025.01
967f8ddbdd98d1077121NaN0defog1.082.01.011.038.042.021.06
968f9efef91fb5d9cae2NaN1defog1.572.00.514.022.539.016.07
969f9fc61ce850405871NaN0defog1.575.00.026.049.572.022.55
\n

970 rows × 14 columns

\n
"},"metadata":{}}]},{"cell_type":"code","source":"tasks.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:02:23.782335Z","iopub.execute_input":"2023-06-09T10:02:23.782889Z","iopub.status.idle":"2023-06-09T10:02:23.799491Z","shell.execute_reply.started":"2023-06-09T10:02:23.782848Z","shell.execute_reply":"2023-06-09T10:02:23.798289Z"},"trusted":true},"execution_count":7,"outputs":[{"execution_count":7,"output_type":"execute_result","data":{"text/plain":" Id Begin End Task Duration t_group\n0 02ab235146 10.00 190.5 0 180.500000 2\n1 02ab235146 211.25 271.5 1 60.312500 2\n2 02ab235146 506.00 522.5 2 16.515625 6\n3 02ab235146 578.00 594.5 3 16.687500 6\n4 02ab235146 701.50 715.5 4 13.960938 1","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
IdBeginEndTaskDurationt_group
002ab23514610.00190.50180.5000002
102ab235146211.25271.5160.3125002
202ab235146506.00522.5216.5156256
302ab235146578.00594.5316.6875006
402ab235146701.50715.5413.9609381
\n
"},"metadata":{}}]},{"cell_type":"markdown","source":"# Load tdcsfog data","metadata":{}},{"cell_type":"code","source":"# list of all tdcsfog csv file path\ntdcs_file_path = glob.glob(os.path.join(pdir, 'train', 'tdcsfog', '*.csv'), recursive=True)\n\n# In this notebook, we limit the number of files to be read in order to reduce the time required for model training.\ntdcs_file_path = tdcs_file_path[:100]\ntdcs_first_file = tdcs_file_path[0].split('/')[-1]\n\nfiles = ['b6c16738e8.csv', '56965f50de.csv', 'd606b42f46.csv', 'a69107b05b.csv', '295040848f.csv', '75a99f625c.csv', '08fbe142f9.csv', 'fc8023cf89.csv', '6c39f4d317.csv', 'f1a1c41c3a.csv', '10efee36af.csv', '1c6d13c09f.csv', '2d481ad987.csv', 'c36c3860e7.csv', '04b449020a.csv', '9fc94b7eb6.csv', '5bb7bbd72e.csv', 'b3ccb04562.csv', '24117478c8.csv', '2e3b57b4e8.csv', 'e8f45c11b3.csv', '4389520d86.csv', 'eac0169d51.csv', 'e13d9996ad.csv', '82fc8e0363.csv', 'feba449e1a.csv', '7468af035f.csv', '8343346bc8.csv', '758ddee3f7.csv', 'ab57c0c2fc.csv', '4fd1175cbd.csv', 'ab0b48f1f9.csv', 'b78d3e162c.csv', '6e258d0083.csv', '36f2e89275.csv', '497d8e25a5.csv', 'a84480ced9.csv', 'ad13e83b8d.csv', '9eaed1e5b7.csv', 'cc91b27f46.csv', '343d04c4ad.csv', '0b26b0d64f.csv', 'eb7a2efee3.csv', '4d593f4d56.csv', '19e4f58e60.csv', '10e83d6550.csv', '2daabdfd67.csv', '9b5626b353.csv', '1c13b1cc60.csv', 'a7a5a87ccf.csv', '4c2efc2da7.csv', '83ce03ef33.csv', '2ea063ca4f.csv', '883f0760eb.csv', 'd98358a75f.csv', 'f936819935.csv', 'dbda844446.csv', '15219ff4dc.csv', '8a5ba92c2e.csv', 'f5f474848e.csv', '9660a58406.csv', 'ea762f5ada.csv', '9cb654d456.csv', 'bb61050681.csv', 'ff92d9244d.csv', '90d67b717a.csv', '8be4dbd97d.csv', '0974bf9e9f.csv', '1e5f843874.csv', '49127986f4.csv', '150e8227fc.csv', '4d89a40d66.csv', '024ba3ebd5.csv', '64dec54859.csv', '6d9b1fc826.csv', 'e44b991c48.csv', '988622d76a.csv', '2b083aaff2.csv', 'bcfd1d3dbf.csv', '35bd42e628.csv', '91990dba8e.csv', '8ea9d29cc4.csv', 'be8fdfa712.csv', '448d4345d4.csv', 'a1396ddf69.csv', '0df04e8431.csv', '13d0015a77.csv', '0b36680dd2.csv', '8fc05fabee.csv', '6a208e14c8.csv', '4c17ea0b75.csv', '9e9a9c4d71.csv', '38323bc013.csv', '59c2a2e1ed.csv', '15671a03ca.csv', '8dd94ab27e.csv', '1e45dd8eab.csv', '8522292a1b.csv', '7a17d4db85.csv', 'd7644a14c3.csv']\ntdcs_file_path = [os.path.join(pdir, 'train', 'tdcsfog', f) for f in files]\ntdcs_first_file = tdcs_file_path[0].split('/')[-1]\n\nprint(f'the number of files to be read: {len(tdcs_file_path)}')\nprint(f'first file: {tdcs_first_file}')","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:02:23.801456Z","iopub.execute_input":"2023-06-09T10:02:23.802332Z","iopub.status.idle":"2023-06-09T10:02:23.973572Z","shell.execute_reply.started":"2023-06-09T10:02:23.802289Z","shell.execute_reply":"2023-06-09T10:02:23.97242Z"},"trusted":true},"execution_count":8,"outputs":[{"name":"stdout","text":"the number of files to be read: 100\nfirst file: b6c16738e8.csv\n","output_type":"stream"}]},{"cell_type":"code","source":"# Initialize a DataFrame to combine data from multiple CSV files.\ndf_tdcs = pd.DataFrame()\n\n# load tdcsfog time series in combination with metadata.\nfor fp in tqdm.tqdm(tdcs_file_path): \n \n # load data into a variable 'tmp'.\n tmp = pd.read_csv(fp, index_col='Time')\n tmp['Id'] = fp.split('/')[-1].split('.')[0]\n dataset = Path(fp).parts[-2]\n tmp['Module'] = dataset\n \n # get file Id from csv file name.\n file_id = os.path.basename(fp).replace(\".csv\", \"\")\n \n # get subject Id.\n subject = df_tdcs_meta.loc[df_tdcs_meta['Id'] == file_id, 'Subject'].iloc[0]\n \n tmp = pd.merge(tmp, tasks, how='left', on='Id').fillna(-1)\n \n tmp = pd.merge(tmp, metadata_w_subjects[['Id', 'Subject', 'Visit_x', 'Test', 'Medication', 'Visit_y', 'Age', 'Sex', 'YearsSinceDx', 'UPDRSIII_On', 'UPDRSIII_Off', 'NFOGQ','s_group']], how='left', on='Id')\n \n # concat the data\n df_tdcs = pd.concat([df_tdcs, tmp]).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:02:23.975111Z","iopub.execute_input":"2023-06-09T10:02:23.97585Z","iopub.status.idle":"2023-06-09T10:02:40.583454Z","shell.execute_reply.started":"2023-06-09T10:02:23.975809Z","shell.execute_reply":"2023-06-09T10:02:40.582228Z"},"trusted":true},"execution_count":9,"outputs":[{"name":"stderr","text":"100%|██████████| 100/100 [00:16<00:00, 6.03it/s]\n","output_type":"stream"}]},{"cell_type":"code","source":"reduce_memory_usage(df_tdcs)","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:02:40.588186Z","iopub.execute_input":"2023-06-09T10:02:40.588581Z","iopub.status.idle":"2023-06-09T10:02:41.640218Z","shell.execute_reply.started":"2023-06-09T10:02:40.588535Z","shell.execute_reply":"2023-06-09T10:02:41.638791Z"},"trusted":true},"execution_count":10,"outputs":[{"name":"stdout","text":"Memory usage of dataframe is 111.30 MB\nMemory usage became: 43.058244705200195 MB\n","output_type":"stream"},{"execution_count":10,"output_type":"execute_result","data":{"text/plain":" AccV AccML AccAP StartHesitation Turn Walking \\\n0 -9.101562 1.147461 -2.937500 0 0 0 \n1 -9.093750 1.150391 -2.939453 0 0 0 \n2 -9.093750 1.157227 -2.951172 0 0 0 \n3 -9.093750 1.150391 -2.949219 0 0 0 \n4 -9.085938 1.134766 -2.949219 0 0 0 \n... ... ... ... ... ... ... \n1100961 -9.664062 1.294922 -1.165039 0 0 0 \n1100962 -9.671875 1.275391 -1.183594 0 0 0 \n1100963 -9.695312 1.244141 -1.180664 0 0 0 \n1100964 -9.687500 1.199219 -1.156250 0 0 0 \n1100965 -9.687500 1.206055 -1.156250 0 0 0 \n\n Id Module Begin End ... Test Medication Visit_y \\\n0 b6c16738e8 tdcsfog -1.0 -1.0 ... 1.0 0 0.0 \n1 b6c16738e8 tdcsfog -1.0 -1.0 ... 1.0 0 0.0 \n2 b6c16738e8 tdcsfog -1.0 -1.0 ... 1.0 0 0.0 \n3 b6c16738e8 tdcsfog -1.0 -1.0 ... 1.0 0 0.0 \n4 b6c16738e8 tdcsfog -1.0 -1.0 ... 1.0 0 0.0 \n... ... ... ... ... ... ... ... ... \n1100961 d7644a14c3 tdcsfog -1.0 -1.0 ... 3.0 1 0.0 \n1100962 d7644a14c3 tdcsfog -1.0 -1.0 ... 3.0 1 0.0 \n1100963 d7644a14c3 tdcsfog -1.0 -1.0 ... 3.0 1 0.0 \n1100964 d7644a14c3 tdcsfog -1.0 -1.0 ... 3.0 1 0.0 \n1100965 d7644a14c3 tdcsfog -1.0 -1.0 ... 3.0 1 0.0 \n\n Age Sex YearsSinceDx UPDRSIII_On UPDRSIII_Off NFOGQ s_group \n0 58.0 1.0 5.0 21.0 27.0 20.0 3 \n1 58.0 1.0 5.0 21.0 27.0 20.0 3 \n2 58.0 1.0 5.0 21.0 27.0 20.0 3 \n3 58.0 1.0 5.0 21.0 27.0 20.0 3 \n4 58.0 1.0 5.0 21.0 27.0 20.0 3 \n... ... ... ... ... ... ... ... \n1100961 56.0 0.0 11.0 27.0 55.0 19.0 7 \n1100962 56.0 0.0 11.0 27.0 55.0 19.0 7 \n1100963 56.0 0.0 11.0 27.0 55.0 19.0 7 \n1100964 56.0 0.0 11.0 27.0 55.0 19.0 7 \n1100965 56.0 0.0 11.0 27.0 55.0 19.0 7 \n\n[1100966 rows x 25 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
AccVAccMLAccAPStartHesitationTurnWalkingIdModuleBeginEnd...TestMedicationVisit_yAgeSexYearsSinceDxUPDRSIII_OnUPDRSIII_OffNFOGQs_group
0-9.1015621.147461-2.937500000b6c16738e8tdcsfog-1.0-1.0...1.000.058.01.05.021.027.020.03
1-9.0937501.150391-2.939453000b6c16738e8tdcsfog-1.0-1.0...1.000.058.01.05.021.027.020.03
2-9.0937501.157227-2.951172000b6c16738e8tdcsfog-1.0-1.0...1.000.058.01.05.021.027.020.03
3-9.0937501.150391-2.949219000b6c16738e8tdcsfog-1.0-1.0...1.000.058.01.05.021.027.020.03
4-9.0859381.134766-2.949219000b6c16738e8tdcsfog-1.0-1.0...1.000.058.01.05.021.027.020.03
..................................................................
1100961-9.6640621.294922-1.165039000d7644a14c3tdcsfog-1.0-1.0...3.010.056.00.011.027.055.019.07
1100962-9.6718751.275391-1.183594000d7644a14c3tdcsfog-1.0-1.0...3.010.056.00.011.027.055.019.07
1100963-9.6953121.244141-1.180664000d7644a14c3tdcsfog-1.0-1.0...3.010.056.00.011.027.055.019.07
1100964-9.6875001.199219-1.156250000d7644a14c3tdcsfog-1.0-1.0...3.010.056.00.011.027.055.019.07
1100965-9.6875001.206055-1.156250000d7644a14c3tdcsfog-1.0-1.0...3.010.056.00.011.027.055.019.07
\n

1100966 rows × 25 columns

\n
"},"metadata":{}}]},{"cell_type":"markdown","source":"# Load defog data","metadata":{}},{"cell_type":"code","source":"# list of all tdcsfog csv file path\ndefog_file_path = glob.glob(os.path.join(pdir, 'train', 'defog', '*.csv'), recursive=True)\n\n# In this notebook, we limit the number of files to be read in order to reduce the time required for model training.\ndefog_file_path = defog_file_path[:10]\n\n# These files were chosen to hopefully improve training performance, since there are more labels in these files.\nfiles = ['38befe9f39.csv', '8db3a7e46b.csv', '8282009100.csv', '0eaac04f17.csv', '4f613ccf88.csv', '77d7d95074.csv', '7030643376.csv', 'e069a57511.csv', 'dd0ddc0bb5.csv', '4c3aa8ea6e.csv']\ndefog_file_path = [os.path.join(pdir, 'train', 'defog', f) for f in files]\n\nfirst_file = defog_file_path[0].split('/')[-1]\n\nprint(f'the number of files to be read: {len(defog_file_path)}')\nprint(f\"first file: {first_file}\")","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:02:41.641622Z","iopub.execute_input":"2023-06-09T10:02:41.642053Z","iopub.status.idle":"2023-06-09T10:02:41.67819Z","shell.execute_reply.started":"2023-06-09T10:02:41.64202Z","shell.execute_reply":"2023-06-09T10:02:41.676938Z"},"trusted":true},"execution_count":11,"outputs":[{"name":"stdout","text":"the number of files to be read: 10\nfirst file: 38befe9f39.csv\n","output_type":"stream"}]},{"cell_type":"code","source":"# Initialize a DataFrame to combine data from multiple CSV files.\ndf_defog = pd.DataFrame()\n\nfor fp in tqdm.tqdm(defog_file_path):\n # load data into a variable 'tmp'.\n tmp = pd.read_csv(fp, index_col='Time')\n tmp['Id'] = fp.split('/')[-1].split('.')[0]\n dataset = Path(fp).parts[-2]\n tmp['Module'] = dataset\n \n # get file Id from csv file name.\n file_id = os.path.basename(fp).replace(\".csv\", \"\")\n \n tmp = pd.merge(tmp, tasks, how='left', on='Id').fillna(-1)\n \n tmp = pd.merge(tmp, metadata_w_subjects[['Id', 'Subject', 'Visit_x', 'Medication', 'Visit_y', 'Age', 'Sex', 'YearsSinceDx', 'UPDRSIII_On', 'UPDRSIII_Off', 'NFOGQ','s_group']], how='left', on='Id')\n \n # extract data from the time period where Valid and Task are both True.\n tmp = tmp[(tmp['Valid'] == True) & (tmp['Task_x']==True)]\n tmp = tmp.drop(['Valid', 'Task_x'], axis=1)\n \n # concat the data\n df_defog = pd.concat([df_defog, tmp]).reset_index(drop=True)\n\ndf_defog.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:02:41.681111Z","iopub.execute_input":"2023-06-09T10:02:41.681815Z","iopub.status.idle":"2023-06-09T10:03:46.119997Z","shell.execute_reply.started":"2023-06-09T10:02:41.681769Z","shell.execute_reply":"2023-06-09T10:03:46.118771Z"},"trusted":true},"execution_count":12,"outputs":[{"name":"stderr","text":"100%|██████████| 10/10 [01:04<00:00, 6.44s/it]\n","output_type":"stream"},{"execution_count":12,"output_type":"execute_result","data":{"text/plain":" AccV AccML AccAP StartHesitation Turn Walking Id \\\n0 -0.980958 0.121826 -0.215828 0 0 0 38befe9f39 \n1 -0.980958 0.121826 -0.215828 0 0 0 38befe9f39 \n2 -0.980958 0.121826 -0.215828 0 0 0 38befe9f39 \n3 -0.980958 0.121826 -0.215828 0 0 0 38befe9f39 \n4 -0.980958 0.121826 -0.215828 0 0 0 38befe9f39 \n\n Module Begin End ... Visit_x Medication Visit_y Age Sex \\\n0 defog 10.0000 21.3125 ... 1 0 1.5 60.0 1.0 \n1 defog 66.8125 70.4375 ... 1 0 1.5 60.0 1.0 \n2 defog 90.8750 97.6875 ... 1 0 1.5 60.0 1.0 \n3 defog 117.0625 124.5000 ... 1 0 1.5 60.0 1.0 \n4 defog 144.7500 167.5000 ... 1 0 1.5 60.0 1.0 \n\n YearsSinceDx UPDRSIII_On UPDRSIII_Off NFOGQ s_group \n0 1.5 18.5 20.0 11.5 3 \n1 1.5 18.5 20.0 11.5 3 \n2 1.5 18.5 20.0 11.5 3 \n3 1.5 18.5 20.0 11.5 3 \n4 1.5 18.5 20.0 11.5 3 \n\n[5 rows x 24 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
AccVAccMLAccAPStartHesitationTurnWalkingIdModuleBeginEnd...Visit_xMedicationVisit_yAgeSexYearsSinceDxUPDRSIII_OnUPDRSIII_OffNFOGQs_group
0-0.9809580.121826-0.21582800038befe9f39defog10.000021.3125...101.560.01.01.518.520.011.53
1-0.9809580.121826-0.21582800038befe9f39defog66.812570.4375...101.560.01.01.518.520.011.53
2-0.9809580.121826-0.21582800038befe9f39defog90.875097.6875...101.560.01.01.518.520.011.53
3-0.9809580.121826-0.21582800038befe9f39defog117.0625124.5000...101.560.01.01.518.520.011.53
4-0.9809580.121826-0.21582800038befe9f39defog144.7500167.5000...101.560.01.01.518.520.011.53
\n

5 rows × 24 columns

\n
"},"metadata":{}}]},{"cell_type":"code","source":"reduce_memory_usage(df_defog)","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:03:46.121664Z","iopub.execute_input":"2023-06-09T10:03:46.122282Z","iopub.status.idle":"2023-06-09T10:03:53.984342Z","shell.execute_reply.started":"2023-06-09T10:03:46.122221Z","shell.execute_reply":"2023-06-09T10:03:53.983403Z"},"trusted":true},"execution_count":13,"outputs":[{"name":"stdout","text":"Memory usage of dataframe is 780.97 MB\nMemory usage became: 321.06938457489014 MB\n","output_type":"stream"},{"execution_count":13,"output_type":"execute_result","data":{"text/plain":" AccV AccML AccAP StartHesitation Turn Walking \\\n0 -0.980957 0.121826 -0.21582 0 0 0 \n1 -0.980957 0.121826 -0.21582 0 0 0 \n2 -0.980957 0.121826 -0.21582 0 0 0 \n3 -0.980957 0.121826 -0.21582 0 0 0 \n4 -0.980957 0.121826 -0.21582 0 0 0 \n... ... ... ... ... ... ... \n9098914 -0.953125 0.109375 -0.16272 0 0 0 \n9098915 -0.953125 0.109375 -0.16272 0 0 0 \n9098916 -0.953125 0.109375 -0.16272 0 0 0 \n9098917 -0.953125 0.109375 -0.16272 0 0 0 \n9098918 -0.953125 0.109375 -0.16272 0 0 0 \n\n Id Module Begin End ... Visit_x Medication \\\n0 38befe9f39 defog 10.0000 21.3125 ... 1 0 \n1 38befe9f39 defog 66.8125 70.4375 ... 1 0 \n2 38befe9f39 defog 90.8750 97.6875 ... 1 0 \n3 38befe9f39 defog 117.0625 124.5000 ... 1 0 \n4 38befe9f39 defog 144.7500 167.5000 ... 1 0 \n... ... ... ... ... ... ... ... \n9098914 4c3aa8ea6e defog 270.0000 287.0000 ... 1 0 \n9098915 4c3aa8ea6e defog 389.0000 423.5000 ... 1 0 \n9098916 4c3aa8ea6e defog 462.2500 509.0000 ... 1 0 \n9098917 4c3aa8ea6e defog 667.0000 677.0000 ... 1 0 \n9098918 4c3aa8ea6e defog 694.5000 731.0000 ... 1 0 \n\n Visit_y Age Sex YearsSinceDx UPDRSIII_On UPDRSIII_Off NFOGQ \\\n0 1.5 60.0 1.0 1.5 18.5 20.0 11.5 \n1 1.5 60.0 1.0 1.5 18.5 20.0 11.5 \n2 1.5 60.0 1.0 1.5 18.5 20.0 11.5 \n3 1.5 60.0 1.0 1.5 18.5 20.0 11.5 \n4 1.5 60.0 1.0 1.5 18.5 20.0 11.5 \n... ... ... ... ... ... ... ... \n9098914 1.5 63.0 0.0 27.0 37.0 39.5 25.0 \n9098915 1.5 63.0 0.0 27.0 37.0 39.5 25.0 \n9098916 1.5 63.0 0.0 27.0 37.0 39.5 25.0 \n9098917 1.5 63.0 0.0 27.0 37.0 39.5 25.0 \n9098918 1.5 63.0 0.0 27.0 37.0 39.5 25.0 \n\n s_group \n0 3 \n1 3 \n2 3 \n3 3 \n4 3 \n... ... \n9098914 7 \n9098915 7 \n9098916 7 \n9098917 7 \n9098918 7 \n\n[9098919 rows x 24 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
AccVAccMLAccAPStartHesitationTurnWalkingIdModuleBeginEnd...Visit_xMedicationVisit_yAgeSexYearsSinceDxUPDRSIII_OnUPDRSIII_OffNFOGQs_group
0-0.9809570.121826-0.2158200038befe9f39defog10.000021.3125...101.560.01.01.518.520.011.53
1-0.9809570.121826-0.2158200038befe9f39defog66.812570.4375...101.560.01.01.518.520.011.53
2-0.9809570.121826-0.2158200038befe9f39defog90.875097.6875...101.560.01.01.518.520.011.53
3-0.9809570.121826-0.2158200038befe9f39defog117.0625124.5000...101.560.01.01.518.520.011.53
4-0.9809570.121826-0.2158200038befe9f39defog144.7500167.5000...101.560.01.01.518.520.011.53
..................................................................
9098914-0.9531250.109375-0.162720004c3aa8ea6edefog270.0000287.0000...101.563.00.027.037.039.525.07
9098915-0.9531250.109375-0.162720004c3aa8ea6edefog389.0000423.5000...101.563.00.027.037.039.525.07
9098916-0.9531250.109375-0.162720004c3aa8ea6edefog462.2500509.0000...101.563.00.027.037.039.525.07
9098917-0.9531250.109375-0.162720004c3aa8ea6edefog667.0000677.0000...101.563.00.027.037.039.525.07
9098918-0.9531250.109375-0.162720004c3aa8ea6edefog694.5000731.0000...101.563.00.027.037.039.525.07
\n

9098919 rows × 24 columns

\n
"},"metadata":{}}]},{"cell_type":"markdown","source":"# Prepare train data","metadata":{}},{"cell_type":"code","source":"# concat tdcs and defog data.\ndf_train = pd.concat([df_tdcs, df_defog]).reset_index(drop=True)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:03:53.985467Z","iopub.execute_input":"2023-06-09T10:03:53.986391Z","iopub.status.idle":"2023-06-09T10:03:56.58232Z","shell.execute_reply.started":"2023-06-09T10:03:53.98635Z","shell.execute_reply":"2023-06-09T10:03:56.581156Z"},"trusted":true},"execution_count":14,"outputs":[{"execution_count":14,"output_type":"execute_result","data":{"text/plain":" AccV AccML AccAP StartHesitation Turn Walking Id \\\n0 -9.101562 1.147461 -2.937500 0 0 0 b6c16738e8 \n1 -9.093750 1.150391 -2.939453 0 0 0 b6c16738e8 \n2 -9.093750 1.157227 -2.951172 0 0 0 b6c16738e8 \n3 -9.093750 1.150391 -2.949219 0 0 0 b6c16738e8 \n4 -9.085938 1.134766 -2.949219 0 0 0 b6c16738e8 \n\n Module Begin End ... Medication Visit_y Age Sex YearsSinceDx \\\n0 tdcsfog -1.0 -1.0 ... 0 0.0 58.0 1.0 5.0 \n1 tdcsfog -1.0 -1.0 ... 0 0.0 58.0 1.0 5.0 \n2 tdcsfog -1.0 -1.0 ... 0 0.0 58.0 1.0 5.0 \n3 tdcsfog -1.0 -1.0 ... 0 0.0 58.0 1.0 5.0 \n4 tdcsfog -1.0 -1.0 ... 0 0.0 58.0 1.0 5.0 \n\n UPDRSIII_On UPDRSIII_Off NFOGQ s_group Task_y \n0 21.0 27.0 20.0 3 NaN \n1 21.0 27.0 20.0 3 NaN \n2 21.0 27.0 20.0 3 NaN \n3 21.0 27.0 20.0 3 NaN \n4 21.0 27.0 20.0 3 NaN \n\n[5 rows x 26 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
AccVAccMLAccAPStartHesitationTurnWalkingIdModuleBeginEnd...MedicationVisit_yAgeSexYearsSinceDxUPDRSIII_OnUPDRSIII_OffNFOGQs_groupTask_y
0-9.1015621.147461-2.937500000b6c16738e8tdcsfog-1.0-1.0...00.058.01.05.021.027.020.03NaN
1-9.0937501.150391-2.939453000b6c16738e8tdcsfog-1.0-1.0...00.058.01.05.021.027.020.03NaN
2-9.0937501.157227-2.951172000b6c16738e8tdcsfog-1.0-1.0...00.058.01.05.021.027.020.03NaN
3-9.0937501.150391-2.949219000b6c16738e8tdcsfog-1.0-1.0...00.058.01.05.021.027.020.03NaN
4-9.0859381.134766-2.949219000b6c16738e8tdcsfog-1.0-1.0...00.058.01.05.021.027.020.03NaN
\n

5 rows × 26 columns

\n
"},"metadata":{}}]},{"cell_type":"code","source":"# encode string columns into 0/1 format\ndf_train['Medication'] = np.where(df_train['Medication']=='on', 1, 0)\ndf_train['Sex'] = np.where(df_train['Sex']=='M', 1, 0)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:03:56.583547Z","iopub.execute_input":"2023-06-09T10:03:56.584337Z","iopub.status.idle":"2023-06-09T10:03:56.809291Z","shell.execute_reply.started":"2023-06-09T10:03:56.584303Z","shell.execute_reply":"2023-06-09T10:03:56.807924Z"},"trusted":true},"execution_count":15,"outputs":[{"execution_count":15,"output_type":"execute_result","data":{"text/plain":" AccV AccML AccAP StartHesitation Turn Walking Id \\\n0 -9.101562 1.147461 -2.937500 0 0 0 b6c16738e8 \n1 -9.093750 1.150391 -2.939453 0 0 0 b6c16738e8 \n2 -9.093750 1.157227 -2.951172 0 0 0 b6c16738e8 \n3 -9.093750 1.150391 -2.949219 0 0 0 b6c16738e8 \n4 -9.085938 1.134766 -2.949219 0 0 0 b6c16738e8 \n\n Module Begin End ... Medication Visit_y Age Sex YearsSinceDx \\\n0 tdcsfog -1.0 -1.0 ... 0 0.0 58.0 0 5.0 \n1 tdcsfog -1.0 -1.0 ... 0 0.0 58.0 0 5.0 \n2 tdcsfog -1.0 -1.0 ... 0 0.0 58.0 0 5.0 \n3 tdcsfog -1.0 -1.0 ... 0 0.0 58.0 0 5.0 \n4 tdcsfog -1.0 -1.0 ... 0 0.0 58.0 0 5.0 \n\n UPDRSIII_On UPDRSIII_Off NFOGQ s_group Task_y \n0 21.0 27.0 20.0 3 NaN \n1 21.0 27.0 20.0 3 NaN \n2 21.0 27.0 20.0 3 NaN \n3 21.0 27.0 20.0 3 NaN \n4 21.0 27.0 20.0 3 NaN \n\n[5 rows x 26 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
AccVAccMLAccAPStartHesitationTurnWalkingIdModuleBeginEnd...MedicationVisit_yAgeSexYearsSinceDxUPDRSIII_OnUPDRSIII_OffNFOGQs_groupTask_y
0-9.1015621.147461-2.937500000b6c16738e8tdcsfog-1.0-1.0...00.058.005.021.027.020.03NaN
1-9.0937501.150391-2.939453000b6c16738e8tdcsfog-1.0-1.0...00.058.005.021.027.020.03NaN
2-9.0937501.157227-2.951172000b6c16738e8tdcsfog-1.0-1.0...00.058.005.021.027.020.03NaN
3-9.0937501.150391-2.949219000b6c16738e8tdcsfog-1.0-1.0...00.058.005.021.027.020.03NaN
4-9.0859381.134766-2.949219000b6c16738e8tdcsfog-1.0-1.0...00.058.005.021.027.020.03NaN
\n

5 rows × 26 columns

\n
"},"metadata":{}}]},{"cell_type":"code","source":"df_train['Test'] = df_train['Test'].fillna(-1)\ndf_train['Task_y'] = df_train['Task_y'].fillna(-1)\ndf_train.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:03:56.810784Z","iopub.execute_input":"2023-06-09T10:03:56.811421Z","iopub.status.idle":"2023-06-09T10:04:03.888323Z","shell.execute_reply.started":"2023-06-09T10:03:56.811387Z","shell.execute_reply":"2023-06-09T10:04:03.88711Z"},"trusted":true},"execution_count":16,"outputs":[{"execution_count":16,"output_type":"execute_result","data":{"text/plain":"AccV 0\nAccML 0\nAccAP 0\nStartHesitation 0\nTurn 0\nWalking 0\nId 0\nModule 0\nBegin 0\nEnd 0\nTask 9098919\nDuration 0\nt_group 0\nSubject 0\nVisit_x 0\nTest 0\nMedication 0\nVisit_y 0\nAge 0\nSex 0\nYearsSinceDx 0\nUPDRSIII_On 0\nUPDRSIII_Off 0\nNFOGQ 0\ns_group 0\nTask_y 0\ndtype: int64"},"metadata":{}}]},{"cell_type":"code","source":"# split data into features and target.\ny = [df_train['StartHesitation'], df_train['Turn'], df_train['Walking']] # target\nX = df_train.drop(['StartHesitation', 'Turn', 'Walking', 'Id', 'Task_y'], axis=1) # feature\ncols = [c for c in X.columns if c not in ['Id','Subject','Module', 'Time','Valid', 'Task','Event']]","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:04:03.889967Z","iopub.execute_input":"2023-06-09T10:04:03.89032Z","iopub.status.idle":"2023-06-09T10:04:04.727081Z","shell.execute_reply.started":"2023-06-09T10:04:03.890291Z","shell.execute_reply":"2023-06-09T10:04:04.725847Z"},"trusted":true},"execution_count":17,"outputs":[]},{"cell_type":"code","source":"# check the contents of the target\ny","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:04:04.728691Z","iopub.execute_input":"2023-06-09T10:04:04.729061Z","iopub.status.idle":"2023-06-09T10:04:04.739183Z","shell.execute_reply.started":"2023-06-09T10:04:04.729023Z","shell.execute_reply":"2023-06-09T10:04:04.737997Z"},"trusted":true},"execution_count":18,"outputs":[{"execution_count":18,"output_type":"execute_result","data":{"text/plain":"[0 0\n 1 0\n 2 0\n 3 0\n 4 0\n ..\n 10199880 0\n 10199881 0\n 10199882 0\n 10199883 0\n 10199884 0\n Name: StartHesitation, Length: 10199885, dtype: int8,\n 0 0\n 1 0\n 2 0\n 3 0\n 4 0\n ..\n 10199880 0\n 10199881 0\n 10199882 0\n 10199883 0\n 10199884 0\n Name: Turn, Length: 10199885, dtype: int8,\n 0 0\n 1 0\n 2 0\n 3 0\n 4 0\n ..\n 10199880 0\n 10199881 0\n 10199882 0\n 10199883 0\n 10199884 0\n Name: Walking, Length: 10199885, dtype: int8]"},"metadata":{}}]},{"cell_type":"code","source":"# check the contents of the feature\nX.head()\nX","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:04:04.741082Z","iopub.execute_input":"2023-06-09T10:04:04.741991Z","iopub.status.idle":"2023-06-09T10:04:08.283597Z","shell.execute_reply.started":"2023-06-09T10:04:04.741948Z","shell.execute_reply":"2023-06-09T10:04:08.282486Z"},"trusted":true},"execution_count":19,"outputs":[{"execution_count":19,"output_type":"execute_result","data":{"text/plain":" AccV AccML AccAP Module Begin End Task \\\n0 -9.101562 1.147461 -2.937500 tdcsfog -1.00 -1.0 -1.0 \n1 -9.093750 1.150391 -2.939453 tdcsfog -1.00 -1.0 -1.0 \n2 -9.093750 1.157227 -2.951172 tdcsfog -1.00 -1.0 -1.0 \n3 -9.093750 1.150391 -2.949219 tdcsfog -1.00 -1.0 -1.0 \n4 -9.085938 1.134766 -2.949219 tdcsfog -1.00 -1.0 -1.0 \n... ... ... ... ... ... ... ... \n10199880 -0.953125 0.109375 -0.162720 defog 270.00 287.0 NaN \n10199881 -0.953125 0.109375 -0.162720 defog 389.00 423.5 NaN \n10199882 -0.953125 0.109375 -0.162720 defog 462.25 509.0 NaN \n10199883 -0.953125 0.109375 -0.162720 defog 667.00 677.0 NaN \n10199884 -0.953125 0.109375 -0.162720 defog 694.50 731.0 NaN \n\n Duration t_group Subject ... Test Medication Visit_y Age \\\n0 -1.000000 -1.0 c8e721 ... 1.0 0 0.0 58.0 \n1 -1.000000 -1.0 c8e721 ... 1.0 0 0.0 58.0 \n2 -1.000000 -1.0 c8e721 ... 1.0 0 0.0 58.0 \n3 -1.000000 -1.0 c8e721 ... 1.0 0 0.0 58.0 \n4 -1.000000 -1.0 c8e721 ... 1.0 0 0.0 58.0 \n... ... ... ... ... ... ... ... ... \n10199880 16.875000 2.0 00f674 ... -1.0 0 1.5 63.0 \n10199881 34.312500 6.0 00f674 ... -1.0 0 1.5 63.0 \n10199882 46.656250 6.0 00f674 ... -1.0 0 1.5 63.0 \n10199883 9.914062 1.0 00f674 ... -1.0 0 1.5 63.0 \n10199884 36.781250 1.0 00f674 ... -1.0 0 1.5 63.0 \n\n Sex YearsSinceDx UPDRSIII_On UPDRSIII_Off NFOGQ s_group \n0 0 5.0 21.0 27.0 20.0 3 \n1 0 5.0 21.0 27.0 20.0 3 \n2 0 5.0 21.0 27.0 20.0 3 \n3 0 5.0 21.0 27.0 20.0 3 \n4 0 5.0 21.0 27.0 20.0 3 \n... ... ... ... ... ... ... \n10199880 0 27.0 37.0 39.5 25.0 7 \n10199881 0 27.0 37.0 39.5 25.0 7 \n10199882 0 27.0 37.0 39.5 25.0 7 \n10199883 0 27.0 37.0 39.5 25.0 7 \n10199884 0 27.0 37.0 39.5 25.0 7 \n\n[10199885 rows x 21 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
AccVAccMLAccAPModuleBeginEndTaskDurationt_groupSubject...TestMedicationVisit_yAgeSexYearsSinceDxUPDRSIII_OnUPDRSIII_OffNFOGQs_group
0-9.1015621.147461-2.937500tdcsfog-1.00-1.0-1.0-1.000000-1.0c8e721...1.000.058.005.021.027.020.03
1-9.0937501.150391-2.939453tdcsfog-1.00-1.0-1.0-1.000000-1.0c8e721...1.000.058.005.021.027.020.03
2-9.0937501.157227-2.951172tdcsfog-1.00-1.0-1.0-1.000000-1.0c8e721...1.000.058.005.021.027.020.03
3-9.0937501.150391-2.949219tdcsfog-1.00-1.0-1.0-1.000000-1.0c8e721...1.000.058.005.021.027.020.03
4-9.0859381.134766-2.949219tdcsfog-1.00-1.0-1.0-1.000000-1.0c8e721...1.000.058.005.021.027.020.03
..................................................................
10199880-0.9531250.109375-0.162720defog270.00287.0NaN16.8750002.000f674...-1.001.563.0027.037.039.525.07
10199881-0.9531250.109375-0.162720defog389.00423.5NaN34.3125006.000f674...-1.001.563.0027.037.039.525.07
10199882-0.9531250.109375-0.162720defog462.25509.0NaN46.6562506.000f674...-1.001.563.0027.037.039.525.07
10199883-0.9531250.109375-0.162720defog667.00677.0NaN9.9140621.000f674...-1.001.563.0027.037.039.525.07
10199884-0.9531250.109375-0.162720defog694.50731.0NaN36.7812501.000f674...-1.001.563.0027.037.039.525.07
\n

10199885 rows × 21 columns

\n
"},"metadata":{}}]},{"cell_type":"markdown","source":"# Train SVMs","metadata":{}},{"cell_type":"code","source":"# Function to import pre-trained models\n\ndef load_models():\n import pickle\n models = []\n svmdir = '/kaggle/input/svm-models2'\n\n for i in range(3):\n if os.path.getsize(os.path.join(svmdir, \"SVM_model_\" + str(i + 3) + '.pkl')) > 0: \n with open(os.path.join(svmdir, \"SVM_model_\" + str(i + 3) + '.pkl'), \"rb\") as f:\n unpickler = pickle.Unpickler(f)\n model = unpickler.load()\n models.append(model)\n return models\n\n# mdoels = load_models()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:04:08.285055Z","iopub.execute_input":"2023-06-09T10:04:08.285888Z","iopub.status.idle":"2023-06-09T10:04:08.293607Z","shell.execute_reply.started":"2023-06-09T10:04:08.285854Z","shell.execute_reply":"2023-06-09T10:04:08.292188Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"code","source":"\ngss = GroupShuffleSplit(n_splits=1, test_size=0.3, random_state=42)\nx_test_list = []\ny_test_list = []\nmodels = []\n\n# Train the models\nfor i, (train_index, test_index) in enumerate(gss.split(X, y[0], X['Subject'])):\n for j in range(3): \n svm_model = LinearSVC(C=0.1, max_iter=5000, verbose=1)\n\n X_train = X.loc[train_index, cols].to_numpy()\n y_train = y[j].loc[train_index].to_numpy()\n\n X_train_scaled = scale(X_train)\n\n print(\"Starting training model nr\", j+1)\n svm_model.fit(X_train_scaled, y_train)\n print(\"Model nr\", j+1, \"fitted on data!\")\n\n models.append(svm_model)\n\n X_test = X.loc[test_index, cols].to_numpy()\n y_test = y[j].loc[test_index].to_numpy()\n \n x_test_list.append(X_test)\n y_test_list.append(y_test)\n X_test_scaled = scale(X_test)\n\n score = metrics.average_precision_score(y_test, svm_model.predict(X_test_scaled).clip(0.0,1.0))\n print(\"Average precision for model nr\", j+1, \":\", score, \"\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-06-09T10:04:08.29532Z","iopub.execute_input":"2023-06-09T10:04:08.295664Z"},"trusted":true},"execution_count":null,"outputs":[{"name":"stdout","text":"Starting training model nr 1\n[LibLinear].............................................................................","output_type":"stream"}]},{"cell_type":"code","source":"print(len(models))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 1==1:\n import pickle\n for i in range(3):\n pickle.dump(models[i], open(\"SVM_model_\" + str(i) + '.pkl', 'wb'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare test data\n\n## Process the test data in the same way as the training data","metadata":{}},{"cell_type":"code","source":"# list of all tdcsfog csv file path\ntdcs_test_file_path = glob.glob(os.path.join(pdir, 'test', 'tdcsfog', '*.csv'), recursive=True)\nprint(f'the number of files to be read: {len(tdcs_test_file_path)}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tasks.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(tdcs_test_file_path)\nfor fp in tdcs_test_file_path:\n tmp = pd.read_csv(fp)\n print(tmp.head())\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize a DataFrame to combine data from multiple CSV files.\ndf_tdcs_test = pd.DataFrame()\n\nfor fp in tqdm.tqdm(tdcs_test_file_path):\n # load data into a variable 'tmp'.\n tmp = pd.read_csv(fp, index_col='Time')\n tmp['Id'] = fp.split('/')[-1].split('.')[0]\n dataset = Path(fp).parts[-2]\n tmp['Module'] = dataset\n \n # get file Id from csv file name.\n file_id = os.path.basename(fp).replace(\".csv\", \"\")\n \n # get subject Id.\n subject = df_tdcs_meta.loc[df_tdcs_meta['Id'] == file_id, 'Subject'].iloc[0]\n \n tmp = pd.merge(tmp, tasks, how='left', on='Id').fillna(-1)\n \n tmp = pd.merge(tmp, metadata_w_subjects[['Id', 'Subject', 'Visit_x', 'Test', 'Medication', 'Visit_y', 'Age', 'Sex', 'YearsSinceDx', 'UPDRSIII_On', 'UPDRSIII_Off', 'NFOGQ','s_group']], how='left', on='Id')\n \n # concat the data\n df_tdcs_test = pd.concat([df_tdcs_test, tmp]).reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the contents of the df_tdcs_test\ndf_tdcs_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# list of all tdcsfog csv file path\ndefog_test_file_path = glob.glob(os.path.join(pdir, 'test', 'defog', '*.csv'), recursive=True)\nprint(f'the number of files to be read: {len(defog_test_file_path)}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize a DataFrame to combine data from multiple CSV files.\ndf_defog_test = pd.DataFrame()\n\nfor fp in tqdm.tqdm(defog_test_file_path):\n # load data into a variable 'tmp'.\n tmp = pd.read_csv(fp, index_col='Time')\n tmp['Id'] = fp.split('/')[-1].split('.')[0]\n dataset = Path(fp).parts[-2]\n tmp['Module'] = dataset\n \n # get file Id from csv file name.\n file_id = os.path.basename(fp).replace(\".csv\", \"\")\n \n tmp = pd.merge(tmp, tasks, how='left', on='Id').fillna(-1)\n \n tmp = pd.merge(tmp, metadata_w_subjects[['Id', 'Subject', 'Visit_x', 'Medication', 'Visit_y', 'Age', 'Sex', 'YearsSinceDx', 'UPDRSIII_On', 'UPDRSIII_Off', 'NFOGQ','s_group']], how='left', on='Id')\n \n # concat the data\n df_defog_test = pd.concat([df_defog_test, tmp]).reset_index(drop=True)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the contents of the df_defog_test\ndf_defog_test.head()\nprint(df_defog_test.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# concat tdcs and defog data.\ndf_test = pd.concat([df_tdcs_test, df_defog_test]).reset_index(drop=True)\n\n# encode string columns into 0/1 format\ndf_test['Medication'] = np.where(df_test['Medication']=='on', 1, 0)\ndf_test['Sex'] = np.where(df_test['Sex']=='M', 1, 0)\ndisplay(df_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split data into submission Id and feature.\nId = df_test['Id'] # Id for submission data\nX_test = df_test.drop(['Id'], axis=1) # feature of test data\nX_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict and submit","metadata":{}},{"cell_type":"code","source":"# Convert columns\nX_test = X_test.loc[:, cols].to_numpy()\nX_test = scale(X_test)\nX_test = np.nan_to_num(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calculate prediction using trained model.\npred_list = []\n\nfor i in range(3):\n svm_model = models[i]\n pred_list.append(svm_model.predict(X_test))\nprediction = np.array(pred_list).transpose()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare submit data\nsubmit = pd.DataFrame(Id, columns=['Id'])\nsubmit['StartHesitation'] = prediction[:, 0]\nsubmit['Turn'] = prediction[:, 1]\nsubmit['Walking'] = prediction[:, 2]","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(submit)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the created submission data.\nsubmit.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}