{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-01T17:41:07.070377Z","iopub.execute_input":"2023-11-01T17:41:07.07099Z","iopub.status.idle":"2023-11-01T17:41:07.107873Z","shell.execute_reply.started":"2023-11-01T17:41:07.07094Z","shell.execute_reply":"2023-11-01T17:41:07.106745Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install phik","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:07.110464Z","iopub.execute_input":"2023-11-01T17:41:07.111186Z","iopub.status.idle":"2023-11-01T17:41:21.383438Z","shell.execute_reply.started":"2023-11-01T17:41:07.111143Z","shell.execute_reply":"2023-11-01T17:41:21.381779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Importing Libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport glob\nimport math\nimport phik\nimport seaborn as sns\nimport lightgbm as lgb\nimport matplotlib.cm as cm\nimport matplotlib.colors as mcolors\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\n\nfrom tqdm.auto import tqdm\nfrom os import path\nfrom pathlib import Path\nfrom phik.phik import phik_matrix\nfrom phik.report import plot_correlation_matrix\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.base import clone\nfrom sklearn.metrics import average_precision_score","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:21.385469Z","iopub.execute_input":"2023-11-01T17:41:21.385952Z","iopub.status.idle":"2023-11-01T17:41:21.396072Z","shell.execute_reply.started":"2023-11-01T17:41:21.385909Z","shell.execute_reply":"2023-11-01T17:41:21.39475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# utility functions\ndef get_num_cols(df):\n numeric_columns = df.select_dtypes(include=[np.number]).columns\n return numeric_columns\n\n\ndef factorize_column(column):\n if column.dtype == \"category\" or column.dtype == \"object\":\n return pd.factorize(column)[0]\n return column","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:21.397604Z","iopub.execute_input":"2023-11-01T17:41:21.398087Z","iopub.status.idle":"2023-11-01T17:41:21.417811Z","shell.execute_reply.started":"2023-11-01T17:41:21.39804Z","shell.execute_reply":"2023-11-01T17:41:21.416151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root = \"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/\"\n\ntrain_defog = glob.glob(os.path.join(root, \"train\", \"defog\", \"**\"))\ntrain_tdcsfog = glob.glob(os.path.join(root, \"train\", \"tdcsfog\", \"**\"))\ntest_defog = glob.glob(os.path.join(root, \"test\", \"defog\", \"**\"))\ntest_tdcsfog = glob.glob(os.path.join(root, \"test\", \"tdcsfog\", \"**\"))\n\nsubjects_df = pd.read_csv(os.path.join(root, \"subjects.csv\"))\n\ndaily_metadata = pd.read_csv(os.path.join(root, \"daily_metadata.csv\"))\ntdcsfog_metadata = pd.read_csv(os.path.join(root, \"tdcsfog_metadata.csv\"))\ndefog_metadata = pd.read_csv(os.path.join(root, \"defog_metadata.csv\"))\n\n# events data which shows information of every FOG episodes\nevents_df = pd.read_csv(os.path.join(root, \"events.csv\"))","metadata":{"execution":{"iopub.status.busy":"2023-11-01T18:05:33.016961Z","iopub.execute_input":"2023-11-01T18:05:33.018398Z","iopub.status.idle":"2023-11-01T18:05:33.076591Z","shell.execute_reply.started":"2023-11-01T18:05:33.018348Z","shell.execute_reply":"2023-11-01T18:05:33.074969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdcsfog_metadata = tdcsfog_metadata.rename(\n columns={\n \"Subject\": \"subject\",\n \"Visit\": \"visit\",\n \"Test\": \"test\",\n \"Medication\": \"medication\",\n }\n)\ndefog_metadata = defog_metadata.rename(\n columns={\"Subject\": \"subject\", \"Visit\": \"visit\", \"Medication\": \"medication\"}\n)\nsubjects_df = subjects_df.rename(\n columns={\n \"Subject\": \"subject\",\n \"Visit\": \"visit\",\n \"Age\": \"age\",\n \"Sex\": \"sex\",\n \"YearsSinceDx\": \"years_since_dx\",\n }\n)\n\ndefog_metadata_subjects = subjects_df.merge(\n defog_metadata, on=[\"subject\", \"visit\"], how=\"inner\"\n)\n\ntdcsfog_subjects = subjects_df.drop(\"visit\", axis=1)\ntdcsfog_metadata_subjects = tdcsfog_subjects.merge(\n tdcsfog_metadata, on=[\"subject\"], how=\"inner\"\n)\ntdcsfog_metadata_subjects = tdcsfog_metadata_subjects.drop(\"visit\", axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:21.473499Z","iopub.execute_input":"2023-11-01T17:41:21.474022Z","iopub.status.idle":"2023-11-01T17:41:21.49932Z","shell.execute_reply.started":"2023-11-01T17:41:21.473979Z","shell.execute_reply":"2023-11-01T17:41:21.4978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Exploration\n","metadata":{}},{"cell_type":"code","source":"subjects_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:21.501123Z","iopub.execute_input":"2023-11-01T17:41:21.502168Z","iopub.status.idle":"2023-11-01T17:41:21.520626Z","shell.execute_reply.started":"2023-11-01T17:41:21.502123Z","shell.execute_reply":"2023-11-01T17:41:21.519002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"PTR about subjects_df:\n- Visit Only available for subjects in the daily and defog datasets.\n- YearsSinceDx Years since Parkinson's diagnosis.\n- UPDRSIIIOn/UPDRSIIIOff Unified Parkinson's Disease Rating Scale score during on/off medication respectively.\n- NFOGQ Self-report FoG questionnaire score. See:\nhttps://pubmed.ncbi.nlm.nih.gov/19660949/","metadata":{}},{"cell_type":"code","source":"defog_metadata_subjects","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:21.52368Z","iopub.execute_input":"2023-11-01T17:41:21.524176Z","iopub.status.idle":"2023-11-01T17:41:21.55426Z","shell.execute_reply.started":"2023-11-01T17:41:21.524133Z","shell.execute_reply":"2023-11-01T17:41:21.552922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdcsfog_metadata_subjects","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:21.555675Z","iopub.execute_input":"2023-11-01T17:41:21.556042Z","iopub.status.idle":"2023-11-01T17:41:21.587164Z","shell.execute_reply.started":"2023-11-01T17:41:21.55601Z","shell.execute_reply":"2023-11-01T17:41:21.584916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"The number of patients in DeFog metada:\" ,\n len(pd.unique(defog_metadata_subjects[\"subject\"])))\nprint(\"The number of patients in TDCSFog metada:\" ,\n len(pd.unique(tdcsfog_metadata_subjects[\"subject\"])))","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:21.588906Z","iopub.execute_input":"2023-11-01T17:41:21.589377Z","iopub.status.idle":"2023-11-01T17:41:21.599661Z","shell.execute_reply.started":"2023-11-01T17:41:21.589343Z","shell.execute_reply":"2023-11-01T17:41:21.597964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For analyzing the information of all 107 patients, both defog and tdcsfog metadata are joined.","metadata":{}},{"cell_type":"code","source":"full_metadata = pd.concat([defog_metadata_subjects, tdcsfog_metadata_subjects], axis=0)\n\nfull_metadata.drop([\"visit\", \"test\"], axis=1, inplace=True)\n\n#mapping binary values to 0 and 1\nfull_metadata[\"sex\"] = full_metadata[\"sex\"].map({\"F\": 0, \"M\": 1})\nfull_metadata[\"medication\"] = full_metadata[\"medication\"].map({\"on\": 0, \"off\": 1})\nfull_metadata","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:21.601863Z","iopub.execute_input":"2023-11-01T17:41:21.602397Z","iopub.status.idle":"2023-11-01T17:41:21.638229Z","shell.execute_reply.started":"2023-11-01T17:41:21.602325Z","shell.execute_reply":"2023-11-01T17:41:21.636769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(full_metadata[[\"age\",\"years_since_dx\",\"UPDRSIII_On\",\"UPDRSIII_Off\",\"NFOGQ\"]].describe())\n\ndf=full_metadata[[\"age\",\"years_since_dx\",\"UPDRSIII_On\",\"UPDRSIII_Off\",\"NFOGQ\"]].copy()\nplt.figure(figsize=(8,5), dpi= 80)\nsns.boxplot( data=df, notch=False)\n\nplt.title('Box Plot of the full metadata', fontsize=10)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:21.642517Z","iopub.execute_input":"2023-11-01T17:41:21.642985Z","iopub.status.idle":"2023-11-01T17:41:22.246242Z","shell.execute_reply.started":"2023-11-01T17:41:21.642951Z","shell.execute_reply":"2023-11-01T17:41:22.244686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.style.use('default')\n\ntemp= full_metadata.copy()\ntemp.dropna(inplace=True)\ntemp=temp[temp['age']!=28]\ntemp.rename(columns={\"years_since_dx\":\"Years Since Diagnosis\"}, inplace=True)\ntemp['Age']=temp['age'].astype(\"int64\")\ntemp['Years Since Diagnosis']=temp['Years Since Diagnosis'].astype(\"int64\")\ncolumns = ['Age', 'Years Since Diagnosis']\n\ncolors_gender = ['#c2c2f0','#ffb3e6']\nfig, axes = plt.subplots(1, 3, figsize=(12, 4))\n\ncounts = temp[\"sex\"].value_counts()\ngender_labels=[\"Male\",\"Female\"]\naxes[0].pie(counts, labels=gender_labels,counterclock=False,startangle=90,autopct='%1.1f%%',\n colors=colors_gender, wedgeprops={'edgecolor': 'white', 'linewidth': 0.7})\naxes[0].set_title('Distribution of Sex', pad=20,fontdict={'fontsize': 10,'fontfamily':'Tahoma'})\n\nfor i, column in enumerate(columns):\n # Get the minimum and maximum values of the column\n min_value = temp[column].min()\n max_value = temp[column].max()\n\n min_value=5 * math.floor(min_value / 5)\n # Create age ranges using pd.cut()\n age_ranges = pd.cut(temp[column], bins=np.arange(min_value, max_value+5, 5), right=False,\n labels=[f'{i}-{i+5}' for i in range(min_value, max_value, 5)])\n\n temp['Age Range'] = age_ranges\n\n counts = temp['Age Range'].value_counts()\n counts = counts.sort_values(ascending=False)\n \n # Sort the sizes_gender variable based on the sorting order of counts\n sizes_gender = temp.groupby('Age Range')['sex'].value_counts().reindex(counts.index, level=0)\n \n gender_counts = sizes_gender.index.get_level_values('sex')\n\n # Define the color map and assign colors based on gender counts\n colors = [colors_gender[0] if gender == 1 else colors_gender[1] for gender in gender_counts]\n\n wedges, texts, autotexts =axes[i+1].pie(counts, labels=counts.index,counterclock=False,startangle=90,\n autopct='%1.0f%%',wedgeprops={'edgecolor': 'white', 'linewidth': 0.5})\n for autotext in autotexts:\n autotext.set_horizontalalignment('center') \n autotext.set_verticalalignment('center') \n autotext.set_position((1.5 * autotext.get_position()[0],1.5*autotext.get_position()[1]))\n autotext.set_fontsize(8)\n autotext.set_color('white')\n \n axes[i+1].set_title(f'Distribution of {column} and \\n Sex ', pad=15, fontdict={'fontsize': 10,'fontfamily':'Tahoma'})\n axes[i+1].pie(sizes_gender,colors=colors,radius=0.75,counterclock=False,startangle=90,\n wedgeprops={'edgecolor': 'white', 'linewidth': 0.5})\n \n centre_circle = plt.Circle((0,0),0.55,fc='white')\n axes[i+1].add_artist(centre_circle)\n # Equal aspect ratio ensures that pie is drawn as a circle\n axes[i+1].axis('equal') \n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:22.24846Z","iopub.execute_input":"2023-11-01T17:41:22.248913Z","iopub.status.idle":"2023-11-01T17:41:23.088287Z","shell.execute_reply.started":"2023-11-01T17:41:22.248879Z","shell.execute_reply":"2023-11-01T17:41:23.087085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Key Insights:\n\n- Approximately 80% of the participants are men.\n- More than 56% of the participants are between 65-75 years old.\n- Around 60% of the participants were diagnosed 5-15 years ago.\n- Among participants aged between 80-85, the majority are women.","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\nimport matplotlib.colors as mcolors\n\nfull_metadata[\"diff\"] = full_metadata[\"UPDRSIII_Off\"]-full_metadata[\"UPDRSIII_On\"]\n\nplt.figure(figsize=(6, 4))\n\n# Compute the correlation matrix\nnumberic_columns= get_num_cols(full_metadata)\ncorr_matrix = full_metadata[numberic_columns].corr()\n\n# Mask the diagonal elements\nmask = np.zeros_like(corr_matrix, dtype=bool)\nmask[np.triu_indices_from(mask)] = True\n\nheatmap = sns.heatmap(corr_matrix, cmap='RdBu', annot=True, fmt='.2f', linewidths=0.15,\n square=True,\n mask=mask\n )\nplt.title('Full metadata pairwise Correlation Heatmap', fontsize=9)\n\nheatmap.set_xticklabels(heatmap.get_xticklabels(), fontsize=8)\nheatmap.set_yticklabels(heatmap.get_yticklabels(), fontsize=8)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:23.09426Z","iopub.execute_input":"2023-11-01T17:41:23.094729Z","iopub.status.idle":"2023-11-01T17:41:23.598374Z","shell.execute_reply.started":"2023-11-01T17:41:23.094692Z","shell.execute_reply":"2023-11-01T17:41:23.595305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A 'diff' column is added to display the difference between UPDRSIII_Off and UPDRSIII_On.\n\n- There is a relationship between diff and age, which will be analyzed.\n- Additionally, it seems reasonable to fill the missing values of UPDRSIII_Off (presented in the preprocessing part) since it has a strong relationship with UPDRSIII_On. It is also slightly correlated with years_since_dx, age, and sex.","metadata":{}},{"cell_type":"code","source":"temp = full_metadata[~full_metadata['UPDRSIII_On'].isna()].sort_values(\"diff\").copy()\n\nage = temp['age']\ndiff = temp['diff']\nfrequency = temp['age'].value_counts()\n\ncmap = plt.colormaps['Oranges']\n\n\nfig = plt.figure(figsize=(10, 4))\n# Define a custom grid layout for the subplots\ngrid = gridspec.GridSpec(1, 2, width_ratios=[1, 1])\n# Create the first subplot with a specific width\nax1 = plt.subplot(grid[0])\n\n\n# Create a subplot for the scatter plot or line plot\nplt.scatter(full_metadata['UPDRSIII_Off'], full_metadata['UPDRSIII_On'], c=full_metadata['age'], cmap='Oranges')\nplt.xlabel('UPDRSIII_Off')\nplt.ylabel('UPDRSIII_On')\nplt.title('Scatter Plot of UPDRSIII_Off vs UPDRSIII_On', fontsize=9, pad=15)\nplt.colorbar(label='Age')\n\nplt.plot([0, 90], [0, 90], 'r--') # Add a red dashed line from (0, 0) to (90, 90)\n\n\n\nax2 = plt.subplot(grid[1])\nplt.subplot(1, 2, 2)\nm=plt.scatter(age, diff, c=frequency[age].values, cmap=cmap, s=frequency[age].values*5, alpha=0.8)\nplt.xlabel('age')\nplt.ylabel('diff')\nplt.title('Relationship between Age and the difference of\\n UPDRSIII_On and UPDRSIII_Off', fontsize=9, pad=15)\nplt.colorbar(mappable=m, label='Frequency')\n\n# Add a horizontal line at y=0\nplt.axhline(0, color='black', linestyle='--') \n\nplt.tight_layout() \nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:23.600326Z","iopub.execute_input":"2023-11-01T17:41:23.60109Z","iopub.status.idle":"2023-11-01T17:41:24.478499Z","shell.execute_reply.started":"2023-11-01T17:41:23.601053Z","shell.execute_reply":"2023-11-01T17:41:24.477056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
As the plot above depicts, for the majority of cases, medication is effective in controlling UPDRSIII. However, in some cases, medication not only fails to be effective but also worsens the situation.
\nIn the next chart, the difference between UPDRSIII, when medication is on and when it is off, is calculated and compared to age. For ages below 60, medication is almost universally effective. However, for ages above 60, the effects vary greatly in a case-by-case manner. The most diversity is observed in the age range of 65-75 (as we saw before, 66% of patients are in this age group). In this age range, medication can either worsen the symptoms or be even more effective compared to other age ranges.
\n\n\nThe following trends can be observed with respect to the role of sex:\n","metadata":{}},{"cell_type":"code","source":"# women full metadata\ntemp = full_metadata.copy()\ntemp = temp[temp[\"sex\"]==0]\ntemp[\"diff\"]=temp[\"UPDRSIII_Off\"]-temp[\"UPDRSIII_On\"]\n\nprint(\"Average of UPDRSIII_Off in women: {:.2f}\".format(temp[\"UPDRSIII_Off\"].mean()))\nprint(\"Average of UPDRSIII_On in women: {:.2f}\".format(temp[\"UPDRSIII_On\"].mean()))\n\nprint(\"Average of difference between UPDRSIII_On and UPDRSIII_Off in women: {:.2f}\".format(temp[\"diff\"].mean()))\nprint(\"Ineffective medication cases in women: {:.2f} \".format((len(temp[temp[\"diff\"]<0])/len(temp))*100))","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:24.480523Z","iopub.execute_input":"2023-11-01T17:41:24.481062Z","iopub.status.idle":"2023-11-01T17:41:24.497255Z","shell.execute_reply.started":"2023-11-01T17:41:24.48101Z","shell.execute_reply":"2023-11-01T17:41:24.495659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# men full metadata\ntemp = full_metadata.copy()\ntemp = temp[temp[\"sex\"]==1]\ntemp[\"diff\"]=temp[\"UPDRSIII_Off\"]-temp[\"UPDRSIII_On\"]\n\nprint(\"Average of UPDRSIII_Off in men: {:.2f}\".format(temp[\"UPDRSIII_Off\"].mean()))\nprint(\"Average of UPDRSIII_On in men: {:.2f}\".format(temp[\"UPDRSIII_On\"].mean()))\nprint(\"Average of difference between UPDRSIII_On and UPDRSIII_Off in men: {:.2f}\".format(temp[\"diff\"].mean()))\nprint(\"Ineffective medication cases in men: {:.2f}\".format((len(temp[temp[\"diff\"]<0])/len(temp))*100))","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:24.499367Z","iopub.execute_input":"2023-11-01T17:41:24.500193Z","iopub.status.idle":"2023-11-01T17:41:24.516013Z","shell.execute_reply.started":"2023-11-01T17:41:24.500137Z","shell.execute_reply":"2023-11-01T17:41:24.514104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This analysis reveals the following insights:\n\n* **Women generally exhibit a slightly better condition in Parkinson's disease.**\n* **On the other hand, medication tends to be trivially more effective in men.**","metadata":{}},{"cell_type":"markdown","source":"Now, we investigate a time series dataset that includes all three types of FOG, i.e., Start Hesitation, Turn, and Walking.","metadata":{}},{"cell_type":"code","source":"# get file series Id that includes all three FOG episodes\ndesired_id = events_df.groupby(\"Id\").filter(lambda x: all(item in x[\"Type\"].values for item in [\"StartHesitation\", \"Turn\", \"Walking\"]))[\"Id\"]\ndesired_id=pd.unique(desired_id)\n \nsample_Id=[]\nfor file in train_tdcsfog:\n Id = file.split('/')[-1].split('.')[0]\n if Id in desired_id:\n sample_Id.append(Id)\n\nsample_train_tdcsfog_series = pd.read_csv(root + '/train/tdcsfog/' + str(sample_Id[0]) + '.csv')\nsample_train_tdcsfog_series","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:24.51796Z","iopub.execute_input":"2023-11-01T17:41:24.518653Z","iopub.status.idle":"2023-11-01T17:41:24.638288Z","shell.execute_reply.started":"2023-11-01T17:41:24.518609Z","shell.execute_reply":"2023-11-01T17:41:24.636905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This dataset is comprised of lower-back 3D accelerometer data which captures the following features related to FOG detection:\n\n* AccV: acceleration measured by a vertical sensor\n* AccML: acceleration measured by a mediolateral sensor\n* AccAP: acceleration measured by an anteroposterior sensor\n\n\"axis\nThese three characteristics are the main feautures to detect FOG.\n","metadata":{}},{"cell_type":"code","source":"sample_train_tdcsfog_series.describe()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:24.639849Z","iopub.execute_input":"2023-11-01T17:41:24.640202Z","iopub.status.idle":"2023-11-01T17:41:24.683052Z","shell.execute_reply.started":"2023-11-01T17:41:24.640173Z","shell.execute_reply":"2023-11-01T17:41:24.681688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since the range of acceleration data is different, a suitable normalization process may improve the model performance (presented in the preprocessing part).","metadata":{}},{"cell_type":"code","source":"# Turn series\nsample_tdcsfog_Turn= sample_train_tdcsfog_series[sample_train_tdcsfog_series[\"Turn\"]==1]\n\n# StartHesitation series\nsample_tdcsfog_StartHesitation= sample_train_tdcsfog_series[sample_train_tdcsfog_series[\"StartHesitation\"]==1]\n\n# Walking series\nsample_tdcsfog_Walking= sample_train_tdcsfog_series[sample_train_tdcsfog_series[\"Walking\"]==1]\n\n# None Event series\nsample_tdcsfog_None= sample_train_tdcsfog_series[~((sample_train_tdcsfog_series[\"Walking\"]==1) |\n (sample_train_tdcsfog_series[\"StartHesitation\"]==1)|\n (sample_train_tdcsfog_series[\"Turn\"]==1))]\n\n# # Create the 4*2 grid of subplots\ngrid = (4, 2)\nfig = plt.figure(figsize=(10, 5))\n\n# # Create the first pie chart spanning cells (1,1) and (2,1) and (3,1) and(4,1)\nax1 = plt.subplot2grid(grid, (0, 0), rowspan=4, colspan=1)\n\nax1.plot(sample_train_tdcsfog_series[\"Time\"], sample_train_tdcsfog_series[\"AccV\"], label=\"AccV\")\nax1.plot(sample_train_tdcsfog_series[\"Time\"], sample_train_tdcsfog_series[\"AccML\"], label=\"AccML\")\nax1.plot(sample_train_tdcsfog_series[\"Time\"], sample_train_tdcsfog_series[\"AccAP\"], label=\"AccAP\")\n\nax1.legend()\n\nax1.set_title(\"TDCSFog series sample\")\n\nax2 = plt.subplot2grid(grid, (0, 1))\n\n# Plot sample_tdcsfog_Turn in the second column\nax2.plot(sample_tdcsfog_Turn[\"Time\"], sample_tdcsfog_Turn[[\"AccV\", \"AccML\", \"AccAP\"]])\nax2.set_title(\"Turn\")\n\nax3 = plt.subplot2grid(grid, (1, 1))\n# Plot sample_tdcsfog_StartHesitation in the second column\nax3.plot(sample_tdcsfog_StartHesitation[\"Time\"], sample_tdcsfog_StartHesitation[[\"AccV\", \"AccML\", \"AccAP\"]])\nax3.set_title(\"StartHesitation\")\n\nax4 = plt.subplot2grid(grid, (2, 1))\n# Plot sample_tdcsfog_Walking in the second column\nax4.plot(sample_tdcsfog_Walking[\"Time\"], sample_tdcsfog_Walking[[\"AccV\", \"AccML\", \"AccAP\"]])\nax4.set_title(\"Walking\")\n\nax4 = plt.subplot2grid(grid, (3, 1))\n# Plot sample_tdcsfog_None in the second column\nax4.plot(sample_tdcsfog_None[\"Time\"], sample_tdcsfog_None[[\"AccV\", \"AccML\", \"AccAP\"]])\nax4.set_title(\"None\")\n\n# Adjust spacing between subplots\nplt.tight_layout()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:24.685158Z","iopub.execute_input":"2023-11-01T17:41:24.685698Z","iopub.status.idle":"2023-11-01T17:41:26.088964Z","shell.execute_reply.started":"2023-11-01T17:41:24.685653Z","shell.execute_reply":"2023-11-01T17:41:26.087524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In a particular time series, we observe that the variation of acceleration data during the Turn episode is at its `maximum` , which is a reasonable expectation.\nHowever, what makes the prediction challenging is the observation of non-trivial variations even in the absence of Turn, Walking, or Start Hesitation episodes. This variability in the data adds complexity to the prediction task.","metadata":{}},{"cell_type":"markdown","source":"Counts of FoG events for different age ranges in the dataset. This will help us to understand if there is any correlation between age and the frequency of FoG events. We can use the pd.qcut() function to bin the age column into different age ranges, then count the number of FoG events in each range and plot it.","metadata":{}},{"cell_type":"code","source":"tdcsfog_metadata['dataset'] = 'tdcsfog'\ndefog_metadata['dataset'] = 'defog'\n\nfull_metadata_plot = pd.concat([tdcsfog_metadata, defog_metadata])\nevents_data = events_df\nevents_data['event_duration'] = events_df['Completion'] - events_df['Init']\n\nall_metadata_df = events_data[['Id', 'Type', 'Kinetic', 'event_duration']] \\\n .merge(full_metadata_plot, on='Id') \\\n .merge(subjects_df.drop('visit', axis=1), on='subject')\nall_metadata_df\n","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:26.09081Z","iopub.execute_input":"2023-11-01T17:41:26.091885Z","iopub.status.idle":"2023-11-01T17:41:26.142246Z","shell.execute_reply.started":"2023-11-01T17:41:26.091848Z","shell.execute_reply":"2023-11-01T17:41:26.140963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_metadata_df[\"sex\"] = all_metadata_df[\"sex\"].map({\"F\": 0, \"M\": 1})\nall_metadata_df[\"medication\"] = all_metadata_df[\"medication\"].map({\"on\": 0, \"off\": 1})\nall_metadata_df","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:26.1438Z","iopub.execute_input":"2023-11-01T17:41:26.144127Z","iopub.status.idle":"2023-11-01T17:41:26.183303Z","shell.execute_reply.started":"2023-11-01T17:41:26.144098Z","shell.execute_reply":"2023-11-01T17:41:26.181808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# bin the Age column based on quantiles\nall_metadata_df['age_bin'] = pd.qcut(all_metadata_df['age'], q=5)","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:26.18519Z","iopub.execute_input":"2023-11-01T17:41:26.186918Z","iopub.status.idle":"2023-11-01T17:41:26.197806Z","shell.execute_reply.started":"2023-11-01T17:41:26.186858Z","shell.execute_reply":"2023-11-01T17:41:26.196313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the frequency of FoG events of each Type for different age ranges\nplt.figure(figsize=(15, 14))\nfor i, age_bin in enumerate(all_metadata_df.age_bin.unique(), 1):\n plt.subplot(3, 3, i)\n sns.barplot(\n x=all_metadata_df[all_metadata_df.age_bin == age_bin].Type\n .value_counts(normalize=True).index,\n y=all_metadata_df[all_metadata_df.age_bin == age_bin].Type\n .value_counts(normalize=True))\n plt.ylabel('The frequency of FoG events')\n plt.title(f\"Age in range {age_bin}\")\n \n # Add frequency values on top of the bars\n for index, value in enumerate(\n all_metadata_df[all_metadata_df.age_bin == age_bin].Type\n .value_counts(normalize=True)):\n plt.annotate(f\"{value * 100:.1f}%\", xy=(index, value),\n ha='center', va='bottom')\nplt.subplots_adjust(wspace=0.3, hspace=0.2)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:26.199664Z","iopub.execute_input":"2023-11-01T17:41:26.200034Z","iopub.status.idle":"2023-11-01T17:41:27.321656Z","shell.execute_reply.started":"2023-11-01T17:41:26.200002Z","shell.execute_reply":"2023-11-01T17:41:27.320819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"frequency of FoG events of each Type for male and female patients in the dataset.","metadata":{}},{"cell_type":"code","source":"# Plot the frequency of FoG events of each Type for male and female patients\nplt.figure(figsize=(12, 5))\ngender_dict = {0: 'Female', 1: 'Male'}\nfor i, gender in enumerate(all_metadata_df.sex.unique(), 1):\n plt.subplot(1, 2, i)\n sns.barplot(\n x=all_metadata_df[all_metadata_df.sex == gender].Type\n .value_counts(normalize=True).index,\n y=all_metadata_df[all_metadata_df.sex == gender].Type\n .value_counts(normalize=True))\n plt.ylabel('The frequency of FoG events')\n plt.title(f\"Gender = {gender_dict[gender]}\")\n \n # Add frequency values on top of the bars\n for index, value in enumerate(\n all_metadata_df[all_metadata_df.sex == gender].Type\n .value_counts(normalize=True)):\n plt.annotate(f\"{value * 100:.1f}%\", xy=(index, value),\n ha='center', va='bottom')\nplt.subplots_adjust(wspace=0.4)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:27.323049Z","iopub.execute_input":"2023-11-01T17:41:27.323578Z","iopub.status.idle":"2023-11-01T17:41:27.875419Z","shell.execute_reply.started":"2023-11-01T17:41:27.323533Z","shell.execute_reply":"2023-11-01T17:41:27.874063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the counts of FoG events for different age ranges\nplt.figure(figsize=(12, 5))\nmed_dict = {0: 'Medication status: off', 1: 'Medication status: on'}\nfor i, med_status in enumerate(all_metadata_df.medication.unique(), 1):\n plt.subplot(1, 2, i)\n sns.barplot(\n x=all_metadata_df[all_metadata_df.medication == med_status].Type\n .value_counts(normalize=True).index,\n y=all_metadata_df[all_metadata_df.medication == med_status].Type\n .value_counts(normalize=True))\n plt.ylabel('The frequency of FoG events')\n plt.title(f\"{med_dict[med_status]}\")\n \n # Add frequency values on top of the bars\n for index, value in enumerate(\n all_metadata_df[all_metadata_df.medication == med_status].Type\n .value_counts(normalize=True)):\n plt.annotate(f\"{value * 100:.1f}%\", xy=(index, value),\n ha='center', va='bottom')\nplt.subplots_adjust(wspace=0.4)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:27.876997Z","iopub.execute_input":"2023-11-01T17:41:27.877584Z","iopub.status.idle":"2023-11-01T17:41:28.755243Z","shell.execute_reply.started":"2023-11-01T17:41:27.877532Z","shell.execute_reply":"2023-11-01T17:41:28.753687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The train Datasets exploring","metadata":{}},{"cell_type":"markdown","source":"### The lab data (`tdcsfog` Dataset)","metadata":{}},{"cell_type":"markdown","source":"The tDCS FOG (tdcsfog) dataset comprises data series collected in the lab, as subjects completed a FOG-provoking protocol.\n\nLet's select only 500 files from the `tdcsfog` folder and take a look at the dataset profile report and correlation matrix.","metadata":{}},{"cell_type":"code","source":"# Read 500 files of the tdcsfog metadata (the total number of files is 833)\ntdcsfog_files = [f'{root}train/tdcsfog/{id}.csv' for id in \\\n tdcsfog_metadata.Id.to_list()[:501]]\n\ntdcsfog_data = pd.concat([pd.read_csv(file) for file in tdcsfog_files])","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:28.756756Z","iopub.execute_input":"2023-11-01T17:41:28.757124Z","iopub.status.idle":"2023-11-01T17:41:35.608325Z","shell.execute_reply.started":"2023-11-01T17:41:28.757094Z","shell.execute_reply":"2023-11-01T17:41:35.607029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mask = np.triu(np.ones_like(tdcsfog_data.corr(), dtype=bool))\nfig, ax = plt.subplots(figsize=(7, 5))\nsns.heatmap(tdcsfog_data.corr(), mask=mask, annot=True, cmap='coolwarm',\n linewidths=0.2, annot_kws={\"size\": 7}, rasterized=True)\nplt.xticks(rotation=35)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:35.610534Z","iopub.execute_input":"2023-11-01T17:41:35.610927Z","iopub.status.idle":"2023-11-01T17:41:37.562448Z","shell.execute_reply.started":"2023-11-01T17:41:35.610895Z","shell.execute_reply":"2023-11-01T17:41:37.560901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"phik_overview = tdcsfog_data.phik_matrix(\n interval_cols=['AccV', 'AccML', 'AccAP','Time'])\n\nplot_correlation_matrix(phik_overview.values, x_labels=phik_overview.columns,\n y_labels=phik_overview.index, vmin=0, vmax=1, \n color_map='Blues', title=r'correlation matrix $\\phi_K$',\n figsize=(7, 5))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:41:37.564658Z","iopub.execute_input":"2023-11-01T17:41:37.56504Z","iopub.status.idle":"2023-11-01T17:42:10.850866Z","shell.execute_reply.started":"2023-11-01T17:41:37.565008Z","shell.execute_reply":"2023-11-01T17:42:10.849828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From dataset report and correlation matrix, it can be concluded that:\n\n - There are no missing values and duplicate rows detected in the dataset.\n - As can be seen from the profiling report, all target variables are highly imbalanced, especially `StartHesitation` (78.5%) and `Walking` (80.1%).\n - Based on histograms and skewness values the distributions of the `AccAP` and `AccV` columns are moderately left-skewed, the `AccML` column seems to have a close to normal distribution.\n - The `AccAP` column have a kurtosis value of less than 3, which indicates that the column is platikurtic. Meanwhile, the `AccV` column has a kurtosis value of more than 3, which indicates that the column is leptokurtic. And a kurtosis value of the`AccML` column is close to 3, which is recognized as mesokurtic column.\n - As can be seen from Pearson's and Phik correlation matrices `Time` column have a moderate positive correlation with two target variables `Turn`, `Walking` and variable of anteroposterior acceleration measurements. However, between the target variable `StartHesitation` and other variables there is no strong or moderate correlation.","metadata":{}},{"cell_type":"markdown","source":"#### Plotting the acceleration measurements for events of each Type","metadata":{}},{"cell_type":"code","source":"# Read one file of the tdcsfog train data with event of Turn type\ntdcsfog_files_turn = [f'{root}train/tdcsfog/{id}.csv' for id in \\\n tdcsfog_metadata.Id.to_list()[16:17]]\n\ntdcsfog_data_turn = pd.concat([pd.read_csv(file) for file in tdcsfog_files_turn])\n\n# Read one file of the tdcsfog train data with event of Walking type\ntdcsfog_files_walk = [f'{root}train/tdcsfog/{id}.csv' for id in \\\n tdcsfog_metadata.Id.to_list()[19:20]]\n\ntdcsfog_data_walk = pd.concat([pd.read_csv(file) for file in tdcsfog_files_walk])\n\n# Read one file of the tdcsfog train data with event of StartHesitation type\ntdcsfog_files_hes = [f'{root}train/tdcsfog/{id}.csv' for id in \\\n tdcsfog_metadata.Id.to_list()[77:78]]\n\ntdcsfog_data_hes = pd.concat([pd.read_csv(file) for file in tdcsfog_files_hes])","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:42:10.85221Z","iopub.execute_input":"2023-11-01T17:42:10.853041Z","iopub.status.idle":"2023-11-01T17:42:10.980085Z","shell.execute_reply.started":"2023-11-01T17:42:10.853006Z","shell.execute_reply":"2023-11-01T17:42:10.978679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since the series in the `tdcsfog` dataset are recorded at a frequency of 128Hz (128 time steps per second), we can plot the acceleration measurements for each axis (V - vertical, ML - mediolateral, AP - anteroposterior) using a rolling mean method with a window size of 128. This helps to smooth out fluctuations in the data and provides a better representation of the overall trend of the acceleration measurements over time.","metadata":{}},{"cell_type":"markdown","source":"Let's plot the overall trends of acceleration measurements for events of each Type.","metadata":{}},{"cell_type":"code","source":"# Plot the overall trends of acceleration measurements for events of each Type\nplt.figure(figsize=(14, 12))\n\ntypes_dict = {1: 'Turn', 2: 'Walking', 3: 'StartHesitation'}\n\nfor i, data in enumerate(\n [tdcsfog_data_turn, tdcsfog_data_walk, tdcsfog_data_hes], 1):\n plt.subplot(2, 2, i)\n window_size = 128\n accv_rolling_mean = data[\"AccV\"].rolling(window_size).mean()\n accap_rolling_mean = data[\"AccAP\"].rolling(window_size).mean()\n accml_rolling_mean = data[\"AccML\"].rolling(window_size).mean()\n\n sns.lineplot(x=data.Time, y=accv_rolling_mean)\n sns.lineplot(x=data.Time, y=accap_rolling_mean)\n sns.lineplot(x=data.Time, y=accml_rolling_mean)\n plt.title(f'The trend of acceleration measurements (\"{types_dict[i]}\" event)')\n plt.xlabel('Time')\n plt.ylabel('Acceleration value (m/s^2)')\n\nplt.legend(['Anteroposterior', 'Mediolateral', 'Vertical'], loc='upper left',\n bbox_to_anchor=(2.3, 2.1), handlelength=0,\n shadow=True, labelcolor=['orange', 'green', 'blue'])\nplt.subplots_adjust(wspace=0.3, hspace=0.2)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:42:10.98251Z","iopub.execute_input":"2023-11-01T17:42:10.983495Z","iopub.status.idle":"2023-11-01T17:42:13.705104Z","shell.execute_reply.started":"2023-11-01T17:42:10.983448Z","shell.execute_reply":"2023-11-01T17:42:13.703864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### The home data (`DeFOG` Dataset)","metadata":{}},{"cell_type":"markdown","source":"The DeFOG (defog) dataset comprises data series collected in the subject's home, as subjects completed a FOG-provoking protocol.\n\nAs we know from the data descriptions, the series in the `notype` folder are from the DeFOG dataset but lack event-type annotations. Let's create a list of indices that correspond to the notype series IDs. Using this list, we can separate the `defog_metadata` set into two metadata sets: one for notype and one for defog series.","metadata":{}},{"cell_type":"code","source":"# Creating a list of indices that correspond to the notype series IDs in the defog_metadata\nnotype_idx = [0, 4, 13, 15, 17, 19, 20, 22, 25, 27, 30, 31, 32, 34, 36, 40, 42,\n 53, 54, 55, 58, 60, 62, 66, 67, 73, 74, 76, 79, 83, 84, 88, 89, 94,\n 96, 97, 98, 101, 102, 105, 113, 114, 115, 121, 124, 126]\n\n# Separating the defog_metadata\ndefog_metadata_only = defog_metadata[~defog_metadata.index.isin(notype_idx)].copy()\nnotype_metadata = defog_metadata[defog_metadata.index.isin(notype_idx)].copy()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:42:13.706738Z","iopub.execute_input":"2023-11-01T17:42:13.707727Z","iopub.status.idle":"2023-11-01T17:42:13.718841Z","shell.execute_reply.started":"2023-11-01T17:42:13.707691Z","shell.execute_reply":"2023-11-01T17:42:13.717531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read one file of the defog metadata (the total number of files is 91)\ndefog_files = [f'{root}train/defog/{id}.csv' for id in \\\n defog_metadata_only.Id.to_list()[:1]]\n\ndefog_data = pd.concat([pd.read_csv(file) for file in defog_files])","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:42:13.720847Z","iopub.execute_input":"2023-11-01T17:42:13.721639Z","iopub.status.idle":"2023-11-01T17:42:13.913222Z","shell.execute_reply.started":"2023-11-01T17:42:13.721593Z","shell.execute_reply":"2023-11-01T17:42:13.912187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.concat([defog_metadata[defog_metadata.Id == '02ea782681'],\n events_data[events_data.Id == '02ea782681']], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:42:13.91508Z","iopub.execute_input":"2023-11-01T17:42:13.915838Z","iopub.status.idle":"2023-11-01T17:42:13.943234Z","shell.execute_reply.started":"2023-11-01T17:42:13.915793Z","shell.execute_reply":"2023-11-01T17:42:13.941484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog_data_valid = defog_data.loc[ \\\n (defog_data.Valid == True) & (defog_data.Task == True)]\ndefog_data_valid.shape","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:42:13.944837Z","iopub.execute_input":"2023-11-01T17:42:13.946942Z","iopub.status.idle":"2023-11-01T17:42:13.961331Z","shell.execute_reply.started":"2023-11-01T17:42:13.946897Z","shell.execute_reply":"2023-11-01T17:42:13.960039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Ploting the acceleration measurements for each axis","metadata":{}},{"cell_type":"markdown","source":"Since the series in the `defog` dataset are recorded at a frequency of 100Hz (100 time steps per second), we can plot the smoothed acceleration measurements for each axis (V - vertical, ML - mediolateral, AP - anteroposterior) using a rolling window technique with a window size of 100 and see the overall trend of acceleration measurements for each axis.","metadata":{}},{"cell_type":"code","source":"# Plot the overall trend of acceleration measurements for each axis\nplt.figure(figsize=(14, 5))\n\ndefogdata_dict = {1: 'All defogdata', 2: 'Valid defog data'}\n\nfor i, data in enumerate([defog_data, defog_data_valid], 1):\n plt.subplot(1, 2, i)\n window_size = 100\n accv_rolling_mean = data[\"AccV\"].rolling(window_size).mean()\n accap_rolling_mean = data[\"AccAP\"].rolling(window_size).mean()\n accml_rolling_mean = data[\"AccML\"].rolling(window_size).mean()\n\n sns.lineplot(x=data.Time, y=accv_rolling_mean)\n sns.lineplot(x=data.Time, y=accap_rolling_mean)\n sns.lineplot(x=data.Time, y=accml_rolling_mean)\n plt.title(f'The trend of acceleration measurements ({defogdata_dict[i]})')\n plt.xlabel('Time')\n plt.ylabel('Acceleration value (g)')\n\nplt.legend(['Mediolateral', 'Anteroposterior', 'Vertical'], loc='upper left',\n bbox_to_anchor=(1.05, 1), shadow=True, handlelength=0,\n labelcolor=['green', 'orange', 'blue'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:42:13.963075Z","iopub.execute_input":"2023-11-01T17:42:13.964035Z","iopub.status.idle":"2023-11-01T17:42:18.5445Z","shell.execute_reply.started":"2023-11-01T17:42:13.963997Z","shell.execute_reply":"2023-11-01T17:42:18.543079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### The daily data (`Daily Living` dataset)","metadata":{}},{"cell_type":"markdown","source":"The Daily Living (daily) dataset comprises one week of continuous 24/7 recordings from sixty-five subjects. Forty-five subjects exhibit FOG symptoms and also have series in the defog dataset, while the other twenty subjects do not exhibit FOG symptoms and do not have series elsewhere in the data.\n","metadata":{}},{"cell_type":"code","source":"daily_metadata.Id.to_list()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:42:18.54604Z","iopub.execute_input":"2023-11-01T17:42:18.546372Z","iopub.status.idle":"2023-11-01T17:42:18.55703Z","shell.execute_reply.started":"2023-11-01T17:42:18.546344Z","shell.execute_reply":"2023-11-01T17:42:18.555673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read one file of the daily metadata (the total number of files is 65)\ndaily_files = [f'{root}unlabeled/{id}.parquet' for id in \\\n daily_metadata.Id.to_list()[:1]]\n\ndaily_data = pd.concat([pd.read_parquet(file) for file in daily_files])\n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:42:18.559173Z","iopub.execute_input":"2023-11-01T17:42:18.560015Z","iopub.status.idle":"2023-11-01T17:42:24.489008Z","shell.execute_reply.started":"2023-11-01T17:42:18.559981Z","shell.execute_reply":"2023-11-01T17:42:24.487153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d = pd.read_parquet(daily_files[0])\nd","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:42:24.491019Z","iopub.execute_input":"2023-11-01T17:42:24.491427Z","iopub.status.idle":"2023-11-01T17:42:28.480131Z","shell.execute_reply.started":"2023-11-01T17:42:24.491393Z","shell.execute_reply":"2023-11-01T17:42:28.478798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog_data.describe(include='float64').T","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:42:28.481606Z","iopub.execute_input":"2023-11-01T17:42:28.482996Z","iopub.status.idle":"2023-11-01T17:42:28.538704Z","shell.execute_reply.started":"2023-11-01T17:42:28.482929Z","shell.execute_reply":"2023-11-01T17:42:28.537277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"## Data Preprocessing","metadata":{}},{"cell_type":"code","source":"print(\"Null values in Subjects dataset:\")\nsubjects_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:42:28.552909Z","iopub.execute_input":"2023-11-01T17:42:28.556589Z","iopub.status.idle":"2023-11-01T17:42:28.578284Z","shell.execute_reply.started":"2023-11-01T17:42:28.556506Z","shell.execute_reply":"2023-11-01T17:42:28.576845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_metadata","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:42:28.580269Z","iopub.execute_input":"2023-11-01T17:42:28.581074Z","iopub.status.idle":"2023-11-01T17:42:28.617978Z","shell.execute_reply.started":"2023-11-01T17:42:28.581028Z","shell.execute_reply":"2023-11-01T17:42:28.616674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"The null percentage of UPDRSIII_Off:{:.2f}%\".format(full_metadata[\"UPDRSIII_Off\"].isna().sum()/len(full_metadata)*100))","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:42:28.620096Z","iopub.execute_input":"2023-11-01T17:42:28.620929Z","iopub.status.idle":"2023-11-01T17:42:28.630098Z","shell.execute_reply.started":"2023-11-01T17:42:28.620879Z","shell.execute_reply":"2023-11-01T17:42:28.62893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_metadata.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:42:28.631737Z","iopub.execute_input":"2023-11-01T17:42:28.632156Z","iopub.status.idle":"2023-11-01T17:42:28.646505Z","shell.execute_reply.started":"2023-11-01T17:42:28.632124Z","shell.execute_reply":"2023-11-01T17:42:28.645255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we saw in the correlation matrix, UPDRSIII_Off strongly correlates with UPDRSIII_On. It is also slightly correlated with years_since_dx, age, and sex. So, filling missing values of UPDRSIII_Off and UPDRSIII_On using an imputation scheme seems to be reasonable. For this porpose, we employ the MICE (Multivariate Imputation by Chained Equations) algorithm that considers relationships between variables.","metadata":{}},{"cell_type":"code","source":"# Imputing using MICE\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\nfrom sklearn import linear_model\n\nfull_metadata_temp= full_metadata.loc[:,['Id','UPDRSIII_Off',\"UPDRSIII_On\",\"years_since_dx\",\"age\",\"sex\"]].copy()\n\ncolumns_to_update=['UPDRSIII_Off','UPDRSIII_On']\ndf_mice = full_metadata_temp.filter(columns_to_update, axis=1).copy()\n\n# Define MICE imputer and fill missing values\nmice_imputer = IterativeImputer(estimator=linear_model.BayesianRidge(), n_nearest_features=None,\n imputation_order='ascending')\n\ndf_mice_imputed = pd.DataFrame(mice_imputer.fit_transform(df_mice), columns=df_mice.columns)\n# Update the columns in the original DataFrame with the imputed values\nfull_metadata_temp.update(df_mice_imputed[columns_to_update])","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:43:54.486394Z","iopub.execute_input":"2023-11-01T17:43:54.488384Z","iopub.status.idle":"2023-11-01T17:43:54.829657Z","shell.execute_reply.started":"2023-11-01T17:43:54.488328Z","shell.execute_reply":"2023-11-01T17:43:54.828285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdcsfog_metadata_subjects.copy()\ntemp=tdcsfog_metadata_subjects.drop([\"UPDRSIII_Off\",\"UPDRSIII_On\"], axis=1)\n\nmerged_df = temp.merge(full_metadata_temp, on='Id', how='left')\n\n# Fill the null values in column 'c1' of df1 with the values from merged_df\ntdcsfog_metadata_subjects['UPDRSIII_Off'].fillna(merged_df['UPDRSIII_Off'], inplace=True)\ntdcsfog_metadata_subjects['UPDRSIII_On'].fillna(merged_df['UPDRSIII_On'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:43:56.483911Z","iopub.execute_input":"2023-11-01T17:43:56.484402Z","iopub.status.idle":"2023-11-01T17:43:56.504348Z","shell.execute_reply.started":"2023-11-01T17:43:56.484365Z","shell.execute_reply":"2023-11-01T17:43:56.503211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdcsfog_metadata_subjects.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:43:59.275047Z","iopub.execute_input":"2023-11-01T17:43:59.275531Z","iopub.status.idle":"2023-11-01T17:43:59.288856Z","shell.execute_reply.started":"2023-11-01T17:43:59.275498Z","shell.execute_reply":"2023-11-01T17:43:59.287183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog_metadata_subjects.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:44:02.102728Z","iopub.execute_input":"2023-11-01T17:44:02.103258Z","iopub.status.idle":"2023-11-01T17:44:02.115621Z","shell.execute_reply.started":"2023-11-01T17:44:02.103222Z","shell.execute_reply":"2023-11-01T17:44:02.114138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Converting Categorical Features","metadata":{}},{"cell_type":"code","source":"full_metadata.dtypes","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:44:05.284229Z","iopub.execute_input":"2023-11-01T17:44:05.284762Z","iopub.status.idle":"2023-11-01T17:44:05.297307Z","shell.execute_reply.started":"2023-11-01T17:44:05.284725Z","shell.execute_reply":"2023-11-01T17:44:05.295337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog_metadata_subjects[\"sex\"] = defog_metadata_subjects[\"sex\"].map({'F' :0, 'M' :1})\ntdcsfog_metadata_subjects[\"sex\"]= tdcsfog_metadata_subjects[\"sex\"].map({'F' :0, 'M' :1})\n\ndefog_metadata_subjects[\"medication\"] = defog_metadata_subjects[\"medication\"].map({'on' :0, 'off' :1})\ntdcsfog_metadata_subjects[\"medication\"] = tdcsfog_metadata_subjects[\"medication\"].map({'on' :0, 'off' :1})","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:44:09.185179Z","iopub.execute_input":"2023-11-01T17:44:09.185735Z","iopub.status.idle":"2023-11-01T17:44:09.203159Z","shell.execute_reply.started":"2023-11-01T17:44:09.185695Z","shell.execute_reply":"2023-11-01T17:44:09.201331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since real values of 'subject' and 'Id' are needed in the next part, they will be converted later.","metadata":{}},{"cell_type":"markdown","source":"## Normalizing Data","metadata":{}},{"cell_type":"markdown","source":"The acceleration data needs to be normalized. The normalization takes place in the model method for the purpose of achieving a better performance.","metadata":{}},{"cell_type":"code","source":"def normalize_columns(df, columns):\n min_values = df[columns].min()\n max_values = df[columns].max()\n\n normalized_df = (df[columns] - min_values) / (max_values - min_values)\n return normalized_df","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:45:00.286292Z","iopub.execute_input":"2023-11-01T17:45:00.286876Z","iopub.status.idle":"2023-11-01T17:45:00.294588Z","shell.execute_reply.started":"2023-11-01T17:45:00.286832Z","shell.execute_reply":"2023-11-01T17:45:00.293483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Outliers","metadata":{}},{"cell_type":"code","source":"events_df['duration'] = events_df['Completion'] - events_df['Init']","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:45:27.311233Z","iopub.execute_input":"2023-11-01T17:45:27.311816Z","iopub.status.idle":"2023-11-01T17:45:27.320322Z","shell.execute_reply.started":"2023-11-01T17:45:27.311771Z","shell.execute_reply":"2023-11-01T17:45:27.31918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.style.use('default')\n\nevents_arr = events_df['Type'].unique()\n# Remove NaN values\nseries = pd.Series(events_arr)\nseries_without_nan = series.dropna()\nevents_arr = series_without_nan.to_numpy()\n\nplt.figure(figsize=(6, 6))\nbox_plot=plt.boxplot([events_df[events_df['Type'] == t]['duration'] for t in events_arr],\n labels=events_arr)\nplt.xlabel('Type')\nplt.ylabel('duration')\nplt.title('Box Plot of Event Durations by Event Type', fontsize=9, pad=10)\n\n# Annotate the minimum and maximum values for each box plot\nfor i, box in enumerate(box_plot['boxes']):\n\n max_val = box_plot['whiskers'][i*2 + 1].get_ydata()[1]\n \n plt.annotate(f\"Max: {max_val:.0f}\", (i+1, max_val), xytext=(10+25, +5), textcoords='offset points', ha='center')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:45:42.042991Z","iopub.execute_input":"2023-11-01T17:45:42.04391Z","iopub.status.idle":"2023-11-01T17:45:42.372175Z","shell.execute_reply.started":"2023-11-01T17:45:42.043867Z","shell.execute_reply":"2023-11-01T17:45:42.37079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# delete outliers\n\n# Get the maximum value for each type from the box plot\nmax_values = {t: item.get_ydata()[1] for t, item in zip(events_df['Type'].unique(), box_plot['whiskers'][1::2])}\n\n# Drop rows with durations greater than the maximum value for each type\nevents_filtered = events_df.groupby('Type').apply(lambda x: x[x['duration'] >= (max_values[x['Type'].unique()[0]])+200]).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:45:57.494652Z","iopub.execute_input":"2023-11-01T17:45:57.495124Z","iopub.status.idle":"2023-11-01T17:45:57.513676Z","shell.execute_reply.started":"2023-11-01T17:45:57.495089Z","shell.execute_reply":"2023-11-01T17:45:57.512272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"def reader(file, module, pre_type=None):\n if module==\"tdcsfog\":\n window_length =4048\n elif module==\"defog\":\n window_length=4048\n try:\n\n df = pd.read_csv(file)\n\n path_split = file.split('/')\n Id = path_split[-1].split('.')[0]\n df['Id'] = Id\n \n if module==\"defog\":\n data= df.merge(defog_metadata_subjects,how=\"left\",on=\"Id\")\n if 'Valid' in data.columns and 'Task' in data.columns and pre_type==\"train\":\n data = data.loc[(data['Valid'] == True) & (data[\"Task\"] == True)]\n data.drop([\"Valid\", \"Task\"], axis=1, inplace=True)\n elif 'Valid' in data.columns and 'Task' in data.columns:\n data.drop([\"Valid\", \"Task\"], axis=1, inplace=True)\n\n elif module == \"tdcsfog\":\n data= df.merge(tdcsfog_metadata_subjects,how=\"left\",on=\"Id\")\n\n data[\"acc_sum\"]= data[\"AccV\"]+data[\"AccML\"]+data[\"AccAP\"]\n data['time_frac']=(data[\"Time\"]/data[\"Time\"].max()).values\n acc_columns=[\"AccV\",\"AccML\",\"AccAP\"]\n \n columns_to_normalize = ['AccV', 'AccML', 'AccAP']\n data[columns_to_normalize] = normalize_columns(data, columns_to_normalize) \n \n for acc in acc_columns:\n data[f'{acc}_cumsum'] = data[acc].cumsum()\n data[f'{acc}_rolling_sum'] = data[acc].rolling(window=window_length, min_periods=1).sum()\n data[f'{acc}_rolling_min'] = data[acc].rolling(window=window_length, min_periods=1).min()\n data[f'{acc}_rolling_max'] = data[acc].rolling(window=window_length, min_periods=1).max()\n data[f'{acc}_rolling_mean'] = data[acc].rolling(window=window_length, min_periods=1).mean()\n data[f'{acc}_rolling_std'] = data[acc].rolling(window=window_length, min_periods=1).std()\n data[f'{acc}_rolling_delta'] = data[f'{acc}_rolling_max'] - data[f'{acc}_rolling_min'] \n data[f'{acc}_EWMA_02'] = data[acc].ewm(alpha=0.2).mean()\n \n for c in acc_columns:\n data[f\"{c}_lead_diff\"] = data[c].diff(-1)\n data[f\"{c}_lag_diff\"] = data[c].diff()\n\n\n data.fillna(method='backfill', inplace=True)\n \n# for train data \n events_id = events_filtered[events_filtered[\"Id\"] == Id]\n\n events_rows = pd.DataFrame()\n if len(events_id)>0:\n return None\n \n return data\n except: pass","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:46:32.961793Z","iopub.execute_input":"2023-11-01T17:46:32.962296Z","iopub.status.idle":"2023-11-01T17:46:32.985188Z","shell.execute_reply.started":"2023-11-01T17:46:32.96226Z","shell.execute_reply":"2023-11-01T17:46:32.983647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_defog_df = pd.concat([reader(f,\"defog\",\"train\") for f in tqdm(train_defog)]).fillna(-1); print(train_defog_df.shape)","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:46:44.634937Z","iopub.execute_input":"2023-11-01T17:46:44.635408Z","iopub.status.idle":"2023-11-01T17:47:31.466235Z","shell.execute_reply.started":"2023-11-01T17:46:44.635372Z","shell.execute_reply":"2023-11-01T17:47:31.46456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_defog_df =reduce_memory_usage(train_defog_df)","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:48:05.047084Z","iopub.execute_input":"2023-11-01T17:48:05.047541Z","iopub.status.idle":"2023-11-01T17:48:05.628936Z","shell.execute_reply.started":"2023-11-01T17:48:05.04751Z","shell.execute_reply":"2023-11-01T17:48:05.626667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tdcsfog_df = pd.concat([reader(f,\"tdcsfog\") for f in tqdm(train_tdcsfog)]).fillna(0); print(train_tdcsfog_df.shape)","metadata":{"execution":{"iopub.status.busy":"2023-11-01T18:06:38.764073Z","iopub.execute_input":"2023-11-01T18:06:38.764608Z","iopub.status.idle":"2023-11-01T18:07:45.573517Z","shell.execute_reply.started":"2023-11-01T18:06:38.764561Z","shell.execute_reply":"2023-11-01T18:07:45.57211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model\n","metadata":{}},{"cell_type":"markdown","source":"### Model Selection\nFor this project, after tuning different ML algorithms, LightGBM algorithm is taken into account for the following reasons.\n\nThe dataset is large including many features, so a highly-effiecient model is required.\nThe task involves multi-class classification.\nThe multi-class problem exhibits imbalanced classes.\nIt is crucial to capture complex and non-linear relationships in the data.","metadata":{}},{"cell_type":"markdown","source":"## Model Training","metadata":{}},{"cell_type":"code","source":"param_grid = {\n 'max_depth': [6,8,10],\n 'learning_rate': [0.15,0.1,0.01],\n 'n_estimators': [50,70,90]\n} ","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:48:00.158804Z","iopub.execute_input":"2023-11-01T17:48:00.159267Z","iopub.status.idle":"2023-11-01T17:48:00.166446Z","shell.execute_reply.started":"2023-11-01T17:48:00.159234Z","shell.execute_reply":"2023-11-01T17:48:00.165178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = [\"StartHesitation\", \"Turn\", 'Walking']","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:48:29.011995Z","iopub.execute_input":"2023-11-01T17:48:29.012425Z","iopub.status.idle":"2023-11-01T17:48:29.018819Z","shell.execute_reply.started":"2023-11-01T17:48:29.012393Z","shell.execute_reply":"2023-11-01T17:48:29.017451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model(df, module):\n \n df = df.apply(factorize_column)\n\n y_train = df.loc[:, targets]\n\n if module == \"defog\":\n X_train = df.drop(targets, axis=1)\n\n elif module == \"tdcsfog\":\n X_train = df.drop([\"StartHesitation\", \"Turn\", 'Walking', 'UPDRSIII_On', 'UPDRSIII_Off'], axis=1)\n \n models = []\n aps=[]\n\n # Get Ids from X_train\n Ids = X_train['Id']\n\n # Create the GroupKFold object based on 'Id' column\n group_cv = GroupKFold(n_splits=5)\n \n for label in targets:\n y_train_l = y_train[label]\n\n # LightGBM classifier\n lgbm = lgb.LGBMClassifier(objective=\"binary\", scale_pos_weight=np.sqrt(y_train_l.value_counts()[0] / y_train_l.value_counts()[1]),\n max_depth=8, learning_rate=0.1, n_estimators=70)\n \n aps_l = []\n # Perform group-based cross-validation\n for train_idx, val_idx in group_cv.split(X_train, y_train_l, groups=Ids):\n X_train_fold, X_val_fold = X_train.iloc[train_idx], X_train.iloc[val_idx]\n y_train_fold, y_val_fold = y_train_l.iloc[train_idx], y_train_l.iloc[val_idx]\n\n # Fit the model on the training fold\n lgbm.fit(X_train_fold, y_train_fold)\n\n # Calculate average precision for the validation fold\n ap_val = average_precision_score(y_val_fold, lgbm.predict_proba(X_val_fold)[:, 1])\n aps_l.append(ap_val)\n print(\"Average precision of {} associated with all folds: {:.4f}\".format(label, np.mean(aps_l)))\n aps.append(aps_l) \n print(\"Overall precision of all folds: {:.4f}\".format( np.mean(aps)))\n# main model trained on entired data\n for label in targets:\n y_train_l = y_train[label]\n\n # LightGBM classifier\n lgbm = lgb.LGBMClassifier(objective=\"binary\", scale_pos_weight=np.sqrt(y_train_l.value_counts()[0] / y_train_l.value_counts()[1]),\n max_depth=8, learning_rate=0.1, n_estimators=70)\n lgbm.fit(X_train, y_train_l)\n models.append(lgbm)\n return models","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:48:45.477426Z","iopub.execute_input":"2023-11-01T17:48:45.479095Z","iopub.status.idle":"2023-11-01T17:48:45.494849Z","shell.execute_reply.started":"2023-11-01T17:48:45.478978Z","shell.execute_reply":"2023-11-01T17:48:45.493425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Evaluation","metadata":{}},{"cell_type":"code","source":"defog_models=model(train_defog_df,\"defog\")","metadata":{"execution":{"iopub.status.busy":"2023-11-01T17:50:42.882754Z","iopub.execute_input":"2023-11-01T17:50:42.883314Z","iopub.status.idle":"2023-11-01T18:03:39.945595Z","shell.execute_reply.started":"2023-11-01T17:50:42.883275Z","shell.execute_reply":"2023-11-01T18:03:39.943521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tdcsfog model\ntdcsfog_models=model(train_tdcsfog_df,\"tdcsfog\")","metadata":{"execution":{"iopub.status.busy":"2023-11-01T18:30:15.947538Z","iopub.execute_input":"2023-11-01T18:30:15.948119Z","iopub.status.idle":"2023-11-01T18:52:08.193278Z","shell.execute_reply.started":"2023-11-01T18:30:15.948079Z","shell.execute_reply":"2023-11-01T18:52:08.191018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prediction","metadata":{}},{"cell_type":"code","source":"def predict(models, test_df, module):\n X_test = test_df.apply(factorize_column)\n\n if module == \"tdcsfog\":\n X_test= X_test.drop(['UPDRSIII_On', 'UPDRSIII_Off'], axis=1)\n \n y_predict = pd.DataFrame()\n \n \n for index, label in enumerate(targets):\n y_predict[label] = models[index].predict_proba(X_test)[:, 1]\n \n test_df[\"Time\"] = test_df[\"Time\"].astype(str)\n test_df=test_df.reset_index(drop=True)\n y_predict.insert(0, 'Id', test_df[\"Id\"] + \"_\" + test_df[\"Time\"])\n return y_predict","metadata":{"execution":{"iopub.status.busy":"2023-11-01T18:56:10.215938Z","iopub.execute_input":"2023-11-01T18:56:10.216511Z","iopub.status.idle":"2023-11-01T18:56:10.234028Z","shell.execute_reply.started":"2023-11-01T18:56:10.216473Z","shell.execute_reply":"2023-11-01T18:56:10.23187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_defog_df = pd.concat([reader(f,\"defog\") for f in tqdm(test_defog)]).fillna(-1); print(test_defog_df.shape)\ntest_tdcsfog_df = pd.concat([reader(f,\"tdcsfog\") for f in tqdm(test_tdcsfog)]).fillna(-1); print(test_tdcsfog_df.shape)\n\n# predict defog test df\ny_defog_predict = predict(defog_models, test_defog_df,\"defog\")\n# predict tdcsfog test df\ny_tdcsfog_predict = predict(tdcsfog_models,test_tdcsfog_df, \"tdcsfog\")","metadata":{"execution":{"iopub.status.busy":"2023-11-01T18:56:14.787773Z","iopub.execute_input":"2023-11-01T18:56:14.789088Z","iopub.status.idle":"2023-11-01T18:56:19.097512Z","shell.execute_reply.started":"2023-11-01T18:56:14.789043Z","shell.execute_reply":"2023-11-01T18:56:19.096592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_result=pd.concat([y_tdcsfog_predict, y_defog_predict])\nfinal_result[\"StartHesitation\"]=np.round(final_result[\"StartHesitation\"].clip(0.0,1.0),8)\nfinal_result[\"Turn\"]=np.round(final_result[\"Turn\"].clip(0.0,1.0),8)\nfinal_result[\"Walking\"]=np.round(final_result[\"Walking\"].clip(0.0,1.0),8)\nfinal_result.to_csv('submission.csv', index=False)\nfinal_result.head(","metadata":{"execution":{"iopub.status.busy":"2023-11-01T18:58:51.659146Z","iopub.execute_input":"2023-11-01T18:58:51.665711Z","iopub.status.idle":"2023-11-01T18:58:53.554026Z","shell.execute_reply.started":"2023-11-01T18:58:51.665661Z","shell.execute_reply":"2023-11-01T18:58:53.55244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}