{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Initial Exploratory Data Analytics for Parkinson's Freezing of Gait Prediction","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport plotly.express as px\nimport altair as alt\n\nalt.data_transformers.enable('default', max_rows=None)\n\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\n\nfrom tqdm.auto import tqdm\nfrom sklearn import *\nimport glob\n\n#References: https://www.kaggle.com/code/dinowun/eda-simplified-parkinson-s-fog-prediction-b","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-25T21:46:09.876877Z","iopub.execute_input":"2023-03-25T21:46:09.877337Z","iopub.status.idle":"2023-03-25T21:46:13.500077Z","shell.execute_reply.started":"2023-03-25T21:46:09.877296Z","shell.execute_reply":"2023-03-25T21:46:13.499132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p = '/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/'\n\ntrain = glob.glob(p+'train/**/**')\ntest = glob.glob(p+'test/**/**')\nsub = pd.read_csv(p+'sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-25T21:46:13.501908Z","iopub.execute_input":"2023-03-25T21:46:13.502540Z","iopub.status.idle":"2023-03-25T21:46:13.933332Z","shell.execute_reply.started":"2023-03-25T21:46:13.502503Z","shell.execute_reply":"2023-03-25T21:46:13.931993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Loading training dataset\nimport pathlib\ndef reader(f):\n    try:\n        df = pd.read_csv(f, index_col=\"Time\")\n        df['Id'] = f.split('/')[-1].split('.')[0]\n        df['Module'] = pathlib.Path(f).parts[-2]\n        return df\n    except: pass\ntrain = pd.concat([reader(f) for f in tqdm(train)]).fillna(0); print(train.shape)\ntrain=train.reset_index(drop=True)\ntrain.shape\ntrain.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T21:46:13.935214Z","iopub.execute_input":"2023-03-25T21:46:13.935972Z","iopub.status.idle":"2023-03-25T21:47:54.631809Z","shell.execute_reply.started":"2023-03-25T21:46:13.935900Z","shell.execute_reply":"2023-03-25T21:47:54.630636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Metadata datasets\ndaily_df = pd.read_csv(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/daily_metadata.csv\")\ndaily_df.shape\ndaily_df.head(2)\ndefog_df = pd.read_csv(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/defog_metadata.csv\")\ndefog_df.shape\ndefog_df.head(2)\ntdcsfog_df = pd.read_csv(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/tdcsfog_metadata.csv\")\ntdcsfog_df.shape\ntdcsfog_df.head(2)\n\n#Events, Subjects and Tasks\nevents_df = pd.read_csv(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/events.csv\")\nevents_df.shape\nevents_df.head(2)\nsubjects_df = pd.read_csv(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/subjects.csv\")\nsubjects_df.shape\nsubjects_df.head(2)\ntasks_df = pd.read_csv(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/tasks.csv\")\ntasks_df.shape\ntasks_df.head(2)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-25T21:47:55.246828Z","iopub.execute_input":"2023-03-25T21:47:55.247686Z","iopub.status.idle":"2023-03-25T21:47:55.364663Z","shell.execute_reply.started":"2023-03-25T21:47:55.247633Z","shell.execute_reply":"2023-03-25T21:47:55.363526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"daily_df.isnull().sum()\ndefog_df.isnull().sum()\ntdcsfog_df.isnull().sum()\ntasks_df.isnull().sum()\nevents_df.isnull().sum()\nsubjects_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T21:47:55.366190Z","iopub.execute_input":"2023-03-25T21:47:55.367585Z","iopub.status.idle":"2023-03-25T21:47:55.405096Z","shell.execute_reply.started":"2023-03-25T21:47:55.367534Z","shell.execute_reply":"2023-03-25T21:47:55.403699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"daily_df.rename(str.lower, axis='columns', inplace = True)\ndefog_df.rename(str.lower, axis='columns', inplace = True)\ntdcsfog_df.rename(str.lower, axis='columns', inplace = True)\ntasks_df.rename(str.lower, axis='columns', inplace = True)\nevents_df.rename(str.lower, axis='columns', inplace = True)\nsubjects_df.rename(str.lower, axis='columns', inplace = True)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T21:47:55.406459Z","iopub.execute_input":"2023-03-25T21:47:55.407259Z","iopub.status.idle":"2023-03-25T21:47:55.416350Z","shell.execute_reply.started":"2023-03-25T21:47:55.407224Z","shell.execute_reply":"2023-03-25T21:47:55.415118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"daily_df.groupby(['visit']).agg({'subject':'nunique', 'id':'nunique'}).reset_index()\nfig = px.histogram(daily_df, x=\"visit\",color_discrete_sequence=['indianred'])\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T21:47:55.418432Z","iopub.execute_input":"2023-03-25T21:47:55.418773Z","iopub.status.idle":"2023-03-25T21:47:57.056867Z","shell.execute_reply.started":"2023-03-25T21:47:55.418733Z","shell.execute_reply":"2023-03-25T21:47:57.055696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"daily_df.dtypes\ndaily_df[\"recording\"] = daily_df[\"beginning of recording [00:00-23:59]\"].str.replace(\":\", \"\").astype(int)\ndaily_df.agg({'recording':'mean'}).reset_index()\ndaily_df.groupby(['visit']).agg({'recording':'mean'}).reset_index()\n\nfig = px.histogram(daily_df, x=\"recording\", marginal=\"box\")\nfig.show()\n#Most recordings start in the morning -- around 8 am - 10 am\n\nfig = px.density_heatmap(daily_df, x=\"visit\", y=\"recording\", text_auto=True)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T21:47:57.058500Z","iopub.execute_input":"2023-03-25T21:47:57.058839Z","iopub.status.idle":"2023-03-25T21:47:57.289105Z","shell.execute_reply.started":"2023-03-25T21:47:57.058805Z","shell.execute_reply":"2023-03-25T21:47:57.287876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog_df.dtypes\n\ndf = defog_df.groupby(['medication','visit']).agg({'subject':'nunique','id':'nunique'}).reset_index()\nfig = px.bar(df, x=\"visit\", y=\"subject\", color=\"medication\", title=\"Medications and Visits\")\nfig.show()\n\nmedication_df = defog_df[\"medication\"].value_counts()\n\nfig = px.pie(medication_df, names=medication_df.index, values=medication_df.values)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T21:47:57.290611Z","iopub.execute_input":"2023-03-25T21:47:57.290930Z","iopub.status.idle":"2023-03-25T21:47:57.460312Z","shell.execute_reply.started":"2023-03-25T21:47:57.290898Z","shell.execute_reply":"2023-03-25T21:47:57.458736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdcsfog_df.dtypes\ndf = tdcsfog_df.groupby(['medication','visit']).agg({'subject':'nunique','id':'nunique'}).reset_index()\nfig = px.bar(df, x=\"visit\", y=\"subject\", color=\"medication\", title=\"Medications and Visits\")\nfig.show()\n\nmedication_df = tdcsfog_df[\"medication\"].value_counts()\n\nfig = px.pie(medication_df, names=medication_df.index, values=medication_df.values)\nfig.show()\n\ntest_df = tdcsfog_df[\"test\"].value_counts()\n\nfig = px.pie(test_df, names=test_df.index, values=test_df.values)\nfig.show()\n\ntdcsfog_df.groupby(['medication','test']).agg({'subject':'nunique'}).reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T21:47:57.461719Z","iopub.execute_input":"2023-03-25T21:47:57.462090Z","iopub.status.idle":"2023-03-25T21:47:57.639106Z","shell.execute_reply.started":"2023-03-25T21:47:57.462053Z","shell.execute_reply":"2023-03-25T21:47:57.637966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.density_heatmap(tdcsfog_df, x=\"test\", y=\"visit\", text_auto=True)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T21:47:57.640273Z","iopub.execute_input":"2023-03-25T21:47:57.640585Z","iopub.status.idle":"2023-03-25T21:47:57.704844Z","shell.execute_reply.started":"2023-03-25T21:47:57.640555Z","shell.execute_reply":"2023-03-25T21:47:57.703943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Tasks, Events and Subjects analysis\n\ntasks_df.dtypes\nevents_df.dtypes\nsubjects_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2023-03-25T21:47:57.705824Z","iopub.execute_input":"2023-03-25T21:47:57.706164Z","iopub.status.idle":"2023-03-25T21:47:57.723247Z","shell.execute_reply.started":"2023-03-25T21:47:57.706132Z","shell.execute_reply":"2023-03-25T21:47:57.721848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tasks_df['duration'] = tasks_df['end'] - tasks_df['begin']\nx1 = tasks_df.groupby(['task']).agg({'id':'nunique','duration':'mean'}).reset_index()\nfig = px.pie(x1, names=x1.task, values=x1.id)\nfig.show()\nfig = px.bar(x1, x=\"task\", y=\"duration\", title=\"Duration on Tasks\")\nfig.show()\n\n#Combined in one graph\nimport plotly.graph_objects as go\nfig = go.Figure()\nfig = fig.add_trace(go.Bar(\n    x=x1['task'].values,\n    y=x1['id'].values,\n    name='Tasks Conducted DEFOG',\n    marker_color='indianred'\n))\nfig = fig.add_trace(go.Bar(\n    x=x1['task'].values,\n    y=x1['duration'].values,\n    name='Duration of Tasks',\n    marker_color='lightsalmon'\n))\nfig = fig.update_layout(barmode='group', xaxis_tickangle=-45)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T21:47:57.728302Z","iopub.execute_input":"2023-03-25T21:47:57.728674Z","iopub.status.idle":"2023-03-25T21:47:57.852670Z","shell.execute_reply.started":"2023-03-25T21:47:57.728638Z","shell.execute_reply":"2023-03-25T21:47:57.851839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events_df['duration'] = events_df['completion'] - events_df['init']\ndf = events_df.groupby(['type']).agg({'id':'nunique','duration':'mean'}).reset_index()\ndf2 = events_df.loc[events_df['kinetic'] == 1].groupby(['type']).agg({'id':'nunique','duration':'mean'}).reset_index()\ndf3 = events_df.loc[events_df['kinetic'] == 0].groupby(['type']).agg({'id':'nunique','duration':'mean'}).reset_index()\n\nx = events_df['type'].value_counts()\nfig = px.pie(x, names=x.index, values=x.values)\nfig.show()\n\n#All Kinds\nfig = go.Figure()\nfig = fig.add_trace(go.Bar(\n    x=df['type'].values,\n    y=df['id'].values,\n    name='Type of Events',\n    marker_color='indianred'\n))\nfig = fig.add_trace(go.Bar(\n    x=df['type'].values,\n    y=df['duration'].values,\n    name='Duration of Event',\n    marker_color='lightsalmon'\n))\nfig = fig.update_layout(barmode='group', xaxis_tickangle=-45)\nfig.show()\n\n#Kinetic\nfig = go.Figure()\nfig = fig.add_trace(go.Bar(\n    x=df2['type'].values,\n    y=df2['id'].values,\n    name='Kinetic - Type of Events',\n    marker_color='darkblue'\n))\nfig = fig.add_trace(go.Bar(\n    x=df2['type'].values,\n    y=df2['duration'].values,\n    name='Kinetic - Duration of Event',\n    marker_color='lightblue'\n))\nfig = fig.update_layout(barmode='group', xaxis_tickangle=-45)\nfig.show()\n\n#Akinetic\nfig = go.Figure()\nfig = fig.add_trace(go.Bar(\n    x=df3['type'].values,\n    y=df3['id'].values,\n    name='Akinetic - Type of Events',\n    marker_color='darkgreen'\n))\nfig = fig.add_trace(go.Bar(\n    x=df3['type'].values,\n    y=df3['duration'].values,\n    name='Akinetic - Duration of Event',\n    marker_color='lightgreen'\n))\nfig = fig.update_layout(barmode='group', xaxis_tickangle=-45)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T21:47:57.853918Z","iopub.execute_input":"2023-03-25T21:47:57.854698Z","iopub.status.idle":"2023-03-25T21:47:57.955585Z","shell.execute_reply.started":"2023-03-25T21:47:57.854664Z","shell.execute_reply":"2023-03-25T21:47:57.954268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subjects_df.isnull().sum()\nnp.unique(subjects_df['nfogq'].values)\nnp.unique([x for x in subjects_df['updrsiii_on'].values if x is not None])\nnp.unique([x for x in subjects_df['updrsiii_off'].values if x is not None])\n\ndf = subjects_df.groupby(['sex']).agg({'subject':'nunique','age':'mean','visit':'mean','yearssincedx':'mean'}).reset_index()\ndf2 = subjects_df.groupby(['nfogq']).agg({'subject':'nunique'}).reset_index().sort_values(by = 'subject', ascending = False)\ndf3 = subjects_df.groupby(['updrsiii_on']).agg({'subject':'nunique'}).reset_index().sort_values(by = 'subject', ascending = False)\ndf4 = subjects_df.groupby(['updrsiii_off']).agg({'subject':'nunique'}).reset_index().sort_values(by = 'subject', ascending = False)\ndf5 = subjects_df['sex'].value_counts()\ndf6 = subjects_df['age'].value_counts()\n\nfig = px.pie(df5, names=df5.index, values=df5.values)\nfig.show()\n\nfig = px.pie(df6, names=df6.index, values=df6.values)\nfig.show()\n\nfig = go.Figure()\nfig = fig.add_trace(go.Bar(\n    x=df['sex'].values,\n    y=df['subject'].values,\n    name='Distribution by gender',\n    marker_color='indianred'\n))\nfig = fig.add_trace(go.Bar(\n    x=df['sex'].values,\n    y=df['age'].values,\n    name='Average Age',\n    marker_color='lightsalmon'\n))\nfig = fig.add_trace(go.Bar(\n    x=df['sex'].values,\n    y=df['visit'].values,\n    name='Average visits',\n    marker_color='lightblue'\n))\nfig = fig.add_trace(go.Bar(\n    x=df['sex'].values,\n    y=df['yearssincedx'].values,\n    name='Average years since Dx',\n    marker_color='lightgreen'\n))\n\nfig = fig.update_layout(barmode='group', xaxis_tickangle=-45)\nfig.show()\n\nfig = px.bar(df2, x='nfogq', y='subject', title='FOG-Q Ratings')\nfig.show()\n\nfig = px.bar(df3, x='updrsiii_on', y='subject', title='Medicine FOG Ratings', color_discrete_sequence=['indianred'])\nfig.show()\n\nfig = px.bar(df4, x='updrsiii_off', y='subject', title='Medicine OFF FOG Ratings', color_discrete_sequence=['darkgreen'])\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T21:47:57.957253Z","iopub.execute_input":"2023-03-25T21:47:57.958162Z","iopub.status.idle":"2023-03-25T21:47:58.278936Z","shell.execute_reply.started":"2023-03-25T21:47:57.958105Z","shell.execute_reply":"2023-03-25T21:47:58.277607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Combine datasets into one training and one test dataset\ntrain.rename(str.lower, axis = 'columns', inplace = True)\ntrain.head(2)\ntdcsfog_df.head(2)\ndefog_df.head(2)\ndaily_df.head(2)\n\nnp.unique(train['module'].values)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T21:47:58.280861Z","iopub.execute_input":"2023-03-25T21:47:58.282249Z","iopub.status.idle":"2023-03-25T21:48:21.565579Z","shell.execute_reply.started":"2023-03-25T21:47:58.282203Z","shell.execute_reply":"2023-03-25T21:48:21.564481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check = defog_df.groupby(['subject']).agg({'id':'nunique'}).reset_index()\ncheck.loc[check['id'] > 1].shape\n\ncheck = defog_df.groupby(['id']).agg({'subject':'nunique'}).reset_index()\ncheck.loc[check['subject'] > 1].shape\n\ncheck = tdcsfog_df.groupby(['subject']).agg({'id':'nunique'}).reset_index()\ncheck.loc[check['id'] > 1].shape\n\ncheck = tdcsfog_df.groupby(['id']).agg({'subject':'nunique'}).reset_index()\ncheck.loc[check['subject'] > 1].shape\n\ncheck = daily_df.groupby(['subject']).agg({'id':'nunique'}).reset_index()\ncheck.loc[check['id'] > 1].shape\n\ncheck = daily_df.groupby(['id']).agg({'subject':'nunique'}).reset_index()\ncheck.loc[check['subject'] > 1].shape\n\n#Rest of the analysis + modeling in separate notebook ******************************************************************","metadata":{"execution":{"iopub.status.busy":"2023-03-25T21:48:21.568741Z","iopub.execute_input":"2023-03-25T21:48:21.569118Z","iopub.status.idle":"2023-03-25T21:48:21.618964Z","shell.execute_reply.started":"2023-03-25T21:48:21.569083Z","shell.execute_reply":"2023-03-25T21:48:21.617770Z"},"trusted":true},"execution_count":null,"outputs":[]}]}