{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Rectangular Dataset Builder\n\n## Author\n\nWill McIntosh\n\n## Purpose\n\nThe main purpose of this notebook is to create large rectangular datasets to either the training (for TCDS or Defog) or the testing (for TCDS of Defog) datasets.\n\n## Use\n\n1. Begin by opening going to the [Parkinsons FOG Competition](https://www.kaggle.com/competitions/tlvmc-parkinsons-freezing-gait-prediction).\n2. Click on the **code** tab > **New Notebook**.\n3. From the new notebook click **(+ Add Data)** > Search for the name of this notebook: \"FOG_RectangularDataset_Utility\" > click the **(+)** plus sign.\n4. From your new notebook create a new cell with the following code:\n    ```python\n        import UtilityFOGRectangularDataset as fr\n    ```\n5. Create a new cell and write the code to make a new `fr` object:\n    ```python\n        frobj = fr.FOGRect()\n    ```\n    This will take somewhere around three or four minutes to complete.\n6. You can now use the `frobj` to:\n    * Collect a list of the csv paths for the testing and training sets for both defog or tcds.\n    * Load various datasets like Events, Tasks, Subjects, Metadata, etc.\n    * Load the 'flattened' versions of Tasks or Events.\n    * Create any rectangular datasets by passing in the list of csv files.\n7. In a new cell write the code `frobj.` (with the dot and no spaces afterward) and press the **tab** key on your keyboard to see a list of available options.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport glob # For importing datasets\nfrom tqdm import tqdm # For progress bar\n# from tqdm.auto import tqdm # For progress bar\nimport pathlib # For parsing path strings","metadata":{"execution":{"iopub.status.busy":"2023-05-24T18:06:41.536475Z","iopub.execute_input":"2023-05-24T18:06:41.536996Z","iopub.status.idle":"2023-05-24T18:06:41.545714Z","shell.execute_reply.started":"2023-05-24T18:06:41.536958Z","shell.execute_reply":"2023-05-24T18:06:41.544314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FOGRect:\n    \n    def __init__(selfobj, include_tasks=False):\n        \n        selfobj.__p = '/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/'\n        selfobj.events = selfobj.__Get_Events()\n        selfobj.subjects = selfobj.__Get_Subjects()\n        selfobj.tasks = selfobj.__Get_Tasks()\n        selfobj.train_tcds_csv_list = selfobj.__Get_Train_TCDS_Csv_List()\n        selfobj.train_defog_csv_list = selfobj.__Get_Train_Defog_Csv_List()\n        selfobj.test_tcds_csv_list = selfobj.__Get_Test_TCDS_Csv_List()\n        selfobj.test_defog_csv_list = selfobj.__Get_Test_Defog_Csv_List()\n        selfobj.include_tasks = include_tasks\n\n        selfobj.events_flat = selfobj.__Get_Flat_Events()\n        if include_tasks:\n            selfobj.tasks_flat = selfobj.__Get_Flat_Tasks()\n\n        selfobj.metadata_complex_tcds = selfobj.__Get_TCDS_Metadata()\n        selfobj.metadata_complex_defog = selfobj.__Get_DEFOG_Metadata()\n    \n    def __Get_Train_TCDS_Csv_List(selfobj):\n        return glob.glob(selfobj.__p+'train/tdcsfog/**')\n\n    def __Get_Train_Defog_Csv_List(selfobj):\n        return glob.glob(selfobj.__p+'train/defog/**')\n    \n    def __Get_Test_TCDS_Csv_List(selfobj):\n        return glob.glob(selfobj.__p+'test/tdcsfog/**')\n\n    def __Get_Test_Defog_Csv_List(selfobj):\n        return glob.glob(selfobj.__p+'test/defog/**')\n    \n    def __Get_Events(selfobj):\n        events = pd.read_csv(selfobj.__p+'events.csv')\n        return events\n    \n    def __Get_Tasks(selfobj):\n        tasks = pd.read_csv(selfobj.__p+'tasks.csv')\n        return tasks\n    \n    def __Get_Subjects(selfobj):\n        subjects = pd.read_csv(selfobj.__p+'subjects.csv')\n        return subjects\n    \n    #############################################################################################\n    ################################# FLATTENING EVENTS & TASKS #################################\n    #############################################################################################\n\n    def __Get_Dataset_Lists(selfobj):\n        train_defog_ids_list = []\n        for path in selfobj.train_defog_csv_list:\n            trial_id = path.split('/')[-1].split('.')[0]\n            train_defog_ids_list.append(trial_id)\n        train_tdcsfog_ids_list = []\n        for path in selfobj.train_tcds_csv_list:\n            trial_id = path.split('/')[-1].split('.')[0]\n            train_tdcsfog_ids_list.append(trial_id)\n        return train_defog_ids_list, train_tdcsfog_ids_list\n    \n    \n    def __Calculate_Timestep_Values(selfobj, row):\n        train_defog_ids_list, train_tdcsfog_ids_list = selfobj.__Get_Dataset_Lists()\n        if row['Id'] in train_defog_ids_list:\n            pTask_Start = int(row['Init'] * 100)\n            pTask_End = int(row['Completion'] * 100)\n        elif row['Id'] in train_tdcsfog_ids_list:\n            pTask_Start = int(row['Init'] * 128)\n            pTask_End = int(row['Completion'] * 128)\n        else:\n            pTask_Start = row['Task_Begin']\n            pTask_End = row['Task_End']\n        return pd.Series({'Task_Begin': pTask_Start, 'pTask_End': pTask_End}).astype(int)\n    \n    \n    def __Get_Event_Timesteps(selfobj):\n        edf = selfobj.events.copy(deep=True)\n        # Set the columns to zeros\n        edf['Task_Begin'] = edf['Init']*0\n        edf['Task_End'] = edf['Completion']*0\n        # Apply the function to the dataframe\n        edf[['Task_Begin', 'Task_End']] = edf.apply(selfobj.__Calculate_Timestep_Values, axis=1)\n        edf = edf.rename(columns={'Type':'Task'})\n        edf['Kinetic'] = edf['Kinetic'].replace(1.0, \"Event_Movement_Kinetic\")\n        edf['Kinetic'] = edf['Kinetic'].replace(0.0, \"Event_Movement_Akinetic\")\n        return edf\n    \n\n    def __Get_Task_Timesteps(selfobj):\n        tdf = selfobj.tasks.copy(deep=True)\n        # # Convert Tasks table information from seconds to time steps for defog dateset\n        tdf['Task_Begin'] = (tdf['Begin']*100).astype(int)\n        tdf['Task_End'] = (tdf['End']*100).astype(int)\n        return tdf\n    \n\n    def __Flatten_Dataframe(selfobj, fdf, d_str):\n        tdf = fdf.copy(deep=True)\n        # create a list of dictionaries to represent the expanded data\n        expanded_data = []\n        desc = f\"Flattening {d_str} Dataset.\"\n        for idx in tqdm(tdf.index, desc=desc):\n            row = tdf.loc[idx]\n            for time in range(row['Task_Begin'], row['Task_End'] + 1):\n                expanded_data.append({'Id': row['Id'], 'Task': row['Task'], 'Time': time})\n        # create a new dataframe from the expanded data\n        expanded_df = pd.DataFrame(expanded_data)\n        # merge the original dataframe with the expanded dataframe\n        flat_df = pd.merge(tdf, expanded_df, on=['Id', 'Task'], how='outer')\n        flat_df = flat_df.drop_duplicates()\n        return flat_df\n    \n    \n    def __Get_Flat_Tasks(selfobj):\n        tt = selfobj.__Get_Task_Timesteps()\n        tasks_flat = selfobj.__Flatten_Dataframe(tt, \"Tasks\")\n        tasks_flat = tasks_flat[['Id', 'Time', 'Task']].copy(deep=True)\n        tasks_flat = tasks_flat.rename(columns={'Task':'Tasks_Type'})\n        return tasks_flat\n    \n    \n    def __Get_Flat_Events(selfobj):\n        et = selfobj.__Get_Event_Timesteps()\n        events_flat = selfobj.__Flatten_Dataframe(et, \"Events\")\n        events_flat = events_flat[['Id', 'Time', 'Task', 'Kinetic']].copy(deep=True)\n        events_flat = events_flat.rename(columns={'Task':'Events_Type'})\n        return events_flat\n    \n    \n    #############################################################################################\n    ################################# MERGE METADATA + SUBJECTS #################################\n    #############################################################################################\n    \n    def __Get_TCDS_Metadata(selfobj):\n        tdcsfog_metadata = pd.read_csv('/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/tdcsfog_metadata.csv')\n        tdcsfog_metadata = tdcsfog_metadata.rename(columns={'Visit':'Subject_Total_Visits'})\n        metadata_complex_tdcs = tdcsfog_metadata.merge(selfobj.subjects,how='left',on='Subject').copy()\n        metadata_complex_tdcs['Medication']= metadata_complex_tdcs['Medication'].factorize()[0]\n        metadata_complex_tdcs = metadata_complex_tdcs.drop(['Visit'], axis=1)\n        return metadata_complex_tdcs\n    \n\n    def __Get_DEFOG_Metadata(selfobj):\n        defog_metadata = pd.read_csv('/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/defog_metadata.csv')\n        defog_metadata = defog_metadata.rename(columns={'Visit':'Subject_Total_Visits'})\n        metadata_complex_defog = defog_metadata.merge(selfobj.subjects,how='left',on='Subject').copy()\n        metadata_complex_defog['Medication'] = metadata_complex_defog['Medication'].factorize()[0]\n        metadata_complex_defog['Visit'] = metadata_complex_defog['Visit'].astype(int)\n        return metadata_complex_defog\n    \n    \n    #############################################################################################\n    ##################################### CSV to DATAFRAMES #####################################\n    #############################################################################################\n    \n    def __TCDS_Csv_To_Dataframe(selfobj, f):\n#         try:\n        df = pd.read_csv(f)\n        df['Id'] = f.split('/')[-1].split('.')[0]\n        df['Dataset'] = pathlib.Path(f).parts[-2]\n        # Merge in the Metadata Dataframe\n        df = pd.merge(df, selfobj.metadata_complex_tcds, on='Id', how='left')\n        df = df.drop_duplicates()\n        # Merge in the Events dataframe\n        df = pd.merge(df, selfobj.events_flat, on=['Id', 'Time'], how='left')\n        df['Events_Type'] = df['Events_Type'].fillna('None')\n        df['Kinetic'] = df['Kinetic'].fillna('Event_Movement_Unknown')\n        df = df.drop_duplicates()\n        return df\n#         except: pass\n        \n    \n    def __DEFOG_Csv_To_Dataframe(selfobj, f):\n#         try:\n        df = pd.read_csv(f)\n        df['Id'] = f.split('/')[-1].split('.')[0]\n        df['Dataset'] = pathlib.Path(f).parts[-2]\n        # Merge in the Metadata Dataframe\n        df = pd.merge(df, selfobj.metadata_complex_defog, how='left', on='Id')\n        df = df.drop_duplicates()\n        # Merge in the Tasks dataframe\n        if selfobj.include_tasks:\n            df = pd.merge(df, selfobj.tasks_flat, on=['Id', 'Time'], how='left')\n            df['Tasks_Type'] = df['Tasks_Type'].fillna('None')\n            df = df.drop_duplicates()\n        # Merge in the Events dataframe\n        df = pd.merge(df, selfobj.events_flat, on=['Id', 'Time'], how='left')\n        df['Events_Type'] = df['Events_Type'].fillna('None')\n        df['Kinetic'] = df['Kinetic'].fillna('Event_Movement_Unknown')\n        df = df.drop_duplicates(subset=['Time','AccV','AccML','AccAP'])\n        return df\n#         except: pass\n        \n        \n    #############################################################################################\n    #################################### RECTANGULAR DATASETS ###################################\n    #############################################################################################\n\n    def Get_TCDS_Rectangular_Dataset(selfobj, tdcsfog_csv_list):\n        \"\"\"Converts the list of csv file paths into a single large, rectangular dataset\n        \n        Parameters\n        ----------\n        tdcsfog_csv_list : list of strings\n            The list of paths to the csv files\n\n        Returns\n        -------\n        pandas dataframe\n            a very large dataframe.\n            First each csv is converted to a dataframe (one per trial/experiment).\n            Next, each trial is merged with metadata, subjects, and events (by time stamp).\n            Lastly, each trial is concatenated together row-wise.\n        \"\"\"\n        tcds_df_list = []\n        count = len(tdcsfog_csv_list)\n        print(\"Creating TCDS Rectangular Dataset.\")\n        for i in tqdm(range(count)):\n            f = tdcsfog_csv_list[i]\n            tcds_df_list.append(selfobj.__TCDS_Csv_To_Dataframe(f))\n        # Concatenates the tcds rows\n        tcds_df = pd.concat(tcds_df_list)\n        tcds_df = tcds_df.reset_index(drop=True)\n        return tcds_df\n    \n\n    def Get_DEFOG_Rectangular_Dataset(selfobj, defog_csv_list):\n        \"\"\"Converts the list of csv file paths into a single large, rectangular dataset\n        \n        Parameters\n        ----------\n        defog_csv_list : list of strings\n            The list of paths to the csv files\n\n        Returns\n        -------\n        pandas dataframe\n            a very large dataframe.\n            First each csv is converted to a dataframe (one per trial/experiment).\n            Next, each trial is merged with metadata, subjects, tasks (by time step) and events (by time step).\n            Lastly, each trial is concatenated together row-wise.\n        \"\"\"\n        defog_df_list = []\n        count = len(defog_csv_list)\n        print(\"Creating Defog Rectangular Dataset.\")\n        for i in tqdm(range(count)):\n            f = defog_csv_list[i]\n            defog_df_list.append(selfobj.__DEFOG_Csv_To_Dataframe(f))\n        # Concatenates the defog rows\n        defog_df = pd.concat(defog_df_list)\n        defog_df = defog_df.reset_index(drop=True)\n        defog_df = defog_df.rename(columns={\"Task\": \"Protocol_Task\"})\n        return defog_df\n    \n    \n    \n    def Get_Short_Long(selfobj, fdf, L):\n        \"\"\"Makes a 'Is_Long' column which determines if the FOG event is 'long' or not based on the 'L' argument.\n            \n            Parameters\n            ----------\n                Takes in a rectangular dataframe. Requires that the dataframe have the columns called 'StartHesitation', 'Turn', 'Walking', and 'None'.\n              \n            Returns\n            -------\n                A copy of the dataframe with the 'Is_Long' column.  \n        \"\"\"\n\n        df = fdf.copy(deep=True)\n\n        df['Is_Long'] = 0  # Initialize 'Is_Long' column with zeros\n\n        # Calculate the rolling sum for each column\n        cumulative_sum = df[['StartHesitation', 'Turn', 'Walking', 'None']].rolling(L, min_periods=1).sum()\n\n        # Check if the rolling sum is greater than or equal to L and set 'Is_Long' column accordingly\n        df['Is_Long'].mask(cumulative_sum.ge(L).any(axis=1), 1, inplace=True)\n\n        return df","metadata":{"execution":{"iopub.status.busy":"2023-05-24T18:06:41.655052Z","iopub.execute_input":"2023-05-24T18:06:41.655405Z","iopub.status.idle":"2023-05-24T18:06:41.701453Z","shell.execute_reply.started":"2023-05-24T18:06:41.655377Z","shell.execute_reply":"2023-05-24T18:06:41.699754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Make an object\n# f = FOGRect(include_tasks=True)\n\n## Make the test rectangular dataframe\n# test_tcds_df = f.Get_TCDS_Rectangular_Dataset(f.test_tcds_csv_list)","metadata":{"execution":{"iopub.status.busy":"2023-05-24T18:07:51.233313Z","iopub.execute_input":"2023-05-24T18:07:51.233705Z","iopub.status.idle":"2023-05-24T18:09:54.694800Z","shell.execute_reply.started":"2023-05-24T18:07:51.233676Z","shell.execute_reply":"2023-05-24T18:09:54.693376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# f1 = FOGRect()","metadata":{"execution":{"iopub.status.busy":"2023-05-24T18:10:45.686069Z","iopub.execute_input":"2023-05-24T18:10:45.686670Z","iopub.status.idle":"2023-05-24T18:11:35.103703Z","shell.execute_reply.started":"2023-05-24T18:10:45.686632Z","shell.execute_reply":"2023-05-24T18:11:35.101807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}