{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Parkinsons' Freezing of Gait Rectangular Data\n\nThe purpose of this notebook is to show how to use the rectangular dataset builder object for the testing dataset.","metadata":{}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import fog_rectangulardataset_utility as fr\nimport pandas as pd\npd.set_option('display.max_columns', None)\nimport numpy as np\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-05-08T00:48:40.468442Z","iopub.execute_input":"2023-05-08T00:48:40.469249Z","iopub.status.idle":"2023-05-08T00:48:40.476156Z","shell.execute_reply.started":"2023-05-08T00:48:40.469195Z","shell.execute_reply":"2023-05-08T00:48:40.474380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"markdown","source":"## Cleanup + Feature Engineering","metadata":{}},{"cell_type":"code","source":"def Clean_Dataset(tdf, dtype_train=True):\n    fdf = tdf.copy(deep=True)\n    # Step 1: convert to numericals\n    fdf['isMale'] = fdf['Sex'].apply(lambda x: 1 if x == 'M' else 0)\n    # Step 2: make the Time column into the trial id and the time stamp\n    fdf['Time'] = fdf['Id'].astype(str) + \"_\" + fdf['Time'].astype(str)\n    # Step 3: Apply the lag to accelerometer data for each group, the concat back\n    fdf = fdf.groupby('Id', group_keys=False).apply(lambda group_df: group_df.assign(\n        AccV_lag1=group_df['AccV'].shift(1),\n        AccML_lag1=group_df['AccML'].shift(1),\n        AccAP_lag1=group_df['AccAP'].shift(1),\n    ))\n    lag_cols = ['AccV_lag1', 'AccML_lag1', 'AccAP_lag1']\n    fdf = fdf.reset_index(drop=True).fillna(0)\n    # Step 4: keep only desired columns\n    target_cols = ['StartHesitation', 'Turn', 'Walking']\n    predictor_cols = ['Time', 'AccV', 'AccML', 'AccAP', 'Medication', 'Age', 'isMale', 'YearsSinceDx']+lag_cols\n    # Only keeps the predictor cols if the datatype is testing\n    keep_cols = target_cols + predictor_cols if dtype_train else predictor_cols\n    fdf = fdf[keep_cols]\n    return fdf","metadata":{"execution":{"iopub.status.busy":"2023-05-08T00:48:40.477993Z","iopub.execute_input":"2023-05-08T00:48:40.479136Z","iopub.status.idle":"2023-05-08T00:48:40.497446Z","shell.execute_reply.started":"2023-05-08T00:48:40.479083Z","shell.execute_reply":"2023-05-08T00:48:40.496347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier\nfrom sklearn.multioutput import MultiOutputClassifier\n\ndef Fit_Model(training, testing):\n    \n    # Variables\n    target_cols = ['StartHesitation', 'Turn', 'Walking']\n    predictor_cols = ['AccV', 'AccML', 'AccAP', 'Medication', 'Age', 'isMale', 'YearsSinceDx']\n    test_index = testing['Time']\n    \n    # Preprocessing\n    X_train = training[predictor_cols]\n    y_train = training[target_cols]\n    X_test = testing[predictor_cols]\n    \n    # Training\n    dtc = DecisionTreeClassifier()    \n    multi_dtc = MultiOutputClassifier(dtc)\n    multi_dtc.fit(X_train, y_train)\n\n    # Prediction\n    y_pred_dtc_prob = multi_dtc.predict_proba(X_test)\n    y_pred_dtc = multi_dtc.predict(X_test)\n    y_pred_dtc = pd.DataFrame(y_pred_dtc, columns=target_cols)\n    y_pred_dtc['Id'] = range(len(X_test))\n    y_pred_dtc.set_index('Id', inplace=True)\n\n    # Create dataframe with predictions for all three columns\n    y_pred_dtc.reset_index(drop=True, inplace=True)\n    y_pred_dtc['Id'] = test_index.values\n    df_pred = pd.DataFrame({\n                'Id': y_pred_dtc['Id'], \n                'StartHesitation': y_pred_dtc['StartHesitation'],\n                'Turn': y_pred_dtc['Turn'],\n                'Walking': y_pred_dtc['Walking'],\n    })\n\n    return df_pred\n","metadata":{"execution":{"iopub.status.busy":"2023-05-08T00:48:40.499439Z","iopub.execute_input":"2023-05-08T00:48:40.500139Z","iopub.status.idle":"2023-05-08T00:48:40.510722Z","shell.execute_reply.started":"2023-05-08T00:48:40.500099Z","shell.execute_reply":"2023-05-08T00:48:40.509530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clean, Train, and Submit","metadata":{}},{"cell_type":"markdown","source":"## 1. Get The Datasets","metadata":{}},{"cell_type":"code","source":"# Create a Freezing of Gait Rectangular Dataset Builder class object\nfrobj = fr.FOGRect()","metadata":{"execution":{"iopub.status.busy":"2023-05-08T00:48:40.512529Z","iopub.execute_input":"2023-05-08T00:48:40.513335Z","iopub.status.idle":"2023-05-08T00:50:53.859195Z","shell.execute_reply.started":"2023-05-08T00:48:40.513301Z","shell.execute_reply":"2023-05-08T00:50:53.857492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Obtain the training through downloading the datasets\ntrain_defog_df = pd.concat([chunk for chunk in tqdm(pd.read_csv(\"/kaggle/input/fog-train-defog/train_defog.csv\", low_memory=False, chunksize=1000), desc='Downloading Train Defog', total=13526)])\ntrain_tcds_df = pd.concat([chunk for chunk in tqdm(pd.read_csv(\"/kaggle/input/fog-train-tcds/train_tcds.csv\", low_memory=False, chunksize=1000), desc='Downloading Train TCDS', total=7102)])\n\n# Obtain the testing through the rectangular dataset builder\ntest_defog_df = frobj.Get_DEFOG_Rectangular_Dataset(frobj.test_defog_csv_list)\ntest_tcds_df = frobj.Get_TCDS_Rectangular_Dataset(frobj.test_tcds_csv_list)","metadata":{"execution":{"iopub.status.busy":"2023-05-08T00:50:53.862117Z","iopub.execute_input":"2023-05-08T00:50:53.863424Z","iopub.status.idle":"2023-05-08T00:53:52.043239Z","shell.execute_reply.started":"2023-05-08T00:50:53.863385Z","shell.execute_reply":"2023-05-08T00:53:52.041886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Preprocess Datasets","metadata":{}},{"cell_type":"code","source":"print(f\"{train_defog_df.shape=}\")\ndisplay(train_defog_df.head(2))\nprint(f\"{train_tcds_df.shape=}\")\ndisplay(train_tcds_df.head(2))\nprint(f\"{test_defog_df.shape=}\")\ndisplay(test_defog_df.head(2))\nprint(f\"{test_tcds_df.shape=}\")\ndisplay(test_tcds_df.head(2))","metadata":{"execution":{"iopub.status.busy":"2023-05-08T00:53:52.047423Z","iopub.execute_input":"2023-05-08T00:53:52.047831Z","iopub.status.idle":"2023-05-08T00:53:52.127687Z","shell.execute_reply.started":"2023-05-08T00:53:52.047790Z","shell.execute_reply":"2023-05-08T00:53:52.126689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_defog_df = Clean_Dataset(train_defog_df)\ntrain_tcds_df = Clean_Dataset(train_tcds_df)\ntest_defog_df = Clean_Dataset(test_defog_df, dtype_train=False)\ntest_tcds_df = Clean_Dataset(test_tcds_df, dtype_train=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-08T00:53:52.129258Z","iopub.execute_input":"2023-05-08T00:53:52.129694Z","iopub.status.idle":"2023-05-08T00:56:04.284936Z","shell.execute_reply.started":"2023-05-08T00:53:52.129652Z","shell.execute_reply":"2023-05-08T00:56:04.283788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Concat Both Datasets","metadata":{}},{"cell_type":"code","source":"train = pd.concat([train_defog_df, train_tcds_df])\ntest = pd.concat([test_defog_df, test_tcds_df])","metadata":{"execution":{"iopub.status.busy":"2023-05-08T00:56:04.286742Z","iopub.execute_input":"2023-05-08T00:56:04.287059Z","iopub.status.idle":"2023-05-08T00:56:07.437863Z","shell.execute_reply.started":"2023-05-08T00:56:04.287032Z","shell.execute_reply":"2023-05-08T00:56:07.436656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train.head(2))\ndisplay(test.head(2))","metadata":{"execution":{"iopub.status.busy":"2023-05-08T00:56:07.439105Z","iopub.execute_input":"2023-05-08T00:56:07.439413Z","iopub.status.idle":"2023-05-08T00:56:07.467079Z","shell.execute_reply.started":"2023-05-08T00:56:07.439387Z","shell.execute_reply":"2023-05-08T00:56:07.465615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Clean Up To Save RAM\n\nDoing so clears up space for only the `train` and `test` dataframes.","metadata":{}},{"cell_type":"code","source":"# Delete original rectangular datasets\ndel train_defog_df, train_tcds_df, test_defog_df, test_tcds_df","metadata":{"execution":{"iopub.status.busy":"2023-05-08T00:56:07.468729Z","iopub.execute_input":"2023-05-08T00:56:07.469058Z","iopub.status.idle":"2023-05-08T00:56:07.662906Z","shell.execute_reply.started":"2023-05-08T00:56:07.469030Z","shell.execute_reply":"2023-05-08T00:56:07.661577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Make Predictions","metadata":{}},{"cell_type":"code","source":"submission = Fit_Model(train, test)","metadata":{"execution":{"iopub.status.busy":"2023-05-08T00:56:07.667047Z","iopub.execute_input":"2023-05-08T00:56:07.667446Z","iopub.status.idle":"2023-05-08T01:11:40.619791Z","shell.execute_reply.started":"2023-05-08T00:56:07.667413Z","shell.execute_reply":"2023-05-08T01:11:40.618394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. Submit Predictions","metadata":{}},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-08T01:11:40.621416Z","iopub.execute_input":"2023-05-08T01:11:40.621798Z","iopub.status.idle":"2023-05-08T01:11:41.173194Z","shell.execute_reply.started":"2023-05-08T01:11:40.621764Z","shell.execute_reply":"2023-05-08T01:11:41.172346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-08T01:11:41.175084Z","iopub.execute_input":"2023-05-08T01:11:41.175952Z","iopub.status.idle":"2023-05-08T01:11:41.187521Z","shell.execute_reply.started":"2023-05-08T01:11:41.175905Z","shell.execute_reply":"2023-05-08T01:11:41.186364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}