{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Notebook is ...","metadata":{}},{"cell_type":"markdown","source":"An attempt to add flag-enabled features to the notebooks in Credits. Practically the more I screwed around, the lower LB score I have got. But I had fun!","metadata":{}},{"cell_type":"markdown","source":"## Parameters","metadata":{}},{"cell_type":"code","source":"WINDOW_SIZE = 3000\nSAMPLES_NUMBER = 2500000\nTIME_FRAC_DECIMALS = 5\nOUTPUT_DECIMALS = 3\nFOLDS_NUMBER = 6\nSUBJECTS_NCLUSTER = 8\nTARGETS_NCLUSTER = 4\nTASKS_NCLUSTER = 4\nEVENTS_NCLUSTER = 8\nCLUSTER_NINIT = 10\nRESCALE_ACC = True\nIGNORE_WITHOUT_SYMPT = False\nADD_EVENTS = False\nROLLING_AVG = False\nROLLING_WINDOW = WINDOW_SIZE\n\nCORR_THRES = 0.95\nENABLE_MI_SCORES = True \nPCT_KEEP_COLS = 0.8\nPCOLS_IDX = [0,1,2]\nSHUFFLE = False\nTRAIN_RESAMPLE = True\nMULT_POS_FACTOR = 4 # Applied only if Resample\n\nREGRESSOR = 'LGBM' # 'RF'\nSTRATEGY =  'Group' #'Stratified' \nTEST_SIZE = 0.2 # Applied only if Stratified\nPARAMS_OPTIMIZATION_ENABLE = True \nCV = 4\nITERATIONS = 25\nEXPANSION_FACTOR = 1.25\nNUM_FI = 12\n\nCVS_FACTOR = 0.5\nDIRTY_HELP = False","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:00.611669Z","iopub.execute_input":"2023-06-05T11:04:00.612135Z","iopub.status.idle":"2023-06-05T11:04:00.621328Z","shell.execute_reply.started":"2023-06-05T11:04:00.612103Z","shell.execute_reply":"2023-06-05T11:04:00.620178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Credits","metadata":{}},{"cell_type":"markdown","source":"- https://www.kaggle.com/code/nickcgray/gait-prediction\n- https://www.kaggle.com/code/xzj19013742/fast-lgbm-groupkfold-tsflex-memory-optimization\n- https://www.kaggle.com/code/averkovanika/parkinson-s-fog-mi-based-feature-selection\n- https://www.kaggle.com/code/tomonorisasaki/comprehensive-eda-and-data-visualization","metadata":{}},{"cell_type":"markdown","source":"## Imports and System Config","metadata":{}},{"cell_type":"code","source":"#\n# Install tsflex and seglearn\n# altrimenti ModuleNotFoundError: No module named 'seglearn'\n#\n!pip install tsflex --no-index --find-links=file:///kaggle/input/time-series-tools\n!pip install seglearn --no-index --find-links=file:///kaggle/input/time-series-tools","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:00.623746Z","iopub.execute_input":"2023-06-05T11:04:00.624200Z","iopub.status.idle":"2023-06-05T11:04:22.869307Z","shell.execute_reply.started":"2023-06-05T11:04:00.624160Z","shell.execute_reply":"2023-06-05T11:04:22.868250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport psutil\nimport glob\nfrom os import path\nfrom pathlib import Path\nimport time\nfrom datetime import datetime","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:22.872163Z","iopub.execute_input":"2023-06-05T11:04:22.872816Z","iopub.status.idle":"2023-06-05T11:04:22.878601Z","shell.execute_reply.started":"2023-06-05T11:04:22.872776Z","shell.execute_reply":"2023-06-05T11:04:22.877463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.auto import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:22.880313Z","iopub.execute_input":"2023-06-05T11:04:22.880684Z","iopub.status.idle":"2023-06-05T11:04:22.889494Z","shell.execute_reply.started":"2023-06-05T11:04:22.880648Z","shell.execute_reply":"2023-06-05T11:04:22.888631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn import *\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.model_selection import StratifiedShuffleSplit\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom sklearn.multioutput import MultiOutputRegressor\nfrom sklearn.base import clone\nfrom sklearn.metrics import average_precision_score","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:22.892263Z","iopub.execute_input":"2023-06-05T11:04:22.892599Z","iopub.status.idle":"2023-06-05T11:04:22.900899Z","shell.execute_reply.started":"2023-06-05T11:04:22.892570Z","shell.execute_reply":"2023-06-05T11:04:22.900022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from seglearn.feature_functions import base_features, emg_features\nfrom tsflex.features import FeatureCollection, MultipleFeatureDescriptors\nfrom tsflex.features.integrations import seglearn_feature_dict_wrapper\nfrom sklearn.feature_selection import mutual_info_classif\nfrom tabulate import tabulate","metadata":{"papermill":{"duration":2.755431,"end_time":"2023-04-16T22:41:25.148066","exception":false,"start_time":"2023-04-16T22:41:22.392635","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T11:04:22.902092Z","iopub.execute_input":"2023-06-05T11:04:22.902526Z","iopub.status.idle":"2023-06-05T11:04:22.915723Z","shell.execute_reply.started":"2023-06-05T11:04:22.902489Z","shell.execute_reply":"2023-06-05T11:04:22.914738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PD_MAX_ROWS = 20\nprocess = psutil.Process(os.getpid())\npd.options.display.float_format = '{:,.5f}'.format\npd.set_option('display.max_rows',PD_MAX_ROWS)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:22.916849Z","iopub.execute_input":"2023-06-05T11:04:22.917262Z","iopub.status.idle":"2023-06-05T11:04:22.927501Z","shell.execute_reply.started":"2023-06-05T11:04:22.917225Z","shell.execute_reply":"2023-06-05T11:04:22.926697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EXPORT_CSV = False\nMB = 1024 * 1024\nRANDOM_STATE = 123","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:22.928653Z","iopub.execute_input":"2023-06-05T11:04:22.929050Z","iopub.status.idle":"2023-06-05T11:04:22.940098Z","shell.execute_reply.started":"2023-06-05T11:04:22.929015Z","shell.execute_reply":"2023-06-05T11:04:22.939005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts = datetime.fromtimestamp(time.time())\nprint('Start Timestamp (after imports) = ',ts)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:22.943352Z","iopub.execute_input":"2023-06-05T11:04:22.943761Z","iopub.status.idle":"2023-06-05T11:04:22.950672Z","shell.execute_reply.started":"2023-06-05T11:04:22.943734Z","shell.execute_reply":"2023-06-05T11:04:22.949641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Preparation","metadata":{"papermill":{"duration":0.009576,"end_time":"2023-04-16T22:41:25.16752","exception":false,"start_time":"2023-04-16T22:41:25.157944","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### Data Import","metadata":{}},{"cell_type":"code","source":"root = '/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/'\n\ntrain = glob.glob(path.join(root, 'train/**/**'))\ntest = glob.glob(path.join(root, 'test/**/**'))\nunlabeled = glob.glob(path.join(root, 'unlabeled/**'))\n\nsubjects = pd.read_csv(path.join(root, 'subjects.csv'))\ntasks = pd.read_csv(path.join(root, 'tasks.csv'))\nevents = pd.read_csv(path.join(root, 'events.csv'))\n\ntdcsfog_metadata = pd.read_csv(path.join(root, 'tdcsfog_metadata.csv'))\ndefog_metadata = pd.read_csv(path.join(root, 'defog_metadata.csv')) \n\ntdcsfog_metadata['Module'] = 'tdcsfog'\ndefog_metadata['Module'] = 'defog'\n\nfull_metadata = pd.concat([tdcsfog_metadata, defog_metadata])","metadata":{"papermill":{"duration":0.170669,"end_time":"2023-04-16T22:41:25.347896","exception":false,"start_time":"2023-04-16T22:41:25.177227","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T11:04:22.951992Z","iopub.execute_input":"2023-06-05T11:04:22.952268Z","iopub.status.idle":"2023-06-05T11:04:23.016125Z","shell.execute_reply.started":"2023-06-05T11:04:22.952246Z","shell.execute_reply":"2023-06-05T11:04:23.014919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# what does glob?\ntrain_to_see = glob.glob(path.join(root, 'train/**/**'))\ntrain_to_see[0:10]","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:23.019622Z","iopub.execute_input":"2023-06-05T11:04:23.019925Z","iopub.status.idle":"2023-06-05T11:04:23.032096Z","shell.execute_reply.started":"2023-06-05T11:04:23.019900Z","shell.execute_reply":"2023-06-05T11:04:23.031250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# full_metadata = tdcsfog_metadata + defog_metadata\nfull_metadata.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:23.033766Z","iopub.execute_input":"2023-06-05T11:04:23.034162Z","iopub.status.idle":"2023-06-05T11:04:23.046605Z","shell.execute_reply.started":"2023-06-05T11:04:23.034127Z","shell.execute_reply":"2023-06-05T11:04:23.045549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visit simple says that two events are in the same visit?\nfull_metadata[full_metadata['Subject']=='f62eec']","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:23.048210Z","iopub.execute_input":"2023-06-05T11:04:23.048871Z","iopub.status.idle":"2023-06-05T11:04:23.064048Z","shell.execute_reply.started":"2023-06-05T11:04:23.048835Z","shell.execute_reply":"2023-06-05T11:04:23.062968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subjects.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:23.065557Z","iopub.execute_input":"2023-06-05T11:04:23.065965Z","iopub.status.idle":"2023-06-05T11:04:23.081390Z","shell.execute_reply.started":"2023-06-05T11:04:23.065931Z","shell.execute_reply":"2023-06-05T11:04:23.080507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subjects[subjects['Subject']=='f62eec']","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:23.082535Z","iopub.execute_input":"2023-06-05T11:04:23.083540Z","iopub.status.idle":"2023-06-05T11:04:23.097497Z","shell.execute_reply.started":"2023-06-05T11:04:23.083502Z","shell.execute_reply":"2023-06-05T11:04:23.096414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subjects['Sex'] = subjects['Sex'].factorize()[0]\nsubjects = subjects.fillna(0).groupby('Subject').median()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:23.099257Z","iopub.execute_input":"2023-06-05T11:04:23.099965Z","iopub.status.idle":"2023-06-05T11:04:23.111373Z","shell.execute_reply.started":"2023-06-05T11:04:23.099927Z","shell.execute_reply":"2023-06-05T11:04:23.110387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:23.112857Z","iopub.execute_input":"2023-06-05T11:04:23.113469Z","iopub.status.idle":"2023-06-05T11:04:23.135895Z","shell.execute_reply.started":"2023-06-05T11:04:23.113434Z","shell.execute_reply":"2023-06-05T11:04:23.134587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tasks","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:23.137462Z","iopub.execute_input":"2023-06-05T11:04:23.137996Z","iopub.status.idle":"2023-06-05T11:04:23.156673Z","shell.execute_reply.started":"2023-06-05T11:04:23.137968Z","shell.execute_reply":"2023-06-05T11:04:23.155904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Subjects Exploration","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nfrom sklearn.preprocessing import MinMaxScaler, StandardScaler\nfrom sklearn.metrics import silhouette_samples, silhouette_score","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:23.158038Z","iopub.execute_input":"2023-06-05T11:04:23.158548Z","iopub.status.idle":"2023-06-05T11:04:23.162588Z","shell.execute_reply.started":"2023-06-05T11:04:23.158520Z","shell.execute_reply":"2023-06-05T11:04:23.161736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = subjects[subjects.columns[1:]]\nfor k in range(2,20):\n    km =  cluster.KMeans(n_clusters = k, random_state = RANDOM_STATE, n_init= CLUSTER_NINIT)\n    km_preds = km.fit_predict(X)\n    sil = silhouette_score(X, km_preds)\n    sse = km.inertia_\n    print(f'k = {k:02d} silhouette = {sil:.2f} sse = {sse:.2f}')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:23.164008Z","iopub.execute_input":"2023-06-05T11:04:23.164282Z","iopub.status.idle":"2023-06-05T11:04:25.385637Z","shell.execute_reply.started":"2023-06-05T11:04:23.164258Z","shell.execute_reply":"2023-06-05T11:04:25.383640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"km_preds = cluster.KMeans(n_clusters = SUBJECTS_NCLUSTER, n_init=CLUSTER_NINIT,\n                          random_state = RANDOM_STATE).fit_predict(subjects[subjects.columns[1:]])","metadata":{"papermill":{"duration":0.110973,"end_time":"2023-04-16T22:41:25.698532","exception":false,"start_time":"2023-04-16T22:41:25.587559","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T11:04:25.387766Z","iopub.execute_input":"2023-06-05T11:04:25.392658Z","iopub.status.idle":"2023-06-05T11:04:25.499674Z","shell.execute_reply.started":"2023-06-05T11:04:25.392620Z","shell.execute_reply":"2023-06-05T11:04:25.498545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def do_pca(n,ds):\n    pca = PCA(n_components=n)\n    scaler = MinMaxScaler()\n    X = scaler.fit_transform(ds)\n    pca.fit(X)\n    print(pca.explained_variance_ratio_)\n    Xg = pca.transform(X)\n    return Xg","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:25.501342Z","iopub.execute_input":"2023-06-05T11:04:25.501892Z","iopub.status.idle":"2023-06-05T11:04:25.507377Z","shell.execute_reply.started":"2023-06-05T11:04:25.501836Z","shell.execute_reply":"2023-06-05T11:04:25.506641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def print_xy_cluster(ds,preds):\n    Xg = do_pca(2,ds)\n    dg = pd.DataFrame(Xg)\n    pca_cols = ['x1','x2']\n    dg.columns = pca_cols\n    dg['Cluster'] = preds\n    fig, ax = plt.subplots(figsize=(10, 6))\n    ax.set_xlabel(pca_cols[0])\n    ax.set_ylabel(pca_cols[1])\n    sns.scatterplot(x=pca_cols[0],y=pca_cols[1],hue='Cluster',palette='bright',data=dg)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:25.508290Z","iopub.execute_input":"2023-06-05T11:04:25.508589Z","iopub.status.idle":"2023-06-05T11:04:25.521094Z","shell.execute_reply.started":"2023-06-05T11:04:25.508563Z","shell.execute_reply":"2023-06-05T11:04:25.520016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print_xy_cluster(subjects,km_preds)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:25.522470Z","iopub.execute_input":"2023-06-05T11:04:25.522906Z","iopub.status.idle":"2023-06-05T11:04:26.153921Z","shell.execute_reply.started":"2023-06-05T11:04:25.522857Z","shell.execute_reply":"2023-06-05T11:04:26.152731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xg = do_pca(2,subjects)\ndg = pd.DataFrame(Xg)\ndg['Subject'] = subjects.index.values","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:26.155624Z","iopub.execute_input":"2023-06-05T11:04:26.156239Z","iopub.status.idle":"2023-06-05T11:04:26.167939Z","shell.execute_reply.started":"2023-06-05T11:04:26.156200Z","shell.execute_reply":"2023-06-05T11:04:26.166820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dg[(dg[0]>0.1)&(dg[0]<0.3)]","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:26.171106Z","iopub.execute_input":"2023-06-05T11:04:26.171388Z","iopub.status.idle":"2023-06-05T11:04:26.182607Z","shell.execute_reply.started":"2023-06-05T11:04:26.171364Z","shell.execute_reply":"2023-06-05T11:04:26.181620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Is this subject an outlier? Sex = 0.5 ?\nsubjects.loc['5d9cae']","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:26.185896Z","iopub.execute_input":"2023-06-05T11:04:26.186190Z","iopub.status.idle":"2023-06-05T11:04:26.197310Z","shell.execute_reply.started":"2023-06-05T11:04:26.186166Z","shell.execute_reply":"2023-06-05T11:04:26.196270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subjects['Sex'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:26.198855Z","iopub.execute_input":"2023-06-05T11:04:26.199157Z","iopub.status.idle":"2023-06-05T11:04:26.211371Z","shell.execute_reply.started":"2023-06-05T11:04:26.199133Z","shell.execute_reply":"2023-06-05T11:04:26.210264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subjects.loc['5d9cae']['Sex']=0","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:26.220075Z","iopub.execute_input":"2023-06-05T11:04:26.220447Z","iopub.status.idle":"2023-06-05T11:04:26.225147Z","shell.execute_reply.started":"2023-06-05T11:04:26.220419Z","shell.execute_reply":"2023-06-05T11:04:26.223991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subjects.columns","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:26.226705Z","iopub.execute_input":"2023-06-05T11:04:26.227011Z","iopub.status.idle":"2023-06-05T11:04:26.239195Z","shell.execute_reply.started":"2023-06-05T11:04:26.226986Z","shell.execute_reply":"2023-06-05T11:04:26.238207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=subjects,x='Age')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:26.240612Z","iopub.execute_input":"2023-06-05T11:04:26.240893Z","iopub.status.idle":"2023-06-05T11:04:26.585286Z","shell.execute_reply.started":"2023-06-05T11:04:26.240861Z","shell.execute_reply":"2023-06-05T11:04:26.584376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anom_age_subjects = subjects[subjects['Age']<40].index.values\nanom_age_subjects","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:26.586748Z","iopub.execute_input":"2023-06-05T11:04:26.587032Z","iopub.status.idle":"2023-06-05T11:04:26.592769Z","shell.execute_reply.started":"2023-06-05T11:04:26.587008Z","shell.execute_reply":"2023-06-05T11:04:26.592083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=subjects,x='YearsSinceDx')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:26.593522Z","iopub.execute_input":"2023-06-05T11:04:26.593776Z","iopub.status.idle":"2023-06-05T11:04:26.942853Z","shell.execute_reply.started":"2023-06-05T11:04:26.593754Z","shell.execute_reply":"2023-06-05T11:04:26.941768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=subjects,x='UPDRSIII_On')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:26.944308Z","iopub.execute_input":"2023-06-05T11:04:26.944622Z","iopub.status.idle":"2023-06-05T11:04:27.286431Z","shell.execute_reply.started":"2023-06-05T11:04:26.944596Z","shell.execute_reply":"2023-06-05T11:04:27.285250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=subjects,x='UPDRSIII_Off')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:27.287839Z","iopub.execute_input":"2023-06-05T11:04:27.288146Z","iopub.status.idle":"2023-06-05T11:04:27.616075Z","shell.execute_reply.started":"2023-06-05T11:04:27.288122Z","shell.execute_reply":"2023-06-05T11:04:27.614984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=subjects,x='NFOGQ')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:27.617628Z","iopub.execute_input":"2023-06-05T11:04:27.618647Z","iopub.status.idle":"2023-06-05T11:04:27.966904Z","shell.execute_reply.started":"2023-06-05T11:04:27.618605Z","shell.execute_reply":"2023-06-05T11:04:27.965693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subjects['s_group'] = km_preds","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:27.968986Z","iopub.execute_input":"2023-06-05T11:04:27.969431Z","iopub.status.idle":"2023-06-05T11:04:27.974921Z","shell.execute_reply.started":"2023-06-05T11:04:27.969375Z","shell.execute_reply":"2023-06-05T11:04:27.973790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_names = {'Visit':'s_visit','Age':'s_age','YearsSinceDx':'s_years','UPDRSIII_On':'s_on','UPDRSIII_Off':'s_off','NFOGQ':'s_NFOGQ', 'Sex': 's_sex'}\nsubjects = subjects.rename(columns = new_names)\nsubjects","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:27.976468Z","iopub.execute_input":"2023-06-05T11:04:27.976887Z","iopub.status.idle":"2023-06-05T11:04:28.000742Z","shell.execute_reply.started":"2023-06-05T11:04:27.976837Z","shell.execute_reply":"2023-06-05T11:04:27.999615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Tasks","metadata":{}},{"cell_type":"code","source":"tasks['Duration'] = tasks['End'] - tasks['Begin']","metadata":{"papermill":{"duration":0.108699,"end_time":"2023-04-16T22:41:25.903139","exception":false,"start_time":"2023-04-16T22:41:25.79444","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T11:04:28.002232Z","iopub.execute_input":"2023-06-05T11:04:28.002578Z","iopub.status.idle":"2023-06-05T11:04:28.012442Z","shell.execute_reply.started":"2023-06-05T11:04:28.002550Z","shell.execute_reply":"2023-06-05T11:04:28.011211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pvt_tasks = pd.pivot_table(tasks, values=['Duration'], index=['Id'], columns=['Task'], aggfunc=[np.sum,np.mean], fill_value=0)\npvt_tasks.head().transpose()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:28.013893Z","iopub.execute_input":"2023-06-05T11:04:28.014188Z","iopub.status.idle":"2023-06-05T11:04:28.065806Z","shell.execute_reply.started":"2023-06-05T11:04:28.014164Z","shell.execute_reply":"2023-06-05T11:04:28.064999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def collapse_columns(df):\n    df = df.copy()\n    if isinstance(df.columns, pd.MultiIndex):\n        df.columns = df.columns.to_series().apply(lambda x: \"_\".join(x))\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:28.067145Z","iopub.execute_input":"2023-06-05T11:04:28.067695Z","iopub.status.idle":"2023-06-05T11:04:28.073016Z","shell.execute_reply.started":"2023-06-05T11:04:28.067661Z","shell.execute_reply":"2023-06-05T11:04:28.072030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pvt_tasks = collapse_columns(pvt_tasks)\npvt_tasks = pvt_tasks.reset_index()\npvt_tasks.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:28.074564Z","iopub.execute_input":"2023-06-05T11:04:28.074911Z","iopub.status.idle":"2023-06-05T11:04:28.099856Z","shell.execute_reply.started":"2023-06-05T11:04:28.074884Z","shell.execute_reply":"2023-06-05T11:04:28.098908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"km_preds = cluster.KMeans(n_clusters = TASKS_NCLUSTER, n_init=CLUSTER_NINIT,\n                                  random_state = RANDOM_STATE).fit_predict(pvt_tasks[pvt_tasks.columns[1:]])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:28.101034Z","iopub.execute_input":"2023-06-05T11:04:28.101330Z","iopub.status.idle":"2023-06-05T11:04:28.123990Z","shell.execute_reply.started":"2023-06-05T11:04:28.101306Z","shell.execute_reply":"2023-06-05T11:04:28.122919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print_xy_cluster(pvt_tasks[pvt_tasks.columns[1:]],km_preds)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:28.125469Z","iopub.execute_input":"2023-06-05T11:04:28.125789Z","iopub.status.idle":"2023-06-05T11:04:28.764025Z","shell.execute_reply.started":"2023-06-05T11:04:28.125762Z","shell.execute_reply":"2023-06-05T11:04:28.762997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pvt_tasks['t_group'] = km_preds","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:28.765502Z","iopub.execute_input":"2023-06-05T11:04:28.765798Z","iopub.status.idle":"2023-06-05T11:04:28.770069Z","shell.execute_reply.started":"2023-06-05T11:04:28.765774Z","shell.execute_reply":"2023-06-05T11:04:28.769362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tasks = pvt_tasks","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:28.771205Z","iopub.execute_input":"2023-06-05T11:04:28.771681Z","iopub.status.idle":"2023-06-05T11:04:28.783028Z","shell.execute_reply.started":"2023-06-05T11:04:28.771653Z","shell.execute_reply":"2023-06-05T11:04:28.782237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tasks.head().transpose()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:28.784153Z","iopub.execute_input":"2023-06-05T11:04:28.784630Z","iopub.status.idle":"2023-06-05T11:04:28.803815Z","shell.execute_reply.started":"2023-06-05T11:04:28.784602Z","shell.execute_reply":"2023-06-05T11:04:28.803100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tasks.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:28.804973Z","iopub.execute_input":"2023-06-05T11:04:28.805441Z","iopub.status.idle":"2023-06-05T11:04:28.827352Z","shell.execute_reply.started":"2023-06-05T11:04:28.805413Z","shell.execute_reply":"2023-06-05T11:04:28.826335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Events","metadata":{}},{"cell_type":"code","source":"events['Duration'] = events['Completion'] - events['Init']\nevents['One'] = 1\nevents.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:28.829014Z","iopub.execute_input":"2023-06-05T11:04:28.829296Z","iopub.status.idle":"2023-06-05T11:04:28.848649Z","shell.execute_reply.started":"2023-06-05T11:04:28.829270Z","shell.execute_reply":"2023-06-05T11:04:28.847729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x=events['Type'])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:28.850242Z","iopub.execute_input":"2023-06-05T11:04:28.850548Z","iopub.status.idle":"2023-06-05T11:04:29.132936Z","shell.execute_reply.started":"2023-06-05T11:04:28.850523Z","shell.execute_reply":"2023-06-05T11:04:29.131859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:29.134368Z","iopub.execute_input":"2023-06-05T11:04:29.134696Z","iopub.status.idle":"2023-06-05T11:04:29.147492Z","shell.execute_reply.started":"2023-06-05T11:04:29.134668Z","shell.execute_reply":"2023-06-05T11:04:29.146216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type_dict = {'StartHesitation':0,'Turn':1,'Walking':2}\nevents['Type'] = events['Type'].map(type_dict)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:29.148838Z","iopub.execute_input":"2023-06-05T11:04:29.149253Z","iopub.status.idle":"2023-06-05T11:04:29.158803Z","shell.execute_reply.started":"2023-06-05T11:04:29.149218Z","shell.execute_reply":"2023-06-05T11:04:29.157922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pvt_events = pd.pivot_table(events, values=['Duration','Kinetic','One','Type'], index=['Id'], aggfunc=[np.sum, np.mean, np.std], fill_value=0)\npvt_events = pvt_events.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:29.160487Z","iopub.execute_input":"2023-06-05T11:04:29.161216Z","iopub.status.idle":"2023-06-05T11:04:29.192190Z","shell.execute_reply.started":"2023-06-05T11:04:29.161176Z","shell.execute_reply":"2023-06-05T11:04:29.191192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pvt_events.fillna(0, inplace=True)\npvt_events.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:29.193899Z","iopub.execute_input":"2023-06-05T11:04:29.194327Z","iopub.status.idle":"2023-06-05T11:04:29.213907Z","shell.execute_reply.started":"2023-06-05T11:04:29.194283Z","shell.execute_reply":"2023-06-05T11:04:29.212825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pvt_events.columns = pvt_events.columns.droplevel(0)\npvt_events.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:29.215805Z","iopub.execute_input":"2023-06-05T11:04:29.216249Z","iopub.status.idle":"2023-06-05T11:04:29.234288Z","shell.execute_reply.started":"2023-06-05T11:04:29.216200Z","shell.execute_reply":"2023-06-05T11:04:29.233261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epvt_cols = ['Id','e_duration_sum','e_kinetic_sum','e_count','e_type_sum',\n            'e_duration_avg','e_kinetic_avg','e_one_avg','e_type_avg',\n            'e_duration_std','e_kinetic_std', 'e_one_std','e_type_std']\n\npvt_events.columns = epvt_cols","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:29.235944Z","iopub.execute_input":"2023-06-05T11:04:29.236252Z","iopub.status.idle":"2023-06-05T11:04:29.243878Z","shell.execute_reply.started":"2023-06-05T11:04:29.236225Z","shell.execute_reply":"2023-06-05T11:04:29.242982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evt_cols = ['Id','e_type_avg','e_count','e_duration_avg','e_kinetic_avg','e_duration_std','e_kinetic_std']\npvt_events = pvt_events[evt_cols]\npvt_events.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:29.245496Z","iopub.execute_input":"2023-06-05T11:04:29.245922Z","iopub.status.idle":"2023-06-05T11:04:29.263681Z","shell.execute_reply.started":"2023-06-05T11:04:29.245884Z","shell.execute_reply":"2023-06-05T11:04:29.262556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pvt_events['Id'].value_counts(ascending=False).head()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:29.265218Z","iopub.execute_input":"2023-06-05T11:04:29.265527Z","iopub.status.idle":"2023-06-05T11:04:29.283496Z","shell.execute_reply.started":"2023-06-05T11:04:29.265501Z","shell.execute_reply":"2023-06-05T11:04:29.282203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pvt_events[pvt_events.columns[1:]].info()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:29.284988Z","iopub.execute_input":"2023-06-05T11:04:29.285312Z","iopub.status.idle":"2023-06-05T11:04:29.302104Z","shell.execute_reply.started":"2023-06-05T11:04:29.285282Z","shell.execute_reply":"2023-06-05T11:04:29.300916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"km_preds = cluster.KMeans(n_clusters = EVENTS_NCLUSTER, n_init=CLUSTER_NINIT,\n                            random_state = RANDOM_STATE).fit_predict(pvt_events[pvt_events.columns[1:]])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:29.303715Z","iopub.execute_input":"2023-06-05T11:04:29.304067Z","iopub.status.idle":"2023-06-05T11:04:29.340281Z","shell.execute_reply.started":"2023-06-05T11:04:29.303981Z","shell.execute_reply":"2023-06-05T11:04:29.339408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print_xy_cluster(pvt_events[pvt_events.columns[1:]],km_preds)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:29.341667Z","iopub.execute_input":"2023-06-05T11:04:29.341973Z","iopub.status.idle":"2023-06-05T11:04:30.101699Z","shell.execute_reply.started":"2023-06-05T11:04:29.341948Z","shell.execute_reply":"2023-06-05T11:04:30.100660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pvt_events['e_group'] = km_preds","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:30.103383Z","iopub.execute_input":"2023-06-05T11:04:30.103833Z","iopub.status.idle":"2023-06-05T11:04:30.110686Z","shell.execute_reply.started":"2023-06-05T11:04:30.103793Z","shell.execute_reply":"2023-06-05T11:04:30.109411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events = pvt_events\nlen(events)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:30.112340Z","iopub.execute_input":"2023-06-05T11:04:30.112781Z","iopub.status.idle":"2023-06-05T11:04:30.124946Z","shell.execute_reply.started":"2023-06-05T11:04:30.112743Z","shell.execute_reply":"2023-06-05T11:04:30.123728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:30.126235Z","iopub.execute_input":"2023-06-05T11:04:30.126678Z","iopub.status.idle":"2023-06-05T11:04:30.143745Z","shell.execute_reply.started":"2023-06-05T11:04:30.126640Z","shell.execute_reply":"2023-06-05T11:04:30.142616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:30.145232Z","iopub.execute_input":"2023-06-05T11:04:30.146300Z","iopub.status.idle":"2023-06-05T11:04:30.637458Z","shell.execute_reply.started":"2023-06-05T11:04:30.146256Z","shell.execute_reply":"2023-06-05T11:04:30.636362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feats Collection & Generation + Joins","metadata":{}},{"cell_type":"code","source":"# merge the subjects with the metadata\nmetadata_w_subjects = full_metadata.merge(subjects, how='left', on='Subject').copy()\nfeatures = metadata_w_subjects.columns","metadata":{"papermill":{"duration":0.060364,"end_time":"2023-04-16T22:41:26.141472","exception":false,"start_time":"2023-04-16T22:41:26.081108","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T11:04:30.638815Z","iopub.execute_input":"2023-06-05T11:04:30.639160Z","iopub.status.idle":"2023-06-05T11:04:30.649660Z","shell.execute_reply.started":"2023-06-05T11:04:30.639130Z","shell.execute_reply":"2023-06-05T11:04:30.648622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_w_subjects['Medication'] = metadata_w_subjects['Medication'].factorize()[0]\nmetadata_w_subjects.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:30.650922Z","iopub.execute_input":"2023-06-05T11:04:30.651290Z","iopub.status.idle":"2023-06-05T11:04:30.669755Z","shell.execute_reply.started":"2023-06-05T11:04:30.651261Z","shell.execute_reply":"2023-06-05T11:04:30.668845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"basic_feats = MultipleFeatureDescriptors(\n    functions=seglearn_feature_dict_wrapper(base_features()),\n    series_names=['AccV', 'AccML', 'AccAP'],\n    windows=[WINDOW_SIZE],\n    strides=[WINDOW_SIZE],\n)\n\nemg_feats = emg_features()\ndel emg_feats['simple square integral'] # is same as abs_energy (which is in base_features)\n\nemg_feats = MultipleFeatureDescriptors(\n    functions=seglearn_feature_dict_wrapper(emg_feats),\n    series_names=['AccV', 'AccML', 'AccAP'],\n    windows=[WINDOW_SIZE],\n    strides=[WINDOW_SIZE],\n)\n\nfc = FeatureCollection([basic_feats, emg_feats])","metadata":{"papermill":{"duration":0.032203,"end_time":"2023-04-16T22:41:26.517639","exception":false,"start_time":"2023-04-16T22:41:26.485436","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T11:04:30.671037Z","iopub.execute_input":"2023-06-05T11:04:30.671317Z","iopub.status.idle":"2023-06-05T11:04:30.679712Z","shell.execute_reply.started":"2023-06-05T11:04:30.671293Z","shell.execute_reply":"2023-06-05T11:04:30.678474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#\n# Not considered columns:\n# CSV: -\n# SUBJECTS: s_visit\n# EVENTS: 'e_duration_std','e_kinetic_std'\n# \nCSV_COLS = ['Time', 'AccV', 'AccML', 'AccAP', 'StartHesitation', 'Turn' , 'Walking']\nMETADATA_W_SUBJECTS_COLS = ['Id','Subject','Test', 'Visit','Medication','s_age','s_NFOGQ','s_years','s_on','s_off','s_group']\nTASK_COLS = ['Time_frac','t_group']\nEVENT_COLS = ['e_count','e_type_avg','e_duration_avg','e_kinetic_avg','e_group']","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:30.681262Z","iopub.execute_input":"2023-06-05T11:04:30.681608Z","iopub.status.idle":"2023-06-05T11:04:30.691756Z","shell.execute_reply.started":"2023-06-05T11:04:30.681572Z","shell.execute_reply":"2023-06-05T11:04:30.690580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# used for join\ntcols = ['Id','t_group']\nevcols = ['Id','e_count','e_type_avg','e_duration_avg','e_kinetic_avg','e_group']","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:30.692915Z","iopub.execute_input":"2023-06-05T11:04:30.693474Z","iopub.status.idle":"2023-06-05T11:04:30.704242Z","shell.execute_reply.started":"2023-06-05T11:04:30.693446Z","shell.execute_reply":"2023-06-05T11:04:30.703233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# this is done because the speeds are at different rates for the datasets\ndef norm_acc(df,dataset):\n    NORM_FACTOR = 9.80665\n    if dataset == 'tdcsfog':\n        df.AccV = df.AccV / NORM_FACTOR\n        df.AccML = df.AccML / NORM_FACTOR\n        df.AccAP = df.AccAP / NORM_FACTOR\n    return df    ","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:30.708063Z","iopub.execute_input":"2023-06-05T11:04:30.708386Z","iopub.status.idle":"2023-06-05T11:04:30.718266Z","shell.execute_reply.started":"2023-06-05T11:04:30.708359Z","shell.execute_reply":"2023-06-05T11:04:30.717271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rolling_average(df,window):\n    df.sort_values(by=['Time_frac'], inplace=True)\n    df['AccV_r'] = df['AccV'].rolling(window).mean()\n    df['AccML_r'] = df['AccML'].rolling(window).mean()\n    df['AccAP_r'] = df['AccAP'].rolling(window).mean()\n    df['AccV_r'].fillna(method=\"bfill\",inplace=True)\n    df['AccML_r'].fillna(method=\"bfill\",inplace=True)\n    df['AccAP_r'].fillna(method=\"bfill\",inplace=True)\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:30.720325Z","iopub.execute_input":"2023-06-05T11:04:30.720645Z","iopub.status.idle":"2023-06-05T11:04:30.729632Z","shell.execute_reply.started":"2023-06-05T11:04:30.720619Z","shell.execute_reply":"2023-06-05T11:04:30.728791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reader(file):\n    \n    fid = ''\n    \n    try:\n        path_split = file.split('/')\n        fid = path_split[-1].split('.')[0]\n       \n        df = pd.read_csv(file, index_col='Time', usecols=CSV_COLS)\n        df['Id'] = fid\n        dataset = Path(file).parts[-2]\n        df['Module'] = dataset\n        \n        if (RESCALE_ACC==True):\n            df = norm_acc(df,dataset)\n                \n        df['Time_frac']=np.round((df.index/df.index.max()).values,TIME_FRAC_DECIMALS)\n        \n        df = pd.merge(df, tasks[tcols], how='left', on='Id').fillna(-1)     \n        df = pd.merge(df, metadata_w_subjects[METADATA_W_SUBJECTS_COLS], how='left', on='Id').fillna(-1)\n        if (ADD_EVENTS == True):\n            df = pd.merge(df, events[evcols], how='left', on='Id').fillna(-1)\n            \n        df_feats = fc.calculate(df, return_df=True, include_final_window=True, approve_sparsity=True, window_idx=\"begin\").astype(np.float32)\n        df = df.merge(df_feats, how=\"left\", left_index=True, right_index=True) \n        df.fillna(method=\"ffill\", inplace=True)\n        \n        if (ROLLING_AVG==True):\n            df = rolling_average(df,ROLLING_WINDOW)\n        \n        return df\n    \n    except Exception as e: \n        print('Error in reading ', fid, \" : \", str(e))\n        pass\n    \n    finally:\n        gc.collect()","metadata":{"papermill":{"duration":533.977331,"end_time":"2023-04-16T22:50:20.513889","exception":false,"start_time":"2023-04-16T22:41:26.536558","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T11:04:30.730721Z","iopub.execute_input":"2023-06-05T11:04:30.731009Z","iopub.status.idle":"2023-06-05T11:04:30.742430Z","shell.execute_reply.started":"2023-06-05T11:04:30.730984Z","shell.execute_reply":"2023-06-05T11:04:30.741633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# often tqdm gives me a widget error: I removed it\ntrain = pd.concat([reader(f) for f in train]).fillna(0)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:04:30.743616Z","iopub.execute_input":"2023-06-05T11:04:30.743899Z","iopub.status.idle":"2023-06-05T11:27:10.018893Z","shell.execute_reply.started":"2023-06-05T11:04:30.743875Z","shell.execute_reply":"2023-06-05T11:27:10.017536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# What happens when it finds an error? The Id is not put into the training set\n# train[['Id','Subject','StartHesitation','Turn','Walking']][train['Id']=='816bd20e5d']","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:10.020757Z","iopub.execute_input":"2023-06-05T11:27:10.021093Z","iopub.status.idle":"2023-06-05T11:27:10.026509Z","shell.execute_reply.started":"2023-06-05T11:27:10.021061Z","shell.execute_reply":"2023-06-05T11:27:10.025039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Original train columns (base + added using feat. engineering and time series tools)\nocols = train.columns\n# Prediction columns\npcols = ['StartHesitation', 'Turn' , 'Walking']\n# Submission columns\nscols = ['Id', 'StartHesitation', 'Turn' , 'Walking']","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:10.027641Z","iopub.execute_input":"2023-06-05T11:27:10.027933Z","iopub.status.idle":"2023-06-05T11:27:10.038225Z","shell.execute_reply.started":"2023-06-05T11:27:10.027907Z","shell.execute_reply":"2023-06-05T11:27:10.037314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def paquet_reader(file):\n    \n    fid = ''\n    \n    try:\n        path_split = file.split('/')\n        fid = path_split[-1].split('.')[0]\n       \n        df = pd.read_parquet(file)\n        df['Id'] = fid\n        dataset = Path(file).parts[-2]\n        df['Module'] = dataset \n        df.fillna(method=\"ffill\", inplace=True)\n        \n        return df\n    \n    except Exception as e: \n        print('Error in reading parquet ', fid, \" : \", str(e))\n        pass\n    \n    finally:\n        gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:10.039665Z","iopub.execute_input":"2023-06-05T11:27:10.039962Z","iopub.status.idle":"2023-06-05T11:27:10.052542Z","shell.execute_reply.started":"2023-06-05T11:27:10.039936Z","shell.execute_reply":"2023-06-05T11:27:10.051179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unlabeled[0:5]","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:10.054045Z","iopub.execute_input":"2023-06-05T11:27:10.054337Z","iopub.status.idle":"2023-06-05T11:27:10.069492Z","shell.execute_reply.started":"2023-06-05T11:27:10.054313Z","shell.execute_reply":"2023-06-05T11:27:10.068535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#\n# what can I do with this?\n#\n#dfpq = paquet_reader(unlabeled[0])\n#dfpq.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:10.070913Z","iopub.execute_input":"2023-06-05T11:27:10.071210Z","iopub.status.idle":"2023-06-05T11:27:10.080694Z","shell.execute_reply.started":"2023-06-05T11:27:10.071186Z","shell.execute_reply":"2023-06-05T11:27:10.079708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfpq = None\nfull_metadata = None\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:10.082458Z","iopub.execute_input":"2023-06-05T11:27:10.082754Z","iopub.status.idle":"2023-06-05T11:27:10.303304Z","shell.execute_reply.started":"2023-06-05T11:27:10.082730Z","shell.execute_reply":"2023-06-05T11:27:10.302065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_rows',None)\ntrain.head().transpose()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:10.304985Z","iopub.execute_input":"2023-06-05T11:27:10.305272Z","iopub.status.idle":"2023-06-05T11:27:10.328333Z","shell.execute_reply.started":"2023-06-05T11:27:10.305246Z","shell.execute_reply":"2023-06-05T11:27:10.327433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_rows',PD_MAX_ROWS)\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:10.331680Z","iopub.execute_input":"2023-06-05T11:27:10.331993Z","iopub.status.idle":"2023-06-05T11:27:10.338915Z","shell.execute_reply.started":"2023-06-05T11:27:10.331966Z","shell.execute_reply":"2023-06-05T11:27:10.337415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:10.357252Z","iopub.execute_input":"2023-06-05T11:27:10.357631Z","iopub.status.idle":"2023-06-05T11:27:10.372633Z","shell.execute_reply.started":"2023-06-05T11:27:10.357601Z","shell.execute_reply":"2023-06-05T11:27:10.371661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### EDA & Feature Selection","metadata":{}},{"cell_type":"code","source":"# are some subjects strange?\nsubject_target_desc = train[['StartHesitation', 'Turn', \n                             'Walking','Subject']].groupby(by='Subject').agg([np.mean,np.std,np.max,np.min])\nsubject_target_desc","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:10.374372Z","iopub.execute_input":"2023-06-05T11:27:10.374693Z","iopub.status.idle":"2023-06-05T11:27:16.544998Z","shell.execute_reply.started":"2023-06-05T11:27:10.374666Z","shell.execute_reply":"2023-06-05T11:27:16.544125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"km_preds = cluster.KMeans(n_clusters = TARGETS_NCLUSTER, random_state = RANDOM_STATE,\n                          n_init= CLUSTER_NINIT).fit_predict(subject_target_desc.values)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:16.546444Z","iopub.execute_input":"2023-06-05T11:27:16.547035Z","iopub.status.idle":"2023-06-05T11:27:16.595066Z","shell.execute_reply.started":"2023-06-05T11:27:16.547002Z","shell.execute_reply":"2023-06-05T11:27:16.594195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print_xy_cluster(subject_target_desc.values,km_preds)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:16.596804Z","iopub.execute_input":"2023-06-05T11:27:16.597595Z","iopub.status.idle":"2023-06-05T11:27:17.170240Z","shell.execute_reply.started":"2023-06-05T11:27:16.597564Z","shell.execute_reply":"2023-06-05T11:27:17.169460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xg = do_pca(2,subject_target_desc.values)\ndg = pd.DataFrame(Xg)\ndg['Subject'] = subject_target_desc.index.values","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:17.171676Z","iopub.execute_input":"2023-06-05T11:27:17.172265Z","iopub.status.idle":"2023-06-05T11:27:17.181390Z","shell.execute_reply.started":"2023-06-05T11:27:17.172235Z","shell.execute_reply":"2023-06-05T11:27:17.180187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subject_target_desc.reset_index(inplace=True)\nsubject_target_desc.columns = ['Subject','Hes_mean','Hes_std','Hes_max','Hes_min',\n                               'Turn_mean','Turn_std','Turn_max','Turn_min','Walk_mean','Walk_std','Walk_max','Walk_min']","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:17.182861Z","iopub.execute_input":"2023-06-05T11:27:17.183291Z","iopub.status.idle":"2023-06-05T11:27:17.191116Z","shell.execute_reply.started":"2023-06-05T11:27:17.183253Z","shell.execute_reply":"2023-06-05T11:27:17.190111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subject_target_desc = pd.merge(subject_target_desc,dg,on='Subject')\nsubject_target_desc[subject_target_desc[0]<-1]","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:17.192372Z","iopub.execute_input":"2023-06-05T11:27:17.192719Z","iopub.status.idle":"2023-06-05T11:27:17.224145Z","shell.execute_reply.started":"2023-06-05T11:27:17.192690Z","shell.execute_reply":"2023-06-05T11:27:17.222872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"without_sympt_list = list(subject_target_desc['Subject'][subject_target_desc[0]<-1])\nwithout_sympt_list","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:17.225519Z","iopub.execute_input":"2023-06-05T11:27:17.225913Z","iopub.status.idle":"2023-06-05T11:27:17.236564Z","shell.execute_reply.started":"2023-06-05T11:27:17.225882Z","shell.execute_reply":"2023-06-05T11:27:17.235264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (IGNORE_WITHOUT_SYMPT==True):\n    train = train[~train['Subject'].isin(without_sympt_list)]","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:17.238150Z","iopub.execute_input":"2023-06-05T11:27:17.238603Z","iopub.status.idle":"2023-06-05T11:27:23.222152Z","shell.execute_reply.started":"2023-06-05T11:27:17.238562Z","shell.execute_reply":"2023-06-05T11:27:23.221103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data=train,x=pcols[0])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:23.223599Z","iopub.execute_input":"2023-06-05T11:27:23.224136Z","iopub.status.idle":"2023-06-05T11:27:24.804319Z","shell.execute_reply.started":"2023-06-05T11:27:23.224103Z","shell.execute_reply":"2023-06-05T11:27:24.803138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_pct_high(df,colname):\n    return sum(df[colname]==1)/len(df)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:24.805707Z","iopub.execute_input":"2023-06-05T11:27:24.806098Z","iopub.status.idle":"2023-06-05T11:27:24.811644Z","shell.execute_reply.started":"2023-06-05T11:27:24.806071Z","shell.execute_reply":"2023-06-05T11:27:24.810618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pct0 = calc_pct_high(train,pcols[0])\npct0","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:24.812949Z","iopub.execute_input":"2023-06-05T11:27:24.813221Z","iopub.status.idle":"2023-06-05T11:27:26.748855Z","shell.execute_reply.started":"2023-06-05T11:27:24.813198Z","shell.execute_reply":"2023-06-05T11:27:26.747774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data=train,x=pcols[1])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:26.750271Z","iopub.execute_input":"2023-06-05T11:27:26.750586Z","iopub.status.idle":"2023-06-05T11:27:28.454573Z","shell.execute_reply.started":"2023-06-05T11:27:26.750559Z","shell.execute_reply":"2023-06-05T11:27:28.453450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pct1 = calc_pct_high(train,pcols[1])\npct1","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:28.456445Z","iopub.execute_input":"2023-06-05T11:27:28.456883Z","iopub.status.idle":"2023-06-05T11:27:30.120781Z","shell.execute_reply.started":"2023-06-05T11:27:28.456833Z","shell.execute_reply":"2023-06-05T11:27:30.119733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data=train,x=pcols[2])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:30.122176Z","iopub.execute_input":"2023-06-05T11:27:30.122617Z","iopub.status.idle":"2023-06-05T11:27:31.688495Z","shell.execute_reply.started":"2023-06-05T11:27:30.122579Z","shell.execute_reply":"2023-06-05T11:27:31.687440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pct2 = calc_pct_high(train,pcols[2])\npct2","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:31.690584Z","iopub.execute_input":"2023-06-05T11:27:31.691018Z","iopub.status.idle":"2023-06-05T11:27:33.480486Z","shell.execute_reply.started":"2023-06-05T11:27:31.690980Z","shell.execute_reply":"2023-06-05T11:27:33.479520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subjects_list = train['Subject'].unique()\nsubjects_list","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:33.481739Z","iopub.execute_input":"2023-06-05T11:27:33.482141Z","iopub.status.idle":"2023-06-05T11:27:34.494382Z","shell.execute_reply.started":"2023-06-05T11:27:33.482113Z","shell.execute_reply":"2023-06-05T11:27:34.493453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_example = np.random.randint(0,len(subjects_list)-1,1)[0]\nexample_subject_data = train[train['Subject']==subjects_list[id_example]].sample(frac=0.1)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:34.495829Z","iopub.execute_input":"2023-06-05T11:27:34.496154Z","iopub.status.idle":"2023-06-05T11:27:35.931415Z","shell.execute_reply.started":"2023-06-05T11:27:34.496126Z","shell.execute_reply":"2023-06-05T11:27:35.930194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_rows',None)\nexample_subject_data.head().transpose()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:35.933183Z","iopub.execute_input":"2023-06-05T11:27:35.933556Z","iopub.status.idle":"2023-06-05T11:27:35.953470Z","shell.execute_reply.started":"2023-06-05T11:27:35.933526Z","shell.execute_reply":"2023-06-05T11:27:35.952428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_rows',PD_MAX_ROWS)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:35.954878Z","iopub.execute_input":"2023-06-05T11:27:35.955235Z","iopub.status.idle":"2023-06-05T11:27:35.961723Z","shell.execute_reply.started":"2023-06-05T11:27:35.955205Z","shell.execute_reply":"2023-06-05T11:27:35.960805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(data=example_subject_data,x='Time_frac',y=pcols[0])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:27:35.962888Z","iopub.execute_input":"2023-06-05T11:27:35.963574Z","iopub.status.idle":"2023-06-05T11:29:26.605784Z","shell.execute_reply.started":"2023-06-05T11:27:35.963545Z","shell.execute_reply":"2023-06-05T11:29:26.604567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(data=example_subject_data,x='Time_frac',y=pcols[1])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:29:26.607750Z","iopub.execute_input":"2023-06-05T11:29:26.608217Z","iopub.status.idle":"2023-06-05T11:31:18.922544Z","shell.execute_reply.started":"2023-06-05T11:29:26.608178Z","shell.execute_reply":"2023-06-05T11:31:18.921228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(data=example_subject_data,x='Time_frac',y=pcols[2])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:31:18.924550Z","iopub.execute_input":"2023-06-05T11:31:18.924983Z","iopub.status.idle":"2023-06-05T11:33:08.931288Z","shell.execute_reply.started":"2023-06-05T11:31:18.924944Z","shell.execute_reply":"2023-06-05T11:33:08.930240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_subject_data = example_subject_data.sort_values(by=['Time_frac'])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:33:08.932772Z","iopub.execute_input":"2023-06-05T11:33:08.933188Z","iopub.status.idle":"2023-06-05T11:33:08.955247Z","shell.execute_reply.started":"2023-06-05T11:33:08.933150Z","shell.execute_reply":"2023-06-05T11:33:08.954154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16,6))\nsns.lineplot(data=example_subject_data,x='Time_frac',y='AccV')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:33:08.956704Z","iopub.execute_input":"2023-06-05T11:33:08.957552Z","iopub.status.idle":"2023-06-05T11:34:59.549836Z","shell.execute_reply.started":"2023-06-05T11:33:08.957520Z","shell.execute_reply":"2023-06-05T11:34:59.548988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16,6))\nsns.lineplot(data=example_subject_data,x='Time_frac',y='AccML')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:34:59.551192Z","iopub.execute_input":"2023-06-05T11:34:59.551730Z","iopub.status.idle":"2023-06-05T11:36:46.634540Z","shell.execute_reply.started":"2023-06-05T11:34:59.551699Z","shell.execute_reply":"2023-06-05T11:36:46.633126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16,6))\nsns.lineplot(data=example_subject_data,x='Time_frac',y='AccAP')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:36:46.636177Z","iopub.execute_input":"2023-06-05T11:36:46.636518Z","iopub.status.idle":"2023-06-05T11:38:40.418236Z","shell.execute_reply.started":"2023-06-05T11:36:46.636487Z","shell.execute_reply":"2023-06-05T11:38:40.417267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (ROLLING_AVG==True):\n    plt.figure(figsize=(16,6))\n    sns.lineplot(data=example_subject_data,x='Time_frac',y='AccV_r')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:38:40.419732Z","iopub.execute_input":"2023-06-05T11:38:40.420052Z","iopub.status.idle":"2023-06-05T11:38:40.425109Z","shell.execute_reply.started":"2023-06-05T11:38:40.420024Z","shell.execute_reply":"2023-06-05T11:38:40.424035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (ROLLING_AVG==True):\n    plt.figure(figsize=(16,6))\n    sns.lineplot(data=example_subject_data,x='Time_frac',y='AccML_r')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:38:40.426434Z","iopub.execute_input":"2023-06-05T11:38:40.426768Z","iopub.status.idle":"2023-06-05T11:38:40.436668Z","shell.execute_reply.started":"2023-06-05T11:38:40.426731Z","shell.execute_reply":"2023-06-05T11:38:40.435588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (ROLLING_AVG==True):\n    plt.figure(figsize=(16,6))\n    sns.lineplot(data=example_subject_data,x='Time_frac',y='AccAP_r')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:38:40.438034Z","iopub.execute_input":"2023-06-05T11:38:40.438312Z","iopub.status.idle":"2023-06-05T11:38:40.448458Z","shell.execute_reply.started":"2023-06-05T11:38:40.438288Z","shell.execute_reply":"2023-06-05T11:38:40.447727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# base columns of train set excluded time-series-tools-added columns\nbcols = CSV_COLS + METADATA_W_SUBJECTS_COLS + TASK_COLS\nif (ADD_EVENTS==True):\n    bcols = bcols + EVENT_COLS\nif (ROLLING_AVG==True):\n    bcols = bcols + ['AccV_r','AccML_r','AccAP_r']\n    bcols = [b for b in bcols if b not in ['AccV','AccML','AccAP']]\nprint(bcols)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:38:40.449597Z","iopub.execute_input":"2023-06-05T11:38:40.450067Z","iopub.status.idle":"2023-06-05T11:38:40.465318Z","shell.execute_reply.started":"2023-06-05T11:38:40.450039Z","shell.execute_reply":"2023-06-05T11:38:40.464237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# parameters only for corr evaluation\nPCT_SAMPLE = 0.2\nCORR_THRES_METADATA = 0.4","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:38:40.466849Z","iopub.execute_input":"2023-06-05T11:38:40.467147Z","iopub.status.idle":"2023-06-05T11:38:40.475803Z","shell.execute_reply.started":"2023-06-05T11:38:40.467122Z","shell.execute_reply":"2023-06-05T11:38:40.475066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr = metadata_w_subjects.corr(numeric_only=True)[abs(metadata_w_subjects.corr(numeric_only=True))>CORR_THRES_METADATA]\ncorr_cols = corr.columns","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:38:40.476968Z","iopub.execute_input":"2023-06-05T11:38:40.477443Z","iopub.status.idle":"2023-06-05T11:38:40.494735Z","shell.execute_reply.started":"2023-06-05T11:38:40.477389Z","shell.execute_reply":"2023-06-05T11:38:40.493937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,8))\nsns.heatmap(corr, xticklabels=corr_cols, yticklabels=corr_cols,cmap='coolwarm', annot=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:38:40.495801Z","iopub.execute_input":"2023-06-05T11:38:40.496743Z","iopub.status.idle":"2023-06-05T11:38:41.023322Z","shell.execute_reply.started":"2023-06-05T11:38:40.496713Z","shell.execute_reply":"2023-06-05T11:38:41.022569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# additional features, float colums added using time series tools\nfadd_cols = [c for c in ocols if c not in bcols]\nfadd_cols = [c for c in fadd_cols if c not in ['Module']]\nfadd_cols","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:38:41.024352Z","iopub.execute_input":"2023-06-05T11:38:41.025272Z","iopub.status.idle":"2023-06-05T11:38:41.033991Z","shell.execute_reply.started":"2023-06-05T11:38:41.025239Z","shell.execute_reply":"2023-06-05T11:38:41.032716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sampled = train.sample(frac=PCT_SAMPLE).copy()\ntrain_for_corr = train_sampled[fadd_cols]\ncorr = train_for_corr.corr(numeric_only=True)[abs(train_for_corr.corr(numeric_only=True))>CORR_THRES]\ncorr_cols = corr.columns","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:38:41.035626Z","iopub.execute_input":"2023-06-05T11:38:41.035940Z","iopub.status.idle":"2023-06-05T11:40:01.704480Z","shell.execute_reply.started":"2023-06-05T11:38:41.035914Z","shell.execute_reply":"2023-06-05T11:40:01.703140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:40:01.706138Z","iopub.execute_input":"2023-06-05T11:40:01.706505Z","iopub.status.idle":"2023-06-05T11:40:01.713173Z","shell.execute_reply.started":"2023-06-05T11:40:01.706474Z","shell.execute_reply":"2023-06-05T11:40:01.712117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_sampled)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:40:01.714604Z","iopub.execute_input":"2023-06-05T11:40:01.715002Z","iopub.status.idle":"2023-06-05T11:40:01.728113Z","shell.execute_reply.started":"2023-06-05T11:40:01.714969Z","shell.execute_reply":"2023-06-05T11:40:01.726830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(18,18))\nsns.heatmap(corr, xticklabels=corr_cols, yticklabels=corr_cols,cmap='coolwarm')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:40:01.729842Z","iopub.execute_input":"2023-06-05T11:40:01.730169Z","iopub.status.idle":"2023-06-05T11:40:03.480690Z","shell.execute_reply.started":"2023-06-05T11:40:01.730142Z","shell.execute_reply":"2023-06-05T11:40:03.479547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr.reset_index(inplace=True)\ncorr_unpivot = pd.melt(corr, id_vars='index', value_vars=corr_cols)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:40:03.482348Z","iopub.execute_input":"2023-06-05T11:40:03.482712Z","iopub.status.idle":"2023-06-05T11:40:03.493569Z","shell.execute_reply.started":"2023-06-05T11:40:03.482684Z","shell.execute_reply":"2023-06-05T11:40:03.492255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_rows',None)\ncond = (corr_unpivot['index']!=corr_unpivot['variable'])&(abs(corr_unpivot['value'])>CORR_THRES)\nhigh_corr = corr_unpivot[cond]\nhigh_corr","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:40:03.495092Z","iopub.execute_input":"2023-06-05T11:40:03.495456Z","iopub.status.idle":"2023-06-05T11:40:03.518897Z","shell.execute_reply.started":"2023-06-05T11:40:03.495420Z","shell.execute_reply":"2023-06-05T11:40:03.517962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#\n# added columns have longer names than originals, so they are excluded as first\n#\ndef FindLongest(a,b):\n    if (len(a)>len(b)):\n        return a\n    else:\n        return b","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:40:03.520564Z","iopub.execute_input":"2023-06-05T11:40:03.520995Z","iopub.status.idle":"2023-06-05T11:40:03.526518Z","shell.execute_reply.started":"2023-06-05T11:40:03.520955Z","shell.execute_reply":"2023-06-05T11:40:03.525344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_rows',PD_MAX_ROWS)\ndel_cols = FindLongest(high_corr['index'],high_corr['variable'])\ndel_cols = del_cols.unique()\ndel_cols","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:40:03.528430Z","iopub.execute_input":"2023-06-05T11:40:03.528844Z","iopub.status.idle":"2023-06-05T11:40:03.540542Z","shell.execute_reply.started":"2023-06-05T11:40:03.528806Z","shell.execute_reply":"2023-06-05T11:40:03.539394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keep_cols = list(set(fadd_cols) - set(del_cols))\nlk = len(keep_cols)\nprint('Kept high corr columns nr = ', lk, ' : pct of added features = ',100*lk/len(fadd_cols))","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:40:03.542125Z","iopub.execute_input":"2023-06-05T11:40:03.542572Z","iopub.status.idle":"2023-06-05T11:40:03.551046Z","shell.execute_reply.started":"2023-06-05T11:40:03.542536Z","shell.execute_reply":"2023-06-05T11:40:03.550091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#\n# keep cols are free from outliers?\n#\nplt.rcParams.update({'figure.max_open_warning': 0})\nfor i,c in enumerate(keep_cols):\n    plt.figure(figsize=(10,1))\n    sns.boxplot(x=train_sampled[c])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:40:03.552424Z","iopub.execute_input":"2023-06-05T11:40:03.552734Z","iopub.status.idle":"2023-06-05T11:40:28.677934Z","shell.execute_reply.started":"2023-06-05T11:40:03.552708Z","shell.execute_reply":"2023-06-05T11:40:28.676720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sampled.columns","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:40:28.679790Z","iopub.execute_input":"2023-06-05T11:40:28.680234Z","iopub.status.idle":"2023-06-05T11:40:28.690758Z","shell.execute_reply.started":"2023-06-05T11:40:28.680195Z","shell.execute_reply":"2023-06-05T11:40:28.689445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_sampled[keep_cols].copy()\ndiscrete_features = X.dtypes == int\nX.columns","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:40:28.692366Z","iopub.execute_input":"2023-06-05T11:40:28.692925Z","iopub.status.idle":"2023-06-05T11:40:29.002363Z","shell.execute_reply.started":"2023-06-05T11:40:28.692896Z","shell.execute_reply":"2023-06-05T11:40:29.001275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_mi_scores(X, y, discrete_features):\n    mi_scores = mutual_info_classif(X, y, discrete_features=discrete_features)\n    mi_scores = pd.Series(mi_scores, name=\"MI Scores\", index=X.columns)\n    mi_scores = mi_scores.sort_values(ascending=False)\n    return mi_scores","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:40:29.003997Z","iopub.execute_input":"2023-06-05T11:40:29.004674Z","iopub.status.idle":"2023-06-05T11:40:29.010386Z","shell.execute_reply.started":"2023-06-05T11:40:29.004633Z","shell.execute_reply":"2023-06-05T11:40:29.009592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_cols_number = int(PCT_KEEP_COLS*len(keep_cols))\nN_HEAD = 10\nmi_cols = []\n\nif ENABLE_MI_SCORES == True:\n    for i in PCOLS_IDX:\n        print('--> Target', pcols[i], 'top mi columns ...')\n        c = pcols[i]\n        y = train_sampled[c]\n        df_mis = make_mi_scores(X, y, discrete_features).to_frame()\n        ix_mis = list(df_mis.index.values)\n        print(tabulate(df_mis.head(N_HEAD), headers='keys', tablefmt='psql'))\n        mi_cols.append(ix_mis)\n    #  Get best columns (nr = top_cols_number) considering all mi scores with selected targets\n    df_mis_pvt = pd.DataFrame(mi_cols).transpose().reset_index()\n    vv = [v for v in df_mis_pvt.columns if v not in ['index']]\n    df_mis_unp = pd.melt(df_mis_pvt,value_vars=vv,id_vars='index')\n    df_mis_top = df_mis_unp[['index','value']].groupby(by='value').sum()\n    df_mis_top = df_mis_top.sort_values(by='index').head(top_cols_number)\n    print('############ Variables and sum of positions #############')\n    print(tabulate(df_mis_top, headers='keys', tablefmt='psql'))\n    mi_cols = list(df_mis_top.index.values)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:40:29.011909Z","iopub.execute_input":"2023-06-05T11:40:29.012571Z","iopub.status.idle":"2023-06-05T12:43:34.641816Z","shell.execute_reply.started":"2023-06-05T11:40:29.012530Z","shell.execute_reply":"2023-06-05T12:43:34.638620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_mis_unp","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:43:34.648589Z","iopub.execute_input":"2023-06-05T12:43:34.649431Z","iopub.status.idle":"2023-06-05T12:43:34.658555Z","shell.execute_reply.started":"2023-06-05T12:43:34.649326Z","shell.execute_reply":"2023-06-05T12:43:34.657769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_mis_unp[df_mis_unp['value']=='AccV__median__w=3000']","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:43:34.660230Z","iopub.execute_input":"2023-06-05T12:43:34.660920Z","iopub.status.idle":"2023-06-05T12:43:34.674196Z","shell.execute_reply.started":"2023-06-05T12:43:34.660890Z","shell.execute_reply":"2023-06-05T12:43:34.673150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if ENABLE_MI_SCORES == False:\n    mi_cols = list(np.random.choice(keep_cols,top_cols_number))","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:43:34.676476Z","iopub.execute_input":"2023-06-05T12:43:34.676946Z","iopub.status.idle":"2023-06-05T12:43:34.687507Z","shell.execute_reply.started":"2023-06-05T12:43:34.676908Z","shell.execute_reply":"2023-06-05T12:43:34.686248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#\n# mi_cols is a part of keep_cols\n#\nprint(mi_cols)\nprint(type(mi_cols))","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:43:34.689222Z","iopub.execute_input":"2023-06-05T12:43:34.689556Z","iopub.status.idle":"2023-06-05T12:43:34.701260Z","shell.execute_reply.started":"2023-06-05T12:43:34.689530Z","shell.execute_reply":"2023-06-05T12:43:34.700360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data=train_sampled,x='Medication')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:43:34.702771Z","iopub.execute_input":"2023-06-05T12:43:34.703425Z","iopub.status.idle":"2023-06-05T12:43:35.349362Z","shell.execute_reply.started":"2023-06-05T12:43:34.703378Z","shell.execute_reply":"2023-06-05T12:43:35.348122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data=train_sampled,x='Test')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:43:35.351301Z","iopub.execute_input":"2023-06-05T12:43:35.351660Z","iopub.status.idle":"2023-06-05T12:43:35.989641Z","shell.execute_reply.started":"2023-06-05T12:43:35.351630Z","shell.execute_reply":"2023-06-05T12:43:35.988478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.barplot(x=np.round(train_sampled['Time_frac'],1),y=train_sampled[pcols[0]])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:43:35.991570Z","iopub.execute_input":"2023-06-05T12:43:35.991912Z","iopub.status.idle":"2023-06-05T12:44:45.465477Z","shell.execute_reply.started":"2023-06-05T12:43:35.991882Z","shell.execute_reply":"2023-06-05T12:44:45.464171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.barplot(x=np.round(train_sampled['Time_frac'],1),y=train_sampled[pcols[1]])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:44:45.467435Z","iopub.execute_input":"2023-06-05T12:44:45.467984Z","iopub.status.idle":"2023-06-05T12:45:54.471496Z","shell.execute_reply.started":"2023-06-05T12:44:45.467948Z","shell.execute_reply":"2023-06-05T12:45:54.470131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.barplot(x=np.round(train_sampled['Time_frac'],1),y=train_sampled[pcols[2]])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:45:54.473846Z","iopub.execute_input":"2023-06-05T12:45:54.474295Z","iopub.status.idle":"2023-06-05T12:47:06.698257Z","shell.execute_reply.started":"2023-06-05T12:45:54.474253Z","shell.execute_reply":"2023-06-05T12:47:06.696914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#\n# ecols are columns to Exclude from training = index & output columns\n#\necols = ['Id', 'Module', 'StartHesitation', 'Turn', 'Walking', 'Time', 'Subject']","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:47:06.700101Z","iopub.execute_input":"2023-06-05T12:47:06.700467Z","iopub.status.idle":"2023-06-05T12:47:06.706305Z","shell.execute_reply.started":"2023-06-05T12:47:06.700438Z","shell.execute_reply":"2023-06-05T12:47:06.705263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#\n# cols here defined are columns used in training = numerical relevant colums, mi_cols is a part of keep_cols\n#\ncols = bcols + mi_cols\ncols = list(set(cols)-set(ecols))\ncols","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:47:06.708016Z","iopub.execute_input":"2023-06-05T12:47:06.708350Z","iopub.status.idle":"2023-06-05T12:47:06.722135Z","shell.execute_reply.started":"2023-06-05T12:47:06.708322Z","shell.execute_reply":"2023-06-05T12:47:06.720914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('A. All Columns                     : ', len(ocols))\nprint('B. Base Columns                    : ', len(bcols))\nprint('C. Feature Added Columns           : ', len(fadd_cols))\nprint('D. Acceptable Related Fadd Columns : ', len(keep_cols))   \nprint('E. Mi Columns                      : ', len(mi_cols))\nprint('F. Not for train Columns           : ', len(ecols))\nprint('----------------------------------------------')\nprint('Train Columns (B,E,F)              : ', len(cols))","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:47:06.724087Z","iopub.execute_input":"2023-06-05T12:47:06.724477Z","iopub.status.idle":"2023-06-05T12:47:06.739490Z","shell.execute_reply.started":"2023-06-05T12:47:06.724444Z","shell.execute_reply":"2023-06-05T12:47:06.738606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"process = psutil.Process(os.getpid())\nprint('End of Data Preparation : Memory = ', process.memory_info().rss/MB, 'MB')\n\ntrain_sampled = None\ntrain_for_corr = None\ndf = None\nsubjects = None\nfull_metadata = None\nX = None\ncorr = None\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:47:06.741043Z","iopub.execute_input":"2023-06-05T12:47:06.741643Z","iopub.status.idle":"2023-06-05T12:47:07.810241Z","shell.execute_reply.started":"2023-06-05T12:47:06.741608Z","shell.execute_reply":"2023-06-05T12:47:07.807853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modelling","metadata":{"execution":{"iopub.status.busy":"2023-04-24T07:22:08.290362Z","iopub.execute_input":"2023-04-24T07:22:08.293579Z","iopub.status.idle":"2023-04-24T07:22:08.33648Z","shell.execute_reply.started":"2023-04-24T07:22:08.293461Z","shell.execute_reply":"2023-04-24T07:22:08.334934Z"}}},{"cell_type":"code","source":"len(train)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:47:07.813754Z","iopub.execute_input":"2023-06-05T12:47:07.814827Z","iopub.status.idle":"2023-06-05T12:47:07.826375Z","shell.execute_reply.started":"2023-06-05T12:47:07.814701Z","shell.execute_reply":"2023-06-05T12:47:07.824759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gcols = cols+['Id','Subject','StartHesitation', 'Turn', 'Walking']\ntrain = train[gcols]\ntrain.drop_duplicates(inplace=True)\nn_all = len(train)\nn_all","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:47:07.827840Z","iopub.execute_input":"2023-06-05T12:47:07.829600Z","iopub.status.idle":"2023-06-05T12:48:43.130179Z","shell.execute_reply.started":"2023-06-05T12:47:07.829546Z","shell.execute_reply":"2023-06-05T12:48:43.129198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_target_1 = train[train['StartHesitation']+train['Turn']+train['Walking']>0]\nn1 = len(train_target_1)\nn1","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:48:43.131385Z","iopub.execute_input":"2023-06-05T12:48:43.132289Z","iopub.status.idle":"2023-06-05T12:48:44.033333Z","shell.execute_reply.started":"2023-06-05T12:48:43.132258Z","shell.execute_reply":"2023-06-05T12:48:44.032108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n1_new = n_all - MULT_POS_FACTOR * n1\ntrain_target_0 = train[train['StartHesitation']+train['Turn']+train['Walking']==0].sample(frac=n1_new/n_all)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:48:44.034873Z","iopub.execute_input":"2023-06-05T12:48:44.035320Z","iopub.status.idle":"2023-06-05T12:48:52.519934Z","shell.execute_reply.started":"2023-06-05T12:48:44.035279Z","shell.execute_reply":"2023-06-05T12:48:52.518483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (TRAIN_RESAMPLE == True):\n    train_r = train_target_0.copy()\n    for m in range(0,MULT_POS_FACTOR):\n        print('Adding copy #', m, 'of training set <<1>> ...')\n        train_r = pd.concat([train_r,train_target_1])\n        gc.collect()\n    print('Resampled Training Set Length = ', len(train_r))\n    train = train_r","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:48:52.521778Z","iopub.execute_input":"2023-06-05T12:48:52.522258Z","iopub.status.idle":"2023-06-05T12:49:01.665868Z","shell.execute_reply.started":"2023-06-05T12:48:52.522212Z","shell.execute_reply":"2023-06-05T12:49:01.664768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.reset_index(drop=True, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:49:01.667260Z","iopub.execute_input":"2023-06-05T12:49:01.667688Z","iopub.status.idle":"2023-06-05T12:49:01.672795Z","shell.execute_reply.started":"2023-06-05T12:49:01.667657Z","shell.execute_reply":"2023-06-05T12:49:01.671729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_target_0 = None\ntrain_target_1 = None\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:49:01.674116Z","iopub.execute_input":"2023-06-05T12:49:01.674440Z","iopub.status.idle":"2023-06-05T12:49:02.007353Z","shell.execute_reply.started":"2023-06-05T12:49:01.674412Z","shell.execute_reply":"2023-06-05T12:49:02.006060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# RandomizedSearchCV: In contrast to GridSearchCV, not all parameter values are tried out, \n# but rather a fixed number of parameter settings is sampled from the specified distributions\n\nlgbm_params_dict = {\n    'colsample_bytree': [0.50, 0.65, 0.80],\n    'learning_rate': [0.10, 0.20, 0.25],\n    'num_leaves': [20, 24, 32],\n    'max_depth': [4, 6, 8],\n    'min_child_weight': [4, 5, 6],\n    'n_estimators': [100, 150, 220],\n    'subsample': [0.80, 0.85, 0.95]\n }","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:49:02.011272Z","iopub.execute_input":"2023-06-05T12:49:02.011662Z","iopub.status.idle":"2023-06-05T12:49:02.024635Z","shell.execute_reply.started":"2023-06-05T12:49:02.011630Z","shell.execute_reply":"2023-06-05T12:49:02.023433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf_params_dict = {\n 'n_estimators': [10,20],\n 'min_samples_split' : [4,6],\n 'min_samples_leaf' : [5, 10, 20]\n}","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:49:02.028605Z","iopub.execute_input":"2023-06-05T12:49:02.028947Z","iopub.status.idle":"2023-06-05T12:49:02.036235Z","shell.execute_reply.started":"2023-06-05T12:49:02.028920Z","shell.execute_reply":"2023-06-05T12:49:02.035453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def custom_average_precision(y_true, y_pred):\n    score = average_precision_score(y_true, y_pred)\n    return 'average_precision', score, True","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:49:02.038047Z","iopub.execute_input":"2023-06-05T12:49:02.038468Z","iopub.status.idle":"2023-06-05T12:49:02.049504Z","shell.execute_reply.started":"2023-06-05T12:49:02.038430Z","shell.execute_reply":"2023-06-05T12:49:02.048385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LGBMMultiOutputRegressor(MultiOutputRegressor):\n    def fit(self, X, y, eval_set=None, **fit_params):\n        self.estimators_ = [clone(self.estimator) for _ in range(y.shape[1])]\n        \n        for i, estimator in enumerate(self.estimators_):\n            if eval_set:\n                fit_params['eval_set'] = [(eval_set[0], eval_set[1][:, i])]\n            estimator.fit(X, y[:, i], **fit_params)\n        \n        return self","metadata":{"papermill":{"duration":0.031497,"end_time":"2023-04-16T22:50:29.071479","exception":false,"start_time":"2023-04-16T22:50:29.039982","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T12:49:02.051105Z","iopub.execute_input":"2023-06-05T12:49:02.052176Z","iopub.status.idle":"2023-06-05T12:49:02.060452Z","shell.execute_reply.started":"2023-06-05T12:49:02.052133Z","shell.execute_reply":"2023-06-05T12:49:02.059358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RFMultiOutputRegressor(MultiOutputRegressor):\n    def fit(self, X, y, eval_set=None, **fit_params):\n        self.estimators_ = [clone(self.estimator) for _ in range(y.shape[1])]\n        \n        for i, estimator in enumerate(self.estimators_):\n            estimator.fit(X, y[:, i], **fit_params)\n        \n        return self","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:49:02.061647Z","iopub.execute_input":"2023-06-05T12:49:02.061953Z","iopub.status.idle":"2023-06-05T12:49:02.071033Z","shell.execute_reply.started":"2023-06-05T12:49:02.061927Z","shell.execute_reply":"2023-06-05T12:49:02.070157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#\n# shuffle if needed\n#\nif (SHUFFLE==True):\n    train = train.sample(frac=1,random_state=RANDOM_STATE).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:49:02.072193Z","iopub.execute_input":"2023-06-05T12:49:02.072622Z","iopub.status.idle":"2023-06-05T12:49:02.082588Z","shell.execute_reply.started":"2023-06-05T12:49:02.072583Z","shell.execute_reply":"2023-06-05T12:49:02.081479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Each fold has 1/FOLDS_NUMBER % samples from the whole dataset (called train)\n# the train set has rows = SAMPLE_NUMBER\n# the test (validation) set has rows = SAMPLE_NUMBER\n# GroupKFold is not randomized at all. The test sets form a complete partition of all the data.\n\nif (STRATEGY=='Group'):\n    kfold = GroupKFold(FOLDS_NUMBER)\n    groups=kfold.split(train, groups=train.Subject)\n    fold_len = len(train)/FOLDS_NUMBER\n    print('Folder Length = ',fold_len)\n    print('Folder Coverage % = ',100*SAMPLES_NUMBER/fold_len)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:49:02.083795Z","iopub.execute_input":"2023-06-05T12:49:02.084223Z","iopub.status.idle":"2023-06-05T12:49:02.097668Z","shell.execute_reply.started":"2023-06-05T12:49:02.084190Z","shell.execute_reply":"2023-06-05T12:49:02.096545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Each fold is the 1-TEST_SIZE % of the whole dataset (called train)\n# the train set has rows = SAMPLE_NUMBER\n# the test (validation) set has rows = TEST_SIZE% * len(train)\n\nif (STRATEGY=='Stratified'):\n    kfold = StratifiedShuffleSplit(n_splits=FOLDS_NUMBER,test_size=TEST_SIZE,random_state=RANDOM_STATE)\n    groups = kfold.split(train[cols], train[pcols], groups=train.Subject)\n    fold_len = len(train)*(1-TEST_SIZE)\n    print('Folder Length = ',fold_len)\n    print('Folder Coverage % = ',100*SAMPLES_NUMBER/fold_len)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:49:02.099021Z","iopub.execute_input":"2023-06-05T12:49:02.099345Z","iopub.status.idle":"2023-06-05T12:49:02.111895Z","shell.execute_reply.started":"2023-06-05T12:49:02.099317Z","shell.execute_reply":"2023-06-05T12:49:02.110308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#\n# Just to have an example of how the kfold works\n#\n\n#for i, (tr_idx, te_idx) in enumerate(groups):\n#    \n#    print('Fold #',i,' len = ', len(tr_idx))\n#    print('  Train len =', len(tr_idx), 'Examples:', tr_idx[0:5])\n#    print('  Test  len =', len(te_idx), 'Examples:', te_idx[0:5])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:49:02.113591Z","iopub.execute_input":"2023-06-05T12:49:02.113999Z","iopub.status.idle":"2023-06-05T12:49:02.125807Z","shell.execute_reply.started":"2023-06-05T12:49:02.113958Z","shell.execute_reply":"2023-06-05T12:49:02.124864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def stratified_sample_df(df, col, n_samples):\n    n = min(n_samples, df[col].value_counts().min())\n    df_ = df.groupby(col).apply(lambda x: x.sample(n))\n    df_.index = df_.index.droplevel(0)\n    return df_","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:49:02.127180Z","iopub.execute_input":"2023-06-05T12:49:02.127604Z","iopub.status.idle":"2023-06-05T12:49:02.137833Z","shell.execute_reply.started":"2023-06-05T12:49:02.127566Z","shell.execute_reply":"2023-06-05T12:49:02.136702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_root = '/kaggle/working/'\nif EXPORT_CSV == True:\n    train_sampled = train.sample(n=SAMPLES_NUMBER)\n    filename = path.join(output_root, 'df_park.csv')\n    train_sampled.to_csv(filename)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:49:02.139375Z","iopub.execute_input":"2023-06-05T12:49:02.140089Z","iopub.status.idle":"2023-06-05T12:49:02.148029Z","shell.execute_reply.started":"2023-06-05T12:49:02.140051Z","shell.execute_reply":"2023-06-05T12:49:02.147091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm_params_ = []\n\nif (PARAMS_OPTIMIZATION_ENABLE==True):\n    \n    rs_best_params = []\n    print('Optimal Regressor parameters from RandomizedSearchCV : samples number =', SAMPLES_NUMBER*EXPANSION_FACTOR)\n    tr_idx = stratified_sample_df(train,'Subject',SAMPLES_NUMBER*EXPANSION_FACTOR).index.values       \n    x_train = train.loc[tr_idx, cols].to_numpy()\n    y_train = train.loc[tr_idx, pcols].to_numpy()\n\n    for j in range(0,len(pcols)):\n        print('Column : ', pcols[j])\n        lgbm_regressor = lgb.LGBMRegressor()\n        reg = RandomizedSearchCV(lgbm_regressor, lgbm_params_dict, random_state=RANDOM_STATE, verbose=2, cv=CV, n_iter=ITERATIONS)\n        search = reg.fit(x_train, y_train[:,j])\n        rs_best_params.append(search.best_params_)\n \n    df_best_params = pd.DataFrame(rs_best_params).mean()\n    lgbm_params_ = {'colsample_bytree': df_best_params['colsample_bytree'],\n     'learning_rate': df_best_params['learning_rate'],\n     'num_leaves': int(df_best_params['num_leaves']),\n     'max_depth': int(df_best_params['max_depth']),\n     'min_child_weight': df_best_params['min_child_weight'],\n     'n_estimators': int(df_best_params['n_estimators']),\n     'subsample': df_best_params['subsample'],\n     'early_stopping_round' : int(df_best_params['n_estimators']/5),\n     'verbose' : 1,\n     'force_col_wise' : True}\n    print('Regressor Parameters After Optimization : ')\n    \nelse:\n    print('Regressor Parameters Default Values : ') \n    lgbm_params_ = {'colsample_bytree': 0.6,\n     'learning_rate': 0.2,\n     'num_leaves': 25,\n     'max_depth': 6,\n     'min_child_weight': 4,\n     'n_estimators': 140,\n     'subsample': 0.85,\n     'early_stopping_round' : 30}\n\nprint(lgbm_params_)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:49:02.149831Z","iopub.execute_input":"2023-06-05T12:49:02.150235Z","iopub.status.idle":"2023-06-05T13:56:40.311792Z","shell.execute_reply.started":"2023-06-05T12:49:02.150197Z","shell.execute_reply":"2023-06-05T13:56:40.310901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('End of Params CV search : Memory = ', process.memory_info().rss/MB, 'MB')\n\ntr_idx = None\nx_train = None\ny_train = None\nreg = None\nsearch = None\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T13:56:40.312981Z","iopub.execute_input":"2023-06-05T13:56:40.313704Z","iopub.status.idle":"2023-06-05T13:56:40.608860Z","shell.execute_reply.started":"2023-06-05T13:56:40.313673Z","shell.execute_reply":"2023-06-05T13:56:40.607634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf_params_ = { 'max_depth': None,\n 'n_estimators': 10,\n 'min_samples_split' : 4,\n 'min_samples_leaf' : 10,\n 'n_jobs' : -1,\n 'verbose' : 2,\n}","metadata":{"execution":{"iopub.status.busy":"2023-06-05T13:56:40.610529Z","iopub.execute_input":"2023-06-05T13:56:40.610998Z","iopub.status.idle":"2023-06-05T13:56:40.616560Z","shell.execute_reply.started":"2023-06-05T13:56:40.610956Z","shell.execute_reply":"2023-06-05T13:56:40.615390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# I don't kwnow why but it loses groups after an enumerate\n\nprint('Strategy = ', STRATEGY)\nif (STRATEGY=='Stratified'):\n    kfold = StratifiedShuffleSplit(n_splits=FOLDS_NUMBER,test_size=TEST_SIZE,random_state=RANDOM_STATE)\n    groups = kfold.split(train[cols], train[pcols], groups=train['Subject'])\nelse:\n    kfold = GroupKFold(FOLDS_NUMBER)\n    groups=kfold.split(train[cols], groups=train['Subject'])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T13:56:40.617997Z","iopub.execute_input":"2023-06-05T13:56:40.619136Z","iopub.status.idle":"2023-06-05T13:56:44.286133Z","shell.execute_reply.started":"2023-06-05T13:56:40.619091Z","shell.execute_reply":"2023-06-05T13:56:44.284831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.columns)\nprint(len(train.columns))","metadata":{"execution":{"iopub.status.busy":"2023-06-05T13:56:44.288048Z","iopub.execute_input":"2023-06-05T13:56:44.288504Z","iopub.status.idle":"2023-06-05T13:56:44.294931Z","shell.execute_reply.started":"2023-06-05T13:56:44.288464Z","shell.execute_reply":"2023-06-05T13:56:44.293639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# - Id, Subject, StartHesitation, Turn, Walking\nprint(cols)\nprint(len(cols))","metadata":{"execution":{"iopub.status.busy":"2023-06-05T13:56:44.296916Z","iopub.execute_input":"2023-06-05T13:56:44.297359Z","iopub.status.idle":"2023-06-05T13:56:44.308774Z","shell.execute_reply.started":"2023-06-05T13:56:44.297317Z","shell.execute_reply":"2023-06-05T13:56:44.307483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ccols = [c for c in cols if c not in ['Subject']]\nprint(ccols)\nprint(len(ccols))","metadata":{"execution":{"iopub.status.busy":"2023-06-05T13:56:44.312392Z","iopub.execute_input":"2023-06-05T13:56:44.312751Z","iopub.status.idle":"2023-06-05T13:56:44.322842Z","shell.execute_reply.started":"2023-06-05T13:56:44.312722Z","shell.execute_reply":"2023-06-05T13:56:44.321545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"regs = []\ncvs = []\n\nfor i, (tr_idx, te_idx) in enumerate(groups):\n    \n    print('Fold #',i,' len = ', len(tr_idx))\n    \n    try:\n        tr_idx = pd.Series(tr_idx).sample(n=SAMPLES_NUMBER,random_state=RANDOM_STATE).values\n\n        x_train = train.loc[tr_idx, cols].to_numpy()\n        y_train = train.loc[tr_idx, pcols].to_numpy()\n\n        x_test = train.loc[te_idx, cols].to_numpy()\n        y_test = train.loc[te_idx, pcols].to_numpy()\n\n        print('  Train len = ', len(tr_idx))\n        print('  Test  len = ', len(te_idx))\n        print('  Subjects  = ',train.loc[tr_idx,'Subject'].unique())\n\n        if (REGRESSOR=='LGBM'):\n            print('Training LGBM..........')\n            multioutput_regressor = LGBMMultiOutputRegressor(lgb.LGBMRegressor(**lgbm_params_))\n            multioutput_regressor.fit(x_train, y_train,eval_set=(x_test, y_test),eval_metric=custom_average_precision)\n        else:\n            print('Training RF............')\n            multioutput_regressor = RFMultiOutputRegressor(RandomForestRegressor(**rf_params_))\n            multioutput_regressor.fit(x_train, y_train)\n            \n        for j,est_ in enumerate(multioutput_regressor.estimators_):\n            print('---------------- Importance for Target#',j,' -------------------------')\n            fi = pd.DataFrame({'Value':est_.feature_importances_,'Feature':cols})\n            dfi = fi.sort_values(by=\"Value\",ascending=False)[0:NUM_FI]\n            print(tabulate(dfi, headers='keys', tablefmt='psql'))\n\n        regs.append(multioutput_regressor)\n\n        cv = metrics.average_precision_score(y_test, multioutput_regressor.predict(x_test).clip(0.0,1.0))\n\n        cvs.append(cv)\n    \n    except Exception as e: \n        print('Error in training : ', str(e))\n        pass\n    \n    finally:\n        gc.collect()","metadata":{"papermill":{"duration":943.279901,"end_time":"2023-04-16T23:06:12.366865","exception":false,"start_time":"2023-04-16T22:50:29.086964","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T13:56:44.324520Z","iopub.execute_input":"2023-06-05T13:56:44.324848Z","iopub.status.idle":"2023-06-05T14:10:27.150113Z","shell.execute_reply.started":"2023-06-05T13:56:44.324821Z","shell.execute_reply":"2023-06-05T14:10:27.148915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(cvs)\ncvs_mean = np.mean(cvs)\ncvs_std = np.std(cvs)\nprint('Precision Mean (Average on Folders) = ', cvs_mean)\nprint('Precision Standard Deviation = ', cvs_std)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:10:27.151933Z","iopub.execute_input":"2023-06-05T14:10:27.152392Z","iopub.status.idle":"2023-06-05T14:10:27.159271Z","shell.execute_reply.started":"2023-06-05T14:10:27.152348Z","shell.execute_reply":"2023-06-05T14:10:27.158085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('End of Data Modelling : Memory = ', process.memory_info().rss/MB, 'MB')\n\ntrain = None\ngroups = None\nx_train = None\ny_train = None\nx_test = None\ny_test = None\nmultioutput_regressor = None\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:10:27.161196Z","iopub.execute_input":"2023-06-05T14:10:27.161655Z","iopub.status.idle":"2023-06-05T14:10:27.429583Z","shell.execute_reply.started":"2023-06-05T14:10:27.161614Z","shell.execute_reply":"2023-06-05T14:10:27.428556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv(path.join(root, 'sample_submission.csv'))\nscols","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:37:55.007767Z","iopub.execute_input":"2023-06-05T14:37:55.008211Z","iopub.status.idle":"2023-06-05T14:37:55.222840Z","shell.execute_reply.started":"2023-06-05T14:37:55.008176Z","shell.execute_reply":"2023-06-05T14:37:55.221492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = []\ndf_test = None","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:38:02.822604Z","iopub.execute_input":"2023-06-05T14:38:02.823023Z","iopub.status.idle":"2023-06-05T14:38:02.832723Z","shell.execute_reply.started":"2023-06-05T14:38:02.822994Z","shell.execute_reply":"2023-06-05T14:38:02.831240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for f in test:\n    \n    fid = ''\n    \n    try:\n        \n        fid = f.split('/')[-1].split('.')[0]\n        print('Predicting fid = ',fid)\n        \n        df = pd.read_csv(f)\n        df.set_index('Time', drop=True, inplace=True)\n        df['Id'] = fid\n        dataset = Path(f).parts[-2]\n         \n        if (RESCALE_ACC==True):\n            df = norm_acc(df,dataset)\n                     \n        df['Time_frac']=np.round((df.index/df.index.max()).values,TIME_FRAC_DECIMALS)\n        \n        df = pd.merge(df, tasks[tcols], how='left', on='Id').fillna(-1)     \n        df = pd.merge(df, metadata_w_subjects[METADATA_W_SUBJECTS_COLS], how='left', on='Id').fillna(-1)\n        if (ADD_EVENTS == True):\n            df = pd.merge(df, events[evcols], how='left', on='Id').fillna(-1)\n            \n        df_feats = fc.calculate(df, return_df=True, include_final_window=True, approve_sparsity=True, window_idx=\"begin\").astype(np.float32)\n        df = df.merge(df_feats, how=\"left\", left_index=True, right_index=True)\n        \n        if (ROLLING_AVG==True):\n            df = rolling_average(df,ROLLING_WINDOW)\n    \n        df.fillna(method=\"ffill\", inplace=True)\n\n        res_vals = []\n        \n        for i_fold in range(FOLDS_NUMBER):\n            \n            if (cvs[i_fold]>cvs_mean*CVS_FACTOR):\n                print('Used regressor from fold #', i_fold)\n                pred = regs[i_fold].predict(df[cols]).clip(0.0,1.0)\n                res_vals.append(np.expand_dims(np.round(pred, 3), axis = 2))\n\n        res_vals = np.round(np.mean(np.concatenate(res_vals, axis = 2), axis = 2),OUTPUT_DECIMALS)\n        res = pd.DataFrame(res_vals, columns=pcols)\n\n        df = pd.concat([df,res], axis=1)\n        df['Id'] = df['Id'].astype(str) + '_' + df.index.astype(str)\n\n        if (df_test is None):\n            df_test = df\n        else:\n            df_test = pd.concat([df_test,df])\n\n    except Exception as e: \n        print('Error in calculating submission : fid = ',fid,' : ', str(e))\n        pass\n    \n    finally:\n        gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:38:04.966100Z","iopub.execute_input":"2023-06-05T14:38:04.966530Z","iopub.status.idle":"2023-06-05T14:38:12.068724Z","shell.execute_reply.started":"2023-06-05T14:38:04.966496Z","shell.execute_reply":"2023-06-05T14:38:12.067532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:38:14.999943Z","iopub.execute_input":"2023-06-05T14:38:15.000385Z","iopub.status.idle":"2023-06-05T14:38:15.008374Z","shell.execute_reply.started":"2023-06-05T14:38:15.000345Z","shell.execute_reply":"2023-06-05T14:38:15.007306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:38:18.520151Z","iopub.execute_input":"2023-06-05T14:38:18.520579Z","iopub.status.idle":"2023-06-05T14:38:18.542863Z","shell.execute_reply.started":"2023-06-05T14:38:18.520543Z","shell.execute_reply":"2023-06-05T14:38:18.541649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_test)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:38:24.996197Z","iopub.execute_input":"2023-06-05T14:38:24.996640Z","iopub.status.idle":"2023-06-05T14:38:25.004772Z","shell.execute_reply.started":"2023-06-05T14:38:24.996602Z","shell.execute_reply":"2023-06-05T14:38:25.003496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_subjects = df_test['Subject'].unique()\ntest_subjects","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:38:27.045283Z","iopub.execute_input":"2023-06-05T14:38:27.045722Z","iopub.status.idle":"2023-06-05T14:38:27.069909Z","shell.execute_reply.started":"2023-06-05T14:38:27.045689Z","shell.execute_reply":"2023-06-05T14:38:27.068607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DG_FRAC = 0.3\ndg0 = df_test[df_test['Subject']==test_subjects[0]].sample(frac=DG_FRAC)\ndg1 = df_test[df_test['Subject']==test_subjects[1]].sample(frac=DG_FRAC)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:38:29.297463Z","iopub.execute_input":"2023-06-05T14:38:29.297884Z","iopub.status.idle":"2023-06-05T14:38:29.482939Z","shell.execute_reply.started":"2023-06-05T14:38:29.297853Z","shell.execute_reply":"2023-06-05T14:38:29.481781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(data=dg0,x ='Time_frac', y='StartHesitation')\nsns.lineplot(data=dg1,x ='Time_frac', y='StartHesitation')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:38:40.380951Z","iopub.execute_input":"2023-06-05T14:38:40.381366Z","iopub.status.idle":"2023-06-05T14:39:55.880593Z","shell.execute_reply.started":"2023-06-05T14:38:40.381334Z","shell.execute_reply":"2023-06-05T14:39:55.879267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(data=dg0,x ='Time_frac', y='Turn')\nsns.lineplot(data=dg1,x ='Time_frac', y='Turn')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:39:55.893578Z","iopub.execute_input":"2023-06-05T14:39:55.894042Z","iopub.status.idle":"2023-06-05T14:41:12.336659Z","shell.execute_reply.started":"2023-06-05T14:39:55.893998Z","shell.execute_reply":"2023-06-05T14:41:12.335559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(data=dg0,x ='Time_frac', y='Walking')\nsns.lineplot(data=dg1,x ='Time_frac', y='Walking')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:41:12.339265Z","iopub.execute_input":"2023-06-05T14:41:12.339612Z","iopub.status.idle":"2023-06-05T14:42:26.897219Z","shell.execute_reply.started":"2023-06-05T14:41:12.339583Z","shell.execute_reply":"2023-06-05T14:42:26.895842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"General trends of pcols are respected in the test results?","metadata":{}},{"cell_type":"code","source":"sns.lineplot(x=np.round(df_test['Time_frac'],1),y=df_test[pcols[0]])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:42:26.899102Z","iopub.execute_input":"2023-06-05T14:42:26.899578Z","iopub.status.idle":"2023-06-05T14:42:30.649156Z","shell.execute_reply.started":"2023-06-05T14:42:26.899537Z","shell.execute_reply":"2023-06-05T14:42:30.648052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(x=np.round(df_test['Time_frac'],1),y=df_test[pcols[1]])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:42:30.650706Z","iopub.execute_input":"2023-06-05T14:42:30.651321Z","iopub.status.idle":"2023-06-05T14:42:34.454673Z","shell.execute_reply.started":"2023-06-05T14:42:30.651280Z","shell.execute_reply":"2023-06-05T14:42:34.453344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(x=np.round(df_test['Time_frac'],1),y=df_test[pcols[2]])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:42:34.456550Z","iopub.execute_input":"2023-06-05T14:42:34.457045Z","iopub.status.idle":"2023-06-05T14:42:38.238785Z","shell.execute_reply.started":"2023-06-05T14:42:34.457000Z","shell.execute_reply":"2023-06-05T14:42:38.237841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"th0 = np.quantile(df_test[pcols[0]],1-pct0)\nth0","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:42:38.240136Z","iopub.execute_input":"2023-06-05T14:42:38.240520Z","iopub.status.idle":"2023-06-05T14:42:38.251876Z","shell.execute_reply.started":"2023-06-05T14:42:38.240488Z","shell.execute_reply":"2023-06-05T14:42:38.250670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"th1 = np.quantile(df_test[pcols[1]],1-pct1)\nth1","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:42:38.253021Z","iopub.execute_input":"2023-06-05T14:42:38.253315Z","iopub.status.idle":"2023-06-05T14:42:38.262849Z","shell.execute_reply.started":"2023-06-05T14:42:38.253289Z","shell.execute_reply":"2023-06-05T14:42:38.261622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"th2 = np.quantile(df_test[pcols[2]],1-pct2)\nth2","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:42:38.267286Z","iopub.execute_input":"2023-06-05T14:42:38.267661Z","iopub.status.idle":"2023-06-05T14:42:38.277556Z","shell.execute_reply.started":"2023-06-05T14:42:38.267629Z","shell.execute_reply":"2023-06-05T14:42:38.276466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def DirtyHelp(df_sub,pcols,th0,th1,th2):\n    df_sub[pcols[0]] = np.where(df_sub[pcols[0]]>th0,1,0)\n    df_sub[pcols[1]] = np.where(df_sub[pcols[1]]>th1,1,0)\n    df_sub[pcols[2]] = np.where(df_sub[pcols[2]]>th2,1,0)\n    return df_sub","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:42:38.278787Z","iopub.execute_input":"2023-06-05T14:42:38.279229Z","iopub.status.idle":"2023-06-05T14:42:38.289210Z","shell.execute_reply.started":"2023-06-05T14:42:38.279196Z","shell.execute_reply":"2023-06-05T14:42:38.288179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#\n#  A (quick and) dirty help?\n#\nif (DIRTY_HELP == True):\n    df_test = DirtyHelp(df_test,pcols,th0,th1,th2)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:42:38.303166Z","iopub.execute_input":"2023-06-05T14:42:38.303611Z","iopub.status.idle":"2023-06-05T14:42:38.350522Z","shell.execute_reply.started":"2023-06-05T14:42:38.303571Z","shell.execute_reply":"2023-06-05T14:42:38.349587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if EXPORT_CSV == True:\n    filename = path.join(output_root, 'df_park_test.csv')\n    df_test.to_csv(filename)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:42:38.352052Z","iopub.execute_input":"2023-06-05T14:42:38.352738Z","iopub.status.idle":"2023-06-05T14:42:38.359872Z","shell.execute_reply.started":"2023-06-05T14:42:38.352697Z","shell.execute_reply":"2023-06-05T14:42:38.357235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = df_test[scols].copy()","metadata":{"papermill":{"duration":0.076052,"end_time":"2023-04-16T23:06:23.331235","exception":false,"start_time":"2023-04-16T23:06:23.255183","status":"completed"},"tags":[],"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-06-05T14:42:38.361803Z","iopub.execute_input":"2023-06-05T14:42:38.362235Z","iopub.status.idle":"2023-06-05T14:42:38.408853Z","shell.execute_reply.started":"2023-06-05T14:42:38.362195Z","shell.execute_reply":"2023-06-05T14:42:38.407791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('End of Submission : Memory = ', process.memory_info().rss/MB)\n\ndg = None\ndf_test = None\ndf = None\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:42:38.410545Z","iopub.execute_input":"2023-06-05T14:42:38.410901Z","iopub.status.idle":"2023-06-05T14:42:38.845350Z","shell.execute_reply.started":"2023-06-05T14:42:38.410873Z","shell.execute_reply":"2023-06-05T14:42:38.844303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.merge(sub[['Id']], submission, how='left', on='Id').fillna(0.0)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:42:38.847043Z","iopub.execute_input":"2023-06-05T14:42:38.847383Z","iopub.status.idle":"2023-06-05T14:42:39.226862Z","shell.execute_reply.started":"2023-06-05T14:42:38.847355Z","shell.execute_reply":"2023-06-05T14:42:39.225489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission[scols].describe()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:42:39.228486Z","iopub.execute_input":"2023-06-05T14:42:39.228927Z","iopub.status.idle":"2023-06-05T14:42:39.277363Z","shell.execute_reply.started":"2023-06-05T14:42:39.228885Z","shell.execute_reply":"2023-06-05T14:42:39.276455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission[scols].to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:42:39.278865Z","iopub.execute_input":"2023-06-05T14:42:39.279152Z","iopub.status.idle":"2023-06-05T14:42:39.792575Z","shell.execute_reply.started":"2023-06-05T14:42:39.279125Z","shell.execute_reply":"2023-06-05T14:42:39.791532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts = datetime.fromtimestamp(time.time())\nprint('End Timestamp = ',ts)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T14:42:39.794324Z","iopub.execute_input":"2023-06-05T14:42:39.794793Z","iopub.status.idle":"2023-06-05T14:42:39.801393Z","shell.execute_reply.started":"2023-06-05T14:42:39.794738Z","shell.execute_reply":"2023-06-05T14:42:39.800077Z"},"trusted":true},"execution_count":null,"outputs":[]}]}