{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Install tsflex and seglearn\n!pip install tsflex --no-index --find-links=file:///kaggle/input/time-series-tools\n!pip install seglearn --no-index --find-links=file:///kaggle/input/time-series-tools","metadata":{"execution":{"iopub.status.busy":"2023-04-09T13:38:49.748121Z","iopub.execute_input":"2023-04-09T13:38:49.748897Z","iopub.status.idle":"2023-04-09T13:39:14.03767Z","shell.execute_reply.started":"2023-04-09T13:38:49.748855Z","shell.execute_reply":"2023-04-09T13:39:14.036203Z"},"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data loading and feature extraction ","metadata":{}},{"cell_type":"markdown","source":"I took help from here https://www.kaggle.com/code/xzj19013742/simple-eda-on-time-for-targets\nTry  to develop some features","metadata":{}},{"cell_type":"markdown","source":"Data Loading","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\nfrom sklearn import *\nimport glob\n\np = '/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/'\n\ntrain = glob.glob(p+'train/**/**')\ntest = glob.glob(p+'test/**/**')\nsubjects = pd.read_csv(p+'subjects.csv')\ntasks = pd.read_csv(p+'tasks.csv')\nsub = pd.read_csv(p+'sample_submission.csv')\n\ntdcsfog_metadata=pd.read_csv('/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/tdcsfog_metadata.csv')\ndefog_metadata=pd.read_csv('/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/defog_metadata.csv')\n# daily_metadata=pd.read_csv('/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/daily_metadata.csv')\ntdcsfog_metadata['Module']='tdcsfog'\ndefog_metadata['Module']='defog'\n# daily_metadata['Module']='daily'\nmetadata=pd.concat([tdcsfog_metadata,defog_metadata])\n# metadata","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-13T15:53:13.575382Z","iopub.execute_input":"2023-04-13T15:53:13.575773Z","iopub.status.idle":"2023-04-13T15:53:15.295441Z","shell.execute_reply.started":"2023-04-13T15:53:13.575741Z","shell.execute_reply":"2023-04-13T15:53:15.294228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-13T15:53:18.542435Z","iopub.execute_input":"2023-04-13T15:53:18.542847Z","iopub.status.idle":"2023-04-13T15:53:18.571734Z","shell.execute_reply.started":"2023-04-13T15:53:18.542813Z","shell.execute_reply":"2023-04-13T15:53:18.570963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/code/jazivxt/familiar-solvs\ntasks['Duration'] = tasks['End'] - tasks['Begin']\ntasks = pd.pivot_table(tasks, values=['Duration'], index=['Id'], columns=['Task'], aggfunc='sum', fill_value=0)\ntasks.columns = [c[-1] for c in tasks.columns]\ntasks = tasks.reset_index()\ntasks['t_kmeans'] = cluster.KMeans(n_clusters=10, random_state=3).fit_predict(tasks[tasks.columns[1:]])\n\nsubjects = subjects.fillna(0).groupby('Subject').median()\nsubjects = subjects.reset_index()\n# subjects.rename(columns={'Subject':'Id'}, inplace=True)\nsubjects['s_kmeans'] = cluster.KMeans(n_clusters=10, random_state=3).fit_predict(subjects[subjects.columns[1:]])\nsubjects=subjects.rename(columns={'Visit':'s_Visit','Age':'s_Age','YearsSinceDx':'s_YearsSinceDx','UPDRSIII_On':'s_UPDRSIII_On','UPDRSIII_Off':'s_UPDRSIII_Off','NFOGQ':'s_NFOGQ'})\n\n# display(tasks)\n# display(subjects)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-13T15:53:22.486737Z","iopub.execute_input":"2023-04-13T15:53:22.487090Z","iopub.status.idle":"2023-04-13T15:53:22.603917Z","shell.execute_reply.started":"2023-04-13T15:53:22.487037Z","shell.execute_reply":"2023-04-13T15:53:22.603126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"complex_featlist=['Visit','Test','Medication','s_Visit','s_Age','s_YearsSinceDx','s_UPDRSIII_On','s_UPDRSIII_Off','s_NFOGQ','s_kmeans']\nmetadata_complex=metadata.merge(subjects,how='left',on='Subject').copy()\nmetadata_complex['Medication']=metadata_complex['Medication'].factorize()[0]\n\n# display(metadata_complex)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-12T10:17:01.786280Z","iopub.execute_input":"2023-04-12T10:17:01.786680Z","iopub.status.idle":"2023-04-12T10:17:01.803511Z","shell.execute_reply.started":"2023-04-12T10:17:01.786644Z","shell.execute_reply":"2023-04-12T10:17:01.801980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install tsflex and seglearn\n!pip install tsflex --no-index --find-links=file:///kaggle/input/time-series-tools\n!pip install seglearn --no-index --find-links=file:///kaggle/input/time-series-tools","metadata":{"execution":{"iopub.status.busy":"2023-04-12T10:17:07.518449Z","iopub.execute_input":"2023-04-12T10:17:07.519421Z","iopub.status.idle":"2023-04-12T10:17:32.822857Z","shell.execute_reply.started":"2023-04-12T10:17:07.519366Z","shell.execute_reply":"2023-04-12T10:17:32.821075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create a tsflex feature collection","metadata":{}},{"cell_type":"code","source":"from seglearn.feature_functions import base_features, emg_features\n\nfrom tsflex.features import FeatureCollection, MultipleFeatureDescriptors\nfrom tsflex.features.integrations import seglearn_feature_dict_wrapper\n\n\nbasic_feats = MultipleFeatureDescriptors(\n    functions=seglearn_feature_dict_wrapper(base_features()),\n    series_names=['AccV', 'AccML', 'AccAP'],\n    windows=[5_000],\n    strides=[5_000],\n)\n\nemg_feats = emg_features()\ndel emg_feats['simple square integral'] # is same as abs_energy (which is in base_features)\n\nemg_feats = MultipleFeatureDescriptors(\n    functions=seglearn_feature_dict_wrapper(emg_feats),\n    series_names=['AccV', 'AccML', 'AccAP'],\n    windows=[5_000],\n    strides=[5_000],\n)\n\nfc = FeatureCollection([basic_feats, emg_feats])","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-12T10:18:02.137334Z","iopub.execute_input":"2023-04-12T10:18:02.137824Z","iopub.status.idle":"2023-04-12T10:18:02.201008Z","shell.execute_reply.started":"2023-04-12T10:18:02.137775Z","shell.execute_reply":"2023-04-12T10:18:02.199804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Extract the features (with Time_frac feature)","metadata":{}},{"cell_type":"code","source":"\nimport pathlib\ndef reader(f):\n    try:\n        df = pd.read_csv(f, index_col=\"Time\", usecols=['Time', 'AccV', 'AccML', 'AccAP', 'StartHesitation', 'Turn' , 'Walking'])\n        \n        df['Id'] = f.split('/')[-1].split('.')[0]\n        df['Module'] = pathlib.Path(f).parts[-2]\n        \n        df['Time_frac']=(df.index/df.index.max()).values#currently the index of data is actually \"Time\"\n        \n        df = pd.merge(df, tasks[['Id','t_kmeans']], how='left', on='Id').fillna(-1)\n#         df = pd.merge(df, subjects[['Id','s_kmeans']], how='left', on='Id').fillna(-1)\n        df = pd.merge(df, metadata_complex[['Id','Subject']+['Visit','Test','Medication','s_kmeans']], how='left', on='Id').fillna(-1)\n        df_feats = fc.calculate(df, return_df=True, include_final_window=True, approve_sparsity=True, window_idx=\"begin\").astype(np.float32)\n        df = df.merge(df_feats, how=\"left\", left_index=True, right_index=True)\n        df.fillna(method=\"ffill\", inplace=True)\n        return df\n    except: pass\ntrain = pd.concat([reader(f) for f in tqdm(train)]).fillna(0); print(train.shape)\ncols = [c for c in train.columns if c not in ['Id','Subject','Module', 'Time', 'StartHesitation', 'Turn' , 'Walking', 'Valid', 'Task','Event']]\npcols = ['StartHesitation', 'Turn' , 'Walking']\nscols = ['Id', 'StartHesitation', 'Turn' , 'Walking']","metadata":{"execution":{"iopub.status.busy":"2023-04-12T10:18:09.954884Z","iopub.execute_input":"2023-04-12T10:18:09.955918Z","iopub.status.idle":"2023-04-12T10:26:22.881962Z","shell.execute_reply.started":"2023-04-12T10:18:09.955866Z","shell.execute_reply":"2023-04-12T10:26:22.880360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=train.reset_index(drop=True)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-12T10:32:20.834261Z","iopub.execute_input":"2023-04-12T10:32:20.836281Z","iopub.status.idle":"2023-04-12T10:32:27.294738Z","shell.execute_reply.started":"2023-04-12T10:32:20.836203Z","shell.execute_reply":"2023-04-12T10:32:27.293458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-12T11:29:12.111758Z","iopub.execute_input":"2023-04-12T11:29:12.112250Z","iopub.status.idle":"2023-04-12T11:29:12.143598Z","shell.execute_reply.started":"2023-04-12T11:29:12.112208Z","shell.execute_reply":"2023-04-12T11:29:12.141865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA: Looking at the distribution of Time for different target variable\n\n1. add time feature: `df['Time_frac']=(df.Time/df.Time.max()).values`(finished in reader() function)\n2. visualized the distribution of `Time_frac` for different target variable[`StartHesitation,Turn,Walking`]\n\n","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfig,ax=plt.subplots(3,1,figsize=(20,10))\ntrain.loc[train['StartHesitation']==1,'Time_frac'].hist(ax=ax[0],bins=100)\ntrain.loc[train['Turn']==1,'Time_frac'].hist(ax=ax[1],bins=100)\ntrain.loc[train['Walking']==1,'Time_frac'].hist(ax=ax[2],bins=100)","metadata":{"execution":{"iopub.status.busy":"2023-04-12T10:32:32.309590Z","iopub.execute_input":"2023-04-12T10:32:32.309992Z","iopub.status.idle":"2023-04-12T10:32:33.959990Z","shell.execute_reply.started":"2023-04-12T10:32:32.309957Z","shell.execute_reply":"2023-04-12T10:32:33.958457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### The distribution betweeen `the 3 target[StartHesitation,Turn,Walking]` is totally different\n\n##### review the Official definition\n> The tDCS FOG (tdcsfog) dataset, comprising data series collected in the lab, as subjects `completed a FOG-provoking protocol`.\n\n> The DeFOG (defog) dataset, comprising data series collected in the subject's home, as subjects `completed a FOG-provoking protocol`\n\n##### EDA Discovery\n\n- StartHesitation frequently occurs in the early stage, and rarely occurs in the late stage\n- Turn frequently occurs in the middle stage\n- Walking frequently occurs in the late stage\n\n##### Assumption: In a `completed FOG-provoking protocol`, the order in which the event occur tends to follow the order `StartHesitation->Turn->Walking`\n\n##### Application example: Add time fraction feature to model","metadata":{}},{"cell_type":"markdown","source":"# Build the model and submit","metadata":{}},{"cell_type":"markdown","source":"## Train the model","metadata":{}},{"cell_type":"code","source":"import xgboost as xgb\nimport lightgbm as lgb\nfrom sklearn.multioutput import MultiOutputRegressor\nfrom sklearn.metrics import mean_squared_error, r2_score\n","metadata":{"execution":{"iopub.status.busy":"2023-04-12T10:32:41.203806Z","iopub.execute_input":"2023-04-12T10:32:41.204895Z","iopub.status.idle":"2023-04-12T10:32:42.901400Z","shell.execute_reply.started":"2023-04-12T10:32:41.204840Z","shell.execute_reply":"2023-04-12T10:32:42.899892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport xgboost as xgb\nfrom sklearn.model_selection import RandomizedSearchCV, train_test_split\nfrom scipy.stats import uniform, randint\nfrom sklearn.metrics import average_precision_score, make_scorer\n\nbest_params_ = {'estimator__colsample_bytree': 0.5282057895135501, \n 'estimator__learning_rate': 0.22659963168004743, \n 'estimator__max_depth': 8, \n 'estimator__min_child_weight': 3.1233911067827616, \n 'estimator__n_estimators': 291, \n 'estimator__subsample': 0.9961057796456088}","metadata":{"execution":{"iopub.status.busy":"2023-04-12T10:32:46.380521Z","iopub.execute_input":"2023-04-12T10:32:46.380946Z","iopub.status.idle":"2023-04-12T10:32:46.388466Z","shell.execute_reply.started":"2023-04-12T10:32:46.380910Z","shell.execute_reply":"2023-04-12T10:32:46.386798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_params_ = {kk: v for k, v in best_params_.items() for kk in k.split('__')}; del best_params_['estimator']","metadata":{"execution":{"iopub.status.busy":"2023-04-12T10:32:53.144092Z","iopub.execute_input":"2023-04-12T10:32:53.144526Z","iopub.status.idle":"2023-04-12T10:32:53.151133Z","shell.execute_reply.started":"2023-04-12T10:32:53.144490Z","shell.execute_reply":"2023-04-12T10:32:53.149967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.base import clone\n\ndef custom_average_precision(y_true, y_pred):\n    score = average_precision_score(y_true, y_pred)\n    return 'average_precision', score, True\n\nclass LGBMMultiOutputRegressor(MultiOutputRegressor):\n    def fit(self, X, y, eval_set=None, **fit_params):\n        self.estimators_ = [clone(self.estimator) for _ in range(y.shape[1])]\n        \n        for i, estimator in enumerate(self.estimators_):\n            if eval_set:\n                fit_params['eval_set'] = [(eval_set[0], eval_set[1][:, i])]\n            estimator.fit(X, y[:, i], **fit_params)\n        \n        return self","metadata":{"execution":{"iopub.status.busy":"2023-04-12T10:33:01.586005Z","iopub.execute_input":"2023-04-12T10:33:01.586466Z","iopub.status.idle":"2023-04-12T10:33:01.596130Z","shell.execute_reply.started":"2023-04-12T10:33:01.586423Z","shell.execute_reply":"2023-04-12T10:33:01.594740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GroupKFold\n\nN_FOLDS=5\nkfold = GroupKFold(N_FOLDS)\ngroup_var = train.Subject\ngroups=kfold.split(train, groups=group_var)\nregs=[]\ncvs=[]\nfor fold, (tr_idx,te_idx ) in enumerate(tqdm(groups, total=N_FOLDS, desc=\"Folds\")):\n    tr_idx=pd.Series(tr_idx).sample(n=2000000,random_state=42).values #2000000\n    \n    # Create a base XGBoost regressor with the common parameters\n    base_regressor = lgb.LGBMRegressor(**best_params_)\n\n    # Wrap the base regressor with the MultiOutputRegressor\n    multioutput_regressor = LGBMMultiOutputRegressor(base_regressor)\n\n    x_tr,y_tr=train.loc[tr_idx,cols].to_numpy(),train.loc[tr_idx,pcols].to_numpy()\n    x_te,y_te=train.loc[te_idx,cols].to_numpy(),train.loc[te_idx,pcols].to_numpy()\n\n    multioutput_regressor.fit(\n    x_tr,y_tr,\n    eval_set=(x_te,y_te),\n    eval_metric=custom_average_precision,\n    early_stopping_rounds=25\n    )\n    regs.append(multioutput_regressor)\n    cv=metrics.average_precision_score(y_te, multioutput_regressor.predict(x_te).clip(0.0,1.0))\n    cvs.append(cv)\nprint(cvs)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-04-12T10:33:13.548847Z","iopub.execute_input":"2023-04-12T10:33:13.549331Z","iopub.status.idle":"2023-04-12T10:48:10.272447Z","shell.execute_reply.started":"2023-04-12T10:33:13.549281Z","shell.execute_reply":"2023-04-12T10:48:10.270729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict for test","metadata":{}},{"cell_type":"code","source":"sub['t'] = 0\nsubmission = []\nfor f in test:\n    df = pd.read_csv(f)\n    df.set_index('Time', drop=True, inplace=True)\n\n    df['Id'] = f.split('/')[-1].split('.')[0]\n#     df = df.fillna(0).reset_index(drop=True)\n    df['Time_frac']=(df.index/df.index.max()).values#currently the index of data is actually \"Time\"\n    df = pd.merge(df, tasks[['Id','t_kmeans']], how='left', on='Id').fillna(-1)\n#     df = pd.merge(df, subjects[['Id','s_kmeans']], how='left', on='Id').fillna(-1)\n    df = pd.merge(df, metadata_complex[['Id','Subject']+['Visit','Test','Medication','s_kmeans']], how='left', on='Id').fillna(-1)\n    df_feats = fc.calculate(df, return_df=True, include_final_window=True, approve_sparsity=True, window_idx=\"begin\")\n    df = df.merge(df_feats, how=\"left\", left_index=True, right_index=True)\n    df.fillna(method=\"ffill\", inplace=True)\n#     res = pd.DataFrame(np.round(reg.predict(df[cols]).clip(0.0,1.0),3), columns=pcols)\n    \n    res_vals=[]\n    for i_fold in range(N_FOLDS):\n        res_val=np.round(regs[i_fold].predict(df[cols]).clip(0.0,1.0),3)\n        res_vals.append(np.expand_dims(res_val,axis=2))\n    res_vals=np.mean(np.concatenate(res_vals,axis=2),axis=2)\n    res = pd.DataFrame(res_vals, columns=pcols)\n    \n    df = pd.concat([df,res], axis=1)\n    df['Id'] = df['Id'].astype(str) + '_' + df.index.astype(str)\n    submission.append(df[scols])\nsubmission = pd.concat(submission)\nsubmission = pd.merge(sub[['Id']], submission, how='left', on='Id').fillna(0.0)\nsubmission[scols].to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-12T10:56:28.076146Z","iopub.execute_input":"2023-04-12T10:56:28.076693Z","iopub.status.idle":"2023-04-12T10:56:37.089546Z","shell.execute_reply.started":"2023-04-12T10:56:28.076646Z","shell.execute_reply":"2023-04-12T10:56:37.087847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2023-04-12T10:56:46.572252Z","iopub.execute_input":"2023-04-12T10:56:46.572719Z","iopub.status.idle":"2023-04-12T10:56:46.591871Z","shell.execute_reply.started":"2023-04-12T10:56:46.572674Z","shell.execute_reply":"2023-04-12T10:56:46.590764Z"},"trusted":true},"execution_count":null,"outputs":[]}]}