{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, gc\nimport numpy as np\nimport pandas as pd\nimport pickle\nfrom collections import deque\nfrom sklearn.linear_model import Ridge, LinearRegression\nfrom sklearn.cluster import MiniBatchKMeans\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nfrom sklearn.metrics import average_precision_score\nfrom lightgbm import LGBMRegressor","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":1.362781,"end_time":"2023-03-18T04:11:41.243800","exception":false,"start_time":"2023-03-18T04:11:39.881019","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:04:38.946991Z","iopub.execute_input":"2023-04-17T00:04:38.947453Z","iopub.status.idle":"2023-04-17T00:04:43.294107Z","shell.execute_reply.started":"2023-04-17T00:04:38.947412Z","shell.execute_reply":"2023-04-17T00:04:43.292636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## listup all data","metadata":{"papermill":{"duration":0.010402,"end_time":"2023-03-18T04:11:41.266255","exception":false,"start_time":"2023-03-18T04:11:41.255853","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_defog = os.listdir(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/train/defog\")\ntest_defog = os.listdir(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/test/defog\")\ntrain_tdcsfog = os.listdir(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/train/tdcsfog\")\ntest_tdcsfog = os.listdir(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/test/tdcsfog\")","metadata":{"papermill":{"duration":0.111312,"end_time":"2023-03-18T04:11:41.389306","exception":false,"start_time":"2023-03-18T04:11:41.277994","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:04:43.296911Z","iopub.execute_input":"2023-04-17T00:04:43.297456Z","iopub.status.idle":"2023-04-17T00:04:43.392434Z","shell.execute_reply.started":"2023-04-17T00:04:43.297399Z","shell.execute_reply":"2023-04-17T00:04:43.391212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are no duplicate users collected on the learning side","metadata":{"papermill":{"duration":0.011851,"end_time":"2023-03-18T04:11:41.411660","exception":false,"start_time":"2023-03-18T04:11:41.399809","status":"completed"},"tags":[]}},{"cell_type":"code","source":"set(train_defog) & set(train_tdcsfog)","metadata":{"papermill":{"duration":0.023268,"end_time":"2023-03-18T04:11:41.445461","exception":false,"start_time":"2023-03-18T04:11:41.422193","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:04:43.393668Z","iopub.execute_input":"2023-04-17T00:04:43.395201Z","iopub.status.idle":"2023-04-17T00:04:43.407081Z","shell.execute_reply.started":"2023-04-17T00:04:43.395136Z","shell.execute_reply":"2023-04-17T00:04:43.405782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Don't know the test side, so just in case I put in the process","metadata":{"papermill":{"duration":0.010379,"end_time":"2023-03-18T04:11:41.466436","exception":false,"start_time":"2023-03-18T04:11:41.456057","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if len(set(test_defog) & set(test_tdcsfog)) > 0: # is there?\n    test_tdcsfog = list(set(test_tdcsfog) - (set(test_defog) & set(test_tdcsfog)))","metadata":{"papermill":{"duration":0.021432,"end_time":"2023-03-18T04:11:41.498416","exception":false,"start_time":"2023-03-18T04:11:41.476984","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:04:43.411134Z","iopub.execute_input":"2023-04-17T00:04:43.412361Z","iopub.status.idle":"2023-04-17T00:04:43.419000Z","shell.execute_reply.started":"2023-04-17T00:04:43.412309Z","shell.execute_reply":"2023-04-17T00:04:43.417634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read Tasks and Meta Data","metadata":{"papermill":{"duration":0.010803,"end_time":"2023-03-18T04:11:41.520322","exception":false,"start_time":"2023-03-18T04:11:41.509519","status":"completed"},"tags":[]}},{"cell_type":"code","source":"task = pd.read_csv(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/tasks.csv\")\ntask_map = {t:i+1 for i,t in enumerate(sorted(list(set(task.Task.values))))}\ntask[\"TaskId\"] = task.Task.apply(lambda x:task_map[x])\nevents = pd.read_csv(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/events.csv\")\nevents = events[events.Kinetic==1]\nsubjects = pd.read_csv(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/subjects.csv\")\nsubjects[\"SexId\"] = (subjects.Sex==\"M\").values.astype(np.uint8)\nsubjects = subjects.fillna(0)\nsubjects = subjects.drop([\"Sex\"], axis=1)\nlen(set(task_map.values()))","metadata":{"papermill":{"duration":0.062608,"end_time":"2023-03-18T04:11:41.593524","exception":false,"start_time":"2023-03-18T04:11:41.530916","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:04:43.420470Z","iopub.execute_input":"2023-04-17T00:04:43.421000Z","iopub.status.idle":"2023-04-17T00:04:43.500462Z","shell.execute_reply.started":"2023-04-17T00:04:43.420949Z","shell.execute_reply":"2023-04-17T00:04:43.499078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata = [pd.read_csv(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/defog_metadata.csv\"),\n            pd.read_csv(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/tdcsfog_metadata.csv\")]\nmetadata[0][\"MedicationId\"] = (metadata[0].Medication==\"on\").values.astype(np.uint8)\nmetadata[0] = metadata[0].drop([\"Medication\"], axis=1)\nmetadata[1][\"MedicationId\"] = (metadata[1].Medication==\"on\").values.astype(np.uint8)\nmetadata[1] = metadata[1].drop([\"Medication\"], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T00:04:43.502655Z","iopub.execute_input":"2023-04-17T00:04:43.503593Z","iopub.status.idle":"2023-04-17T00:04:43.529972Z","shell.execute_reply.started":"2023-04-17T00:04:43.503540Z","shell.execute_reply":"2023-04-17T00:04:43.528241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ext_columns = list(subjects.columns)[1:]","metadata":{"execution":{"iopub.status.busy":"2023-04-17T00:04:43.531410Z","iopub.execute_input":"2023-04-17T00:04:43.532165Z","iopub.status.idle":"2023-04-17T00:04:43.538276Z","shell.execute_reply.started":"2023-04-17T00:04:43.532119Z","shell.execute_reply":"2023-04-17T00:04:43.536892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## function to read train/test csv file and marge to task","metadata":{"papermill":{"duration":0.010661,"end_time":"2023-03-18T04:11:41.615795","exception":false,"start_time":"2023-03-18T04:11:41.605134","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def read_csv_with_task(csv):\n    global task, events, defog_metadata, tdcsfog_metadata\n    fn = csv.split(\"/\")[-1]\n    idf = fn.split(\".\")[0]\n    tdf = task[task.Id==idf]\n    edf = events[events.Id==idf]\n    df = pd.read_csv(csv)\n    taskids = np.zeros(len(df), dtype=np.uint8)\n    for b,e,t in zip(tdf.Begin,tdf.End,tdf.TaskId):\n        taskids[int(b):int(e)] = t\n    for b,e,t in zip(edf.Init,edf.Completion,edf.Type):\n        if t==\"Turn\":\n            taskids[int(b):int(e)] = taskids[int(b):int(e)] + 32\n        else:\n            taskids[int(b):int(e)] = taskids[int(b):int(e)] + 64\n    df[\"TaskId\"] = taskids\n    met = metadata[0 if \"defog\" in csv else 1]\n    subId = met[met.Id==idf].values.flatten()[1]\n    sub = subjects[subjects.Subject==subId].mean().values.astype(np.uint8)\n    for i,c in enumerate(ext_columns):\n        df[c] = sub[i]\n    return df","metadata":{"papermill":{"duration":0.025351,"end_time":"2023-03-18T04:11:41.651985","exception":false,"start_time":"2023-03-18T04:11:41.626634","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:04:43.539947Z","iopub.execute_input":"2023-04-17T00:04:43.541001Z","iopub.status.idle":"2023-04-17T00:04:43.555796Z","shell.execute_reply.started":"2023-04-17T00:04:43.540937Z","shell.execute_reply":"2023-04-17T00:04:43.554418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pre-Feature Engineering Learning Columns and Targets","metadata":{"papermill":{"duration":0.010607,"end_time":"2023-03-18T04:11:41.673373","exception":false,"start_time":"2023-03-18T04:11:41.662766","status":"completed"},"tags":[]}},{"cell_type":"code","source":"target_cols = [\"StartHesitation\",\"Turn\",\"Walking\"]\ntrain_cols = [\"Time\",\"AccV\",\"AccML\",\"AccAP\",\"TaskId\"]","metadata":{"papermill":{"duration":0.021245,"end_time":"2023-03-18T04:11:41.705779","exception":false,"start_time":"2023-03-18T04:11:41.684534","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:04:43.557720Z","iopub.execute_input":"2023-04-17T00:04:43.558081Z","iopub.status.idle":"2023-04-17T00:04:43.569448Z","shell.execute_reply.started":"2023-04-17T00:04:43.558047Z","shell.execute_reply":"2023-04-17T00:04:43.568186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering\n\nfunction to increase learning columns","metadata":{"papermill":{"duration":0.010431,"end_time":"2023-03-18T04:11:41.727031","exception":false,"start_time":"2023-03-18T04:11:41.716600","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def feature_engineering(val, clfs, target=None):\n    # Cluster and Dimensional mapping analysis for each data\n    if clfs[0] is None:\n        clfs[0] = MiniBatchKMeans(n_clusters=8, random_state=0, init=\"random\").fit(val[:,1:4])\n    km = clfs[0].predict(val[:,1:4])\n    km_oh = np.zeros((val.shape[0],8), dtype=np.uint8) # discrete value change to One-hot\n    for i in range(8):\n        idx = np.where(km==0)[0]\n        km_oh[idx,i] = 1\n    if clfs[1] is None:\n        clfs[1] = TruncatedSVD(n_components=2, n_iter=10, random_state=0).fit(val[:,1:4])\n    svd = clfs[1].transform(val[:,1:4])\n    # Per-user statistics\n    print(\"Per-user statistics\")\n    cp = 0\n    sp = 0\n    usrm = np.zeros((val.shape[0], 5*val.shape[1]-10), dtype=np.float16)\n    for i in range(val.shape[0]):\n        if cp > val[i,0]:\n            for t in range(val.shape[1]-2):\n                usrm[sp:i,5*t] = np.mean(val[sp:i,t+1])\n                usrm[sp:i,5*t+1] = np.std(val[sp:i,t+1])\n                usrm[sp:i,5*t+2] = np.max(val[sp:i,t+1])\n                usrm[sp:i,5*t+3] = np.min(val[sp:i,t+1])\n                usrm[sp:i,5*t+4] = (i-sp)/val.shape[0]\n            sp = i\n        cp = val[i,0]\n    for t in range(val.shape[1]-2):\n        usrm[sp:,5*t] = np.mean(val[sp:,t+1])\n        usrm[sp:,5*t+1] = np.std(val[sp:,t+1])\n        usrm[sp:,5*t+2] = np.max(val[sp:,t+1])\n        usrm[sp:,5*t+3] = np.min(val[sp:,t+1])\n        usrm[sp:,5*t+4] = (val.shape[0]-sp)/val.shape[0]\n    iskinetic = np.stack([(val[:,4]>=32).astype(np.uint8), (val[:,4]>=64).astype(np.uint8)]).transpose((1,0))\n    # Cluster and Dimensional mapping analysis for each user/task\n    print(\"Cluster and Dimensional mapping analysis for each user/task\")\n    if clfs[2] is None:\n        clfs[2] = MiniBatchKMeans(n_clusters=8, random_state=0, init=\"random\").fit(usrm)\n    kmu = clfs[2].predict(usrm)\n    kmu_oh = np.zeros((val.shape[0],8), dtype=np.uint8) # discrete value change to One-hot\n    for i in range(8):\n        idx = np.where(kmu==0)[0]\n        kmu_oh[idx,i] = 1\n    del kmu\n    gc.collect()\n    if clfs[3] is None:\n        clfs[3] = TruncatedSVD(n_components=2, n_iter=10, random_state=0).fit(usrm)\n    svdu = clfs[3].transform(usrm)\n    gc.collect()\n    # Merge waypoints\n    marged = np.hstack([val[:,1:4],km_oh,svd])\n    # Moving average and variance within the same user\n    print(\"Moving average and variance within the same user/task\")\n    wnd = np.zeros((val.shape[0],52), dtype=np.float16)\n    cp = val[0,0]\n    window = deque([marged[0,:15]] * 5)\n    for i in range(val.shape[0]):\n        if cp > val[i,0]:\n            window = deque([marged[i,:15]] * 5)\n        else:\n            window.popleft()\n            window.append(marged[i,:15])\n        cp = val[i,0]\n        wnd[i] = np.hstack([np.mean(window, axis=0),np.std(window, axis=0),np.min(window, axis=0),np.max(window, axis=0)])\n    # Analyze the entire merge data\n    print(\"Analyze the entire merge data\")\n    usrv = np.hstack([svd,svdu])\n    if clfs[4] is None:\n        clfs[4] = [LinearRegression().fit(usrv, target[:,i]) for i in range(3)]\n    reg = np.stack([clfs[4][i].predict(usrv) for i in range(3)]).transpose((1,0))\n    del usrv\n    gc.collect()\n    if clfs[5] is None:\n        clfs[5] = TruncatedSVD(n_components=2, n_iter=10, random_state=0).fit(marged)\n    svdm = clfs[5].transform(marged)\n    # Marge all\n    return np.hstack([marged,wnd,reg,svdm,usrm,kmu_oh,svdu,iskinetic])","metadata":{"papermill":{"duration":0.052851,"end_time":"2023-03-18T04:11:41.790510","exception":false,"start_time":"2023-03-18T04:11:41.737659","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:04:43.575609Z","iopub.execute_input":"2023-04-17T00:04:43.576023Z","iopub.status.idle":"2023-04-17T00:04:43.607074Z","shell.execute_reply.started":"2023-04-17T00:04:43.575983Z","shell.execute_reply":"2023-04-17T00:04:43.605550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training and Prediction Function","metadata":{"papermill":{"duration":0.010444,"end_time":"2023-03-18T04:11:41.811673","exception":false,"start_time":"2023-03-18T04:11:41.801229","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#from sklearn.tree import DecisionTreeRegressor\ndef get_regressor(lgb=False):\n    return Ridge(max_iter=1000,random_state=0) if not lgb else LGBMRegressor(random_state=0)\n\ndef training(val, target, lgb=False):\n    return get_regressor(lgb).fit(val, target)\n\ndef predict(clfs, val):\n    return clfs.predict(val)","metadata":{"papermill":{"duration":0.02404,"end_time":"2023-03-18T04:11:41.846528","exception":false,"start_time":"2023-03-18T04:11:41.822488","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:04:43.611480Z","iopub.execute_input":"2023-04-17T00:04:43.612565Z","iopub.status.idle":"2023-04-17T00:04:43.627392Z","shell.execute_reply.started":"2023-04-17T00:04:43.612516Z","shell.execute_reply":"2023-04-17T00:04:43.625803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get training datas","metadata":{"papermill":{"duration":0.010372,"end_time":"2023-03-18T04:11:41.867593","exception":false,"start_time":"2023-03-18T04:11:41.857221","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_dfs = [read_csv_with_task(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/train/defog/\"+i)[train_cols+target_cols] for i in train_defog]\ntrain_val = [i[train_cols].values for i in train_dfs]\ntrain_tgt = [i[target_cols].values for i in train_dfs]\ndel train_dfs\ngc.collect()","metadata":{"papermill":{"duration":35.797392,"end_time":"2023-03-18T04:12:17.675637","exception":false,"start_time":"2023-03-18T04:11:41.878245","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:04:43.629390Z","iopub.execute_input":"2023-04-17T00:04:43.629941Z","iopub.status.idle":"2023-04-17T00:05:12.986957Z","shell.execute_reply.started":"2023-04-17T00:04:43.629888Z","shell.execute_reply":"2023-04-17T00:05:12.985847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_val = np.vstack(train_val)\ntrain_tgt = np.vstack(train_tgt)\ngc.collect()","metadata":{"papermill":{"duration":0.571988,"end_time":"2023-03-18T04:12:18.258267","exception":false,"start_time":"2023-03-18T04:12:17.686279","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:05:12.988258Z","iopub.execute_input":"2023-04-17T00:05:12.988602Z","iopub.status.idle":"2023-04-17T00:05:13.587244Z","shell.execute_reply.started":"2023-04-17T00:05:12.988570Z","shell.execute_reply":"2023-04-17T00:05:13.585827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog_trans = [None, None, None, None, None ,None]\ntrain_val = feature_engineering(train_val, defog_trans, target=train_tgt)","metadata":{"papermill":{"duration":622.398743,"end_time":"2023-03-18T04:22:40.668052","exception":false,"start_time":"2023-03-18T04:12:18.269309","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:05:13.591149Z","iopub.execute_input":"2023-04-17T00:05:13.591502Z","iopub.status.idle":"2023-04-17T00:33:48.150248Z","shell.execute_reply.started":"2023-04-17T00:05:13.591467Z","shell.execute_reply":"2023-04-17T00:33:48.144951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Learning","metadata":{"papermill":{"duration":0.011212,"end_time":"2023-03-18T04:22:40.691442","exception":false,"start_time":"2023-03-18T04:22:40.680230","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_val = train_val.astype(np.float16) # reduce memory\ngc.collect()\ndefog_clf = [training(train_val,train_tgt[:,i]) for i in range(len(target_cols))]","metadata":{"papermill":{"duration":419.844363,"end_time":"2023-03-18T04:29:40.547215","exception":false,"start_time":"2023-03-18T04:22:40.702852","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:33:48.160135Z","iopub.execute_input":"2023-04-17T00:33:48.168182Z","iopub.status.idle":"2023-04-17T00:35:19.330082Z","shell.execute_reply.started":"2023-04-17T00:33:48.167849Z","shell.execute_reply":"2023-04-17T00:35:19.328491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_val, train_tgt, train_defog\ngc.collect()","metadata":{"papermill":{"duration":0.256561,"end_time":"2023-03-18T04:29:40.833175","exception":false,"start_time":"2023-03-18T04:29:40.576614","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:35:19.331949Z","iopub.execute_input":"2023-04-17T00:35:19.333261Z","iopub.status.idle":"2023-04-17T00:35:19.482069Z","shell.execute_reply.started":"2023-04-17T00:35:19.333200Z","shell.execute_reply":"2023-04-17T00:35:19.480346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get training datas","metadata":{"papermill":{"duration":0.019478,"end_time":"2023-03-18T04:29:40.866420","exception":false,"start_time":"2023-03-18T04:29:40.846942","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_dfs = [read_csv_with_task(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/train/tdcsfog/\"+i)[train_cols+target_cols] for i in train_tdcsfog]\ntrain_val = [i[train_cols].values for i in train_dfs]\ntrain_tgt = [i[target_cols].values for i in train_dfs]\ndel train_dfs\ngc.collect()","metadata":{"papermill":{"duration":45.023958,"end_time":"2023-03-18T04:30:25.905816","exception":false,"start_time":"2023-03-18T04:29:40.881858","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:35:19.486441Z","iopub.execute_input":"2023-04-17T00:35:19.486914Z","iopub.status.idle":"2023-04-17T00:35:44.632612Z","shell.execute_reply.started":"2023-04-17T00:35:19.486871Z","shell.execute_reply":"2023-04-17T00:35:44.631251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_val = np.vstack(train_val)\ntrain_tgt = np.vstack(train_tgt)\ngc.collect()","metadata":{"papermill":{"duration":0.368661,"end_time":"2023-03-18T04:30:26.286019","exception":false,"start_time":"2023-03-18T04:30:25.917358","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:35:44.634583Z","iopub.execute_input":"2023-04-17T00:35:44.635083Z","iopub.status.idle":"2023-04-17T00:35:44.950276Z","shell.execute_reply.started":"2023-04-17T00:35:44.635041Z","shell.execute_reply":"2023-04-17T00:35:44.948835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdcsfog_trans = [None, None, None, None, None, None]\ntrain_val = feature_engineering(train_val, tdcsfog_trans, target=train_tgt)","metadata":{"papermill":{"duration":326.609361,"end_time":"2023-03-18T04:35:52.907746","exception":false,"start_time":"2023-03-18T04:30:26.298385","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:35:44.951969Z","iopub.execute_input":"2023-04-17T00:35:44.952332Z","iopub.status.idle":"2023-04-17T00:50:19.231460Z","shell.execute_reply.started":"2023-04-17T00:35:44.952293Z","shell.execute_reply":"2023-04-17T00:50:19.230184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Learning","metadata":{"papermill":{"duration":0.01261,"end_time":"2023-03-18T04:35:52.933397","exception":false,"start_time":"2023-03-18T04:35:52.920787","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_val = train_val.astype(np.float16) # reduce memory\ngc.collect()\ntdcsfog_clf = [training(train_val,train_tgt[:,0]),\n               training(train_val,train_tgt[:,1], lgb=True),\n               training(train_val,train_tgt[:,2])]","metadata":{"papermill":{"duration":198.858414,"end_time":"2023-03-18T04:39:11.804737","exception":false,"start_time":"2023-03-18T04:35:52.946323","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:50:19.232988Z","iopub.execute_input":"2023-04-17T00:50:19.233338Z","iopub.status.idle":"2023-04-17T00:52:56.551164Z","shell.execute_reply.started":"2023-04-17T00:50:19.233302Z","shell.execute_reply":"2023-04-17T00:52:56.548389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_val, train_tgt, train_tdcsfog\ngc.collect()","metadata":{"papermill":{"duration":0.208209,"end_time":"2023-03-18T04:39:12.045445","exception":false,"start_time":"2023-03-18T04:39:11.837236","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:52:56.552845Z","iopub.execute_input":"2023-04-17T00:52:56.553253Z","iopub.status.idle":"2023-04-17T00:52:56.709192Z","shell.execute_reply.started":"2023-04-17T00:52:56.553213Z","shell.execute_reply":"2023-04-17T00:52:56.707801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get Prediction Datas","metadata":{"papermill":{"duration":0.012418,"end_time":"2023-03-18T04:39:12.070667","exception":false,"start_time":"2023-03-18T04:39:12.058249","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_dfs = [read_csv_with_task(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/test/defog/\"+i)[train_cols] for i in test_defog]\ntest_val = [i.values for i in test_dfs]\ngc.collect()","metadata":{"papermill":{"duration":0.607926,"end_time":"2023-03-18T04:39:12.691058","exception":false,"start_time":"2023-03-18T04:39:12.083132","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:52:56.710735Z","iopub.execute_input":"2023-04-17T00:52:56.711121Z","iopub.status.idle":"2023-04-17T00:52:57.302448Z","shell.execute_reply.started":"2023-04-17T00:52:56.711085Z","shell.execute_reply":"2023-04-17T00:52:57.301059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_val = np.vstack(test_val)\ntest_val = feature_engineering(test_val, defog_trans)","metadata":{"papermill":{"duration":1.728185,"end_time":"2023-03-18T04:39:14.431828","exception":false,"start_time":"2023-03-18T04:39:12.703643","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:52:57.304397Z","iopub.execute_input":"2023-04-17T00:52:57.304888Z","iopub.status.idle":"2023-04-17T00:53:26.192832Z","shell.execute_reply.started":"2023-04-17T00:52:57.304835Z","shell.execute_reply":"2023-04-17T00:53:26.191604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Run Predict","metadata":{"papermill":{"duration":0.018418,"end_time":"2023-03-18T04:39:14.463400","exception":false,"start_time":"2023-03-18T04:39:14.444982","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_val = test_val.astype(np.float16) # reduce memory\ngc.collect()\ntest_defog_preds = [np.clip(predict(c, test_val), 0, 1) for i,c in enumerate(defog_clf)]","metadata":{"papermill":{"duration":6.171498,"end_time":"2023-03-18T04:39:20.649990","exception":false,"start_time":"2023-03-18T04:39:14.478492","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:53:26.194211Z","iopub.execute_input":"2023-04-17T00:53:26.194536Z","iopub.status.idle":"2023-04-17T00:53:27.363870Z","shell.execute_reply.started":"2023-04-17T00:53:26.194504Z","shell.execute_reply":"2023-04-17T00:53:27.361873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog_ids = []\nfor f,d in zip(test_defog,test_dfs):\n    fid = f.split(\".\")[0]\n    for t in d.Time.values:\n        sid = f\"{fid}_{t}\"\n        defog_ids.append(sid)","metadata":{"papermill":{"duration":0.232727,"end_time":"2023-03-18T04:39:20.917068","exception":false,"start_time":"2023-03-18T04:39:20.684341","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:53:27.366244Z","iopub.execute_input":"2023-04-17T00:53:27.366723Z","iopub.status.idle":"2023-04-17T00:53:27.596329Z","shell.execute_reply.started":"2023-04-17T00:53:27.366674Z","shell.execute_reply":"2023-04-17T00:53:27.594959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_defog, test_dfs, test_val\ngc.collect()","metadata":{"papermill":{"duration":0.153462,"end_time":"2023-03-18T04:39:21.083439","exception":false,"start_time":"2023-03-18T04:39:20.929977","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:53:27.598030Z","iopub.execute_input":"2023-04-17T00:53:27.598397Z","iopub.status.idle":"2023-04-17T00:53:27.758979Z","shell.execute_reply.started":"2023-04-17T00:53:27.598359Z","shell.execute_reply":"2023-04-17T00:53:27.757469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get Prediction Datas","metadata":{"papermill":{"duration":0.012697,"end_time":"2023-03-18T04:39:21.109069","exception":false,"start_time":"2023-03-18T04:39:21.096372","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_dfs = [read_csv_with_task(\"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/test/tdcsfog/\"+i)[train_cols] for i in test_tdcsfog]\ntest_val = [i.values for i in test_dfs]\ngc.collect()","metadata":{"papermill":{"duration":0.181847,"end_time":"2023-03-18T04:39:21.304753","exception":false,"start_time":"2023-03-18T04:39:21.122906","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:53:27.760629Z","iopub.execute_input":"2023-04-17T00:53:27.761026Z","iopub.status.idle":"2023-04-17T00:53:27.931318Z","shell.execute_reply.started":"2023-04-17T00:53:27.760991Z","shell.execute_reply":"2023-04-17T00:53:27.930134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_val = np.vstack(test_val)\ntest_val = feature_engineering(test_val, tdcsfog_trans)","metadata":{"papermill":{"duration":0.570917,"end_time":"2023-03-18T04:39:21.888766","exception":false,"start_time":"2023-03-18T04:39:21.317849","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:53:27.936972Z","iopub.execute_input":"2023-04-17T00:53:27.937362Z","iopub.status.idle":"2023-04-17T00:53:28.925584Z","shell.execute_reply.started":"2023-04-17T00:53:27.937326Z","shell.execute_reply":"2023-04-17T00:53:28.924335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Run Predict","metadata":{"papermill":{"duration":0.013145,"end_time":"2023-03-18T04:39:21.915072","exception":false,"start_time":"2023-03-18T04:39:21.901927","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_val = test_val.astype(np.float16) # reduce memory\ngc.collect()\ntest_tdcsfog_preds = [np.clip(predict(c, test_val), 0, 1) for i,c in enumerate(tdcsfog_clf)]","metadata":{"papermill":{"duration":0.325548,"end_time":"2023-03-18T04:39:22.253760","exception":false,"start_time":"2023-03-18T04:39:21.928212","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:53:28.927309Z","iopub.execute_input":"2023-04-17T00:53:28.927733Z","iopub.status.idle":"2023-04-17T00:53:29.151059Z","shell.execute_reply.started":"2023-04-17T00:53:28.927697Z","shell.execute_reply":"2023-04-17T00:53:29.148932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdcsfog_ids = []\nfor f,d in zip(test_tdcsfog,test_dfs):\n    fid = f.split(\".\")[0]\n    for t in d.Time.values:\n        sid = f\"{fid}_{t}\"\n        tdcsfog_ids.append(sid)","metadata":{"papermill":{"duration":0.06764,"end_time":"2023-03-18T04:39:22.356623","exception":false,"start_time":"2023-03-18T04:39:22.288983","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:53:29.154816Z","iopub.execute_input":"2023-04-17T00:53:29.155221Z","iopub.status.idle":"2023-04-17T00:53:29.186146Z","shell.execute_reply.started":"2023-04-17T00:53:29.155178Z","shell.execute_reply":"2023-04-17T00:53:29.183942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_tdcsfog, test_dfs, test_val\ngc.collect()","metadata":{"papermill":{"duration":0.159983,"end_time":"2023-03-18T04:39:22.534924","exception":false,"start_time":"2023-03-18T04:39:22.374941","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:53:29.189882Z","iopub.execute_input":"2023-04-17T00:53:29.190265Z","iopub.status.idle":"2023-04-17T00:53:29.389639Z","shell.execute_reply.started":"2023-04-17T00:53:29.190229Z","shell.execute_reply":"2023-04-17T00:53:29.388245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make Submission File","metadata":{"papermill":{"duration":0.012815,"end_time":"2023-03-18T04:39:22.561454","exception":false,"start_time":"2023-03-18T04:39:22.548639","status":"completed"},"tags":[]}},{"cell_type":"code","source":"all_ids = defog_ids + tdcsfog_ids\nall_starts = list(test_defog_preds[0]) + list(test_tdcsfog_preds[0])\nall_turns = list(test_defog_preds[1]) + list(test_tdcsfog_preds[1])\nall_walkings = list(test_defog_preds[2]) + list(test_tdcsfog_preds[2])","metadata":{"papermill":{"duration":0.090646,"end_time":"2023-03-18T04:39:22.665288","exception":false,"start_time":"2023-03-18T04:39:22.574642","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:53:29.391742Z","iopub.execute_input":"2023-04-17T00:53:29.392246Z","iopub.status.idle":"2023-04-17T00:53:29.467825Z","shell.execute_reply.started":"2023-04-17T00:53:29.392195Z","shell.execute_reply":"2023-04-17T00:53:29.466321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({\"Id\":all_ids,\"StartHesitation\":all_starts,\"Turn\":all_turns,\"Walking\":all_walkings})\ndf","metadata":{"papermill":{"duration":0.265093,"end_time":"2023-03-18T04:39:22.943731","exception":false,"start_time":"2023-03-18T04:39:22.678638","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:53:29.469422Z","iopub.execute_input":"2023-04-17T00:53:29.469810Z","iopub.status.idle":"2023-04-17T00:53:29.703861Z","shell.execute_reply.started":"2023-04-17T00:53:29.469773Z","shell.execute_reply":"2023-04-17T00:53:29.702539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(\"submission.csv\", index=False)","metadata":{"papermill":{"duration":0.685153,"end_time":"2023-03-18T04:39:23.642534","exception":false,"start_time":"2023-03-18T04:39:22.957381","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-17T00:53:29.705461Z","iopub.execute_input":"2023-04-17T00:53:29.705884Z","iopub.status.idle":"2023-04-17T00:53:30.407007Z","shell.execute_reply.started":"2023-04-17T00:53:29.705845Z","shell.execute_reply":"2023-04-17T00:53:30.405685Z"},"trusted":true},"execution_count":null,"outputs":[]}]}