{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":41880,"databundleVersionId":5677426,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n \n

Parkinson's Freezing of Gait Prediction

\n
","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nimport plotly.express as px\nimport cufflinks as cf\nfrom plotly.offline import download_plotlyjs,init_notebook_mode,iplot\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler,MinMaxScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom catboost import Pool, CatBoostClassifier, cv\nfrom xgboost import XGBClassifier\nfrom lightgbm import LGBMClassifier\nfrom sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV, cross_validate, StratifiedKFold, RepeatedStratifiedKFold\nfrom sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix\nfrom sklearn.metrics import average_precision_score,classification_report\nfrom sklearn.ensemble import RandomForestClassifier\nimport optuna\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-22T20:00:23.310129Z","iopub.execute_input":"2023-11-22T20:00:23.310532Z","iopub.status.idle":"2023-11-22T20:00:23.326533Z","shell.execute_reply.started":"2023-11-22T20:00:23.310504Z","shell.execute_reply":"2023-11-22T20:00:23.325401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

\n đź“ť Table of Contents\n

\n
\n
\n 1. Dataset Overview
\n 2. EDA
\n 3. Simple Data Preprocessing
\n 4. Training Models
\n 4.1 Logistic Regression Model
\n 4.2 Baseline CatBoost Model
\n 4.3 Baseline LGBM Model
\n\n","metadata":{}},{"cell_type":"markdown","source":"
\n

\n đź“š Dataset Overview\n

\n
","metadata":{}},{"cell_type":"markdown","source":"**We have two major datasets here - tdcsfog and defog. Lets see the description and content of these datasets.**","metadata":{}},{"cell_type":"code","source":"tdcsfog_003f117e14 = pd.read_csv('/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/train/tdcsfog/003f117e14.csv')\ntdcsfog_003f117e14.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:08:41.665487Z","iopub.execute_input":"2023-11-22T17:08:41.666096Z","iopub.status.idle":"2023-11-22T17:08:41.718041Z","shell.execute_reply.started":"2023-11-22T17:08:41.666062Z","shell.execute_reply":"2023-11-22T17:08:41.717169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* AccV: Vertical\n* AccML: Mediolateral\n* AccAP: Anteroposterior ","metadata":{}},{"cell_type":"code","source":"tdcsfog_003f117e14.info()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:08:41.719175Z","iopub.execute_input":"2023-11-22T17:08:41.719533Z","iopub.status.idle":"2023-11-22T17:08:41.740649Z","shell.execute_reply.started":"2023-11-22T17:08:41.719496Z","shell.execute_reply":"2023-11-22T17:08:41.739765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdcsfog_003f117e14.describe()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:08:41.742326Z","iopub.execute_input":"2023-11-22T17:08:41.742587Z","iopub.status.idle":"2023-11-22T17:08:41.773238Z","shell.execute_reply.started":"2023-11-22T17:08:41.742564Z","shell.execute_reply":"2023-11-22T17:08:41.772402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog_02ea782681 = pd.read_csv('/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/train/defog/02ea782681.csv')\ndefog_02ea782681.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:08:59.971044Z","iopub.execute_input":"2023-11-22T17:08:59.971722Z","iopub.status.idle":"2023-11-22T17:09:00.145620Z","shell.execute_reply.started":"2023-11-22T17:08:59.971688Z","shell.execute_reply":"2023-11-22T17:09:00.144491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog_02ea782681.info()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:09:00.463211Z","iopub.execute_input":"2023-11-22T17:09:00.463554Z","iopub.status.idle":"2023-11-22T17:09:00.475011Z","shell.execute_reply.started":"2023-11-22T17:09:00.463527Z","shell.execute_reply":"2023-11-22T17:09:00.474057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog_02ea782681.describe()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:09:00.979751Z","iopub.execute_input":"2023-11-22T17:09:00.980798Z","iopub.status.idle":"2023-11-22T17:09:01.030972Z","shell.execute_reply.started":"2023-11-22T17:09:00.980762Z","shell.execute_reply":"2023-11-22T17:09:01.030045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#combining all tdcsfog '.csv' train files\ntdcsfog_path= '/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/train/tdcsfog'\ntdcsfog_list= []\n\nfor file_name in os.listdir(tdcsfog_path):\n if file_name.endswith('.csv'):\n file_path= os.path.join(tdcsfog_path,file_name)\n df= pd.read_csv(file_path)\n df['Time']= df['Time']/(len(df)-1) \n tdcsfog_list.append(df)\n \ntdcsfog= pd.concat(tdcsfog_list,axis= 0)\ntdcsfog","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:09:01.489714Z","iopub.execute_input":"2023-11-22T17:09:01.490587Z","iopub.status.idle":"2023-11-22T17:09:21.906287Z","shell.execute_reply.started":"2023-11-22T17:09:01.490552Z","shell.execute_reply":"2023-11-22T17:09:21.905072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdcsfog.info()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:09:31.281696Z","iopub.execute_input":"2023-11-22T17:09:31.282376Z","iopub.status.idle":"2023-11-22T17:09:31.290949Z","shell.execute_reply.started":"2023-11-22T17:09:31.282339Z","shell.execute_reply":"2023-11-22T17:09:31.290090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdcsfog.describe()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:09:35.891874Z","iopub.execute_input":"2023-11-22T17:09:35.892219Z","iopub.status.idle":"2023-11-22T17:09:37.446212Z","shell.execute_reply.started":"2023-11-22T17:09:35.892195Z","shell.execute_reply":"2023-11-22T17:09:37.445243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

\n đź“Š Exploratory Data Analysis\n

\n
","metadata":{}},{"cell_type":"code","source":"#reducing memory usage of dataset\n\ndef reduce_memory_usage(df):\n \n init_mem = df.memory_usage().sum() / 1024**2\n print('Memory usage of dataframe is {:.2f} MB'.format(init_mem))\n \n for col in df.columns:\n col_type = df[col].dtype.name\n if ((col_type != 'datetime64[ns]') & (col_type != 'category')):\n if (col_type != 'object'):\n c_min = df[col].min()\n c_max = df[col].max()\n\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64)\n\n else:\n# if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n# df[col] = df[col].astype(np.float16)\n if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n pass\n else:\n df[col] = df[col].astype('category')\n mem_usg = df.memory_usage().sum() / 1024**2 \n print(\"Memory usage became: \",mem_usg,\" MB\")\n \n return df","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:12:51.768591Z","iopub.execute_input":"2023-11-22T17:12:51.769024Z","iopub.status.idle":"2023-11-22T17:12:51.780866Z","shell.execute_reply.started":"2023-11-22T17:12:51.768974Z","shell.execute_reply":"2023-11-22T17:12:51.779888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tdcsfog = reduce_memory_usage(tdcsfog)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:12:52.399977Z","iopub.execute_input":"2023-11-22T17:12:52.400826Z","iopub.status.idle":"2023-11-22T17:12:52.544361Z","shell.execute_reply.started":"2023-11-22T17:12:52.400790Z","shell.execute_reply":"2023-11-22T17:12:52.543421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(tdcsfog.corr(),annot= True,cmap='magma')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:12:55.211378Z","iopub.execute_input":"2023-11-22T17:12:55.211738Z","iopub.status.idle":"2023-11-22T17:12:56.740541Z","shell.execute_reply.started":"2023-11-22T17:12:55.211710Z","shell.execute_reply":"2023-11-22T17:12:56.739648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2= pd.DataFrame(np.concatenate([\n ['total entries'] * len(tdcsfog),\n ['Start Hesitation'] * int(tdcsfog['StartHesitation'].mean() * len(tdcsfog)),\n ['Turn'] * int(tdcsfog['Turn'].mean() * len(tdcsfog)),\n ['Walking'] * int(tdcsfog['Walking'].mean() * len(tdcsfog))]),\n columns= ['Number of 1s'] \n )\n\nsns.countplot(data= df2, x='Number of 1s')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:13:10.439317Z","iopub.execute_input":"2023-11-22T17:13:10.440105Z","iopub.status.idle":"2023-11-22T17:13:23.972327Z","shell.execute_reply.started":"2023-11-22T17:13:10.440072Z","shell.execute_reply":"2023-11-22T17:13:23.971453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Majority of target variables are 0.**","metadata":{}},{"cell_type":"code","source":"sns.pairplot(tdcsfog[['AccV','AccML','AccAP']])","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:15:24.043238Z","iopub.execute_input":"2023-11-22T17:15:24.043589Z","iopub.status.idle":"2023-11-22T17:18:12.541771Z","shell.execute_reply.started":"2023-11-22T17:15:24.043561Z","shell.execute_reply":"2023-11-22T17:18:12.540828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize = (10, 6))\n\nax.plot(tdcsfog['Time'], tdcsfog['StartHesitation'], label = 'StartHesitation')\nax.plot(tdcsfog['Time'], tdcsfog['Turn'], label = 'Turn')\nax.plot(tdcsfog['Time'], tdcsfog['Walking'], label = 'Walking')\n\nax.set_xlabel('Time')\nax.set_ylabel('Binary Status(0 or 1)')\nax.set_title('Relationship between Time and Movement Status')\n\nax.legend(loc='upper left',bbox_to_anchor=(1,0.5))","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:57:46.464196Z","iopub.execute_input":"2023-11-22T17:57:46.464495Z","iopub.status.idle":"2023-11-22T17:57:49.583247Z","shell.execute_reply.started":"2023-11-22T17:57:46.464470Z","shell.execute_reply":"2023-11-22T17:57:49.582339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**It can be seen that 'StartHesitation' has mostly occurred during specific times, mainly between 'Time' 0 to 0.1 whereas 'Walking' seems to have a positive correlation with 'Time' as it increases with 'Time'. 'Turn' seems to have the least correlation with 'Time' among the other 2 features with mostly concentrated below 'Time'= 0.5**","metadata":{}},{"cell_type":"code","source":"defog_path = '/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/train/defog'\n\ndefog_list = []\n\nfor file_name in os.listdir(defog_path):\n if file_name.endswith('.csv'):\n file_path = os.path.join(defog_path, file_name)\n file = pd.read_csv(file_path)\n file.Time = file.Time / (len(file) - 1)\n defog_list.append(file)\n\ndefog = pd.concat(defog_list, axis = 0)\n\ndefog.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:19:24.048947Z","iopub.execute_input":"2023-11-22T17:19:24.049810Z","iopub.status.idle":"2023-11-22T17:19:52.284329Z","shell.execute_reply.started":"2023-11-22T17:19:24.049777Z","shell.execute_reply":"2023-11-22T17:19:52.283372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog.info()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:19:52.286385Z","iopub.execute_input":"2023-11-22T17:19:52.287101Z","iopub.status.idle":"2023-11-22T17:19:52.296990Z","shell.execute_reply.started":"2023-11-22T17:19:52.287063Z","shell.execute_reply":"2023-11-22T17:19:52.295918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog= reduce_memory_usage(defog)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:20:17.422574Z","iopub.execute_input":"2023-11-22T17:20:17.423196Z","iopub.status.idle":"2023-11-22T17:20:17.971435Z","shell.execute_reply.started":"2023-11-22T17:20:17.423164Z","shell.execute_reply":"2023-11-22T17:20:17.970483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

\n 🧪 Simple Data Preprocessing\n

\n
\n

\n\n

","metadata":{}},{"cell_type":"code","source":"defog= defog[(defog['Valid']==1) & (defog['Task']==1)]\ndefog.dropna()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:20:21.271382Z","iopub.execute_input":"2023-11-22T17:20:21.272269Z","iopub.status.idle":"2023-11-22T17:20:21.657874Z","shell.execute_reply.started":"2023-11-22T17:20:21.272234Z","shell.execute_reply":"2023-11-22T17:20:21.656959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defog= defog.iloc[:,:7]","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:20:21.889389Z","iopub.execute_input":"2023-11-22T17:20:21.889706Z","iopub.status.idle":"2023-11-22T17:20:21.969026Z","shell.execute_reply.started":"2023-11-22T17:20:21.889679Z","shell.execute_reply":"2023-11-22T17:20:21.968111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged= pd.concat([tdcsfog,defog],axis=0)\nmerged","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:20:23.178510Z","iopub.execute_input":"2023-11-22T17:20:23.179199Z","iopub.status.idle":"2023-11-22T17:20:23.280913Z","shell.execute_reply.started":"2023-11-22T17:20:23.179165Z","shell.execute_reply":"2023-11-22T17:20:23.280036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_merged = merged.iloc[:,0:4] \nX = tdcsfog.iloc[:,0:4] \ny1 = merged['StartHesitation'] # target variable for StartHesitation\ny2 = merged['Turn'] # target variable for Turn\ny3 = tdcsfog['Walking'] # target variable for Walking\nX_merged.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:20:24.626219Z","iopub.execute_input":"2023-11-22T17:20:24.626579Z","iopub.status.idle":"2023-11-22T17:20:24.863592Z","shell.execute_reply.started":"2023-11-22T17:20:24.626550Z","shell.execute_reply":"2023-11-22T17:20:24.862721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y1.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:20:25.946730Z","iopub.execute_input":"2023-11-22T17:20:25.947406Z","iopub.status.idle":"2023-11-22T17:20:25.953973Z","shell.execute_reply.started":"2023-11-22T17:20:25.947372Z","shell.execute_reply":"2023-11-22T17:20:25.953059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**As majority of target variables are 0, we create 3 balanced datasets with equal number of 0s and 1s to get better results.**","metadata":{}},{"cell_type":"code","source":"y1_ones= np.where(y1==1)[0]\n\nn1_ones= (y1==1).sum()\ny1_zeros= np.random.choice(np.where(y1==0)[0],size= n1_ones,replace= False)\n\ny1_balanced_idx= np.sort(np.concatenate([y1_zeros,y1_ones]))\ny1_balanced_idx","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:20:34.482809Z","iopub.execute_input":"2023-11-22T17:20:34.483559Z","iopub.status.idle":"2023-11-22T17:20:34.978548Z","shell.execute_reply.started":"2023-11-22T17:20:34.483525Z","shell.execute_reply":"2023-11-22T17:20:34.977510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X1_balanced= X_merged.iloc[y1_balanced_idx,:]\ny1_balanced= y1.iloc[y1_balanced_idx]","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:20:34.980226Z","iopub.execute_input":"2023-11-22T17:20:34.980552Z","iopub.status.idle":"2023-11-22T17:20:35.024329Z","shell.execute_reply.started":"2023-11-22T17:20:34.980524Z","shell.execute_reply":"2023-11-22T17:20:35.023520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X1_balanced.shape","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:20:36.213339Z","iopub.execute_input":"2023-11-22T17:20:36.213691Z","iopub.status.idle":"2023-11-22T17:20:36.219821Z","shell.execute_reply.started":"2023-11-22T17:20:36.213661Z","shell.execute_reply":"2023-11-22T17:20:36.218975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y2_ones= np.where(y2==1)[0]\n\nn2_ones= (y2==1).sum()\ny2_zeros= np.random.choice(np.where(y2==0)[0],size= n2_ones,replace= False)\n\ny2_balanced_idx= np.sort(np.concatenate([y2_zeros,y2_ones]))\n\nX2_balanced= X_merged.iloc[y2_balanced_idx,:]\ny2_balanced= y2.iloc[y2_balanced_idx]","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:20:43.803825Z","iopub.execute_input":"2023-11-22T17:20:43.804659Z","iopub.status.idle":"2023-11-22T17:20:44.710493Z","shell.execute_reply.started":"2023-11-22T17:20:43.804592Z","shell.execute_reply":"2023-11-22T17:20:44.709591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y3_ones= np.where(y3==1)[0]\n\nn3_ones= (y3==1).sum()\ny3_zeros= np.random.choice(np.where(y3==0)[0],size= n3_ones,replace= False)\n\ny3_balanced_idx= np.sort(np.concatenate([y3_zeros,y3_ones]))\n\nX3_balanced= X.iloc[y3_balanced_idx,:]\ny3_balanced= y3.iloc[y3_balanced_idx]","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:20:44.712100Z","iopub.execute_input":"2023-11-22T17:20:44.712438Z","iopub.status.idle":"2023-11-22T17:20:45.019046Z","shell.execute_reply.started":"2023-11-22T17:20:44.712409Z","shell.execute_reply":"2023-11-22T17:20:45.018164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

\n 🤖 Training Models\n

\n
","metadata":{}},{"cell_type":"code","source":"X1_train, X1_test, y1_train, y1_test = train_test_split(X1_balanced, y1_balanced, test_size = 0.2, random_state = 42)\nX2_train, X2_test, y2_train, y2_test = train_test_split(X2_balanced, y2_balanced, test_size = 0.2, random_state = 42)\nX3_train, X3_test, y3_train, y3_test = train_test_split(X3_balanced, y3_balanced, test_size = 0.2, random_state = 42)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:22:32.298440Z","iopub.execute_input":"2023-11-22T17:22:32.298808Z","iopub.status.idle":"2023-11-22T17:22:32.939521Z","shell.execute_reply.started":"2023-11-22T17:22:32.298779Z","shell.execute_reply":"2023-11-22T17:22:32.938745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler1 = MinMaxScaler()\nX1_train = scaler1.fit_transform(X1_train)\nX1_test = scaler1.transform(X1_test)\n\nscaler2 = MinMaxScaler()\nX2_train = scaler2.fit_transform(X2_train)\nX2_test = scaler2.transform(X2_test)\n\nscaler3 = MinMaxScaler()\nX3_train = scaler3.fit_transform(X3_train)\nX3_test = scaler3.transform(X3_test)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T21:05:48.670355Z","iopub.execute_input":"2023-11-22T21:05:48.671151Z","iopub.status.idle":"2023-11-22T21:05:48.753758Z","shell.execute_reply.started":"2023-11-22T21:05:48.671119Z","shell.execute_reply":"2023-11-22T21:05:48.752968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

\n 4.1 Logistic Regression Model\n

\n
\n
","metadata":{}},{"cell_type":"markdown","source":"**Logistic regression is a supervised machine learning algorithm mainly used for classification tasks where the goal is to predict the probability that an instance of belonging to a given class or not. It is a kind of statistical algorithm, which analyze the relationship between a set of independent variables and the dependent binary variables**\n\n","metadata":{}},{"cell_type":"code","source":"model1 = LogisticRegression()\nmodel2 = LogisticRegression()\nmodel3 = LogisticRegression()\n","metadata":{"execution":{"iopub.status.busy":"2023-11-22T20:18:43.261602Z","iopub.execute_input":"2023-11-22T20:18:43.262534Z","iopub.status.idle":"2023-11-22T20:18:43.267050Z","shell.execute_reply.started":"2023-11-22T20:18:43.262497Z","shell.execute_reply":"2023-11-22T20:18:43.265985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model1.fit(X1_train,y1_train)\nmodel2.fit(X2_train,y2_train)\nmodel3.fit(X3_train,y3_train)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T19:57:39.784957Z","iopub.execute_input":"2023-11-22T19:57:39.785345Z","iopub.status.idle":"2023-11-22T19:57:43.722403Z","shell.execute_reply.started":"2023-11-22T19:57:39.785314Z","shell.execute_reply":"2023-11-22T19:57:43.720909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y1_pred = model1.predict(X1_test)\ny2_pred = model2.predict(X2_test)\ny3_pred = model3.predict(X3_test)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T19:58:38.316389Z","iopub.execute_input":"2023-11-22T19:58:38.317024Z","iopub.status.idle":"2023-11-22T19:58:38.376646Z","shell.execute_reply.started":"2023-11-22T19:58:38.316979Z","shell.execute_reply":"2023-11-22T19:58:38.375278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Start Hesitation \\n',classification_report(y1_test,y1_pred))\nprint('Turn \\n',classification_report(y2_test,y2_pred))\nprint('Walking \\n',classification_report(y3_test,y3_pred))","metadata":{"execution":{"iopub.status.busy":"2023-11-22T20:01:36.900889Z","iopub.execute_input":"2023-11-22T20:01:36.901777Z","iopub.status.idle":"2023-11-22T20:01:38.813900Z","shell.execute_reply.started":"2023-11-22T20:01:36.901743Z","shell.execute_reply":"2023-11-22T20:01:38.812971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sh = confusion_matrix(y1_test,y1_pred)\ntu = confusion_matrix(y2_test,y2_pred)\nwa = confusion_matrix(y3_test,y3_pred)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-22T20:22:22.177787Z","iopub.execute_input":"2023-11-22T20:22:22.178498Z","iopub.status.idle":"2023-11-22T20:22:22.370672Z","shell.execute_reply.started":"2023-11-22T20:22:22.178463Z","shell.execute_reply":"2023-11-22T20:22:22.369730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title('Start Hesitation')\nsns.heatmap(sh,annot=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T20:23:11.267160Z","iopub.execute_input":"2023-11-22T20:23:11.267530Z","iopub.status.idle":"2023-11-22T20:23:11.523829Z","shell.execute_reply.started":"2023-11-22T20:23:11.267502Z","shell.execute_reply":"2023-11-22T20:23:11.522945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title('Turn')\nsns.heatmap(tu,annot=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T20:23:58.774117Z","iopub.execute_input":"2023-11-22T20:23:58.774980Z","iopub.status.idle":"2023-11-22T20:23:59.094936Z","shell.execute_reply.started":"2023-11-22T20:23:58.774945Z","shell.execute_reply":"2023-11-22T20:23:59.094023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title('Walikng')\nsns.heatmap(wa,annot=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T20:24:12.847479Z","iopub.execute_input":"2023-11-22T20:24:12.848230Z","iopub.status.idle":"2023-11-22T20:24:13.169002Z","shell.execute_reply.started":"2023-11-22T20:24:12.848196Z","shell.execute_reply":"2023-11-22T20:24:13.168096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

\n 4.2 Baseline CatBoost Model\n

\n
\n
","metadata":{}},{"cell_type":"markdown","source":"* **CatBoost is a supervised machine learning method that is used by the Train Using AutoML tool and uses decision trees for classification and regression. As its name suggests, CatBoost has two main features, it works with categorical data (the Cat) and it uses gradient boosting (the Boost)**\n\n* **unique characteristic of CatBoost is that it uses symmetric trees. This means that at every depth level, all the decision nodes use the same split condition.CatBoost can also be faster than other methods such as XGBoost. It retains certain features—such as cross-validation, regularization, and missing value support—from the prior algorithms.**","metadata":{}},{"cell_type":"code","source":"seed = np.random.seed(6)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:22:43.274460Z","iopub.execute_input":"2023-11-22T17:22:43.274824Z","iopub.status.idle":"2023-11-22T17:22:43.279397Z","shell.execute_reply.started":"2023-11-22T17:22:43.274794Z","shell.execute_reply":"2023-11-22T17:22:43.278282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train1_dataset = Pool(data=X1_train,label=y1_train)\neval1_dataset = Pool(data=X1_test,label=y1_test)\ncatmodel = CatBoostClassifier(iterations=1000,verbose=100, od_type=\"Iter\",eval_metric=\"PRAUC\", random_seed=seed, early_stopping_rounds=100)\ncatmodel.fit(train1_dataset, use_best_model=True, eval_set=eval1_dataset);","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:23:01.478152Z","iopub.execute_input":"2023-11-22T17:23:01.479050Z","iopub.status.idle":"2023-11-22T17:24:49.436599Z","shell.execute_reply.started":"2023-11-22T17:23:01.479011Z","shell.execute_reply":"2023-11-22T17:24:49.435623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train2_dataset = Pool(data=X2_train,label=y2_train)\neval2_dataset = Pool(data=X2_test,label=y2_test)\ncatmodel = CatBoostClassifier(iterations=1000,verbose=100, od_type=\"Iter\",eval_metric=\"PRAUC\", random_seed=seed, early_stopping_rounds=100)\ncatmodel.fit(train2_dataset, use_best_model=True, eval_set=eval2_dataset);","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:25:22.769811Z","iopub.execute_input":"2023-11-22T17:25:22.770686Z","iopub.status.idle":"2023-11-22T17:39:38.744484Z","shell.execute_reply.started":"2023-11-22T17:25:22.770650Z","shell.execute_reply":"2023-11-22T17:39:38.743512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train3_dataset = Pool(data=X3_train,label=y3_train)\neval3_dataset = Pool(data=X3_test,label=y3_test)\ncatmodel = CatBoostClassifier(iterations=1000,verbose=100, od_type=\"Iter\",eval_metric=\"PRAUC\", random_seed=seed, early_stopping_rounds=100)\ncatmodel.fit(train3_dataset, use_best_model=True, eval_set=eval3_dataset);","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:44:40.673232Z","iopub.execute_input":"2023-11-22T17:44:40.673749Z","iopub.status.idle":"2023-11-22T17:45:55.409639Z","shell.execute_reply.started":"2023-11-22T17:44:40.673698Z","shell.execute_reply":"2023-11-22T17:45:55.408517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SPLITS = 6\ncat1_auc_score = []\nCAT1_MODELS = []\n\nfor i,(tr,val) in enumerate(StratifiedKFold(n_splits=SPLITS,random_state=seed).split(X1_balanced,y1_balanced)):\n \n print(\"-\"*30,f\"FOLD {i+1}/{SPLITS}\",\"-\"*30)\n\n train_dataset1 = Pool(data=X1_balanced.iloc[tr,:],label=y1_balanced.iloc[tr])\n eval_dataset1 = Pool(data=X1_balanced.iloc[val,:],label=y1_balanced.iloc[val])\n \n catmodel = CatBoostClassifier(iterations=10000,verbose=1000,\n od_type=\"Iter\",eval_metric=\"PRAUC\",\n random_seed=seed,early_stopping_rounds=1000)\n catmodel.fit(train_dataset1, use_best_model=True, eval_set=eval_dataset1)\n CAT1_MODELS.append(catmodel)\n \n auc = roc_auc_score(y1_balanced.iloc[val], catmodel.predict_proba(X1_balanced.iloc[val,:])[:,1])\n cat1_auc_score.append(auc)\n\n \n","metadata":{"execution":{"iopub.status.busy":"2023-11-22T17:46:38.824693Z","iopub.execute_input":"2023-11-22T17:46:38.825564Z","iopub.status.idle":"2023-11-22T17:57:46.462549Z","shell.execute_reply.started":"2023-11-22T17:46:38.825531Z","shell.execute_reply":"2023-11-22T17:57:46.461543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nprint(\"CV score of CAT is \",np.array(cat1_auc_score).mean())","metadata":{"execution":{"iopub.status.busy":"2023-11-22T18:20:09.755141Z","iopub.execute_input":"2023-11-22T18:20:09.756013Z","iopub.status.idle":"2023-11-22T18:20:09.760877Z","shell.execute_reply.started":"2023-11-22T18:20:09.755968Z","shell.execute_reply":"2023-11-22T18:20:09.760032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

\n 4.3 LGBM Model\n

\n
\n
","metadata":{}},{"cell_type":"markdown","source":"* **Light GBM grows tree vertically while other algorithm grows trees horizontally meaning that Light GBM grows tree leaf-wise while other algorithm grows level-wise. It will choose the leaf with max delta loss to grow. When growing the same leaf, Leaf-wise algorithm can reduce more loss than a level-wise algorithm.**\n\n","metadata":{}},{"cell_type":"code","source":"lgbmmodel = LGBMClassifier(random_state=seed, device=\"gpu\")\nprint(\"CV score of LGBM is \",cross_val_score(lgbmmodel,X1_balanced,y1_balanced,cv=7, scoring = 'average_precision').mean())","metadata":{"execution":{"iopub.status.busy":"2023-11-22T18:20:12.374443Z","iopub.execute_input":"2023-11-22T18:20:12.374795Z","iopub.status.idle":"2023-11-22T18:20:25.348961Z","shell.execute_reply.started":"2023-11-22T18:20:12.374765Z","shell.execute_reply":"2023-11-22T18:20:25.347906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* **It is almost impossible to make a machine learning algorithm do the job without tuning hyperparameters. The number of hyperparameters tends to be high, especially in deep learning, and it is believed that performance largely depends on how we tune them.**\n* **Optuna is a software framework for automating the optimization process of these hyperparameters. It automatically searches for and finds optimal hyperparameter values by trial and error for excellent performance. Currently, the software can be used in Python.**","metadata":{}},{"cell_type":"code","source":"# def objective(trial):\n# params = {\n# 'n_estimators' : trial.suggest_int('n_estimators',500,1000),\n# \"max_depth\":trial.suggest_int('max_depth',3,50),\n# \"learning_rate\" : trial.suggest_float('learning_rate',1e-4, 0.25, log=True),\n# \"min_child_weight\" : trial.suggest_float('min_child_weight', 0.5,4),\n# \"min_child_samples\" : trial.suggest_int('min_child_samples',1,250),\n# \"subsample\" : trial.suggest_float('subsample', 0.2, 1),\n# \"subsample_freq\" : trial.suggest_int('subsample_freq',0,5),\n# \"colsample_bytree\" : trial.suggest_float('colsample_bytree',0.2,1),\n# 'num_leaves' : trial.suggest_int('num_leaves', 2, 128),\n# }\n# lgbmmodel_optuna = LGBMClassifier(**params,random_state=seed,device=\"gpu\")\n# cv = cross_val_score(lgbmmodel_optuna, X1_balanced, y1_balanced, cv = 3,scoring='average_precision').mean()\n# return cv\n\n# study = optuna.create_study(direction='maximize')\n# study.optimize(objective, n_trials=10,timeout=2000)","metadata":{"execution":{"iopub.status.busy":"2023-11-16T11:56:14.124591Z","iopub.execute_input":"2023-11-16T11:56:14.125210Z","iopub.status.idle":"2023-11-16T12:07:35.122316Z","shell.execute_reply.started":"2023-11-16T11:56:14.125179Z","shell.execute_reply":"2023-11-16T12:07:35.121303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm_params= {\n 'max_depth': 8,\n 'learning_rate': 0.009975674358731229,\n 'min_child_weight': 0.7116049927720877,\n 'min_child_samples': 35,\n 'subsample': 0.44087392623500254,\n 'subsample_freq': 3,\n 'colsample_bytree': 0.7172664346925739,\n 'num_leaves': 64\n}\nlgbm_opt = LGBMClassifier(**lgbm_params,n_estimators=643,random_state=seed,device=\"gpu\")\nprint(\"CV score of LGBM Optuna is \",cross_val_score(lgbm_opt,X1_balanced,y1_balanced,cv=4, scoring = 'average_precision').mean()) ","metadata":{"execution":{"iopub.status.busy":"2023-11-22T18:21:47.612011Z","iopub.execute_input":"2023-11-22T18:21:47.612383Z","iopub.status.idle":"2023-11-22T18:23:12.229609Z","shell.execute_reply.started":"2023-11-22T18:21:47.612354Z","shell.execute_reply":"2023-11-22T18:23:12.228605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_estimators = [int(x) for x in np.linspace(start=200, stop=2000,num=10)]\nmax_features = ['sqrt', 'log2']\nmax_depth = [int(x) for x in np.linspace(10,1000,40)]\nmin_samples_split = [1,3,5,9]\nmin_samples_leaf = [2,4,8]\nparam_grid = {\n \"n_estimators\":n_estimators,\n \"max_features\":max_features,\n \"max_depth\":max_depth,\n \"min_samples_split\":min_samples_split,\n \"min_samples_leaf\":min_samples_leaf,\n \"criterion\":['gini', 'entropy', 'log_loss']\n}","metadata":{"execution":{"iopub.status.busy":"2023-11-22T19:36:17.163363Z","iopub.execute_input":"2023-11-22T19:36:17.164167Z","iopub.status.idle":"2023-11-22T19:36:17.171038Z","shell.execute_reply.started":"2023-11-22T19:36:17.164126Z","shell.execute_reply":"2023-11-22T19:36:17.170066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Thank You**","metadata":{}}]}