{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":8540,"databundleVersionId":862041,"sourceType":"competition"},{"sourceId":7624767,"sourceType":"datasetVersion","datasetId":4441829},{"sourceId":7788617,"sourceType":"datasetVersion","datasetId":4558812}],"dockerImageVersionId":30665,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.linear_model import LogisticRegression\nfrom tqdm import tqdm\nfrom imblearn.over_sampling import SMOTE\nfrom sklearn.utils import resample\nfrom sklearn.model_selection import cross_val_score\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import AdaBoostClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.svm import LinearSVC\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.ensemble import StackingClassifier\nfrom scipy.stats import mode\nfrom catboost import CatBoostClassifier, Pool\nfrom skopt import BayesSearchCV\nimport torch\nimport numpy as np\nfrom sklearn.preprocessing import MinMaxScaler, LabelEncoder, StandardScaler\nfrom sklearn.metrics import log_loss\nfrom sklearn.pipeline import make_pipeline\n\nimport gc \nfrom sklearn.model_selection import train_test_split, RandomizedSearchCV\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import roc_auc_score\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"700af4b2-ca93-4a2f-b7de-0fef3c212eb6","_cell_guid":"34ea337a-c686-4618-bfe8-ed89858725b8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-03-25T10:26:57.384067Z","iopub.execute_input":"2024-03-25T10:26:57.384589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_original = pd.read_csv('/kaggle/input/khois-adclick-data/test.csv')\ntrain_original = pd.read_csv('/kaggle/input/khois-adclick-data/train_resampled.csv')","metadata":{"_uuid":"45ea90fc-a81b-4a0f-a334-148d814c683f","_cell_guid":"9ddfadf3-666d-43b5-9be1-9d10a05c95c4","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train_original.copy()\ntest = test_original.copy()","metadata":{"_uuid":"de2fa6c9-bdd3-4dbb-aa6c-7e04f4cbf072","_cell_guid":"36cb5987-aee7-4e57-8b0c-3dba906aff9f","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"_uuid":"eedbd5f1-ab35-40b7-adbe-a41b8a17f449","_cell_guid":"0d6c387c-ef0d-4e41-b0ee-ac2c2c2a6ffd","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(labels='Unnamed: 0', axis=1, inplace=True)\ntest.drop(labels='Unnamed: 0', axis=1, inplace=True)","metadata":{"_uuid":"164d1413-ceb7-4175-8f13-0a52f68bc183","_cell_guid":"a5cd4d34-b3fd-40fc-98f6-d0261741cbb7","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Initial Processing","metadata":{"_uuid":"d1af0853-06e6-4242-ae8a-2041a3cdb0a5","_cell_guid":"9e64cb4f-d3cf-4a32-b29e-dd968b6d2040","trusted":true}},{"cell_type":"markdown","source":"### Missing Values","metadata":{"_uuid":"3572b4c1-f150-4e5f-bfbc-ed65f038e19d","_cell_guid":"437f7759-ca8d-4907-b4fb-7731ce373ebe","trusted":true}},{"cell_type":"code","source":"train.isnull().sum(axis = 0)","metadata":{"_uuid":"6fea26ac-727f-4957-9b5f-76b0c2f1e834","_cell_guid":"a72f3c65-72d8-4301-983e-6a48182d68b6","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### What happened to attributed_time?","metadata":{"_uuid":"113c9fba-de23-4cfd-a312-98d7aec6ef47","_cell_guid":"e4d0b636-77ed-4e3e-b7f8-4d7770485f0f","trusted":true}},{"cell_type":"code","source":"temp1 = train.loc[~train.attributed_time.isnull(),'is_attributed']\nif np.sum(temp1.values) == len(temp1):\n  print('Same')\nelse:\n  print('No')\nprint('-'*50)\ntemp2 = train.loc[train.attributed_time.isnull(),'is_attributed']\nif np.sum(temp2.values) == 0:\n  print('Same')\nelse:\n  print('No')\n\n# whenever is_attributed == 1, then there is a corresponding time in attributed_time","metadata":{"_uuid":"c0f4d662-425f-4345-9541-7f59657df17b","_cell_guid":"db2c7f54-156e-48b7-a094-c25333a1240d","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Label Distribution","metadata":{"_uuid":"682e36fb-c15d-46e7-9b78-178594cea178","_cell_guid":"4f8904f0-2954-41f9-8980-7317c968f426","trusted":true}},{"cell_type":"code","source":"fig = plt.figure(figsize=(6,5))\nplt.bar(train.is_attributed.value_counts().index, train.is_attributed.value_counts().values)\nplt.xlabel('labels')\nplt.ylabel('counts')\nplt.xticks([0,1])\nplt.show()","metadata":{"_uuid":"f224d6b9-a9ad-44c6-bcd1-8aa5df4b7775","_cell_guid":"d342da90-ac37-41e3-9294-d672f00cd87c","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Distribution","metadata":{"_uuid":"fdf820c9-4d18-4f8c-8d14-c0a54eee4a50","_cell_guid":"bd4fd867-7054-4e8d-92cc-f6c2533558ef","trusted":true}},{"cell_type":"code","source":"fig = plt.figure(figsize=(15,12))\nfor i, col in enumerate(['ip','app','device','os','channel']):\n  plt.subplot(3,2,i+1)\n  sns.histplot(data=train, x=col, label = col, kde=True)\nfig.tight_layout(pad=1.0)","metadata":{"_uuid":"75015512-e1b6-4001-97c2-6ae5f679cecb","_cell_guid":"f0dace91-8bf7-45c7-bb1d-6327a8b17e9e","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Click Time Transformation","metadata":{"_uuid":"159a8433-3a4f-4cc4-a776-bda29467c229","_cell_guid":"814a15e8-6b7f-4831-8495-45a46a9e5d2c","trusted":true}},{"cell_type":"code","source":"def transformation(df):\n  df['click_time'] = pd.to_datetime(df.click_time)\n  df['day_of_week'] = df.click_time.dt.day_of_week\n  df['day'] = df.click_time.dt.day\n  df['hour'] = df.click_time.dt.hour\n  #df.drop(labels='click_time', axis=1, inplace = True)\n  return df\n\n# transformation\ntrain = transformation(train)\ntest = transformation(test)","metadata":{"_uuid":"b65c101a-3fff-491f-acd4-fe1aa56b6289","_cell_guid":"fd47a937-1dbd-4db2-8d8e-2efbe247a9dd","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Deleting variables","metadata":{"_uuid":"75dbec6f-fcc8-4f63-8d36-eed61443e9d2","_cell_guid":"5157c386-511c-404d-9492-13652a50e54e","trusted":true}},{"cell_type":"code","source":"# store the label\ntrain_labels = train.is_attributed.values\ntest_labels = test.is_attributed.values\n\n# drop labels and attributed_time since it represnets the same info as the is_attributed\ntrain.drop(labels = ['attributed_time', 'is_attributed'], axis = 1, inplace = True)\ntest.drop(labels = ['attributed_time', 'is_attributed'], axis = 1, inplace = True)","metadata":{"_uuid":"317ed048-bd30-4fa0-b938-bf68d66963a8","_cell_guid":"100f348c-dc24-4ac3-8277-5b89379dde11","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Baseline Model - Logistic Regression","metadata":{"_uuid":"b7c3705c-0459-49df-97b0-db93ccdae2c0","_cell_guid":"dce875e1-bbf5-4081-9458-7346d36e507a","trusted":true}},{"cell_type":"code","source":"results = []","metadata":{"_uuid":"564204b7-0375-4655-a1b7-0c2d49ae0d7d","_cell_guid":"ecf937e6-22cb-447a-ac1c-94ed06f01ee0","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\n\ntemp_train = train.drop(labels='click_time', axis=1)\ntemp_test = test.drop(labels='click_time', axis=1)\n\n# Initialize logistic regression model with 'sag' solver\nmodel = LogisticRegression(solver='sag', C=1.0, max_iter=10000, penalty='l2')\n\n# Train the model\nmodel.fit(temp_train, train_labels)\n\n# Predictions\ny_pred = model.predict(temp_test)\ny_pred_proba = model.predict_proba(temp_test)[:, 1]\n\n# Evaluate the model\naccuracy = accuracy_score(test_labels, y_pred)\nauc_score = roc_auc_score(test_labels, y_pred_proba)\n\nprint(f\"Accuracy: {accuracy * 100.0}%\")\nprint(f\"AUC Score: {auc_score}\")","metadata":{"_uuid":"2401988b-9f6b-4cd9-a4eb-42146eeaccc0","_cell_guid":"2a7b16c3-2e0f-459b-9fad-d02d148a3d6e","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'Logistic Regression', \"Accuracy\": accuracy, \"AUC\": auc_score})","metadata":{"_uuid":"2b72c7e3-b89e-49a5-bf50-85cb73e391f7","_cell_guid":"173a3d47-c258-455b-91f4-f154249d5337","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{"_uuid":"ea51b032-4611-4b22-8c3c-c412f33356b9","_cell_guid":"21014213-b229-4259-83cb-e1fea8fb2ad4","trusted":true}},{"cell_type":"markdown","source":"### Statistic Features","metadata":{"_uuid":"1fa97815-10b1-4c04-89d6-1b00544dcd26","_cell_guid":"f8e68432-d21c-4e41-ba11-aa05f84142e2","trusted":true}},{"cell_type":"code","source":"from tqdm import tqdm\n\ndef transform_data(df):\n    # Initialize list of column groups\n    column_groups = [['ip'], ['ip', 'os', 'device'], ['ip', 'day', 'hour']]\n\n    # Process each column group\n    for cols in tqdm(column_groups, desc=\"Count\"):\n        name = '_'.join(cols)\n        df[name + '_cnts'] = df.groupby(cols)['click_time'].transform('count').astype('uint16')\n\n    # Process nunique\n    for f1 in ['ip']:\n        for f2 in tqdm(['app', 'device', 'os', 'channel'], desc=\"Nunique\"):\n            nunique_col_name = f\"{f1}_{f2}_nuni\"\n            df[nunique_col_name] = df.groupby([f1])[f2].transform('nunique').astype('uint16')\n\n    return df\n\n# Example usage:\ntrain = transform_data(train)\ntest = transform_data(test)","metadata":{"_uuid":"eabfad62-d2a4-480c-a337-a3188023a397","_cell_guid":"a348bb1e-983f-4d28-9268-1b0e4c5c9b53","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Next click features","metadata":{}},{"cell_type":"code","source":"# function to calculate next click\ndef do_next_Click( df,agg_suffix='nextClick', agg_type='float32'):\n    \n    print(f\">> \\nExtracting {agg_suffix} time calculation features...\\n\")\n    \n    GROUP_BY_NEXT_CLICKS = [\n    \n    # V1\n    # {'groupby': ['ip']},\n    # {'groupby': ['ip', 'app']},\n    # {'groupby': ['ip', 'channel']},\n    # {'groupby': ['ip', 'os']},\n    \n    # V3\n    {'groupby': ['ip', 'app', 'device', 'os', 'channel']},\n    {'groupby': ['ip', 'os', 'device']},\n    {'groupby': ['ip', 'os', 'device', 'app']}\n    ]\n\n    # Calculate the time to next click for each group\n    for spec in GROUP_BY_NEXT_CLICKS:\n    \n       # Name of new feature\n        new_feature = '{}_{}'.format('_'.join(spec['groupby']),agg_suffix)    \n    \n        # Unique list of features to select\n        all_features = spec['groupby'] + ['click_time']\n\n        # Run calculation\n        print(f\">> Grouping by {spec['groupby']}, and saving time to {agg_suffix} in: {new_feature}\")\n        df[new_feature] = (df[all_features].groupby(spec[\n            'groupby']).click_time.shift(-1) - df.click_time).dt.seconds.astype(agg_type)\n        \n        gc.collect()\n    return (df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = do_next_Click(train,agg_suffix='nextClick', agg_type='float32'  )\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = do_next_Click(test,agg_suffix='nextClick', agg_type='float32'  )\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prev click features","metadata":{}},{"cell_type":"code","source":"def do_prev_Click( df,agg_suffix='prevClick', agg_type='float32'):\n\n    print(f\">> \\nExtracting {agg_suffix} time calculation features...\\n\")\n    \n    GROUP_BY_NEXT_CLICKS = [\n    \n    # V1\n    # {'groupby': ['ip']},\n    # {'groupby': ['ip', 'app']},\n    {'groupby': ['ip', 'channel']},\n    {'groupby': ['ip', 'os']},\n    \n    # V3\n    #{'groupby': ['ip', 'app', 'device', 'os', 'channel']},\n    #{'groupby': ['ip', 'os', 'device']},\n    #{'groupby': ['ip', 'os', 'device', 'app']}\n    ]\n\n    # Calculate the time to next click for each group\n    for spec in GROUP_BY_NEXT_CLICKS:\n    \n       # Name of new feature\n        new_feature = '{}_{}'.format('_'.join(spec['groupby']),agg_suffix)    \n    \n        # Unique list of features to select\n        all_features = spec['groupby'] + ['click_time']\n\n        # Run calculation\n        print(f\">> Grouping by {spec['groupby']}, and saving time to {agg_suffix} in: {new_feature}\")\n        df[new_feature] = (df.click_time - df[all_features].groupby(spec[\n                'groupby']).click_time.shift(+1) ).dt.seconds.astype(agg_type)\n        \n        gc.collect()\n    return (df)    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = do_prev_Click(train,agg_suffix='prevClick', agg_type='float32'  )\ngc.collect()\ntest = do_prev_Click(test,agg_suffix='prevClick', agg_type='float32'  )\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### DAY & HOUR","metadata":{}},{"cell_type":"code","source":"# train\ntrain['day'] = train['click_time'].dt.day.astype('uint8')\ntrain['hour'] = train['click_time'].dt.hour.astype('uint8')\n\n# test\ntest['day'] = test['click_time'].dt.day.astype('uint8')\ntest['hour'] = test['click_time'].dt.hour.astype('uint8')\n\ndf_concat = pd.concat([train, test], ignore_index=True)\ndf_concat['day'] = df_concat['click_time'].dt.day.astype('uint8')\ndf_concat['hour'] = df_concat['click_time'].dt.hour.astype('uint8')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### in_test_hh","metadata":{}},{"cell_type":"code","source":"most_freq_hours_in_test_data = [4, 5, 9, 10, 13, 14]\nleast_freq_hours_in_test_data = [6, 11, 15]\n\ntrain['in_test_hh'] = (\n    3 - 2 * train.hour.isin(most_freq_hours_in_test_data)\n      - 1 * train.hour.isin(least_freq_hours_in_test_data)\n).astype('uint8')\n\ntest['in_test_hh'] = (\n    3 - 2 * test.hour.isin(most_freq_hours_in_test_data)\n      - 1 * test.hour.isin(least_freq_hours_in_test_data)\n).astype('uint8')\n\ndf_concat['in_test_hh'] = (\n    3 - 2 * df_concat.hour.isin(most_freq_hours_in_test_data)\n      - 1 * df_concat.hour.isin(least_freq_hours_in_test_data)\n).astype('uint8')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### per day per hour count features","metadata":{}},{"cell_type":"code","source":"group = ['ip', 'day', 'hour']\ndf = df_concat.groupby(group).size().astype('uint16')\ndf = pd.DataFrame(df, columns=['ip_day_hour_clicks']).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)\n# ip binded with 'app', 'os', 'device', 'channel'\nothers_list = ['app', 'os', 'device']\nfor col in others_list:\n    group = ['ip', col, 'day', 'hour']\n    df = df_concat.groupby(group).size().astype('uint16')\n    df = pd.DataFrame(df, columns=['ip_{}_day_hour_clicks'.format(col)]).reset_index()    # noqa\n    train = train.merge(df, how='left', on=group)\n    test = test.merge(df, how='left', on=group)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ip_day_test_hh_clicks","metadata":{}},{"cell_type":"code","source":"group = ['ip', 'day', 'in_test_hh']\ndf = df_concat.groupby(group).size().astype('uint16')\ndf = pd.DataFrame(df, columns=['ip_day_test_hh_clicks']).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Counts groupby 3+ features","metadata":{}},{"cell_type":"code","source":"# ip_app_device_clicks\ngroup = ['ip', 'app', 'device']\ndf = df_concat.groupby(group).size().astype('uint16')\ndf = pd.DataFrame(df, columns=['ip_app_device_clicks']).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)\n\n# ip_app_device_day_clicks\ngroup = ['ip', 'app', 'device', 'day']\ndf = df_concat.groupby(group).size().astype('uint16')\ndf = pd.DataFrame(df, columns=['ip_app_device_day_clicks']).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Nunique","metadata":{}},{"cell_type":"code","source":"# single 'ip' grouper for ['app', 'device', 'channel']\ncount_cols = ['app', 'device', 'channel', 'hour']\ngroup = 'ip'\nfor col in count_cols:\n    df = df_concat.groupby(group)[col].nunique().astype('uint16')\n    df.name = 'ip_nunique_{}'.format(col)\n    df = pd.DataFrame(df).reset_index()\n    train = train.merge(df, how='left', on=group)\n    test = test.merge(df, how='left', on=group)\n    #####################\n    # self-add\n    group_day = ['ip', 'day']\n    df = df_concat.groupby(group_day)[col].nunique().astype('uint16')\n    df.name = 'ip_day_nunique_{}'.format(col)\n    df = pd.DataFrame(df).reset_index()\n    train = train.merge(df, how='left', on=group_day)\n    test = test.merge(df, how='left', on=group_day)\n    #####################\n    \n# single 'app' grouper for 'channel'\ngroup = 'app'\ncol = 'channel'\ndf = df_concat.groupby(group)[col].nunique().astype('uint16')\ndf.name = 'app_nunique_{}'.format(col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)\n#####################\n\n# self-add\ngroup_day = ['app', 'day']\ncol = 'channel'\ndf = df_concat.groupby(group_day)[col].nunique().astype('uint16')\ndf.name = 'app_day_nunique_{}'.format(col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=group_day)\ntest = test.merge(df, how='left', on=group_day)\n#####################\n\n# duble ['ip', 'app'] grouper for 'os'\ngroup = ['ip', 'app']\ncol = 'os'\ndf = df_concat.groupby(group)[col].nunique().astype('uint16')\ndf.name = 'ip_app_nunique_{}'.format(col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)\n#####################\n\n# self-add\ngroup_day = ['ip', 'app', 'day']\ncol = 'os'\ndf = df_concat.groupby(group_day)[col].nunique().astype('uint16')\ndf.name = 'ip_app_day_nunique_{}'.format(col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=group_day)\ntest = test.merge(df, how='left', on=group_day)\n#####################\n\n# triple ['ip', 'device', 'os'] grouper for 'app'\ngroup = ['ip', 'device', 'os']\ncol = 'app'\ndf = df_concat.groupby(group)[col].nunique().astype('uint16')\ndf.name = 'ip_device_os_nunique_{}'.format(col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)\n#####################\n\n# self-add\ngroup_day = ['ip', 'device', 'os'] + ['day']\ncol = 'app'\ndf = df_concat.groupby(group_day)[col].nunique().astype('uint16')\ndf.name = 'ip_device_os_day_nunique_{}'.format(col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=group_day)\ntest = test.merge(df, how='left', on=group_day)\n#####################","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Var & Mean","metadata":{}},{"cell_type":"code","source":"# ip_?_day_var_hour\ngroup_day_cols = ['app', 'device', 'os', 'channel']\ncol = 'hour'\nfor group in group_day_cols:\n    df = df_concat.groupby(['ip', group, 'day'])[col].var().fillna(0).astype('float32')      # noqa\n    df.name = 'ip_{}_day_var_hour'.format(group)\n    df = pd.DataFrame(df).reset_index()\n    train = train.merge(df, how='left', on=['ip', group, 'day'])\n    test = test.merge(df, how='left', on=['ip', group, 'day'])           # noqa\n\n# ip_app_os_var_hour\ngroup = 'os'\ncol = 'hour'\ndf = df_concat.groupby(['ip', 'app', group])[col].var().fillna(0).astype('float32')          # noqa\ndf.name = 'ip_app_{}_var_{}'.format(group, col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=['ip', 'app', group])\ntest = test.merge(df, how='left', on=['ip', 'app', group])\n\n# ip_app_channel_var_day\ngroup = 'channel'\ncol = 'day'\ndf = df_concat.groupby(['ip', 'app', group])[col].var().fillna(0).astype('float32')          # noqa\ndf.name = 'ip_app_{}_var_{}'.format(group, col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=['ip', 'app', group])\ntest = test.merge(df, how='left', on=['ip', 'app', group])\n\n# ip_app_channel_mean_hour\ngroup = 'channel'\ncol = 'hour'\ndf = df_concat.groupby(['ip', 'app', group])[col].mean().fillna(0).astype('float32')         # noqa\ndf.name = 'ip_app_{}_mean_{}'.format(group, col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=['ip', 'app', group])\ntest = test.merge(df, how='left', on=['ip', 'app', group])\n\ndel df_concat, df\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ranking Features","metadata":{"_uuid":"2af781ec-63e2-4b65-ba56-ed22bcc9a289","_cell_guid":"42cec1c4-9d0c-4be5-9063-8402c49f9d51","trusted":true}},{"cell_type":"code","source":"def calculate_click_ranks(df):\n    # Define column combinations\n    column_combinations = [['ip', 'os', 'device', 'app'], ['ip', 'os', 'device', 'app', 'day']]\n\n    for cols in tqdm(column_combinations, desc=\"Click Ranks\"):\n        name_asc = '{}_click_asc_rank'.format('_'.join(cols))\n        name_dec = '{}_click_dec_rank'.format('_'.join(cols))\n\n        df[name_asc] = df.groupby(cols)['click_time'].rank(ascending=True)\n        df[name_dec] = df.groupby(cols)['click_time'].rank(ascending=False)\n\n    return df\n\n# Example usage:\ntrain = calculate_click_ranks(train)\ntest = calculate_click_ranks(test)","metadata":{"_uuid":"9f1c73f3-49f3-4f39-8cc8-7e75618fd2b3","_cell_guid":"162b4e14-c05a-4ef0-910e-fdb457ee077a","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# delete features\ntrain.drop(labels='click_time',axis=1,inplace=True)\ntrain.head()","metadata":{"_uuid":"9ed72864-eda5-4c6c-8710-fd349e44f68e","_cell_guid":"f1510118-10bb-4846-bff8-9ba232b0419b","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.drop(labels='click_time',axis=1,inplace=True)\ntest.head()","metadata":{"_uuid":"3b788841-3e50-4443-88f0-8502786297f8","_cell_guid":"b2ef672b-cc2b-4873-99d6-878888cb6099","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(labels='ip',axis=1,inplace=True)\ntrain.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.drop(labels='ip',axis=1,inplace=True)\ntest.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(labels='day',axis=1,inplace=True)\ntest.drop(labels='day',axis=1,inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fill NaN values","metadata":{}},{"cell_type":"code","source":"train.isnull().mean()*100","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.isnull().mean()*100","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.fillna(train.max())\ntest = test.fillna(test.max())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().mean()*100","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### SMOTE","metadata":{"_uuid":"02706aaa-df7d-41f6-9829-80f81724f17c","_cell_guid":"62838d0f-5374-4565-821e-066aa3fc257d","trusted":true}},{"cell_type":"code","source":"def balance_data(features, labels, sampling_strategy=0.5, random_state=42):\n    # Positive oversampling (SMOTE)\n    smote = SMOTE(sampling_strategy=sampling_strategy, random_state=random_state)\n    features_resampled, labels_resampled = smote.fit_resample(features, labels)\n\n    # Negative downsampling (reduce negative samples to match positive samples)\n    negative_samples = features_resampled[labels_resampled == 0]\n    positive_samples = features_resampled[labels_resampled == 1]\n\n    # Perform negative downsampling, e.g., reduce negative samples to match twice the number of positive samples\n    negative_samples_downsampled = resample(negative_samples, n_samples=len(positive_samples)*2, random_state=random_state)\n\n    # Combine positive and downsampled negative samples\n    balanced_features = pd.concat([positive_samples, negative_samples_downsampled], ignore_index=True)\n    balanced_labels = pd.Series([1] * len(positive_samples) + [0] * len(negative_samples_downsampled))\n\n    return balanced_features, balanced_labels\n\n# Example usage:\ntrain_balanced, train_labels_balanced = balance_data(train, train_labels, sampling_strategy=0.5, random_state=42)","metadata":{"_uuid":"de79da0e-c82a-4cc7-873a-921f64a43090","_cell_guid":"12b5f51d-ec0c-4e79-b7bf-a24514c30412","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Count the number of occurrences of each label\nlabel_counts = train_labels_balanced.value_counts()\n\n# Display the counts\nprint(\"Number of '0' labels:\", label_counts[0])\nprint(\"Number of '1' labels:\", label_counts[1])","metadata":{"_uuid":"dbb6b8c8-ec4b-4ce8-859a-2896f9a99c5e","_cell_guid":"94b67441-19ef-4dd9-83d5-30e268c8939b","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model - Gradient Boosting Tree","metadata":{"_uuid":"1a63eec8-787c-4e1b-802f-41e6d37d68b0","_cell_guid":"4124e2b8-2e00-46d9-a302-27dd101f8445","trusted":true}},{"cell_type":"code","source":"# Manually select parameters from the parameter grid\nparams = {\n    'n_estimators': 200,\n    'learning_rate': 0.1,\n    'max_depth': 5,\n    'min_samples_split': 4,\n    'min_samples_leaf': 2,\n    'max_features': 'sqrt'\n}\n\n# Initialize Gradient Boosting Classifier with selected parameters\ngb_model = GradientBoostingClassifier(**params)\n\n# Perform cross-validation\ncv_scores = cross_val_score(gb_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\ngb_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions on the test set\ngb_y_pred = gb_model.predict(test)\ny_pred_proba = gb_model.predict_proba(test)[:, 1]\n\n# Calculate accuracy and AUC Score on the test set\ngb_accuracy = accuracy_score(test_labels, gb_y_pred)\ngb_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\nprint(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\nprint(f\"Average Cross-Validation ROC-AUC Score: {cv_scores.mean()}\")\nprint(f\"Test Set Accuracy: {gb_accuracy * 100.0}%\")\nprint(f\"Test Set AUC Score: {gb_auc_score}\")","metadata":{"_uuid":"73b6e9fa-efce-48ca-b6ab-3caa9e2559d4","_cell_guid":"bf2b78d5-b7f7-42dd-b2f6-1217d8f4c9db","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'Gradient Boosting Tree', \"Accuracy\": gb_accuracy, \"AUC\": gb_auc_score})","metadata":{"_uuid":"09010870-f99f-4cc4-92bf-c62d4d9d44ac","_cell_guid":"9c47b950-29c1-4e74-ab80-a93e7edf0a2b","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model - XGBoost","metadata":{"_uuid":"382b8437-8f09-4f67-b981-cc7751fd4d02","_cell_guid":"751a1b0f-abf8-4f83-b041-83da3e477ce9","trusted":true}},{"cell_type":"code","source":"params = {'eta': 0.2,\n          'max_depth': 15,\n          'subsample': 0.7,\n          'colsample_bytree': 0.9,\n          'objective': 'binary:logistic',\n          'scale_pos_weight': 9,\n          'eval_metric': 'auc',\n          'n_jobs': 24,\n          'n_estimators': 400\n          }\n\n# Initialize XGBoost Classifier with selected parameters\nxgb_model = xgb.XGBClassifier(**params)\n\n# Perform cross-validation\ncv_scores = cross_val_score(xgb_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\nxgb_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions on the test set\nxgb_y_pred = xgb_model.predict(test)\ny_pred_proba = xgb_model.predict_proba(test)[:, 1]\n\n# Calculate accuracy and AUC Score on the test set\nxgb_accuracy = accuracy_score(test_labels, xgb_y_pred)\nxgb_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\nprint(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\nprint(f\"Average Cross-Validation ROC-AUC Score: {np.mean(cv_scores)}\")\nprint(f\"Test Set Accuracy: {xgb_accuracy * 100.0}%\")\nprint(f\"Test Set AUC Score: {xgb_auc_score}\")\n\n# Display feature importance\nfeature_importance = xgb_model.feature_importances_\nsorted_idx = np.argsort(feature_importance)[::-1]\n\nprint(\"\\nTop 10 Feature Importance:\")\nfor i in range(10):\n    print(f\"{train_balanced.columns[sorted_idx[i]]}: {feature_importance[sorted_idx[i]]}\")","metadata":{"_uuid":"94177d6f-c9dc-474f-8ded-8210624f8434","_cell_guid":"a295632d-9338-48bc-86cc-d8ec9bee3484","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'XGBoost', \"Accuracy\": xgb_accuracy, \"AUC\": xgb_auc_score})","metadata":{"_uuid":"508afe8f-3004-4b71-8ae5-a1c904f0f22d","_cell_guid":"b433ef88-1290-4035-8f2a-47e9415288c3","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model - LightGBM","metadata":{"_uuid":"986082d9-8475-424c-8964-1923b9fa22c8","_cell_guid":"5035b192-dce5-411a-b0ab-7b30271839a5","trusted":true}},{"cell_type":"code","source":"params = {'num_leaves': 127,\n          'min_data_in_leaf': 32,\n          'objective': 'binary',\n          'max_depth': -1,\n          'learning_rate': 0.1,\n          'min_child_samples': 20,\n          'boosting': 'gbdt',\n          'feature_fraction': 0.8,\n          'bagging_freq': 1,\n          'bagging_fraction': 0.8,\n          'bagging_seed': 11,\n          'metric': 'auc',\n          'lambda_l1': 0.1,\n          'verbosity': -1,\n          'n_estimators': 200\n          }\n\n# Initialize LightGBM Classifier with selected parameters\nlgb_model = lgb.LGBMClassifier(**params)\n\n# Perform cross-validation\ncv_scores = cross_val_score(lgb_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\nlgb_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions on the test set\nlgb_y_pred = lgb_model.predict(test)\ny_pred_proba = lgb_model.predict_proba(test)[:, 1]\n\n# Calculate accuracy and AUC Score on the test set\nlgb_accuracy = accuracy_score(test_labels, lgb_y_pred)\nlgb_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\nprint(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\nprint(f\"Average Cross-Validation ROC-AUC Score: {np.mean(cv_scores)}\")\nprint(f\"Test Set Accuracy: {lgb_accuracy * 100.0}%\")\nprint(f\"Test Set AUC Score: {lgb_auc_score}\")\n\n# Display feature importances\nfeature_importance = lgb_model.feature_importances_\nsorted_idx = np.argsort(feature_importance)[::-1]\n\nprint(\"\\nTop 10 Feature Importance:\")\nfor i in range(10):\n    print(f\"{train_balanced.columns[sorted_idx[i]]}: {feature_importance[sorted_idx[i]]}\")","metadata":{"_uuid":"25afb764-b956-45d3-97f4-9e1229686608","_cell_guid":"669043b3-889f-42d0-8fae-409e9908bf47","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'Light GBM', \"Accuracy\": lgb_accuracy, \"AUC\": lgb_auc_score})","metadata":{"_uuid":"27b03d74-a54c-45e9-a0c5-b0e9a2231a52","_cell_guid":"55d17844-b063-48b6-926c-7e67bb8908dd","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model - CatBoost","metadata":{"_uuid":"4b8b000b-9a0f-480a-98ad-4f870fb348e5","_cell_guid":"dbe174d3-8c90-4b21-ae35-5d0fba6dae5a","trusted":true}},{"cell_type":"code","source":"from catboost import CatBoostClassifier\n\n# Manually select parameters from the parameter grid\nparams = {\n    'depth': 10,\n    'learning_rate': 0.2,\n    'iterations': 500,\n    'l2_leaf_reg': 3,\n    'border_count': 50,\n    'bagging_temperature': 0.04019881483063329,\n    'loss_function': 'Logloss',\n    'od_type': 'Iter',\n    'od_wait': 50,\n    'verbose': False\n}\n\n# Initialize CatBoost Classifier with selected parameters\ncat_model = CatBoostClassifier(**params)\n\n# Perform cross-validation\ncv_scores = cross_val_score(cat_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\ncat_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions on the test set\ncat_y_pred = cat_model.predict(test)\ny_pred_proba = cat_model.predict_proba(test)[:, 1]\n\n# Calculate accuracy and AUC Score on the test set\ncat_accuracy = accuracy_score(test_labels, cat_y_pred)\ncat_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\nprint(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\nprint(f\"Average Cross-Validation ROC-AUC Score: {np.mean(cv_scores)}\")\nprint(f\"Test Set Accuracy: {cat_accuracy * 100.0}%\")\nprint(f\"Test Set AUC Score: {cat_auc_score}\")","metadata":{"_uuid":"6f7d6d84-a0ab-4c3f-8619-73ad04802bfa","_cell_guid":"6d050bd9-e069-4349-9e91-7bbf35c3582f","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'CatBoost', \"Accuracy\": cat_accuracy, \"AUC\": cat_auc_score})","metadata":{"_uuid":"8af66b70-758f-4dd7-bf2e-304969f76723","_cell_guid":"46ea66d9-d925-45a3-8200-e8b765e0f60a","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model - Random Forest","metadata":{"_uuid":"50f59850-c645-4ea8-a736-2feda46cb80c","_cell_guid":"7edc1ec3-5c38-4f24-a96e-36b7734502dc","trusted":true}},{"cell_type":"code","source":"# Initialize Random Forest Classifier\nrf_model = RandomForestClassifier(\n    n_estimators=100,\n    max_depth=20,\n    min_samples_split=2,\n    min_samples_leaf=1,\n    max_features='sqrt',\n    bootstrap=False,\n    criterion='gini'\n)\n\n# Perform cross-validation\ncv_scores = cross_val_score(rf_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\nrf_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions on the test set\nrf_y_pred = rf_model.predict(test)\ny_pred_proba = rf_model.predict_proba(test)[:, 1]\n\n# Calculate accuracy and AUC Score on the test set\nrf_accuracy = accuracy_score(test_labels, rf_y_pred)\nrf_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\nprint(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\nprint(f\"Average Cross-Validation ROC-AUC Score: {np.mean(cv_scores)}\")\nprint(f\"Test Set Accuracy: {rf_accuracy * 100.0}%\")\nprint(f\"Test Set AUC Score: {rf_auc_score}\")","metadata":{"_uuid":"94cc51dc-b29c-48ce-8b1f-6d2f6afb4464","_cell_guid":"a3777945-2215-4c76-ba88-c78d392d901f","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'Random Forest', \"Accuracy\": rf_accuracy, \"AUC\": rf_auc_score})","metadata":{"_uuid":"a421381e-be68-4a6a-bf59-6adce1aecbd1","_cell_guid":"0f7265e8-3773-433f-86bf-f2cb9ccf0cf0","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model - AdaBoost","metadata":{"_uuid":"96e54bf8-348b-49f8-8f3c-88c6a31ad616","_cell_guid":"2841ff6a-d89b-41bd-8d15-6b0d26841ec1","trusted":true}},{"cell_type":"code","source":"# model performance on test data with chosen hyperparameters\n\n# base estimator\ntree = DecisionTreeClassifier(max_depth=2)\n\n# adaboost with the tree as base estimator\n# learning rate is arbitrarily set, we'll discuss learning_rate below\nABC_model = AdaBoostClassifier(\n    base_estimator=tree,\n    learning_rate=1.0,\n    n_estimators=300,\n    algorithm=\"SAMME\")\n\n# Perform cross-validation\ncv_scores = cross_val_score(ABC_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\nABC_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions on the test set\nABC_y_pred = ABC_model.predict(test)\ny_pred_proba = ABC_model.predict_proba(test)[:, 1]\n\n# Calculate accuracy and AUC Score on the test set\nABC_accuracy = accuracy_score(test_labels, ABC_y_pred)\nABC_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\nprint(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\nprint(f\"Average Cross-Validation ROC-AUC Score: {np.mean(cv_scores)}\")\nprint(f\"Test Set Accuracy: {ABC_accuracy * 100.0}%\")\nprint(f\"Test Set AUC Score: {ABC_auc_score}\")","metadata":{"_uuid":"4394718d-3321-4748-bf3a-bde0d5ee76f1","_cell_guid":"7ac318b1-ff04-4bd1-b12c-08dcaa0c6a41","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'AdaBoost', \"Accuracy\": ABC_accuracy, \"AUC\": ABC_auc_score})","metadata":{"_uuid":"5b95e0b1-5ba3-4bc9-8736-d3d5ce2f3dee","_cell_guid":"c65d81e6-f1a2-4cea-a018-b279ccee75ea","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model SVM","metadata":{"_uuid":"8cd8dc75-4978-4c50-9dd7-77ba911001a3","_cell_guid":"a4058be5-0271-431f-9400-dbb32cf2fef8","trusted":true}},{"cell_type":"code","source":"# LinearSVC\nfrom sklearn.svm import LinearSVC\nfrom sklearn.calibration import CalibratedClassifierCV\n\n# Initialize LinearSVC\nlinear_svc = LinearSVC()\n\n# Use CalibratedClassifierCV to enable probability predictions\nsvm_model = make_pipeline(StandardScaler(), CalibratedClassifierCV(linear_svc))\n\n# Perform cross-validation\ncv_scores = cross_val_score(svm_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\nsvm_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions on the test set\nsvm_y_pred = svm_model.predict(test)\ny_pred_proba = svm_model.predict_proba(test)[:, 1]\n\n# Calculate accuracy and AUC Score on the test set\nsvm_accuracy = accuracy_score(test_labels, svm_y_pred)\nsvm_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\nprint(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\nprint(f\"Average Cross-Validation ROC-AUC Score: {np.mean(cv_scores)}\")\nprint(f\"Test Set Accuracy: {svm_accuracy * 100.0}%\")\nprint(f\"Test Set AUC Score: {svm_auc_score}\")","metadata":{"_uuid":"da82daf8-03a3-4678-85a2-8583c0a8538d","_cell_guid":"7e64858f-ed06-42b8-8396-66862b6a8b74","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'SVM', \"Accuracy\": svm_accuracy, \"AUC\": svm_auc_score})","metadata":{"_uuid":"254760c4-e1e1-4bb4-b03a-64063c13f73c","_cell_guid":"26f2a753-2056-4826-9d9c-0a2af78b5171","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Stacking Generalization Ensembling","metadata":{"_uuid":"5c181af6-0ece-4167-868e-6a2af17e790d","_cell_guid":"617c1db0-6400-4dc8-98f3-147af0058a55","trusted":true}},{"cell_type":"code","source":"from sklearn.ensemble import StackingClassifier\n\n# Define base models\nbase_models = [\n    ('GradientBoosting', gb_model),\n    ('XGBoost', xgb_model),\n    ('LightGBM', lgb_model),\n    ('CatBoost', cat_model),\n    ('RF', rf_model),\n    ('AdaBoost', ABC_model)\n]\n\n# Define meta-model\nmeta_model = LogisticRegression()\n\n# Initialize Stacking Classifier\nstacked_model = StackingClassifier(estimators=base_models, final_estimator=meta_model, cv=5)\n\n# Train the stacked model (Note: The pre-trained model won't be re-trained)\nstacked_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions and evaluation\ny_pred = stacked_model.predict(test)\ny_pred_proba = stacked_model.predict_proba(test)[:, 1]  # Probabilities for the positive class\n\n# Calculate accuracy and AUC Score\naccuracy = accuracy_score(test_labels, y_pred)\nauc_score = roc_auc_score(test_labels, y_pred_proba)\n\nprint(f\"Stacked Model Accuracy: {accuracy * 100.0}%\")\nprint(f\"AUC Score: {auc_score}\")","metadata":{"_uuid":"e7d307d8-6c73-4616-84cc-db48b0fbd784","_cell_guid":"441f7c2b-7ef4-42a0-8bad-07629ebdfd38","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'Stacked Model', \"Accuracy\": accuracy, \"AUC\": auc_score})","metadata":{"_uuid":"7f9eebaa-4217-4882-a544-35f081a09ab6","_cell_guid":"1617ee7b-232c-4f5d-94db-8454a514be33","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Majority Vote Ensembling","metadata":{"_uuid":"4c4d5b41-4876-4d4d-a11c-966c0e4bcc95","_cell_guid":"3f77fab7-ffb9-4e65-87e5-3295531e4e08","trusted":true}},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((gb_y_pred, xgb_y_pred, lgb_y_pred, cat_y_pred, ABC_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\naccuracy = accuracy_score(test_labels, final_predictions)\nprint(f\"Majority Vote Accuracy: {accuracy * 100.0}%\")\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")","metadata":{"_uuid":"14578ead-d54f-43e3-85f1-095f51fc2ed4","_cell_guid":"6cbb3c3b-7d47-4ce0-8b57-955af15866a9","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'Majority Vote', \"Accuracy\": accuracy, \"AUC\": auc_score})","metadata":{"_uuid":"cc86f48e-e19d-4598-8ccd-98041d7b1912","_cell_guid":"721ce3e9-4725-4287-ab04-603e4c10e076","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Neuron Network","metadata":{"_uuid":"922acd08-ccfa-48d1-bb5b-f62f794fec11","_cell_guid":"1e75bf28-6707-48cd-840f-12ae8dc1b8c6","trusted":true}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers, models\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import roc_auc_score","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = train_balanced\ny_train = train_labels_balanced\nX_val = test\ny_val = test_labels\n\nscaler = StandardScaler()\n\n# Fit on training data and transform both training and validation data\nX_train_scaled = scaler.fit_transform(X_train)\nX_val_scaled = scaler.transform(X_val)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = keras.Sequential([\n    layers.Input(shape=(X_train_scaled.shape[1],)),  # Input layer\n    layers.Dense(128, activation='relu'),            # Hidden layer 1\n    layers.Dropout(0.5),                             # Dropout for regularization\n    layers.Dense(64, activation='relu'),             # Hidden layer 2\n    layers.Dropout(0.5),                             # Dropout for regularization\n    layers.BatchNormalization(),\n    layers.Dense(32, activation='relu'),\n    layers.Dense(1, activation='sigmoid')            # Output layer\n])\n\nmodel.compile(optimizer='adam',\n              loss='binary_crossentropy',\n              metrics=[tf.keras.metrics.AUC(name='auc')])\n\n\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(X_train_scaled, y_train,\n                    validation_data=(X_val_scaled, y_val),\n                    epochs=50,\n                    batch_size=256,\n                    verbose=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate the model on the validation set\nval_preds = model.predict(X_val_scaled)\nval_preds_binary = np.where(val_preds > 0.5, 1, 0)\nval_acc = accuracy_score(y_val, val_preds_binary)\nprint(f'Accuracy: {val_acc}')\nval_auc = roc_auc_score(y_val, val_preds)\nprint(f'Validation AUC: {val_auc}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'NN', \"Accuracy\": val_acc, \"AUC\": val_auc})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Cascade Forest Reimplementation","metadata":{}},{"cell_type":"code","source":"\nimport itertools\nimport numpy as np\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\nclass gcForest(object):\n\n    def __init__(self, shape_1X=None, n_mgsRFtree=30, window=None, stride=1,\n                 cascade_test_size=0.2, n_cascadeRF=2, n_cascadeRFtree=101, cascade_layer=np.inf,\n                 min_samples_mgs=0.1, min_samples_cascade=0.05, tolerance=0.0, n_jobs=1):\n        \"\"\" gcForest Classifier.\n\n        :param shape_1X: int or tuple list or np.array (default=None)\n            Shape of a single sample element [n_lines, n_cols]. Required when calling mg_scanning!\n            For sequence data a single int can be given.\n\n        :param n_mgsRFtree: int (default=30)\n            Number of trees in a Random Forest during Multi Grain Scanning.\n\n        :param window: int (default=None)\n            List of window sizes to use during Multi Grain Scanning.\n            If 'None' no slicing will be done.\n\n        :param stride: int (default=1)\n            Step used when slicing the data.\n\n        :param cascade_test_size: float or int (default=0.2)\n            Split fraction or absolute number for cascade training set splitting.\n\n        :param n_cascadeRF: int (default=2)\n            Number of Random Forests in a cascade layer.\n            For each pseudo Random Forest a complete Random Forest is created, hence\n            the total numbe of Random Forests in a layer will be 2*n_cascadeRF.\n\n        :param n_cascadeRFtree: int (default=101)\n            Number of trees in a single Random Forest in a cascade layer.\n\n        :param min_samples_mgs: float or int (default=0.1)\n            Minimum number of samples in a node to perform a split\n            during the training of Multi-Grain Scanning Random Forest.\n            If int number_of_samples = int.\n            If float, min_samples represents the fraction of the initial n_samples to consider.\n\n        :param min_samples_cascade: float or int (default=0.1)\n            Minimum number of samples in a node to perform a split\n            during the training of Cascade Random Forest.\n            If int number_of_samples = int.\n            If float, min_samples represents the fraction of the initial n_samples to consider.\n\n        :param cascade_layer: int (default=np.inf)\n            mMximum number of cascade layers allowed.\n            Useful to limit the contruction of the cascade.\n\n        :param tolerance: float (default=0.0)\n            Accuracy tolerance for the casacade growth.\n            If the improvement in accuracy is not better than the tolerance the construction is\n            stopped.\n\n        :param n_jobs: int (default=1)\n            The number of jobs to run in parallel for any Random Forest fit and predict.\n            If -1, then the number of jobs is set to the number of cores.\n        \"\"\"\n        setattr(self, 'shape_1X', shape_1X)\n        setattr(self, 'n_layer', 0)\n        setattr(self, '_n_samples', 0)\n        setattr(self, 'n_cascadeRF', int(n_cascadeRF))\n        if isinstance(window, int):\n            setattr(self, 'window', [window])\n        elif isinstance(window, list):\n            setattr(self, 'window', window)\n        setattr(self, 'stride', stride)\n        setattr(self, 'cascade_test_size', cascade_test_size)\n        setattr(self, 'n_mgsRFtree', int(n_mgsRFtree))\n        setattr(self, 'n_cascadeRFtree', int(n_cascadeRFtree))\n        setattr(self, 'cascade_layer', cascade_layer)\n        setattr(self, 'min_samples_mgs', min_samples_mgs)\n        setattr(self, 'min_samples_cascade', min_samples_cascade)\n        setattr(self, 'tolerance', tolerance)\n        setattr(self, 'n_jobs', n_jobs)\n\n    def fit(self, X, y):\n        \"\"\" Training the gcForest on input data X and associated target y.\n\n        :param X: np.array\n            Array containing the input samples.\n            Must be of shape [n_samples, data] where data is a 1D array.\n\n        :param y: np.array\n            1D array containing the target values.\n            Must be of shape [n_samples]\n        \"\"\"\n        if np.shape(X)[0] != len(y):\n            raise ValueError('Sizes of y and X do not match.')\n\n        mgs_X = self.mg_scanning(X, y)\n        _ = self.cascade_forest(mgs_X, y)\n\n    def predict_proba(self, X):\n        \"\"\" Predict the class probabilities of unknown samples X.\n\n        :param X: np.array\n            Array containing the input samples.\n            Must be of the same shape [n_samples, data] as the training inputs.\n\n        :return: np.array\n            1D array containing the predicted class probabilities for each input sample.\n        \"\"\"\n        mgs_X = self.mg_scanning(X)\n        cascade_all_pred_prob = self.cascade_forest(mgs_X)\n        predict_proba = np.mean(cascade_all_pred_prob, axis=0)\n\n        return predict_proba\n\n    def predict(self, X):\n        \"\"\" Predict the class of unknown samples X.\n\n        :param X: np.array\n            Array containing the input samples.\n            Must be of the same shape [n_samples, data] as the training inputs.\n\n        :return: np.array\n            1D array containing the predicted class for each input sample.\n        \"\"\"\n        pred_proba = self.predict_proba(X=X)\n        predictions = np.argmax(pred_proba, axis=1)\n\n        return predictions\n\n    def mg_scanning(self, X, y=None):\n        \"\"\" Performs a Multi Grain Scanning on input data.\n\n        :param X: np.array\n            Array containing the input samples.\n            Must be of shape [n_samples, data] where data is a 1D array.\n\n        :param y: np.array (default=None)\n\n        :return: np.array\n            Array of shape [n_samples, .. ] containing Multi Grain Scanning sliced data.\n        \"\"\"\n        setattr(self, '_n_samples', np.shape(X)[0])\n        shape_1X = getattr(self, 'shape_1X')\n        if isinstance(shape_1X, int):\n            shape_1X = [1,shape_1X]\n        if not getattr(self, 'window'):\n            setattr(self, 'window', [shape_1X[1]])\n\n        mgs_pred_prob = []\n\n        for wdw_size in getattr(self, 'window'):\n            wdw_pred_prob = self.window_slicing_pred_prob(X, wdw_size, shape_1X, y=y)\n            mgs_pred_prob.append(wdw_pred_prob)\n\n        return np.concatenate(mgs_pred_prob, axis=1)\n\n    def window_slicing_pred_prob(self, X, window, shape_1X, y=None):\n        \"\"\" Performs a window slicing of the input data and send them through Random Forests.\n        If target values 'y' are provided sliced data are then used to train the Random Forests.\n\n        :param X: np.array\n            Array containing the input samples.\n            Must be of shape [n_samples, data] where data is a 1D array.\n\n        :param window: int\n            Size of the window to use for slicing.\n\n        :param shape_1X: list or np.array\n            Shape of a single sample.\n\n        :param y: np.array (default=None)\n            Target values. If 'None' no training is done.\n\n        :return: np.array\n            Array of size [n_samples, ..] containing the Random Forest.\n            prediction probability for each input sample.\n        \"\"\"\n        n_tree = getattr(self, 'n_mgsRFtree')\n        min_samples = getattr(self, 'min_samples_mgs')\n        stride = getattr(self, 'stride')\n\n        if shape_1X[0] > 1:\n            print('Slicing Images...')\n            sliced_X, sliced_y = self._window_slicing_img(X, window, shape_1X, y=y, stride=stride)\n        else:\n            print('Slicing Sequence...')\n            sliced_X, sliced_y = self._window_slicing_sequence(X, window, shape_1X, y=y, stride=stride)\n\n        if y is not None:\n            n_jobs = getattr(self, 'n_jobs')\n            prf = RandomForestClassifier(n_estimators=n_tree, max_features='sqrt',\n                                         min_samples_split=min_samples, oob_score=True, n_jobs=n_jobs)\n            crf = RandomForestClassifier(n_estimators=n_tree, max_features=1,\n                                         min_samples_split=min_samples, oob_score=True, n_jobs=n_jobs)\n            print('Training MGS Random Forests...')\n            prf.fit(sliced_X, sliced_y)\n            crf.fit(sliced_X, sliced_y)\n            setattr(self, '_mgsprf_{}'.format(window), prf)\n            setattr(self, '_mgscrf_{}'.format(window), crf)\n            pred_prob_prf = prf.oob_decision_function_\n            pred_prob_crf = crf.oob_decision_function_\n\n        if hasattr(self, '_mgsprf_{}'.format(window)) and y is None:\n            prf = getattr(self, '_mgsprf_{}'.format(window))\n            crf = getattr(self, '_mgscrf_{}'.format(window))\n            pred_prob_prf = prf.predict_proba(sliced_X)\n            pred_prob_crf = crf.predict_proba(sliced_X)\n\n        pred_prob = np.c_[pred_prob_prf, pred_prob_crf]\n\n        return pred_prob.reshape([getattr(self, '_n_samples'), -1])\n\n    def _window_slicing_img(self, X, window, shape_1X, y=None, stride=1):\n        \"\"\" Slicing procedure for images\n\n        :param X: np.array\n            Array containing the input samples.\n            Must be of shape [n_samples, data] where data is a 1D array.\n\n        :param window: int\n            Size of the window to use for slicing.\n\n        :param shape_1X: list or np.array\n            Shape of a single sample [n_lines, n_cols].\n\n        :param y: np.array (default=None)\n            Target values.\n\n        :param stride: int (default=1)\n            Step used when slicing the data.\n\n        :return: np.array and np.array\n            Arrays containing the sliced images and target values (empty if 'y' is None).\n        \"\"\"\n        if any(s < window for s in shape_1X):\n            raise ValueError('window must be smaller than both dimensions for an image')\n\n        len_iter_x = np.floor_divide((shape_1X[1] - window), stride) + 1\n        len_iter_y = np.floor_divide((shape_1X[0] - window), stride) + 1\n        iterx_array = np.arange(0, stride*len_iter_x, stride)\n        itery_array = np.arange(0, stride*len_iter_y, stride)\n\n        ref_row = np.arange(0, window)\n        ref_ind = np.ravel([ref_row + shape_1X[1] * i for i in range(window)])\n        inds_to_take = [ref_ind + ix + shape_1X[1] * iy\n                        for ix, iy in itertools.product(iterx_array, itery_array)]\n\n        sliced_imgs = np.take(X, inds_to_take, axis=1).reshape(-1, window**2)\n\n        if y is not None:\n            sliced_target = np.repeat(y, len_iter_x * len_iter_y)\n        elif y is None:\n            sliced_target = None\n\n        return sliced_imgs, sliced_target\n\n    def _window_slicing_sequence(self, X, window, shape_1X, y=None, stride=1):\n        \"\"\" Slicing procedure for sequences (aka shape_1X = [.., 1]).\n\n        :param X: np.array\n            Array containing the input samples.\n            Must be of shape [n_samples, data] where data is a 1D array.\n\n        :param window: int\n            Size of the window to use for slicing.\n\n        :param shape_1X: list or np.array\n            Shape of a single sample [n_lines, n_col].\n\n        :param y: np.array (default=None)\n            Target values.\n\n        :param stride: int (default=1)\n            Step used when slicing the data.\n\n        :return: np.array and np.array\n            Arrays containing the sliced sequences and target values (empty if 'y' is None).\n        \"\"\"\n        if shape_1X[1] < window:\n            raise ValueError('window must be smaller than the sequence dimension')\n\n        len_iter = np.floor_divide((shape_1X[1] - window), stride) + 1\n        iter_array = np.arange(0, stride*len_iter, stride)\n\n        ind_1X = np.arange(np.prod(shape_1X))\n        inds_to_take = [ind_1X[i:i+window] for i in iter_array]\n        sliced_sqce = np.take(X, inds_to_take, axis=1).reshape(-1, window)\n\n        if y is not None:\n            sliced_target = np.repeat(y, len_iter)\n        elif y is None:\n            sliced_target = None\n\n        return sliced_sqce, sliced_target\n\n    def cascade_forest(self, X, y=None):\n        \"\"\" Perform (or train if 'y' is not None) a cascade forest estimator.\n\n        :param X: np.array\n            Array containing the input samples.\n            Must be of shape [n_samples, data] where data is a 1D array.\n\n        :param y: np.array (default=None)\n            Target values. If 'None' perform training.\n\n        :return: np.array\n            1D array containing the predicted class for each input sample.\n        \"\"\"\n        if y is not None:\n            setattr(self, 'n_layer', 0)\n            test_size = getattr(self, 'cascade_test_size')\n            max_layers = getattr(self, 'cascade_layer')\n            tol = getattr(self, 'tolerance')\n\n            X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size)\n\n            self.n_layer += 1\n            prf_crf_pred_ref = self._cascade_layer(X_train, y_train)\n            accuracy_ref = self._cascade_evaluation(X_test, y_test)\n            feat_arr = self._create_feat_arr(X_train, prf_crf_pred_ref)\n\n            self.n_layer += 1\n            prf_crf_pred_layer = self._cascade_layer(feat_arr, y_train)\n            accuracy_layer = self._cascade_evaluation(X_test, y_test)\n\n            while accuracy_layer > (accuracy_ref + tol) and self.n_layer <= max_layers:\n                accuracy_ref = accuracy_layer\n                prf_crf_pred_ref = prf_crf_pred_layer\n                feat_arr = self._create_feat_arr(X_train, prf_crf_pred_ref)\n                self.n_layer += 1\n                prf_crf_pred_layer = self._cascade_layer(feat_arr, y_train)\n                accuracy_layer = self._cascade_evaluation(X_test, y_test)\n\n            if accuracy_layer < accuracy_ref :\n                n_cascadeRF = getattr(self, 'n_cascadeRF')\n                for irf in range(n_cascadeRF):\n                    delattr(self, '_casprf{}_{}'.format(self.n_layer, irf))\n                    delattr(self, '_cascrf{}_{}'.format(self.n_layer, irf))\n                self.n_layer -= 1\n\n        elif y is None:\n            at_layer = 1\n            prf_crf_pred_ref = self._cascade_layer(X, layer=at_layer)\n            while at_layer < getattr(self, 'n_layer'):\n                at_layer += 1\n                feat_arr = self._create_feat_arr(X, prf_crf_pred_ref)\n                prf_crf_pred_ref = self._cascade_layer(feat_arr, layer=at_layer)\n\n        return prf_crf_pred_ref\n\n    def _cascade_layer(self, X, y=None, layer=0):\n        \"\"\" Cascade layer containing Random Forest estimators.\n        If y is not None the layer is trained.\n\n        :param X: np.array\n            Array containing the input samples.\n            Must be of shape [n_samples, data] where data is a 1D array.\n\n        :param y: np.array (default=None)\n            Target values. If 'None' perform training.\n\n        :param layer: int (default=0)\n            Layer indice. Used to call the previously trained layer.\n\n        :return: list\n            List containing the prediction probabilities for all samples.\n        \"\"\"\n        n_tree = getattr(self, 'n_cascadeRFtree')\n        n_cascadeRF = getattr(self, 'n_cascadeRF')\n        min_samples = getattr(self, 'min_samples_cascade')\n\n        n_jobs = getattr(self, 'n_jobs')\n        prf = RandomForestClassifier(n_estimators=n_tree, max_features='sqrt',\n                                     min_samples_split=min_samples, oob_score=True, n_jobs=n_jobs)\n        crf = RandomForestClassifier(n_estimators=n_tree, max_features=1,\n                                     min_samples_split=min_samples, oob_score=True, n_jobs=n_jobs)\n\n        prf_crf_pred = []\n        if y is not None:\n            print('Adding/Training Layer, n_layer={}'.format(self.n_layer))\n            for irf in range(n_cascadeRF):\n                prf.fit(X, y)\n                crf.fit(X, y)\n                setattr(self, '_casprf{}_{}'.format(self.n_layer, irf), prf)\n                setattr(self, '_cascrf{}_{}'.format(self.n_layer, irf), crf)\n                prf_crf_pred.append(prf.oob_decision_function_)\n                prf_crf_pred.append(crf.oob_decision_function_)\n        elif y is None:\n            for irf in range(n_cascadeRF):\n                prf = getattr(self, '_casprf{}_{}'.format(layer, irf))\n                crf = getattr(self, '_cascrf{}_{}'.format(layer, irf))\n                prf_crf_pred.append(prf.predict_proba(X))\n                prf_crf_pred.append(crf.predict_proba(X))\n\n        return prf_crf_pred\n\n    def _cascade_evaluation(self, X_test, y_test):\n        \"\"\" Evaluate the accuracy of the cascade using X and y.\n\n        :param X_test: np.array\n            Array containing the test input samples.\n            Must be of the same shape as training data.\n\n        :param y_test: np.array\n            Test target values.\n\n        :return: float\n            the cascade accuracy.\n        \"\"\"\n        casc_pred_prob = np.mean(self.cascade_forest(X_test), axis=0)\n        casc_pred = np.argmax(casc_pred_prob, axis=1)\n        casc_accuracy = accuracy_score(y_true=y_test, y_pred=casc_pred)\n        print('Layer validation accuracy = {}'.format(casc_accuracy))\n\n        return casc_accuracy\n\n    def _create_feat_arr(self, X, prf_crf_pred):\n        \"\"\" Concatenate the original feature vector with the predicition probabilities\n        of a cascade layer.\n\n        :param X: np.array\n            Array containing the input samples.\n            Must be of shape [n_samples, data] where data is a 1D array.\n\n        :param prf_crf_pred: list\n            Prediction probabilities by a cascade layer for X.\n\n        :return: np.array\n            Concatenation of X and the predicted probabilities.\n            To be used for the next layer in a cascade forest.\n        \"\"\"\n        swap_pred = np.swapaxes(prf_crf_pred, 0, 1)\n        add_feat = swap_pred.reshape([np.shape(X)[0], -1])\n        feat_arr = np.concatenate([add_feat, X], axis=1)\n\n        return feat_arr\n    \nrows, cols = train_balanced.shape\ntemp = train_balanced.values.tolist()  # Convert data to a list of lists\ntarget_list = train_labels_balanced.tolist()\n\nX_train = temp\ny_train = target_list\n\nmodel = gcForest(shape_1X=cols, window=2, tolerance=0.0)\nmodel.fit(X_train, y_train)\n\n# Predictions\ntemp_test = test.values.tolist()  # Convert data to a list of lists\ny_pred = model.predict(temp_test)\ny_pred_proba = model.predict_proba(temp_test)[:, 1]\n\n# Evaluate the model\naccuracy = accuracy_score(test_labels, y_pred)\nauc_score = roc_auc_score(test_labels, y_pred_proba)\n\nprint(f\"Accuracy: {accuracy * 100.0}%\")\nprint(f\"AUC Score: {auc_score}\")\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'Cascade Random Forest', \"Accuracy\": accuracy, \"AUC\": auc_score})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Final Results","metadata":{}},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Light GBM both achieved good results on Accuracy and AUC. Our method \"Stacking\" also achieve good results. ","metadata":{}}]}