{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":8540,"databundleVersionId":862041,"sourceType":"competition"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.linear_model import LogisticRegression\nfrom tqdm import tqdm\nfrom imblearn.over_sampling import SMOTE\nfrom sklearn.utils import resample\nfrom sklearn.model_selection import cross_val_score\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import AdaBoostClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.svm import LinearSVC\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.ensemble import StackingClassifier\nfrom scipy.stats import mode\nfrom catboost import CatBoostClassifier, Pool\nfrom skopt import BayesSearchCV\nimport torch\nimport numpy as np\nfrom sklearn.preprocessing import MinMaxScaler, LabelEncoder, StandardScaler\nfrom sklearn.metrics import log_loss\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.metrics import roc_curve\nimport sklearn.metrics as metrics\nimport matplotlib.pyplot as plt\n# !pip install optbinning\n# from optbinning import OptimalBinning, Scorecard, BinningProcess\nimport gc \nimport math\nfrom sklearn.model_selection import train_test_split, RandomizedSearchCV\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import roc_auc_score\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:30:42.997288Z","iopub.execute_input":"2024-04-23T09:30:42.99768Z","iopub.status.idle":"2024-04-23T09:30:43.010754Z","shell.execute_reply.started":"2024-04-23T09:30:42.99765Z","shell.execute_reply":"2024-04-23T09:30:43.009473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.ensemble import RandomForestClassifier\nimport xgboost as xgb\nfrom sklearn.metrics import f1_score, roc_auc_score  # Import f1_score\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.ensemble import RandomForestClassifier, StackingClassifier, GradientBoostingClassifier, AdaBoostClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import SVC\nfrom sklearn.neighbors import KNeighborsClassifier","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dask\nimport dask.dataframe as dd\nimport numpy as np\nimport pandas as pd\nimport os\nimport time\n\nimport matplotlib.pyplot as plt\nplt.rcParams[\"figure.figsize\"] = [16,9]\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:30:43.013239Z","iopub.execute_input":"2024-04-23T09:30:43.013629Z","iopub.status.idle":"2024-04-23T09:30:43.02746Z","shell.execute_reply.started":"2024-04-23T09:30:43.013597Z","shell.execute_reply":"2024-04-23T09:30:43.026086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtypes = {\n        'ip'            : 'uint32',\n        'app'           : 'object',\n        'device'        : 'object',\n        'os'            : 'object',\n        'channel'       : 'object',\n        'click_time'    : 'object',\n        'is_attributed' : 'uint8',\n        }","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:30:43.028777Z","iopub.execute_input":"2024-04-23T09:30:43.029097Z","iopub.status.idle":"2024-04-23T09:30:43.039159Z","shell.execute_reply.started":"2024-04-23T09:30:43.02907Z","shell.execute_reply":"2024-04-23T09:30:43.037945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Sample data - (100000, 8)\n#click_data = pd.read_csv('../input/talkingdata-adtracking-fraud-detection/train_sample.csv', parse_dates=['click_time'])\n\n#Full data - No idea how large it is, this notebook can not handle its size in RAM\n\n#Read only first limit rows\n#limit = 20_000_000\n\n#Read only these columns - skip attributed_time \nusecols = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:30:43.041748Z","iopub.execute_input":"2024-04-23T09:30:43.04275Z","iopub.status.idle":"2024-04-23T09:30:43.058538Z","shell.execute_reply.started":"2024-04-23T09:30:43.042713Z","shell.execute_reply":"2024-04-23T09:30:43.057485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Competition data","metadata":{}},{"cell_type":"code","source":"competition_data = dd.read_csv('../input/talkingdata-adtracking-fraud-detection/train.csv', \n                               dtype=dtypes,\n                               #nrows=limit, #not supported by `dd.read_csv`\n                               usecols=usecols, \n                               parse_dates=['click_time'])","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:30:43.059954Z","iopub.execute_input":"2024-04-23T09:30:43.060573Z","iopub.status.idle":"2024-04-23T09:30:43.29852Z","shell.execute_reply.started":"2024-04-23T09:30:43.060538Z","shell.execute_reply":"2024-04-23T09:30:43.297292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"competition_data.info()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:30:43.299913Z","iopub.execute_input":"2024-04-23T09:30:43.300256Z","iopub.status.idle":"2024-04-23T09:30:43.307122Z","shell.execute_reply.started":"2024-04-23T09:30:43.300225Z","shell.execute_reply":"2024-04-23T09:30:43.305971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"competition_data_postitive = competition_data[competition_data.is_attributed == 1] ","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:30:43.308597Z","iopub.execute_input":"2024-04-23T09:30:43.308932Z","iopub.status.idle":"2024-04-23T09:30:43.412357Z","shell.execute_reply.started":"2024-04-23T09:30:43.308904Z","shell.execute_reply":"2024-04-23T09:30:43.411422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_time = time.time()\ntype(competition_data_postitive)\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:30:43.4135Z","iopub.execute_input":"2024-04-23T09:30:43.414342Z","iopub.status.idle":"2024-04-23T09:30:43.420201Z","shell.execute_reply.started":"2024-04-23T09:30:43.414308Z","shell.execute_reply":"2024-04-23T09:30:43.418998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_time = time.time()\ncompetition_data_postitive = competition_data_postitive.compute()\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:30:43.422166Z","iopub.execute_input":"2024-04-23T09:30:43.422745Z","iopub.status.idle":"2024-04-23T09:41:53.657458Z","shell.execute_reply.started":"2024-04-23T09:30:43.422705Z","shell.execute_reply":"2024-04-23T09:41:53.656085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"competition_data_postitive.sample(10)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:41:53.663312Z","iopub.execute_input":"2024-04-23T09:41:53.664326Z","iopub.status.idle":"2024-04-23T09:41:53.69659Z","shell.execute_reply.started":"2024-04-23T09:41:53.664259Z","shell.execute_reply":"2024-04-23T09:41:53.692836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"competition_data_negative = competition_data[competition_data.is_attributed == 0] ","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:41:53.712633Z","iopub.execute_input":"2024-04-23T09:41:53.713318Z","iopub.status.idle":"2024-04-23T09:41:53.720954Z","shell.execute_reply.started":"2024-04-23T09:41:53.713278Z","shell.execute_reply":"2024-04-23T09:41:53.719824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"competition_data_negative = competition_data_negative.sample(frac=0.0025) #number","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:41:53.72622Z","iopub.execute_input":"2024-04-23T09:41:53.726545Z","iopub.status.idle":"2024-04-23T09:41:53.743063Z","shell.execute_reply.started":"2024-04-23T09:41:53.726516Z","shell.execute_reply":"2024-04-23T09:41:53.741594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_time = time.time()\ncompetition_data_negative = competition_data_negative.compute()\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:41:53.744261Z","iopub.execute_input":"2024-04-23T09:41:53.744605Z","iopub.status.idle":"2024-04-23T09:56:14.269392Z","shell.execute_reply.started":"2024-04-23T09:41:53.744574Z","shell.execute_reply":"2024-04-23T09:56:14.26809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"competition_data_negative.sample(10)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:56:14.270836Z","iopub.execute_input":"2024-04-23T09:56:14.271157Z","iopub.status.idle":"2024-04-23T09:56:14.300374Z","shell.execute_reply.started":"2024-04-23T09:56:14.27113Z","shell.execute_reply":"2024-04-23T09:56:14.299112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"competition_data = pd.concat([competition_data_postitive, competition_data_negative])","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:56:14.302322Z","iopub.execute_input":"2024-04-23T09:56:14.302635Z","iopub.status.idle":"2024-04-23T09:56:14.400381Z","shell.execute_reply.started":"2024-04-23T09:56:14.302609Z","shell.execute_reply":"2024-04-23T09:56:14.399156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train, test, train_labels, test_labels = train_test_split(competition_data, competition_data.is_attributed, test_size=0.3, random_state=42)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:26:53.941521Z","iopub.execute_input":"2024-04-23T10:26:53.941988Z","iopub.status.idle":"2024-04-23T10:26:54.63524Z","shell.execute_reply.started":"2024-04-23T10:26:53.941952Z","shell.execute_reply":"2024-04-23T10:26:54.634177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering 1 Construct features from timestamps\n\nNotice that the `click_data` DataFrame has a `'click_time'` column with timestamp data.\n\nUse this column to create features for the coresponding day, hour, minute and second. \n\nStore these as new integer columns `day`, `hour`, `minute`, and `second` in a new DataFrame `clicks`.","metadata":{}},{"cell_type":"code","source":"# Add new columns for timestamp features day, hour, minute, and second\ntrain['click_time'] = pd.to_datetime(train['click_time'])\ntrain['day'] = train['click_time'].dt.day.astype('uint8')\n# Fill in the rest\ntrain['hour'] = train['click_time'].dt.hour.astype('uint8')\ntrain['minute'] = train['click_time'].dt.minute.astype('uint8')\ntrain['second'] = train['click_time'].dt.second.astype('uint8')\n","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:26:54.637031Z","iopub.execute_input":"2024-04-23T10:26:54.637356Z","iopub.status.idle":"2024-04-23T10:26:55.026402Z","shell.execute_reply.started":"2024-04-23T10:26:54.637329Z","shell.execute_reply":"2024-04-23T10:26:55.025232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Add new columns for timestamp features day, hour, minute, and second\ntest['click_time'] = pd.to_datetime(test['click_time'])\ntest['day'] = test['click_time'].dt.day.astype('uint8')\n# Fill in the rest\ntest['hour'] = test['click_time'].dt.hour.astype('uint8')\ntest['minute'] = test['click_time'].dt.minute.astype('uint8')\ntest['second'] = test['click_time'].dt.second.astype('uint8')","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:26:55.028489Z","iopub.execute_input":"2024-04-23T10:26:55.028863Z","iopub.status.idle":"2024-04-23T10:26:55.204817Z","shell.execute_reply.started":"2024-04-23T10:26:55.028832Z","shell.execute_reply":"2024-04-23T10:26:55.203588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# store the label\ntrain_labels = train.is_attributed.values\ntest_labels = test.is_attributed.values\n\n# drop labels and attributed_time since it represnets the same info as the is_attributed\ntrain.drop(labels = ['click_time','is_attributed'], axis = 1, inplace = True)\ntest.drop(labels = ['click_time', 'is_attributed'], axis = 1, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:26:55.206269Z","iopub.execute_input":"2024-04-23T10:26:55.206704Z","iopub.status.idle":"2024-04-23T10:26:55.431686Z","shell.execute_reply.started":"2024-04-23T10:26:55.206665Z","shell.execute_reply":"2024-04-23T10:26:55.430589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.fillna(train.max())\ntest = test.fillna(test.max())","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:26:55.533704Z","iopub.execute_input":"2024-04-23T10:26:55.534099Z","iopub.status.idle":"2024-04-23T10:26:57.405832Z","shell.execute_reply.started":"2024-04-23T10:26:55.53407Z","shell.execute_reply":"2024-04-23T10:26:57.404707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate():\n  \n  fpr, tpr, _ = metrics.roc_curve(test_labels,  y_pred_proba)\n\n  # Plot the ROC curve\n  plt.plot(fpr, tpr, color='blue', label='ROC Curve')\n\n  # Fill below the ROC curve with color\n  plt.fill_between(fpr, 0, tpr, color='skyblue', alpha=0.2)\n\n  # Plot the diagonal line (y=x) for reference\n  plt.plot([0, 1], [0, 1], color='red', linestyle='--', label='Random Chance')\n\n  # Add labels and title\n  plt.ylabel('True Positive Rate')\n  plt.xlabel('False Positive Rate')\n  plt.title('ROC Curve')\n    \n\n  # Show the plot\n  plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:07:10.464219Z","iopub.execute_input":"2024-04-23T10:07:10.464555Z","iopub.status.idle":"2024-04-23T10:07:10.4766Z","shell.execute_reply.started":"2024-04-23T10:07:10.464526Z","shell.execute_reply":"2024-04-23T10:07:10.475001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = []","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:27:17.560429Z","iopub.execute_input":"2024-04-23T10:27:17.560832Z","iopub.status.idle":"2024-04-23T10:27:17.569492Z","shell.execute_reply.started":"2024-04-23T10:27:17.560802Z","shell.execute_reply":"2024-04-23T10:27:17.568166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# from sklearn.linear_model import LogisticRegression\n\n# # temp_train = train.drop(labels='click_time', axis=1)\n# # temp_test = test.drop(labels='click_time', axis=1)\n\n# # Initialize logistic regression model with 'sag' solver\n# model = LogisticRegression(class_weight = 'balanced')\n\n# # Train the model\n# model.fit(train, train_labels)\n\n# # Predictions\n# y_pred = model.predict(test)\n# y_pred_proba = model.predict_proba(test)[:, 1]\n\n# from sklearn.metrics import f1_score\n\n# # Calculate F1-score\n# f1 = f1_score(test_labels, y_pred)\n# auc_score = roc_auc_score(test_labels, y_pred_proba)\n\n\n# # Print the F1-score (Note: No need for a probability array for F1-score)\n# print(f\"F1 Score: {f1}\")\n# print(f\"AUC Score: {auc_score}\")\n\n# evaluate()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:27:18.435074Z","iopub.execute_input":"2024-04-23T10:27:18.435451Z","iopub.status.idle":"2024-04-23T10:27:26.947985Z","shell.execute_reply.started":"2024-04-23T10:27:18.435422Z","shell.execute_reply":"2024-04-23T10:27:26.94682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# results.append({\"Model\": 'Baseline Logistic Regression before label encoding', \"F1 score\": f1, \"AUC\": auc_score})","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:29:30.660035Z","iopub.execute_input":"2024-04-23T10:29:30.660496Z","iopub.status.idle":"2024-04-23T10:29:30.669339Z","shell.execute_reply.started":"2024-04-23T10:29:30.660465Z","shell.execute_reply":"2024-04-23T10:29:30.668194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_feats = ['app', 'device', 'os', 'channel']\nlabelencoder = LabelEncoder()\n\nfor col in cat_feats:\n    train[col] = labelencoder.fit_transform(train[col])\n    test[col] = labelencoder.fit_transform(test[col])\n# for col in cat_feats:\n#     train[col] = train[col].astype('int')\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:31:59.757358Z","iopub.execute_input":"2024-04-23T10:31:59.75792Z","iopub.status.idle":"2024-04-23T10:32:01.514594Z","shell.execute_reply.started":"2024-04-23T10:31:59.757881Z","shell.execute_reply":"2024-04-23T10:32:01.513565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:32:17.618975Z","iopub.execute_input":"2024-04-23T10:32:17.619826Z","iopub.status.idle":"2024-04-23T10:32:17.645535Z","shell.execute_reply.started":"2024-04-23T10:32:17.619756Z","shell.execute_reply":"2024-04-23T10:32:17.643683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.linear_model import LogisticRegression\n\n# # temp_train = train.drop(labels='click_time', axis=1)\n# # temp_test = test.drop(labels='click_time', axis=1)\n\n# # Initialize logistic regression model with 'sag' solver\n# model = LogisticRegression(class_weight = 'balanced')\n\n# # Train the model\n# model.fit(train, train_labels)\n\n# # Predictions\n# y_pred = model.predict(test)\n# y_pred_proba = model.predict_proba(test)[:, 1]\n\n# from sklearn.metrics import f1_score\n\n# # Calculate F1-score\n# f1 = f1_score(test_labels, y_pred)\n# auc_score = roc_auc_score(test_labels, y_pred_proba)\n\n\n# # Print the F1-score (Note: No need for a probability array for F1-score)\n# print(f\"F1 Score: {f1}\")\n# print(f\"AUC Score: {auc_score}\")\n\n# evaluate()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:32:27.616861Z","iopub.execute_input":"2024-04-23T10:32:27.617355Z","iopub.status.idle":"2024-04-23T10:32:37.569858Z","shell.execute_reply.started":"2024-04-23T10:32:27.617308Z","shell.execute_reply":"2024-04-23T10:32:37.568865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# results.append({\"Model\": 'Baseline Logistic Regression after label encoding', \"F1 score\": f1, \"AUC\": auc_score})","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:32:37.571407Z","iopub.execute_input":"2024-04-23T10:32:37.572406Z","iopub.status.idle":"2024-04-23T10:32:37.577042Z","shell.execute_reply.started":"2024-04-23T10:32:37.572372Z","shell.execute_reply":"2024-04-23T10:32:37.576075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# # Initialize LightGBM Classifier with selected parameters\n# lgb_model = lgb.LGBMClassifier(class_weight = 'balanced')\n\n# # Perform cross-validation\n# # cv_scores = cross_val_score(lgb_model, train, train_labels, cv=5, scoring='roc_auc')\n\n# # Train the model on the entire training set\n# lgb_model.fit(train, train_labels)\n\n# # Predictions on the test set\n# lgb_y_pred = lgb_model.predict(test)\n# y_pred_proba = lgb_model.predict_proba(test)[:, 1]\n# # Calculate accuracy and AUC Score on the test set\n# lgb_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\n# # print(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\n# # print(f\"Average Cross-Validation ROC-AUC Score: {np.mean(cv_scores)}\")\n# # Calculate F1-score\n# f1 = f1_score(test_labels, lgb_y_pred) \n# auc_score = roc_auc_score(test_labels, y_pred_proba)\n\n\n# # Print the F1-score (Note: No need for a probability array for F1-score)\n# print(f\"F1 Score: {f1}\")\n# print(f\"AUC Score: {auc_score}\")\n\n# evaluate()\n","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:32:44.304469Z","iopub.execute_input":"2024-04-23T10:32:44.305544Z","iopub.status.idle":"2024-04-23T10:33:02.255992Z","shell.execute_reply.started":"2024-04-23T10:32:44.305507Z","shell.execute_reply":"2024-04-23T10:33:02.254539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# features = list(train.columns)\n# feature_importances = pd.DataFrame({'feature': features, 'importance': lgb_model.feature_importances_})\n# feature_importances.head()\n# def plot_feature_importances(df, n = 9, threshold = None):\n#     plt.style.use('fivethirtyeight')\n    \n#     # Sort features with most important at the head\n#     df = df.sort_values('importance', ascending = False).reset_index(drop = True)\n    \n#     # Normalize the feature importances to add up to one and calculate cumulative importance\n#     df['importance_normalized'] = df['importance'] / df['importance'].sum()\n#     df['cumulative_importance'] = np.cumsum(df['importance_normalized'])\n    \n#     plt.rcParams['font.size'] = 12\n    \n#     # Bar plot of n most important features\n#     df.loc[:n, :].plot.barh(y = 'importance_normalized', \n#                             x = 'feature', color = 'darkgreen', \n#                             edgecolor = 'k', figsize = (20, 13),\n#                             legend = False, linewidth = 2)\n\n#     plt.xlabel('Normalized Importance', size = 18); plt.ylabel(''); \n#     plt.title(f'{n} Most Important Features', size = 18)\n#     plt.gca().invert_yaxis()\n#     if threshold:\n#         # Cumulative importance plot\n#         plt.figure(figsize = (8, 6))\n#         plt.plot(list(range(len(df))), df['cumulative_importance'], 'b-')\n#         plt.xlabel('Number of Features', size = 16); plt.ylabel('Cumulative Importance', size = 16); \n#         plt.title('Cumulative Feature Importance', size = 18);\n        \n#         # Number of features needed for threshold cumulative importance\n#         # This is the index (will need to add 1 for the actual number)\n#         importance_index = np.min(np.where(df['cumulative_importance'] > threshold))\n        \n#         # Add vertical line to plot\n#         plt.vlines(importance_index + 1, ymin = 0, ymax = 1.05, linestyles = '--', colors = 'red')\n#         plt.show();\n        \n#         print('{} features required for {:.0f}% of cumulative importance.'.format(importance_index + 1, \n#                                                                                   100 * threshold))\n    \n#     return df\n# norm_fi = plot_feature_importances(feature_importances, threshold=0.95)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:34:16.971721Z","iopub.execute_input":"2024-04-23T10:34:16.972189Z","iopub.status.idle":"2024-04-23T10:34:17.740493Z","shell.execute_reply.started":"2024-04-23T10:34:16.972155Z","shell.execute_reply":"2024-04-23T10:34:17.739235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# results.append({\"Model\": 'Light GBM Baseline', \"F1 score\": f1, \"AUC\": auc_score})\n","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:34:24.664595Z","iopub.execute_input":"2024-04-23T10:34:24.665027Z","iopub.status.idle":"2024-04-23T10:34:24.671492Z","shell.execute_reply.started":"2024-04-23T10:34:24.664993Z","shell.execute_reply":"2024-04-23T10:34:24.670181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# params = {'eta': 0.2,\n#           'max_depth': 15,\n#           'subsample': 0.7,\n#           'colsample_bytree': 0.9,\n#           'objective': 'binary:logistic',\n#           'scale_pos_weight': 9,\n#           'eval_metric': 'auc',\n#           'n_jobs': 24,\n#           'n_estimators': 400\n#           }\n\n# # Initialize XGBoost Classifier with selected parameters\n# class_weight={0:1,1:19}\n# pos_weight = sum(class_weight.values())/sum(class_weight.keys())\n# xgb_model = xgb.XGBClassifier(scale_pos_weight = pos_weight, class_weight = class_weight)\n\n\n# # Perform cross-validation\n# # cv_scores = cross_val_score(xgb_model, train, train_labels, cv=5, scoring='roc_auc')\n\n# # Train the model on the entire training set\n# xgb_model.fit(train, train_labels)\n\n# # Predictions on the test set\n# xgb_y_pred = xgb_model.predict(test)\n# y_pred_proba = xgb_model.predict_proba(test)[:, 1]\n\n# f1 = f1_score(test_labels, xgb_y_pred)\n# auc_score = roc_auc_score(test_labels, y_pred_proba)\n\n\n# # Print the F1-score (Note: No need for a probability array for F1-score)\n# print(f\"F1 Score: {f1}\")\n# print(f\"AUC Score: {auc_score}\")\n# # Calculate accuracy and AUC Score on the test set\n\n# # print(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\n# # print(f\"Average Cross-Validation ROC-AUC Score: {np.mean(cv_scores)}\")\n\n\n\n# evaluate()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:34:25.093038Z","iopub.execute_input":"2024-04-23T10:34:25.093442Z","iopub.status.idle":"2024-04-23T10:34:37.48066Z","shell.execute_reply.started":"2024-04-23T10:34:25.09339Z","shell.execute_reply":"2024-04-23T10:34:37.479465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Display feature importance\n# feature_importance = xgb_model.feature_importances_\n# sorted_idx = np.argsort(feature_importance)[::-1]\n\n# print(\"\\nTop 9 Feature Importance:\")\n# for i in range(9):\n#     print(f\"{train.columns[sorted_idx[i]]}: {feature_importance[sorted_idx[i]]}\")","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:34:51.271479Z","iopub.execute_input":"2024-04-23T10:34:51.271966Z","iopub.status.idle":"2024-04-23T10:34:51.282577Z","shell.execute_reply.started":"2024-04-23T10:34:51.271931Z","shell.execute_reply":"2024-04-23T10:34:51.281132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train, test, train_labels, test_labels","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:39:35.340174Z","iopub.execute_input":"2024-04-23T10:39:35.340608Z","iopub.status.idle":"2024-04-23T10:39:35.442517Z","shell.execute_reply.started":"2024-04-23T10:39:35.340575Z","shell.execute_reply":"2024-04-23T10:39:35.441606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"train, test, train_labels, test_labels = train_test_split(competition_data, competition_data.is_attributed, test_size=0.3, random_state=42)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:39:38.044557Z","iopub.execute_input":"2024-04-23T10:39:38.045364Z","iopub.status.idle":"2024-04-23T10:39:38.847619Z","shell.execute_reply.started":"2024-04-23T10:39:38.04532Z","shell.execute_reply":"2024-04-23T10:39:38.846559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:42:22.568566Z","iopub.execute_input":"2024-04-23T10:42:22.569008Z","iopub.status.idle":"2024-04-23T10:42:23.178027Z","shell.execute_reply.started":"2024-04-23T10:42:22.568972Z","shell.execute_reply":"2024-04-23T10:42:23.177014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add new columns for timestamp features day, hour, minute, and second\ntrain['click_time'] = pd.to_datetime(train['click_time'])\ntrain['day'] = train['click_time'].dt.day.astype('uint8')\n# Fill in the rest\ntrain['hour'] = train['click_time'].dt.hour.astype('uint8')\ntrain['minute'] = train['click_time'].dt.minute.astype('uint8')\ntrain['second'] = train['click_time'].dt.second.astype('uint8')\n\n# Add new columns for timestamp features day, hour, minute, and second\ntest['click_time'] = pd.to_datetime(test['click_time'])\ntest['day'] = test['click_time'].dt.day.astype('uint8')\n# Fill in the rest\ntest['hour'] = test['click_time'].dt.hour.astype('uint8')\ntest['minute'] = test['click_time'].dt.minute.astype('uint8')\ntest['second'] = test['click_time'].dt.second.astype('uint8')","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:44:08.920292Z","iopub.execute_input":"2024-04-23T10:44:08.920722Z","iopub.status.idle":"2024-04-23T10:44:09.49626Z","shell.execute_reply.started":"2024-04-23T10:44:08.920691Z","shell.execute_reply":"2024-04-23T10:44:09.495071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def do_prev_Click( df,agg_suffix='prevClick', agg_type='float32'):\n\n    print(f\">> \\nExtracting {agg_suffix} time calculation features...\\n\")\n    \n    GROUP_BY_NEXT_CLICKS = [\n \n    {'groupby': ['ip', 'channel']},\n    {'groupby': ['ip', 'os']},\n  \n    ]\n\n    # Calculate the time to next click for each group\n    for spec in GROUP_BY_NEXT_CLICKS:\n    \n       # Name of new feature\n        new_feature = '{}_{}'.format('_'.join(spec['groupby']),agg_suffix)    \n    \n        # Unique list of features to select\n        all_features = spec['groupby'] + ['click_time']\n\n        # Run calculation\n        print(f\">> Grouping by {spec['groupby']}, and saving time to {agg_suffix} in: {new_feature}\")\n        df[new_feature] = (df.click_time - df[all_features].groupby(spec[\n                'groupby']).click_time.shift(+1) ).dt.seconds.astype(agg_type)\n        \n        gc.collect()\n    return (df)   ","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:44:11.881152Z","iopub.execute_input":"2024-04-23T10:44:11.881584Z","iopub.status.idle":"2024-04-23T10:44:11.890323Z","shell.execute_reply.started":"2024-04-23T10:44:11.88155Z","shell.execute_reply":"2024-04-23T10:44:11.889032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train\ntrain['day'] = train['click_time'].dt.day.astype('uint8')\ntrain['hour'] = train['click_time'].dt.hour.astype('uint8')\n\n# test\ntest['day'] = test['click_time'].dt.day.astype('uint8')\ntest['hour'] = test['click_time'].dt.hour.astype('uint8')\n\ndf_concat = pd.concat([train, test], ignore_index=True)\ndf_concat['day'] = df_concat['click_time'].dt.day.astype('uint8')\ndf_concat['hour'] = df_concat['click_time'].dt.hour.astype('uint8')","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:44:12.069909Z","iopub.execute_input":"2024-04-23T10:44:12.070308Z","iopub.status.idle":"2024-04-23T10:44:12.342372Z","shell.execute_reply.started":"2024-04-23T10:44:12.070277Z","shell.execute_reply":"2024-04-23T10:44:12.341171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_freq_hours_in_test_data = [4, 5, 9, 10, 13, 14]\nleast_freq_hours_in_test_data = [6, 11, 15]\n\ntrain['in_test_hh'] = (\n    3 - 2 * train.hour.isin(most_freq_hours_in_test_data)\n      - 1 * train.hour.isin(least_freq_hours_in_test_data)\n).astype('uint8')\n\ntest['in_test_hh'] = (\n    3 - 2 * test.hour.isin(most_freq_hours_in_test_data)\n      - 1 * test.hour.isin(least_freq_hours_in_test_data)\n).astype('uint8')\n\ndf_concat['in_test_hh'] = (\n    3 - 2 * df_concat.hour.isin(most_freq_hours_in_test_data)\n      - 1 * df_concat.hour.isin(least_freq_hours_in_test_data)\n).astype('uint8')","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:44:12.3738Z","iopub.execute_input":"2024-04-23T10:44:12.374205Z","iopub.status.idle":"2024-04-23T10:44:12.769523Z","shell.execute_reply.started":"2024-04-23T10:44:12.374174Z","shell.execute_reply":"2024-04-23T10:44:12.768268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Binning\n# !pip install optbinning\n# from optbinning import BinningProcess\n# cat_cols = train.select_dtypes(include='object').columns.tolist()\n# num_cols = train.select_dtypes(exclude='object').columns.tolist()\n\n# variable_names = train.columns.tolist()\n# binning_process = BinningProcess(variable_names, categorical_variables=cat_cols, \n#                                  max_n_prebins=30)\n\n# binning_process.fit(train, train_labels)\n\n# # Transform train and test\n# train_binned = binning_process.transform(train, metric_missing=0.05)\n# train_binned.columns = [f'{col}_BINNED' for col in train_binned.columns]\n# train_binned.index = train.index\n# test_binned = binning_process.transform(test, metric_missing=0.05)\n# test_binned.columns = [f'{col}_BINNED' for col in test_binned.columns]\n# test_binned.index = test.index\n# # Concat original and binned\n# train = train.select_dtypes('number')\n# train = pd.concat([train, train_binned], axis=1)\n# test = test.select_dtypes('number')\n# test = pd.concat([test, test_binned], axis=1)\n# print(f'Train shape: {train.shape}, Test shape: {test.shape}')\n# train =train.drop(['ip_channel_prevClick_BINNED','ip_os_prevClick_BINNED', 'day_BINNED','date_BINNED','click_time_BINNED','day_of_week_BINNED','tm_hour_BINNED','in_test_hh_BINNED'], axis=1)\n# test =test.drop(['ip_channel_prevClick_BINNED','ip_os_prevClick_BINNED', 'day_BINNED','date_BINNED','click_time_BINNED','day_of_week_BINNED','tm_hour_BINNED','in_test_hh_BINNED'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:45:34.974037Z","iopub.status.idle":"2024-04-23T10:45:34.974476Z","shell.execute_reply.started":"2024-04-23T10:45:34.974267Z","shell.execute_reply":"2024-04-23T10:45:34.974287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"group = ['ip', 'day', 'in_test_hh']\ndf = df_concat.groupby(group).size().astype('uint16')\ndf = pd.DataFrame(df, columns=['ip_day_test_hh_clicks']).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:47:23.307669Z","iopub.execute_input":"2024-04-23T10:47:23.30814Z","iopub.status.idle":"2024-04-23T10:47:24.433267Z","shell.execute_reply.started":"2024-04-23T10:47:23.308103Z","shell.execute_reply":"2024-04-23T10:47:24.432183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ip_app_device_clicks\ngroup = ['ip', 'app', 'device']\ndf = df_concat.groupby(group).size().astype('uint16')\ndf = pd.DataFrame(df, columns=['ip_app_device_clicks']).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)\n\n# ip_app_device_day_clicks\ngroup = ['ip', 'app', 'device', 'day']\ndf = df_concat.groupby(group).size().astype('uint16')\ndf = pd.DataFrame(df, columns=['ip_app_device_day_clicks']).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:47:24.790883Z","iopub.execute_input":"2024-04-23T10:47:24.793635Z","iopub.status.idle":"2024-04-23T10:47:29.041006Z","shell.execute_reply.started":"2024-04-23T10:47:24.793597Z","shell.execute_reply":"2024-04-23T10:47:29.039833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# single 'ip' grouper for ['app', 'device', 'channel']\ncount_cols = ['app', 'device', 'channel', 'hour']\ngroup = 'ip'\nfor col in count_cols:\n    df = df_concat.groupby(group)[col].nunique().astype('uint16')\n    df.name = 'ip_nunique_{}'.format(col)\n    df = pd.DataFrame(df).reset_index()\n    train = train.merge(df, how='left', on=group)\n    test = test.merge(df, how='left', on=group)\n    #####################\n    # self-add\n    group_day = ['ip', 'day']\n    df = df_concat.groupby(group_day)[col].nunique().astype('uint16')\n    df.name = 'ip_day_nunique_{}'.format(col)\n    df = pd.DataFrame(df).reset_index()\n    train = train.merge(df, how='left', on=group_day)\n    test = test.merge(df, how='left', on=group_day)\n    #####################\n    \n# single 'app' grouper for 'channel'\ngroup = 'app'\ncol = 'channel'\ndf = df_concat.groupby(group)[col].nunique().astype('uint16')\ndf.name = 'app_nunique_{}'.format(col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)\n#####################\n\n# self-add\ngroup_day = ['app', 'day']\ncol = 'channel'\ndf = df_concat.groupby(group_day)[col].nunique().astype('uint16')\ndf.name = 'app_day_nunique_{}'.format(col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=group_day)\ntest = test.merge(df, how='left', on=group_day)\n#####################\n\n# duble ['ip', 'app'] grouper for 'os'\ngroup = ['ip', 'app']\ncol = 'os'\ndf = df_concat.groupby(group)[col].nunique().astype('uint16')\ndf.name = 'ip_app_nunique_{}'.format(col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)\n#####################\n\n# self-add\ngroup_day = ['ip', 'app', 'day']\ncol = 'os'\ndf = df_concat.groupby(group_day)[col].nunique().astype('uint16')\ndf.name = 'ip_app_day_nunique_{}'.format(col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=group_day)\ntest = test.merge(df, how='left', on=group_day)\n#####################\n\n# triple ['ip', 'device', 'os'] grouper for 'app'\ngroup = ['ip', 'device', 'os']\ncol = 'app'\ndf = df_concat.groupby(group)[col].nunique().astype('uint16')\ndf.name = 'ip_device_os_nunique_{}'.format(col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)\n#####################\n\n# self-add\ngroup_day = ['ip', 'device', 'os'] + ['day']\ncol = 'app'\ndf = df_concat.groupby(group_day)[col].nunique().astype('uint16')\ndf.name = 'ip_device_os_day_nunique_{}'.format(col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=group_day)\ntest = test.merge(df, how='left', on=group_day)\n#####################","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:48:31.130988Z","iopub.execute_input":"2024-04-23T10:48:31.131417Z","iopub.status.idle":"2024-04-23T10:48:45.981077Z","shell.execute_reply.started":"2024-04-23T10:48:31.131387Z","shell.execute_reply":"2024-04-23T10:48:45.979889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define all the groupby transformations\nGROUPBY_AGGREGATIONS = [\n    # Variance in day, for ip-app-channel\n    {'groupby': ['ip','app','channel'], 'select': 'day', 'agg': 'var', 'type': 'float32'},\n    # Variance in day, for ip-app-device\n    {'groupby': ['ip','app','device'], 'select': 'day', 'agg': 'var', 'type': 'float32'},\n    # Variance in day, for ip-app-os\n    {'groupby': ['ip','app','os'], 'select': 'day', 'agg': 'var', 'type': 'float32'},\n   # Count, for ip-day-hour\n#     {'groupby': ['ip','day','hour'], 'select': 'channel', 'agg': 'count', 'type': 'float32'},\n    # Count, for ip-day-hour\n     {'groupby': ['channel'], 'select': 'day', 'agg': 'var', 'type': 'float32'},\n    # Count, for ip-day-hour\n   {'groupby': ['os'], 'select': 'hour', 'agg': 'var', 'type': 'float32'},\n    {'groupby': ['ip','app','os','device'], 'select': 'hour', 'agg': 'var', 'type': 'float32'},\n    {'groupby': ['ip','app','os'], 'select': 'hour', 'agg': 'var', 'type': 'float32'},\n\n    # Mean hour, for ip-app-channel\n    {'groupby': ['ip','app','channel'], 'select': 'hour', 'agg': 'mean', 'type': 'float32', 'type': 'float32'},\n    {'groupby': ['channel'], 'select': 'hour', 'agg': 'mean', 'type': 'float32', 'type': 'float32'}\n\n]\n# Apply all the groupby transformations\nfor spec in GROUPBY_AGGREGATIONS:\n    print(f\"Grouping by {spec['groupby']}, and aggregating {spec['select']} with {spec['agg']}\")\n    \n    # Unique list of features to select\n    all_features = list(set(spec['groupby'] + [spec['select']]))\n    # Name of new feature\n    new_feature = '{}_{}'.format('_'.join(spec['groupby']), spec['agg'])\n     # Perform the groupby\n    gp = train[all_features]. \\\n        groupby(spec['groupby'])[spec['select']]. \\\n        agg(spec['agg']). \\\n        reset_index(). \\\n        rename(index=str, columns={spec['select']: new_feature}).astype(spec['type'])\n    gp = test[all_features]. \\\n        groupby(spec['groupby'])[spec['select']]. \\\n        agg(spec['agg']). \\\n        reset_index(). \\\n        rename(index=str, columns={spec['select']: new_feature}).astype(spec['type'])\n","metadata":{"execution":{"iopub.status.busy":"2024-04-23T11:03:33.41863Z","iopub.execute_input":"2024-04-23T11:03:33.419207Z","iopub.status.idle":"2024-04-23T11:03:43.138822Z","shell.execute_reply.started":"2024-04-23T11:03:33.419168Z","shell.execute_reply":"2024-04-23T11:03:43.137486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gp['channel'] = gp['channel'].astype('object')\ngp['channel_mean'] = gp['channel_mean'].astype('object')\n# Merge back to X_train\ntrain = train.merge(gp, on=spec['groupby'], how='left')\ntest = test.merge(gp, on=spec['groupby'], how='left')\ndel gp\nprint(\"End\")","metadata":{"execution":{"iopub.status.busy":"2024-04-23T11:07:03.638283Z","iopub.execute_input":"2024-04-23T11:07:03.638672Z","iopub.status.idle":"2024-04-23T11:07:04.154188Z","shell.execute_reply.started":"2024-04-23T11:07:03.638643Z","shell.execute_reply":"2024-04-23T11:07:04.152837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.decomposition import LatentDirichletAllocation\n\ndef modelTopicsWithDevice(df, ip_column, app_column, device_column):\n    \"\"\"Model LDA topics for each IP based on apps and devices.\"\"\"\n    # Assuming vocab selection is based on app usage frequency\n    vocab = df[app_column].value_counts().nlargest(10).index.tolist()\n\n    # Aggregate app and device values for each IP\n    ip_activities = {}\n    for _, row in tqdm(df.iterrows(), total=len(df)):\n        ip = row[ip_column]\n        app = row[app_column]\n        device = str(row[device_column])  # Convert device to string if it's not already\n        activity = f\"{app}_{device}\"  # Concatenate app and device information\n\n        if app in vocab:\n            ip_activities.setdefault(ip, []).append(activity)\n\n    ips = list(ip_activities.keys())\n    sentences = [' '.join(ip_activities[ip]) for ip in ips]\n    \n    # Vectorize the app and device sentences\n    cv = CountVectorizer()\n    dt_matrix = cv.fit_transform(sentences)\n    \n    # Apply LDA\n    lda = LatentDirichletAllocation(n_components=5, random_state=42)\n    ip_topics = lda.fit_transform(dt_matrix)\n    \n    # Create a DataFrame for the topics\n    topic_cols = [f\"{ip_column}_{app_column}_{device_column}_topic{i+1}\" for i in range(5)]\n    topic_df = pd.DataFrame(ip_topics, columns=topic_cols)\n    topic_df[ip_column] = ips\n    \n    # Merge the topics back to the original DataFrame\n    df = df.merge(topic_df, on=ip_column, how='left')\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-23T11:07:50.561399Z","iopub.execute_input":"2024-04-23T11:07:50.562084Z","iopub.status.idle":"2024-04-23T11:07:50.607777Z","shell.execute_reply.started":"2024-04-23T11:07:50.562015Z","shell.execute_reply":"2024-04-23T11:07:50.606493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = modelTopicsWithDevice(train, 'ip', 'app', 'device')\ntest = modelTopicsWithDevice(test, 'ip', 'app', 'device')","metadata":{"execution":{"iopub.status.busy":"2024-04-23T11:07:52.480294Z","iopub.execute_input":"2024-04-23T11:07:52.480734Z","iopub.status.idle":"2024-04-23T11:14:05.118005Z","shell.execute_reply.started":"2024-04-23T11:07:52.480699Z","shell.execute_reply":"2024-04-23T11:14:05.11685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(['channel_mean'], axis=1, inplace=True)\ntest.drop(['channel_mean'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T11:51:52.464356Z","iopub.execute_input":"2024-04-23T11:51:52.465053Z","iopub.status.idle":"2024-04-23T11:51:52.810837Z","shell.execute_reply.started":"2024-04-23T11:51:52.465001Z","shell.execute_reply":"2024-04-23T11:51:52.809756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.fillna(train.max())\ntest = test.fillna(test.max())","metadata":{"execution":{"iopub.status.busy":"2024-04-23T11:52:17.401402Z","iopub.execute_input":"2024-04-23T11:52:17.401853Z","iopub.status.idle":"2024-04-23T11:52:19.540204Z","shell.execute_reply.started":"2024-04-23T11:52:17.401818Z","shell.execute_reply":"2024-04-23T11:52:19.538972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop('click_time', axis=1, inplace=True)\ntest.drop('click_time', axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T11:53:52.232512Z","iopub.execute_input":"2024-04-23T11:53:52.235637Z","iopub.status.idle":"2024-04-23T11:53:52.353675Z","shell.execute_reply.started":"2024-04-23T11:53:52.235594Z","shell.execute_reply":"2024-04-23T11:53:52.351673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def balance_data(features, labels, sampling_strategy=0.5, random_state=42):\n#     # Positive oversampling (SMOTE)\n#     smote = SMOTE(sampling_strategy=sampling_strategy, random_state=random_state)\n#     features_resampled, labels_resampled = smote.fit_resample(features, labels)\n\n#     # Negative downsampling (reduce negative samples to match positive samples)\n#     negative_samples = features_resampled[labels_resampled == 0]\n#     positive_samples = features_resampled[labels_resampled == 1]\n\n#     # Perform negative downsampling, e.g., reduce negative samples to match twice the number of positive samples\n#     negative_samples_downsampled = resample(negative_samples, n_samples=len(positive_samples)*2, random_state=random_state)\n\n#     # Combine positive and downsampled negative samples\n#     balanced_features = pd.concat([positive_samples, negative_samples_downsampled], ignore_index=True)\n#     balanced_labels = pd.Series([1] * len(positive_samples) + [0] * len(negative_samples_downsampled))\n\n#     return balanced_features, balanced_labels\n\n# # Example usage:\n# train_balanced, train_labels_balanced = balance_data(train, train_labels, sampling_strategy=0.5, random_state=42)\n# initialize SMOTE method\nsm = SMOTE(random_state=42)\n# ytrain = train.drop('is_attributed',axis=1)\ntrain_balanced,train_labels_balanced = sm.fit_resample(train.drop('is_attributed',axis=1),train['is_attributed'])\nprint(\"Dimension of X_train_sm Shape:\", train_balanced.shape)\nprint(\"Dimension of y_train_sm Shape:\", train_labels_balanced.shape)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T12:06:36.349575Z","iopub.execute_input":"2024-04-23T12:06:36.35084Z","iopub.status.idle":"2024-04-23T12:19:25.150107Z","shell.execute_reply.started":"2024-04-23T12:06:36.350673Z","shell.execute_reply":"2024-04-23T12:19:25.148818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Count the number of occurrences of each label\nlabel_counts = train_labels_balanced.value_counts()\n\n# Display the counts\nprint(\"Number of '0' labels:\", label_counts[0])\nprint(\"Number of '1' labels:\", label_counts[1])","metadata":{"execution":{"iopub.status.busy":"2024-04-23T12:21:44.291371Z","iopub.execute_input":"2024-04-23T12:21:44.29186Z","iopub.status.idle":"2024-04-23T12:21:44.311463Z","shell.execute_reply.started":"2024-04-23T12:21:44.291824Z","shell.execute_reply":"2024-04-23T12:21:44.310231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.columns","metadata":{"execution":{"iopub.status.busy":"2024-04-23T12:47:32.605206Z","iopub.execute_input":"2024-04-23T12:47:32.605692Z","iopub.status.idle":"2024-04-23T12:47:32.614566Z","shell.execute_reply.started":"2024-04-23T12:47:32.605658Z","shell.execute_reply":"2024-04-23T12:47:32.613372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = StandardScaler()\ntrain_balanced = scaler.fit_transform(train_balanced)\ntest = scaler.transform(test.drop('is_attributed',axis=1))","metadata":{"execution":{"iopub.status.busy":"2024-04-23T12:47:56.348336Z","iopub.execute_input":"2024-04-23T12:47:56.348798Z","iopub.status.idle":"2024-04-23T12:47:59.606207Z","shell.execute_reply.started":"2024-04-23T12:47:56.348749Z","shell.execute_reply":"2024-04-23T12:47:59.604862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train","metadata":{"execution":{"iopub.status.busy":"2024-04-23T12:48:05.770958Z","iopub.execute_input":"2024-04-23T12:48:05.772022Z","iopub.status.idle":"2024-04-23T12:48:05.777324Z","shell.execute_reply.started":"2024-04-23T12:48:05.771981Z","shell.execute_reply":"2024-04-23T12:48:05.776226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Initialize logistic regression model with 'sag' solver\n# lr_model = LogisticRegression()\n# # (solver='sag', C=1.0, max_iter=10000, penalty='l2',class_weight = 'balanced' )\n\n# # Train the model\n# lr_model.fit(train_balanced, train_labels_balanced)\n\n# # Predictions\n# y_pred = lr_model.predict(test)\n# y_pred_proba = lr_model.predict_proba(test)[:, 1]\n# # Calculate F1-score\n# f1 = f1_score(test_labels, y_pred)\n# auc_score = roc_auc_score(test_labels, y_pred_proba)\n\n\n# # Print the F1-score (Note: No need for a probability array for F1-score)\n# print(f\"F1 Score: {f1}\")\n# print(f\"AUC Score: {auc_score}\")\n\n# evaluate()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T13:47:03.895687Z","iopub.execute_input":"2024-04-23T13:47:03.896131Z","iopub.status.idle":"2024-04-23T13:47:24.685555Z","shell.execute_reply.started":"2024-04-23T13:47:03.896097Z","shell.execute_reply":"2024-04-23T13:47:24.684267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# results.append({\"Model\": 'Logistic Regression', \"F1 Score\": f1, \"AUC\": auc_score})\n","metadata":{"execution":{"iopub.status.busy":"2024-04-23T12:50:33.459296Z","iopub.execute_input":"2024-04-23T12:50:33.460862Z","iopub.status.idle":"2024-04-23T12:50:33.46719Z","shell.execute_reply.started":"2024-04-23T12:50:33.46081Z","shell.execute_reply":"2024-04-23T12:50:33.465755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# params = {'eta': 0.2,\n#           'max_depth': 15,\n#           'subsample': 0.7,\n#           'colsample_bytree': 0.9,\n#           'objective': 'binary:logistic',\n#           'eval_metric': 'auc',\n#           'n_jobs': 24,\n#           'n_estimators': 400\n#           }\n\n# Initialize XGBoost Classifier with selected parameters\nclass_weight={0:1,1:19}\npos_weight = sum(class_weight.values())/sum(class_weight.keys())\nxgb_model = xgb.XGBClassifier()\n# (scale_pos_weight = pos_weight, class_weight = class_weight, **params)\n\n# Perform cross-validation\ncv_scores = cross_val_score(xgb_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\nxgb_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions on the test set\nxgb_y_pred = xgb_model.predict(test)\ny_pred_proba = xgb_model.predict_proba(test)[:, 1]\n\n\n# Calculate F1-score\nf1 = f1_score(test_labels, xgb_y_pred)\nxgb_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\n\n# Print the F1-score (Note: No need for a probability array for F1-score)\nprint(f\"F1 Score: {f1}\")\nprint(f\"AUC Score: {xgb_auc_score}\")\n\nevaluate()\n\nresults.append({\"Model\": 'XGBoost', \"F1 Score\": f1, \"AUC\": xgb_auc_score})","metadata":{"execution":{"iopub.status.busy":"2024-04-23T13:44:17.564707Z","iopub.execute_input":"2024-04-23T13:44:17.565309Z","iopub.status.idle":"2024-04-23T13:47:03.893162Z","shell.execute_reply.started":"2024-04-23T13:44:17.56527Z","shell.execute_reply":"2024-04-23T13:47:03.891868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfinal_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# params = {'num_leaves': 127,\n#           'min_data_in_leaf': 32,\n#           'objective': 'binary',\n#           'max_depth': -1,\n#           'learning_rate': 0.1,\n#           'min_child_samples': 20,\n#           'boosting': 'gbdt',\n#           'feature_fraction': 0.8,\n#           'bagging_freq': 1,\n#           'bagging_fraction': 0.8,\n#           'bagging_seed': 11,\n#           'metric': 'auc',\n#           'lambda_l1': 0.1,\n#           'verbosity': -1,\n#           'n_estimators': 200,\n#           'class_weight' : 'balanced'\n#           }\n\n# Initialize LightGBM Classifier with selected parameters\nlgb_model = lgb.LGBMClassifier()\n# (**params)\n\n# Perform cross-validation\ncv_scores = cross_val_score(lgb_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\nlgb_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions on the test set\nlgb_y_pred = lgb_model.predict(test)\ny_pred_proba = lgb_model.predict_proba(test)[:, 1]\n\n# Calculate F1-score\nf1 = f1_score(test_labels, lgb_y_pred)\nlgb_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\n\n# Print the F1-score (Note: No need for a probability array for F1-score)\nprint(f\"F1 Score: {f1}\")\nprint(f\"AUC Score: {lgb_auc_score}\")\n\nprint(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\nprint(f\"Average Cross-Validation ROC-AUC Score: {np.mean(cv_scores)}\")\n\nevaluate()\nresults.append({\"Model\": 'Light GBM', \"F1 Score\": f1, \"AUC\": lgb_auc_score})","metadata":{"execution":{"iopub.status.busy":"2024-04-23T13:48:42.678094Z","iopub.execute_input":"2024-04-23T13:48:42.678517Z","iopub.status.idle":"2024-04-23T13:52:49.438113Z","shell.execute_reply.started":"2024-04-23T13:48:42.678485Z","shell.execute_reply":"2024-04-23T13:52:49.436797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfinal_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Manually select parameters from the parameter grid\n# params = {\n#     'n_estimators': 200,\n#     'learning_rate': 0.1,\n#     'max_depth': 5,\n#     'min_samples_split': 4,\n#     'min_samples_leaf': 2,\n#     'max_features': 'sqrt'\n# }\n\n# Initialize Gradient Boosting Classifier with selected parameters\ngb_model = GradientBoostingClassifier()\n# (**params)\n\n# Perform cross-validation\n# cv_scores = cross_val_score(gb_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\ngb_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions on the test set\ngb_y_pred = gb_model.predict(test)\ny_pred_proba = gb_model.predict_proba(test)[:, 1]\n\n# Calculate F1-score\nf1 = f1_score(test_labels, gb_y_pred)\ngb_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\n\n# Print the F1-score (Note: No need for a probability array for F1-score)\nprint(f\"F1 Score: {f1}\")\nprint(f\"AUC Score: {gb_auc_score}\")\n\nprint(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\nprint(f\"Average Cross-Validation ROC-AUC Score: {cv_scores.mean()}\")\n\nevaluate()\nresults.append({\"Model\": 'Gradient Boosting Tree', \"F1 Score\": f1, \"AUC\": gb_auc_score})","metadata":{"execution":{"iopub.status.busy":"2024-04-23T14:21:06.270616Z","iopub.execute_input":"2024-04-23T14:21:06.271032Z","iopub.status.idle":"2024-04-23T14:29:39.436039Z","shell.execute_reply.started":"2024-04-23T14:21:06.271Z","shell.execute_reply":"2024-04-23T14:29:39.434669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfinal_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import optuna\n# from sklearn.ensemble import GradientBoostingClassifier\n\n# def objective(trial):\n#     hyperparameters = {\n#         'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),\n#         'n_estimators': trial.suggest_int('n_estimators', 50, 500),\n#         'max_depth': trial.suggest_int('max_depth', 3, 10),\n#         'min_samples_split': trial.suggest_int('min_samples_split', 2, 20),\n#         'min_samples_leaf': trial.suggest_int('min_samples_leaf', 1, 20),\n#         'subsample': trial.suggest_uniform('subsample', 0.5, 1.0)\n#     }\n    \n#     model = GradientBoostingClassifier(**hyperparameters)\n#     scores = cross_val_score(model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n#     return np.mean(scores)\n\n# study = optuna.create_study(direction='maximize')\n# study.optimize(objective, n_trials=10) \n\n\n# best_params = study.best_params\n# print(\"Best params found:\", best_params)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T11:52:30.061667Z","iopub.status.idle":"2024-04-23T11:52:30.062226Z","shell.execute_reply.started":"2024-04-23T11:52:30.061988Z","shell.execute_reply":"2024-04-23T11:52:30.06201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import CatBoostClassifier\n\n# Manually select parameters from the parameter grid\nparams = {\n    'depth': 10,\n    'learning_rate': 0.2,\n    'iterations': 500,\n    'l2_leaf_reg': 3,\n    'border_count': 50,\n    'bagging_temperature': 0.04019881483063329,\n    'loss_function': 'Logloss',\n    'od_type': 'Iter',\n    'od_wait': 50,\n    'verbose': False\n}\n\n# Initialize CatBoost Classifier with selected parameters\ncat_model = CatBoostClassifier()\n# (**params)\n\n# Perform cross-validation\ncv_scores = cross_val_score(cat_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\ncat_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions on the test set\ncat_y_pred = cat_model.predict(test)\ny_pred_proba = cat_model.predict_proba(test)[:, 1]\n\n# Calculate F1-score\nf1 = f1_score(test_labels, cat_y_pred)\ncat_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\n\n# Print the F1-score (Note: No need for a probability array for F1-score)\nprint(f\"F1 Score: {f1}\")\nprint(f\"AUC Score: {cat_auc_score}\")\nprint(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\nprint(f\"Average Cross-Validation ROC-AUC Score: {np.mean(cv_scores)}\")\n\nevaluate()\nresults.append({\"Model\": 'CatBoost', \"F1 Score\": f1, \"AUC\": cat_auc_score})","metadata":{"execution":{"iopub.status.busy":"2024-04-23T14:29:39.438726Z","iopub.execute_input":"2024-04-23T14:29:39.439514Z","iopub.status.idle":"2024-04-23T14:44:31.694425Z","shell.execute_reply.started":"2024-04-23T14:29:39.439469Z","shell.execute_reply":"2024-04-23T14:44:31.693103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfinal_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Initialize Random Forest Classifier\n# rf_model = RandomForestClassifier()\n# #     n_estimators=100,\n# #     max_depth=20,\n# #     min_samples_split=2,\n# #     min_samples_leaf=1,\n# #     max_features='sqrt',\n# #     bootstrap=False,\n# #     criterion='gini'\n# # )\n\n# # Perform cross-validation\n# cv_scores = cross_val_score(rf_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# # Train the model on the entire training set\n# rf_model.fit(train_balanced, train_labels_balanced)\n\n# # Predictions on the test set\n# rf_y_pred = rf_model.predict(test)\n# y_pred_proba = rf_model.predict_proba(test)[:, 1]\n\n# # Calculate accuracy and AUC Score on the test set\n# f1 = f1_score(test_labels, cat_y_pred)\n# rf_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\n# print(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\n# print(f\"Average Cross-Validation ROC-AUC Score: {np.mean(cv_scores)}\")\n\n# evaluate()\n# results.append({\"Model\": 'Random Forest', \"F1 Score\": f1, \"AUC\": rf_auc_score})","metadata":{"execution":{"iopub.status.busy":"2024-04-23T14:44:31.696513Z","iopub.execute_input":"2024-04-23T14:44:31.697009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# final_results = pd.DataFrame(results)\n# print(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model performance on test data with chosen hyperparameters\n\n# base estimator\ntree = DecisionTreeClassifier()\n# (max_depth=2)\n\n# adaboost with the tree as base estimator\n# learning rate is arbitrarily set, we'll discuss learning_rate below\nABC_model = AdaBoostClassifier()\n#     base_estimator=tree,\n#     learning_rate=1.0,\n#     n_estimators=300,\n#     algorithm=\"SAMME\")\n\n# Perform cross-validation\ncv_scores = cross_val_score(ABC_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\nABC_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions on the test set\nABC_y_pred = ABC_model.predict(test)\ny_pred_proba = ABC_model.predict_proba(test)[:, 1]\n\n# Calculate accuracy and AUC Score on the test set\nf1 = f1_score(test_labels, ABC_y_pred)\n\nABC_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\n\nprint(f\"F1 Score: {f1}\")\nprint(f\"AUC Score: {ABC_auc_score}\")\n\nprint(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\nprint(f\"Average Cross-Validation ROC-AUC Score: {np.mean(cv_scores)}\")\n\nevaluate()\nresults.append({\"Model\": 'AdaBoost', \"F1 Score\": f1, \"AUC\": ABC_auc_score})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.ensemble import StackingClassifier\n\n# # Define base models\n# base_models = [\n#     ('Logistic Regression', lr_model),\n#     ('XGBoost', xgb_model),\n#     ('Gradient Boosting', gb_model),\n#     ('LightGBM', lgb_model),\n#     ('CatBoost', cat_model),\n#     ('RF', rf_model),\n#     ('AdaBoost', ABC_model)\n# ]\n\n# # Define meta-model\n# meta_model = LogisticRegression()\n\n# # Initialize Stacking Classifier\n# stacked_model = StackingClassifier(estimators=base_models, final_estimator=meta_model, cv=5)\n\n# # Train the stacked model (Note: The pre-trained model won't be re-trained)\n# stacked_model.fit(train_balanced, train_labels_balanced)\n\n# # Predictions and evaluation\n# y_pred = stacked_model.predict(test)\n# y_pred_proba = stacked_model.predict_proba(test)[:, 1]  # Probabilities for the positive class\n\n# # Calculate accuracy and AUC Score\n# f1 = f1_score(test_labels, y_pred)\n# auc_score = roc_auc_score(test_labels, y_pred_proba)\n\n# print(f\"AUC Score: {auc_score}\")\n# evaluate()\n# results.append({\"Model\": 'Stacked Model', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((xgb_y_pred, lgb_y_pred, cat_y_pred, ABC_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote XGB + LGB + CAT + ADA ', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((gb_y_pred, lgb_y_pred, cat_y_pred, ABC_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote GB + LGB + CAT + ADA ', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((gb_y_pred, xgb_y_pred, cat_y_pred, ABC_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote GB + XGB + CAT + ADA', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((gb_y_pred, xgb_y_pred, cat_y_pred, ABC_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote GB + XGB + CAT + ADA', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((gb_y_pred, xgb_y_pred, lgb_y_pred, ABC_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote GB + XGB + LGB + ADA', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((gb_y_pred, xgb_y_pred, lgb_y_pred, cat_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote GB + XGB + LGB + CAT', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((lgb_y_pred, cat_y_pred, ABC_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote LGB + CAT + ADA', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((xgb_y_pred, lgb_y_pred, ABC_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote XGB + LGB + ADA', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((xgb_y_pred, cat_y_pred, ABC_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote XGB + CAT + ADA', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((xgb_y_pred, lgb_y_pred, cat_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote XGB + LGB + CAT', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((gb_y_pred, lgb_y_pred, ABC_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote GB + LGB + ADA', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((gb_y_pred, cat_y_pred, ABC_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote GB + CAT + ADA', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((gb_y_pred,lgb_y_pred, cat_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote GB + LGB + CAT', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((gb_y_pred, xgb_y_pred,  ABC_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote GB + XGB + ADA', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((gb_y_pred, xgb_y_pred,  cat_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote GB + XGB + CAT', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((gb_y_pred, xgb_y_pred, lgb_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote GB + XGB + LGB', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((cat_y_pred, ABC_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote CAT + ADA', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((lgb_y_pred,ABC_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote LGB + ADA', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((lgb_y_pred, cat_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote LGB + CAT', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((xgb_y_pred,ABC_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote XGB + ADA', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((xgb_y_pred,cat_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote XGB + CAT', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((xgb_y_pred, lgb_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote XGB + LGB', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((gb_y_pred, ABC_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote GB + ADA', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((gb_y_pred, cat_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote GB + CAT', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((gb_y_pred, lgb_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote GB + LGB', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((gb_y_pred, xgb_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\nresults.append({\"Model\": 'Majority Vote GB + XGB', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Define base models\n# from sklearn.ensemble import RandomForestClassifier\n# import xgboost as xgb\n\n# # Define and train XGBoost model\n# xgb_model = xgb.XGBClassifier(n_estimators=100, random_state=42)\n# xgb_model.fit(train_balanced, train_labels_balanced)  \n\n# # Define and train Random Forest model\n# rf_model = RandomForestClassifier(n_estimators=100, random_state=42)\n# rf_model.fit(train_balanced, train_labels_balanced) \n\n# base_models = [\n#     ('XGBoost', xgb_model),\n#     ('Random Forest', rf_model),\n# ]\n\n# # Define meta-model\n# meta_model = LogisticRegression()\n\n# # Initialize Stacking Classifier\n# stacked_model = StackingClassifier(estimators=base_models, final_estimator=meta_model, cv=5)\n\n# # Train the stacked model (Note: The pre-trained model won't be re-trained)\n# stacked_model.fit(train_balanced, train_labels_balanced)\n\n# # Predictions and evaluation\n# y_pred = stacked_model.predict(test)\n# y_pred_proba = stacked_model.predict_proba(test)[:, 1]  # Probabilities for the positive class\n\n# # Calculate accuracy and AUC Score\n# f1 = f1_score(test_labels, y_pred)\n# auc_score = roc_auc_score(test_labels, y_pred_proba)\n\n# print(f\"AUC Score: {auc_score}\")\n# results.append({\"Model\": 'XGBoost + Random Forest', \"F1 Score\": f1, \"AUC\": auc_score})\n# evaluate()\n","metadata":{},"execution_count":null,"outputs":[]}]}