{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":8540,"databundleVersionId":862041,"sourceType":"competition"},{"sourceId":7624767,"sourceType":"datasetVersion","datasetId":4441829},{"sourceId":7788617,"sourceType":"datasetVersion","datasetId":4558812}],"dockerImageVersionId":30648,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.linear_model import LogisticRegression\nfrom tqdm import tqdm\nfrom imblearn.over_sampling import SMOTE\nfrom sklearn.utils import resample\nfrom sklearn.model_selection import cross_val_score\nimport xgboost as xgb\nfrom sklearn.metrics import accuracy_score\n\nimport lightgbm as lgb\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import AdaBoostClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.svm import LinearSVC\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.ensemble import StackingClassifier\nfrom scipy.stats import mode\nfrom catboost import CatBoostClassifier, Pool\nfrom skopt import BayesSearchCV\nimport torch\nimport numpy as np\n!pip install optbinning\n\nfrom optbinning import BinningProcess\n\nfrom sklearn.preprocessing import MinMaxScaler, LabelEncoder, StandardScaler\nfrom sklearn.metrics import log_loss\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.metrics import roc_curve\nimport sklearn.metrics as metrics\n\nimport gc \nfrom sklearn.model_selection import train_test_split, RandomizedSearchCV\nfrom sklearn.metrics import roc_auc_score\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"700af4b2-ca93-4a2f-b7de-0fef3c212eb6","_cell_guid":"34ea337a-c686-4618-bfe8-ed89858725b8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:16:44.981650Z","iopub.execute_input":"2024-05-01T03:16:44.982592Z","iopub.status.idle":"2024-05-01T03:16:57.208991Z","shell.execute_reply.started":"2024-05-01T03:16:44.982533Z","shell.execute_reply":"2024-05-01T03:16:57.207838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_original = pd.read_csv('/kaggle/input/khois-adclick-data/test.csv')\ntrain_original = pd.read_csv('/kaggle/input/khois-adclick-data/train_resampled.csv')","metadata":{"_uuid":"45ea90fc-a81b-4a0f-a334-148d814c683f","_cell_guid":"9ddfadf3-666d-43b5-9be1-9d10a05c95c4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:16:57.211022Z","iopub.execute_input":"2024-05-01T03:16:57.211330Z","iopub.status.idle":"2024-05-01T03:16:57.406570Z","shell.execute_reply.started":"2024-05-01T03:16:57.211302Z","shell.execute_reply":"2024-05-01T03:16:57.405681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train_original.copy()\ntest = test_original.copy()","metadata":{"_uuid":"de2fa6c9-bdd3-4dbb-aa6c-7e04f4cbf072","_cell_guid":"36cb5987-aee7-4e57-8b0c-3dba906aff9f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:16:57.407850Z","iopub.execute_input":"2024-05-01T03:16:57.408149Z","iopub.status.idle":"2024-05-01T03:16:57.415350Z","shell.execute_reply.started":"2024-05-01T03:16:57.408122Z","shell.execute_reply":"2024-05-01T03:16:57.414422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"_uuid":"eedbd5f1-ab35-40b7-adbe-a41b8a17f449","_cell_guid":"0d6c387c-ef0d-4e41-b0ee-ac2c2c2a6ffd","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:16:57.417364Z","iopub.execute_input":"2024-05-01T03:16:57.417664Z","iopub.status.idle":"2024-05-01T03:16:57.434769Z","shell.execute_reply.started":"2024-05-01T03:16:57.417641Z","shell.execute_reply":"2024-05-01T03:16:57.433854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(labels='Unnamed: 0', axis=1, inplace=True)\ntest.drop(labels='Unnamed: 0', axis=1, inplace=True)","metadata":{"_uuid":"164d1413-ceb7-4175-8f13-0a52f68bc183","_cell_guid":"a5cd4d34-b3fd-40fc-98f6-d0261741cbb7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:16:57.435818Z","iopub.execute_input":"2024-05-01T03:16:57.436087Z","iopub.status.idle":"2024-05-01T03:16:57.449713Z","shell.execute_reply.started":"2024-05-01T03:16:57.436064Z","shell.execute_reply":"2024-05-01T03:16:57.448887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Initial Processing","metadata":{"_uuid":"d1af0853-06e6-4242-ae8a-2041a3cdb0a5","_cell_guid":"9e64cb4f-d3cf-4a32-b29e-dd968b6d2040","trusted":true}},{"cell_type":"markdown","source":"### Missing Values","metadata":{"_uuid":"3572b4c1-f150-4e5f-bfbc-ed65f038e19d","_cell_guid":"437f7759-ca8d-4907-b4fb-7731ce373ebe","trusted":true}},{"cell_type":"code","source":"train.isnull().sum(axis = 0)","metadata":{"_uuid":"6fea26ac-727f-4957-9b5f-76b0c2f1e834","_cell_guid":"a72f3c65-72d8-4301-983e-6a48182d68b6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:16:57.451044Z","iopub.execute_input":"2024-05-01T03:16:57.451879Z","iopub.status.idle":"2024-05-01T03:16:57.474344Z","shell.execute_reply.started":"2024-05-01T03:16:57.451842Z","shell.execute_reply":"2024-05-01T03:16:57.473346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### What happened to attributed_time?","metadata":{"_uuid":"113c9fba-de23-4cfd-a312-98d7aec6ef47","_cell_guid":"e4d0b636-77ed-4e3e-b7f8-4d7770485f0f","trusted":true}},{"cell_type":"code","source":"temp1 = train.loc[~train.attributed_time.isnull(),'is_attributed']\nif np.sum(temp1.values) == len(temp1):\n print('Same')\nelse:\n print('No')\nprint('-'*50)\ntemp2 = train.loc[train.attributed_time.isnull(),'is_attributed']\nif np.sum(temp2.values) == 0:\n print('Same')\nelse:\n print('No')\n\n# whenever is_attributed == 1, then there is a corresponding time in attributed_time","metadata":{"_uuid":"c0f4d662-425f-4345-9541-7f59657df17b","_cell_guid":"db2c7f54-156e-48b7-a094-c25333a1240d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:16:57.475671Z","iopub.execute_input":"2024-05-01T03:16:57.475939Z","iopub.status.idle":"2024-05-01T03:16:57.501780Z","shell.execute_reply.started":"2024-05-01T03:16:57.475917Z","shell.execute_reply":"2024-05-01T03:16:57.500924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Label Distribution","metadata":{"_uuid":"682e36fb-c15d-46e7-9b78-178594cea178","_cell_guid":"4f8904f0-2954-41f9-8980-7317c968f426","trusted":true}},{"cell_type":"code","source":"fig = plt.figure(figsize=(6,5))\nplt.bar(train.is_attributed.value_counts().index, train.is_attributed.value_counts().values)\nplt.xlabel('labels')\nplt.ylabel('counts')\nplt.xticks([0,1])\nplt.show()","metadata":{"_uuid":"f224d6b9-a9ad-44c6-bcd1-8aa5df4b7775","_cell_guid":"d342da90-ac37-41e3-9294-d672f00cd87c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:16:57.502963Z","iopub.execute_input":"2024-05-01T03:16:57.503238Z","iopub.status.idle":"2024-05-01T03:16:57.641945Z","shell.execute_reply.started":"2024-05-01T03:16:57.503215Z","shell.execute_reply":"2024-05-01T03:16:57.641122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Distribution","metadata":{"_uuid":"fdf820c9-4d18-4f8c-8d14-c0a54eee4a50","_cell_guid":"bd4fd867-7054-4e8d-92cc-f6c2533558ef","trusted":true}},{"cell_type":"code","source":"fig = plt.figure(figsize=(15,12))\nfor i, col in enumerate(['ip','app','device','os','channel']):\n plt.subplot(3,2,i+1)\n sns.histplot(data=train, x=col, label = col, kde=True)\nfig.tight_layout(pad=1.0)","metadata":{"_uuid":"75015512-e1b6-4001-97c2-6ae5f679cecb","_cell_guid":"f0dace91-8bf7-45c7-bb1d-6327a8b17e9e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:16:57.643140Z","iopub.execute_input":"2024-05-01T03:16:57.643459Z","iopub.status.idle":"2024-05-01T03:17:07.184037Z","shell.execute_reply.started":"2024-05-01T03:16:57.643429Z","shell.execute_reply":"2024-05-01T03:17:07.183137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Click Time Transformation","metadata":{"_uuid":"159a8433-3a4f-4cc4-a776-bda29467c229","_cell_guid":"814a15e8-6b7f-4831-8495-45a46a9e5d2c","trusted":true}},{"cell_type":"code","source":"def transformation(df):\n df['click_time'] = pd.to_datetime(df.click_time)\n df['day_of_week'] = df.click_time.dt.day_of_week\n df['day'] = df.click_time.dt.day\n df['hour'] = df.click_time.dt.hour\n return df\n\n# transformation\ntrain = transformation(train)\ntest = transformation(test)","metadata":{"_uuid":"b65c101a-3fff-491f-acd4-fe1aa56b6289","_cell_guid":"fd47a937-1dbd-4db2-8d8e-2efbe247a9dd","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:07.188520Z","iopub.execute_input":"2024-05-01T03:17:07.189173Z","iopub.status.idle":"2024-05-01T03:17:07.247338Z","shell.execute_reply.started":"2024-05-01T03:17:07.189136Z","shell.execute_reply":"2024-05-01T03:17:07.246586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Deleting variables","metadata":{"_uuid":"75dbec6f-fcc8-4f63-8d36-eed61443e9d2","_cell_guid":"5157c386-511c-404d-9492-13652a50e54e","trusted":true}},{"cell_type":"code","source":"# store the label\ntrain_labels = train.is_attributed.values\ntest_labels = test.is_attributed.values\n\n# drop labels and attributed_time since it represnets the same info as the is_attributed\ntrain.drop(labels = ['attributed_time', 'is_attributed'], axis = 1, inplace = True)\ntest.drop(labels = ['attributed_time', 'is_attributed'], axis = 1, inplace = True)","metadata":{"_uuid":"317ed048-bd30-4fa0-b938-bf68d66963a8","_cell_guid":"100f348c-dc24-4ac3-8277-5b89379dde11","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:07.248356Z","iopub.execute_input":"2024-05-01T03:17:07.248672Z","iopub.status.idle":"2024-05-01T03:17:07.258920Z","shell.execute_reply.started":"2024-05-01T03:17:07.248646Z","shell.execute_reply":"2024-05-01T03:17:07.257954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ROC curve","metadata":{}},{"cell_type":"code","source":"def evaluate():\n \n fpr, tpr, _ = metrics.roc_curve(test_labels, y_pred_proba)\n\n # Plot the ROC curve\n plt.plot(fpr, tpr, color='blue', label='ROC Curve')\n\n # Fill below the ROC curve with color\n plt.fill_between(fpr, 0, tpr, color='skyblue', alpha=0.2)\n\n # Plot the diagonal line (y=x) for reference\n plt.plot([0, 1], [0, 1], color='red', linestyle='--', label='Random Chance')\n\n # Add labels and title\n plt.ylabel('True Positive Rate')\n plt.xlabel('False Positive Rate')\n plt.title('ROC Curve')\n\n # Show the plot\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:07.260097Z","iopub.execute_input":"2024-05-01T03:17:07.260390Z","iopub.status.idle":"2024-05-01T03:17:07.267329Z","shell.execute_reply.started":"2024-05-01T03:17:07.260365Z","shell.execute_reply":"2024-05-01T03:17:07.266584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Baseline Model - Logistic Regression","metadata":{"_uuid":"b7c3705c-0459-49df-97b0-db93ccdae2c0","_cell_guid":"dce875e1-bbf5-4081-9458-7346d36e507a","trusted":true}},{"cell_type":"code","source":"results = []","metadata":{"_uuid":"564204b7-0375-4655-a1b7-0c2d49ae0d7d","_cell_guid":"ecf937e6-22cb-447a-ac1c-94ed06f01ee0","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:07.268336Z","iopub.execute_input":"2024-05-01T03:17:07.268625Z","iopub.status.idle":"2024-05-01T03:17:07.278254Z","shell.execute_reply.started":"2024-05-01T03:17:07.268601Z","shell.execute_reply":"2024-05-01T03:17:07.277306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\n\n# temp_train = train.drop(labels='click_time', axis=1)\n# temp_test = test.drop(labels='click_time', axis=1)\n\n# Initialize logistic regression model with 'sag' solver\nmodel = LogisticRegression(class_weight = 'balanced')\ntemp_train = train.drop(labels='click_time', axis=1)\ntemp_test = test.drop(labels='click_time', axis=1)\n# Train the model\nmodel.fit(temp_train, train_labels)\n\n# Predictions\ny_pred = model.predict(temp_test)\ny_pred_proba = model.predict_proba(temp_test)[:, 1]\n\nfrom sklearn.metrics import f1_score\n\n# Calculate F1-score\nf1 = f1_score(test_labels, y_pred)\nauc_score = roc_auc_score(test_labels, y_pred_proba)\n\n\n# Print the F1-score (Note: No need for a probability array for F1-score)\nprint(f\"F1 Score: {f1}\")\nprint(f\"AUC Score: {auc_score}\")\n\nevaluate()","metadata":{"_uuid":"2401988b-9f6b-4cd9-a4eb-42146eeaccc0","_cell_guid":"2a7b16c3-2e0f-459b-9fad-d02d148a3d6e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:07.279256Z","iopub.execute_input":"2024-05-01T03:17:07.279607Z","iopub.status.idle":"2024-05-01T03:17:08.689936Z","shell.execute_reply.started":"2024-05-01T03:17:07.279577Z","shell.execute_reply":"2024-05-01T03:17:08.688984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'Baseline Logistic Regression', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{"_uuid":"2b72c7e3-b89e-49a5-bf50-85cb73e391f7","_cell_guid":"173a3d47-c258-455b-91f4-f154249d5337","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:08.691346Z","iopub.execute_input":"2024-05-01T03:17:08.691806Z","iopub.status.idle":"2024-05-01T03:17:08.696666Z","shell.execute_reply.started":"2024-05-01T03:17:08.691769Z","shell.execute_reply":"2024-05-01T03:17:08.695753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Baseline Model - LightGBM","metadata":{}},{"cell_type":"code","source":"# Initialize LightGBM Classifier with selected parameters\nlgb_model = lgb.LGBMClassifier(class_weight = 'balanced')\n\n# Perform cross-validation\ncv_scores = cross_val_score(lgb_model, temp_train, train_labels, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\nlgb_model.fit(temp_train, train_labels)\n\n# Predictions on the test set\nlgb_y_pred = lgb_model.predict(temp_test)\ny_pred_proba = lgb_model.predict_proba(temp_test)[:, 1]\n# Calculate accuracy and AUC Score on the test set\nlgb_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\nprint(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\nprint(f\"Average Cross-Validation ROC-AUC Score: {np.mean(cv_scores)}\")\n# Calculate F1-score\nf1 = f1_score(test_labels, lgb_y_pred) \nauc_score = roc_auc_score(test_labels, y_pred_proba)\n\n\n# Print the F1-score (Note: No need for a probability array for F1-score)\nprint(f\"F1 Score: {f1}\")\nprint(f\"AUC Score: {auc_score}\")\n\nevaluate()","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:08.697897Z","iopub.execute_input":"2024-05-01T03:17:08.698202Z","iopub.status.idle":"2024-05-01T03:17:11.166865Z","shell.execute_reply.started":"2024-05-01T03:17:08.698177Z","shell.execute_reply":"2024-05-01T03:17:11.162842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'Light GBM Baseline', \"F1 Score\": f1, \"AUC\": auc_score})\n","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.167831Z","iopub.status.idle":"2024-05-01T03:17:11.168209Z","shell.execute_reply.started":"2024-05-01T03:17:11.168035Z","shell.execute_reply":"2024-05-01T03:17:11.168050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Baseline Model - XGBoost","metadata":{}},{"cell_type":"code","source":"class_weight={0:1,1:19}\npos_weight = sum(class_weight.values())/sum(class_weight.keys())\nxgb_model = xgb.XGBClassifier(scale_pos_weight = pos_weight, class_weight = class_weight)\n\n\n# Perform cross-validation\ncv_scores = cross_val_score(xgb_model, temp_train, train_labels, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\nxgb_model.fit(temp_train, train_labels)\n\n# Predictions on the test set\nxgb_y_pred = xgb_model.predict(temp_test)\ny_pred_proba = xgb_model.predict_proba(temp_test)[:, 1]\n\n\nf1 = f1_score(test_labels, xgb_y_pred)\nauc_score = roc_auc_score(test_labels, y_pred_proba)\n\n\n# Print the F1-score (Note: No need for a probability array for F1-score)\nprint(f\"F1 Score: {f1}\")\nprint(f\"AUC Score: {auc_score}\")\n# Calculate accuracy and AUC Score on the test set\n\nprint(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\nprint(f\"Average Cross-Validation ROC-AUC Score: {np.mean(cv_scores)}\")\n\n\n\nevaluate()","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.169287Z","iopub.status.idle":"2024-05-01T03:17:11.169627Z","shell.execute_reply.started":"2024-05-01T03:17:11.169445Z","shell.execute_reply":"2024-05-01T03:17:11.169457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'XGBoost Baseline', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.170467Z","iopub.status.idle":"2024-05-01T03:17:11.170807Z","shell.execute_reply.started":"2024-05-01T03:17:11.170645Z","shell.execute_reply":"2024-05-01T03:17:11.170659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{"_uuid":"ea51b032-4611-4b22-8c3c-c412f33356b9","_cell_guid":"21014213-b229-4259-83cb-e1fea8fb2ad4","trusted":true}},{"cell_type":"markdown","source":"### Time features","metadata":{}},{"cell_type":"code","source":"\ntrain['date'] = pd.to_datetime(train['click_time'], format=format)\ntest['date'] = pd.to_datetime(test['click_time'], format=format)\ntrain['minute'] = train['date'].dt.minute.astype('uint8')\ntest['minute'] = test['date'].dt.minute.astype('uint8')\n\ntrain['tm_hour'] = (train['hour'] + train['minute'] / 60.0).astype('float32')\ntest['tm_hour'] = (test['hour'] + test['minute'] / 60.0).astype('float32')","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.172158Z","iopub.status.idle":"2024-05-01T03:17:11.172508Z","shell.execute_reply.started":"2024-05-01T03:17:11.172339Z","shell.execute_reply":"2024-05-01T03:17:11.172360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prev click features","metadata":{}},{"cell_type":"code","source":"def do_prev_Click( df,agg_suffix='prevClick', agg_type='float32'):\n\n print(f\">> \\nExtracting {agg_suffix} time calculation features...\\n\")\n \n GROUP_BY_NEXT_CLICKS = [\n \n {'groupby': ['ip', 'channel']},\n {'groupby': ['ip', 'os']},\n \n ]\n\n # Calculate the time to next click for each group\n for spec in GROUP_BY_NEXT_CLICKS:\n \n # Name of new feature\n new_feature = '{}_{}'.format('_'.join(spec['groupby']),agg_suffix) \n \n # Unique list of features to select\n all_features = spec['groupby'] + ['click_time']\n\n # Run calculation\n print(f\">> Grouping by {spec['groupby']}, and saving time to {agg_suffix} in: {new_feature}\")\n df[new_feature] = (df.click_time - df[all_features].groupby(spec[\n 'groupby']).click_time.shift(+1) ).dt.seconds.astype(agg_type)\n \n gc.collect()\n return (df) ","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.174336Z","iopub.status.idle":"2024-05-01T03:17:11.174726Z","shell.execute_reply.started":"2024-05-01T03:17:11.174520Z","shell.execute_reply":"2024-05-01T03:17:11.174535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = do_prev_Click(train,agg_suffix='prevClick', agg_type='float32' )\ngc.collect()\ntest = do_prev_Click(test,agg_suffix='prevClick', agg_type='float32' )\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.175700Z","iopub.status.idle":"2024-05-01T03:17:11.176017Z","shell.execute_reply.started":"2024-05-01T03:17:11.175856Z","shell.execute_reply":"2024-05-01T03:17:11.175869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### DAY & HOUR\n","metadata":{}},{"cell_type":"code","source":"# train\ntrain['day'] = train['click_time'].dt.day.astype('uint8')\ntrain['hour'] = train['click_time'].dt.hour.astype('uint8')\n\n# test\ntest['day'] = test['click_time'].dt.day.astype('uint8')\ntest['hour'] = test['click_time'].dt.hour.astype('uint8')\n\ndf_concat = pd.concat([train, test], ignore_index=True)\ndf_concat['day'] = df_concat['click_time'].dt.day.astype('uint8')\ndf_concat['hour'] = df_concat['click_time'].dt.hour.astype('uint8')","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.177086Z","iopub.status.idle":"2024-05-01T03:17:11.177415Z","shell.execute_reply.started":"2024-05-01T03:17:11.177248Z","shell.execute_reply":"2024-05-01T03:17:11.177261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### in_test_hh","metadata":{}},{"cell_type":"code","source":"most_freq_hours_in_test_data = [4, 5, 9, 10, 13, 14]\nleast_freq_hours_in_test_data = [6, 11, 15]\n\ntrain['in_test_hh'] = (\n 3 - 2 * train.hour.isin(most_freq_hours_in_test_data)\n - 1 * train.hour.isin(least_freq_hours_in_test_data)\n).astype('uint8')\n\ntest['in_test_hh'] = (\n 3 - 2 * test.hour.isin(most_freq_hours_in_test_data)\n - 1 * test.hour.isin(least_freq_hours_in_test_data)\n).astype('uint8')\n\ndf_concat['in_test_hh'] = (\n 3 - 2 * df_concat.hour.isin(most_freq_hours_in_test_data)\n - 1 * df_concat.hour.isin(least_freq_hours_in_test_data)\n).astype('uint8')","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.178609Z","iopub.status.idle":"2024-05-01T03:17:11.179021Z","shell.execute_reply.started":"2024-05-01T03:17:11.178799Z","shell.execute_reply":"2024-05-01T03:17:11.178818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Binning\ncat_cols = train.select_dtypes(include='object').columns.tolist()\nnum_cols = train.select_dtypes(exclude='object').columns.tolist()\n\nvariable_names = train.columns.tolist()\nbinning_process = BinningProcess(variable_names, categorical_variables=cat_cols, \n max_n_prebins=30)\n\nbinning_process.fit(train, train_labels)\n\n# Transform train and test\ntrain_binned = binning_process.transform(train, metric_missing=0.05)\ntrain_binned.columns = [f'{col}_BINNED' for col in train_binned.columns]\ntrain_binned.index = train.index\ntest_binned = binning_process.transform(test, metric_missing=0.05)\ntest_binned.columns = [f'{col}_BINNED' for col in test_binned.columns]\ntest_binned.index = test.index\n# Concat original and binned\ntrain = train.select_dtypes('number')\ntrain = pd.concat([train, train_binned], axis=1)\ntest = test.select_dtypes('number')\ntest = pd.concat([test, test_binned], axis=1)\nprint(f'Train shape: {train.shape}, Test shape: {test.shape}')\n","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.180452Z","iopub.status.idle":"2024-05-01T03:17:11.180814Z","shell.execute_reply.started":"2024-05-01T03:17:11.180653Z","shell.execute_reply":"2024-05-01T03:17:11.180667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train =train.drop(['ip_channel_prevClick_BINNED','ip_os_prevClick_BINNED', 'day_BINNED','date_BINNED','click_time_BINNED','day_of_week_BINNED','tm_hour_BINNED','in_test_hh_BINNED'], axis=1)\ntest =test.drop(['ip_channel_prevClick_BINNED','ip_os_prevClick_BINNED', 'day_BINNED','date_BINNED','click_time_BINNED','day_of_week_BINNED','tm_hour_BINNED','in_test_hh_BINNED'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.182009Z","iopub.status.idle":"2024-05-01T03:17:11.182328Z","shell.execute_reply.started":"2024-05-01T03:17:11.182159Z","shell.execute_reply":"2024-05-01T03:17:11.182172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ip_day_test_hh_clicks","metadata":{}},{"cell_type":"code","source":"group = ['ip', 'day', 'in_test_hh']\ndf = df_concat.groupby(group).size().astype('uint16')\ndf = pd.DataFrame(df, columns=['ip_day_test_hh_clicks']).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.183698Z","iopub.status.idle":"2024-05-01T03:17:11.184015Z","shell.execute_reply.started":"2024-05-01T03:17:11.183860Z","shell.execute_reply":"2024-05-01T03:17:11.183874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Counts groupby 3+ features","metadata":{}},{"cell_type":"code","source":"# ip_app_device_clicks\ngroup = ['ip', 'app', 'device']\ndf = df_concat.groupby(group).size().astype('uint16')\ndf = pd.DataFrame(df, columns=['ip_app_device_clicks']).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)\n\n# ip_app_device_day_clicks\ngroup = ['ip', 'app', 'device', 'day']\ndf = df_concat.groupby(group).size().astype('uint16')\ndf = pd.DataFrame(df, columns=['ip_app_device_day_clicks']).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.185150Z","iopub.status.idle":"2024-05-01T03:17:11.185505Z","shell.execute_reply.started":"2024-05-01T03:17:11.185311Z","shell.execute_reply":"2024-05-01T03:17:11.185324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Interaction variables","metadata":{}},{"cell_type":"code","source":"def calc_entropy(df, group, subgrp, tgt_vn_prefix):\n # Calculate counts of subgroups within each group\n sum1 = df.groupby([group, subgrp]).size().reset_index(name='subgrpcnt')\n \n # Calculate total counts of each group\n sum2 = df.groupby(group).size().reset_index(name='cnt')\n \n # Merge counts of subgroups and total counts\n sum3 = pd.merge(sum2, sum1, on=group, how='left')\n \n # Calculate entropy\n sum3['entropy'] = -(np.log(sum3['subgrpcnt'] * 1.0 / sum3['cnt']) * (sum3['subgrpcnt'] * 1.0 / sum3['cnt']))\n sum3['entropy'].fillna(0, inplace=True)\n \n # Sum up entropy for each group\n sum4 = sum3.groupby(group)['entropy'].sum().reset_index(name=tgt_vn_prefix + '_entropy')\n \n return sum4","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.186804Z","iopub.status.idle":"2024-05-01T03:17:11.187160Z","shell.execute_reply.started":"2024-05-01T03:17:11.186983Z","shell.execute_reply":"2024-05-01T03:17:11.187009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"app_entropy_train = calc_entropy(train, 'app', 'ip', 'app_ip')\nprint(app_entropy_train)\ntrain = pd.merge(train, app_entropy_train, on='app', how='left')\n\napp_entropy_test = calc_entropy(test, 'app', 'ip', 'app_ip')\n# print(app_entropy_test)\ntest = pd.merge(test, app_entropy_test, on='app', how='left')","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.188235Z","iopub.status.idle":"2024-05-01T03:17:11.188536Z","shell.execute_reply.started":"2024-05-01T03:17:11.188387Z","shell.execute_reply":"2024-05-01T03:17:11.188399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ip_entropy_train = calc_entropy(train, 'ip', 'app', 'ip_app')\nprint(ip_entropy_train)\ntrain = pd.merge(train, ip_entropy_train, on='ip', how='left')\n\nip_entropy_test = calc_entropy(test, 'ip', 'app', 'ip_app')\n# print(app_entropy_test)\ntest = pd.merge(test, ip_entropy_test, on='ip', how='left')","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.189816Z","iopub.status.idle":"2024-05-01T03:17:11.190278Z","shell.execute_reply.started":"2024-05-01T03:17:11.190044Z","shell.execute_reply":"2024-05-01T03:17:11.190063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Nunique","metadata":{}},{"cell_type":"code","source":"# single 'ip' grouper for ['app', 'device', 'channel']\ncount_cols = ['app', 'device', 'channel', 'hour']\ngroup = 'ip'\nfor col in count_cols:\n df = df_concat.groupby(group)[col].nunique().astype('uint16')\n df.name = 'ip_nunique_{}'.format(col)\n df = pd.DataFrame(df).reset_index()\n train = train.merge(df, how='left', on=group)\n test = test.merge(df, how='left', on=group)\n #####################\n # self-add\n group_day = ['ip', 'day']\n df = df_concat.groupby(group_day)[col].nunique().astype('uint16')\n df.name = 'ip_day_nunique_{}'.format(col)\n df = pd.DataFrame(df).reset_index()\n train = train.merge(df, how='left', on=group_day)\n test = test.merge(df, how='left', on=group_day)\n #####################\n \n# single 'app' grouper for 'channel'\ngroup = 'app'\ncol = 'channel'\ndf = df_concat.groupby(group)[col].nunique().astype('uint16')\ndf.name = 'app_nunique_{}'.format(col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)\n#####################\n\n# self-add\ngroup_day = ['app', 'day']\ncol = 'channel'\ndf = df_concat.groupby(group_day)[col].nunique().astype('uint16')\ndf.name = 'app_day_nunique_{}'.format(col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=group_day)\ntest = test.merge(df, how='left', on=group_day)\n#####################\n\n# duble ['ip', 'app'] grouper for 'os'\ngroup = ['ip', 'app']\ncol = 'os'\ndf = df_concat.groupby(group)[col].nunique().astype('uint16')\ndf.name = 'ip_app_nunique_{}'.format(col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)\n#####################\n\n# self-add\ngroup_day = ['ip', 'app', 'day']\ncol = 'os'\ndf = df_concat.groupby(group_day)[col].nunique().astype('uint16')\ndf.name = 'ip_app_day_nunique_{}'.format(col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=group_day)\ntest = test.merge(df, how='left', on=group_day)\n#####################\n\n# triple ['ip', 'device', 'os'] grouper for 'app'\ngroup = ['ip', 'device', 'os']\ncol = 'app'\ndf = df_concat.groupby(group)[col].nunique().astype('uint16')\ndf.name = 'ip_device_os_nunique_{}'.format(col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=group)\ntest = test.merge(df, how='left', on=group)\n#####################\n\n# self-add\ngroup_day = ['ip', 'device', 'os'] + ['day']\ncol = 'app'\ndf = df_concat.groupby(group_day)[col].nunique().astype('uint16')\ndf.name = 'ip_device_os_day_nunique_{}'.format(col)\ndf = pd.DataFrame(df).reset_index()\ntrain = train.merge(df, how='left', on=group_day)\ntest = test.merge(df, how='left', on=group_day)\n#####################","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.192080Z","iopub.status.idle":"2024-05-01T03:17:11.192438Z","shell.execute_reply.started":"2024-05-01T03:17:11.192259Z","shell.execute_reply":"2024-05-01T03:17:11.192273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define all the groupby transformations\nGROUPBY_AGGREGATIONS = [\n # Variance in day, for ip-app-channel\n {'groupby': ['ip','app','channel'], 'select': 'day', 'agg': 'var', 'type': 'float32'},\n # Variance in day, for ip-app-device\n {'groupby': ['ip','app','device'], 'select': 'day', 'agg': 'var', 'type': 'float32'},\n # Variance in day, for ip-app-os\n {'groupby': ['ip','app','os'], 'select': 'day', 'agg': 'var', 'type': 'float32'},\n # Count, for ip-day-hour\n# {'groupby': ['ip','day','hour'], 'select': 'channel', 'agg': 'count', 'type': 'float32'},\n # Count, for ip-day-hour\n {'groupby': ['channel'], 'select': 'day', 'agg': 'var', 'type': 'float32'},\n # Count, for ip-day-hour\n {'groupby': ['os'], 'select': 'hour', 'agg': 'var', 'type': 'float32'},\n {'groupby': ['ip','app','os','device'], 'select': 'hour', 'agg': 'var', 'type': 'float32'},\n {'groupby': ['ip','app','os'], 'select': 'hour', 'agg': 'var', 'type': 'float32'},\n\n # Mean hour, for ip-app-channel\n {'groupby': ['ip','app','channel'], 'select': 'hour', 'agg': 'mean', 'type': 'float32', 'type': 'float32'},\n {'groupby': ['channel'], 'select': 'hour', 'agg': 'mean', 'type': 'float32', 'type': 'float32'}\n\n]\n# Apply all the groupby transformations\nfor spec in GROUPBY_AGGREGATIONS:\n print(f\"Grouping by {spec['groupby']}, and aggregating {spec['select']} with {spec['agg']}\")\n \n # Unique list of features to select\n all_features = list(set(spec['groupby'] + [spec['select']]))\n # Name of new feature\n new_feature = '{}_{}'.format('_'.join(spec['groupby']), spec['agg'])\n # Perform the groupby\n gp = train[all_features]. \\\n groupby(spec['groupby'])[spec['select']]. \\\n agg(spec['agg']). \\\n reset_index(). \\\n rename(index=str, columns={spec['select']: new_feature}).astype(spec['type'])\n gp = test[all_features]. \\\n groupby(spec['groupby'])[spec['select']]. \\\n agg(spec['agg']). \\\n reset_index(). \\\n rename(index=str, columns={spec['select']: new_feature}).astype(spec['type'])\n # Merge back to X_train\n train = train.merge(gp, on=spec['groupby'], how='left')\n test = test.merge(gp, on=spec['groupby'], how='left')\ndel gp\nprint(\"End\")","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.194006Z","iopub.status.idle":"2024-05-01T03:17:11.194311Z","shell.execute_reply.started":"2024-05-01T03:17:11.194161Z","shell.execute_reply":"2024-05-01T03:17:11.194173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.decomposition import LatentDirichletAllocation\n\ndef modelTopicsWithDevice(df, ip_column, app_column, device_column):\n \"\"\"Model LDA topics for each IP based on apps and devices.\"\"\"\n # Assuming vocab selection is based on app usage frequency\n vocab = df[app_column].value_counts().nlargest(10).index.tolist()\n\n # Aggregate app and device values for each IP\n ip_activities = {}\n for _, row in tqdm(df.iterrows(), total=len(df)):\n ip = row[ip_column]\n app = row[app_column]\n device = str(row[device_column]) # Convert device to string if it's not already\n activity = f\"{app}_{device}\" # Concatenate app and device information\n\n if app in vocab:\n ip_activities.setdefault(ip, []).append(activity)\n\n ips = list(ip_activities.keys())\n sentences = [' '.join(ip_activities[ip]) for ip in ips]\n \n # Vectorize the app and device sentences\n cv = CountVectorizer()\n dt_matrix = cv.fit_transform(sentences)\n \n # Apply LDA\n lda = LatentDirichletAllocation(n_components=5, random_state=42)\n ip_topics = lda.fit_transform(dt_matrix)\n \n # Create a DataFrame for the topics\n topic_cols = [f\"{ip_column}_{app_column}_{device_column}_topic{i+1}\" for i in range(5)]\n topic_df = pd.DataFrame(ip_topics, columns=topic_cols)\n topic_df[ip_column] = ips\n \n # Merge the topics back to the original DataFrame\n df = df.merge(topic_df, on=ip_column, how='left')\n \n return df","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.195945Z","iopub.status.idle":"2024-05-01T03:17:11.196369Z","shell.execute_reply.started":"2024-05-01T03:17:11.196149Z","shell.execute_reply":"2024-05-01T03:17:11.196166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = modelTopicsWithDevice(train, 'ip', 'app', 'device')\ntest = modelTopicsWithDevice(test, 'ip', 'app', 'device')","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.197425Z","iopub.status.idle":"2024-05-01T03:17:11.197759Z","shell.execute_reply.started":"2024-05-01T03:17:11.197598Z","shell.execute_reply":"2024-05-01T03:17:11.197612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fill NaN values","metadata":{}},{"cell_type":"code","source":"train = train.fillna(train.max())\ntest = test.fillna(test.max())","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.198856Z","iopub.status.idle":"2024-05-01T03:17:11.199191Z","shell.execute_reply.started":"2024-05-01T03:17:11.199033Z","shell.execute_reply":"2024-05-01T03:17:11.199046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().mean()*100","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.200482Z","iopub.status.idle":"2024-05-01T03:17:11.200819Z","shell.execute_reply.started":"2024-05-01T03:17:11.200663Z","shell.execute_reply":"2024-05-01T03:17:11.200677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### SMOTE","metadata":{"_uuid":"02706aaa-df7d-41f6-9829-80f81724f17c","_cell_guid":"62838d0f-5374-4565-821e-066aa3fc257d","trusted":true}},{"cell_type":"code","source":"def balance_data(features, labels, sampling_strategy=0.5, random_state=42):\n # Positive oversampling (SMOTE)\n smote = SMOTE(sampling_strategy=sampling_strategy, random_state=random_state)\n features_resampled, labels_resampled = smote.fit_resample(features, labels)\n\n # Negative downsampling (reduce negative samples to match positive samples)\n negative_samples = features_resampled[labels_resampled == 0]\n positive_samples = features_resampled[labels_resampled == 1]\n\n # Perform negative downsampling, e.g., reduce negative samples to match twice the number of positive samples\n negative_samples_downsampled = resample(negative_samples, n_samples=len(positive_samples)*2, random_state=random_state)\n\n # Combine positive and downsampled negative samples\n balanced_features = pd.concat([positive_samples, negative_samples_downsampled], ignore_index=True)\n balanced_labels = pd.Series([1] * len(positive_samples) + [0] * len(negative_samples_downsampled))\n\n return balanced_features, balanced_labels\n\n# Example usage:\ntrain_balanced, train_labels_balanced = balance_data(train, train_labels, sampling_strategy=0.5, random_state=42)","metadata":{"_uuid":"de79da0e-c82a-4cc7-873a-921f64a43090","_cell_guid":"12b5f51d-ec0c-4e79-b7bf-a24514c30412","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:11.201886Z","iopub.status.idle":"2024-05-01T03:17:11.202206Z","shell.execute_reply.started":"2024-05-01T03:17:11.202046Z","shell.execute_reply":"2024-05-01T03:17:11.202059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Count the number of occurrences of each label\nlabel_counts = train_labels_balanced.value_counts()\n\n# Display the counts\nprint(\"Number of '0' labels:\", label_counts[0])\nprint(\"Number of '1' labels:\", label_counts[1])","metadata":{"_uuid":"dbb6b8c8-ec4b-4ce8-859a-2896f9a99c5e","_cell_guid":"94b67441-19ef-4dd9-83d5-30e268c8939b","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2024-05-01T03:17:11.203391Z","iopub.status.idle":"2024-05-01T03:17:11.203749Z","shell.execute_reply.started":"2024-05-01T03:17:11.203580Z","shell.execute_reply":"2024-05-01T03:17:11.203595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Standard scaler","metadata":{}},{"cell_type":"code","source":"scaler = StandardScaler()\ntrain_balanced = scaler.fit_transform(train_balanced)\ntest = scaler.transform(test)","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.205163Z","iopub.status.idle":"2024-05-01T03:17:11.205463Z","shell.execute_reply.started":"2024-05-01T03:17:11.205312Z","shell.execute_reply":"2024-05-01T03:17:11.205325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model - Logistic Regression","metadata":{}},{"cell_type":"code","source":"# Initialize logistic regression model with 'sag' solver\nlr_model = LogisticRegression(solver='sag', C=1.0, max_iter=10000, penalty='l2',class_weight = 'balanced' )\n\n# Train the model\nlr_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions\ny_pred = lr_model.predict(test)\ny_pred_proba = lr_model.predict_proba(test)[:, 1]\n# Calculate F1-score\nf1 = f1_score(test_labels, y_pred)\nauc_score = roc_auc_score(test_labels, y_pred_proba)\n\n\n# Print the F1-score (Note: No need for a probability array for F1-score)\nprint(f\"F1 Score: {f1}\")\nprint(f\"AUC Score: {auc_score}\")\n\nevaluate()","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.206370Z","iopub.status.idle":"2024-05-01T03:17:11.206704Z","shell.execute_reply.started":"2024-05-01T03:17:11.206517Z","shell.execute_reply":"2024-05-01T03:17:11.206529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'Logistic Regression', \"F1 Score\": f1, \"AUC\": auc_score})\n","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.208284Z","iopub.status.idle":"2024-05-01T03:17:11.208727Z","shell.execute_reply.started":"2024-05-01T03:17:11.208488Z","shell.execute_reply":"2024-05-01T03:17:11.208506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model - XGBoost","metadata":{"_uuid":"382b8437-8f09-4f67-b981-cc7751fd4d02","_cell_guid":"751a1b0f-abf8-4f83-b041-83da3e477ce9","trusted":true}},{"cell_type":"code","source":"params = {'eta': 0.2,\n 'max_depth': 15,\n 'subsample': 0.7,\n 'colsample_bytree': 0.9,\n 'objective': 'binary:logistic',\n 'eval_metric': 'auc',\n 'n_jobs': 24,\n 'n_estimators': 400\n }\n\n# Initialize XGBoost Classifier with selected parameters\nclass_weight={0:1,1:19}\npos_weight = sum(class_weight.values())/sum(class_weight.keys())\nxgb_model = xgb.XGBClassifier(scale_pos_weight = pos_weight, class_weight = class_weight, **params)\n\n# Perform cross-validation\ncv_scores = cross_val_score(xgb_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\nxgb_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions on the test set\nxgb_y_pred = xgb_model.predict(test)\ny_pred_proba = xgb_model.predict_proba(test)[:, 1]\n\n\n# Calculate F1-score\nf1 = f1_score(test_labels, xgb_y_pred)\nauc_score = roc_auc_score(test_labels, y_pred_proba)\n\n\n# Print the F1-score (Note: No need for a probability array for F1-score)\nprint(f\"F1 Score: {f1}\")\nprint(f\"AUC Score: {auc_score}\")\n\nevaluate()","metadata":{"_uuid":"94177d6f-c9dc-474f-8ded-8210624f8434","_cell_guid":"a295632d-9338-48bc-86cc-d8ec9bee3484","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:11.210184Z","iopub.status.idle":"2024-05-01T03:17:11.210630Z","shell.execute_reply.started":"2024-05-01T03:17:11.210388Z","shell.execute_reply":"2024-05-01T03:17:11.210405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'XGBoost', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{"_uuid":"508afe8f-3004-4b71-8ae5-a1c904f0f22d","_cell_guid":"b433ef88-1290-4035-8f2a-47e9415288c3","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:11.212628Z","iopub.status.idle":"2024-05-01T03:17:11.213095Z","shell.execute_reply.started":"2024-05-01T03:17:11.212846Z","shell.execute_reply":"2024-05-01T03:17:11.212866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model - LightGBM","metadata":{"_uuid":"986082d9-8475-424c-8964-1923b9fa22c8","_cell_guid":"5035b192-dce5-411a-b0ab-7b30271839a5","trusted":true}},{"cell_type":"code","source":"params = {'num_leaves': 127,\n 'min_data_in_leaf': 32,\n 'objective': 'binary',\n 'max_depth': -1,\n 'learning_rate': 0.1,\n 'min_child_samples': 20,\n 'boosting': 'gbdt',\n 'feature_fraction': 0.8,\n 'bagging_freq': 1,\n 'bagging_fraction': 0.8,\n 'bagging_seed': 11,\n 'metric': 'auc',\n 'lambda_l1': 0.1,\n 'verbosity': -1,\n 'n_estimators': 200,\n 'class_weight' : 'balanced'\n }\n\n# Initialize LightGBM Classifier with selected parameters\nlgb_model = lgb.LGBMClassifier(**params)\n\n# Perform cross-validation\ncv_scores = cross_val_score(lgb_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\nlgb_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions on the test set\nlgb_y_pred = lgb_model.predict(test)\ny_pred_proba = lgb_model.predict_proba(test)[:, 1]\n\n# Calculate F1-score\nf1 = f1_score(test_labels, lgb_y_pred)\nauc_score = roc_auc_score(test_labels, y_pred_proba)\n\n\n# Print the F1-score (Note: No need for a probability array for F1-score)\nprint(f\"F1 Score: {f1}\")\nprint(f\"AUC Score: {auc_score}\")\n\nprint(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\nprint(f\"Average Cross-Validation ROC-AUC Score: {np.mean(cv_scores)}\")\n","metadata":{"_uuid":"25afb764-b956-45d3-97f4-9e1229686608","_cell_guid":"669043b3-889f-42d0-8fae-409e9908bf47","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:11.214172Z","iopub.status.idle":"2024-05-01T03:17:11.214650Z","shell.execute_reply.started":"2024-05-01T03:17:11.214388Z","shell.execute_reply":"2024-05-01T03:17:11.214406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'Light GBM', \"F1 Score\": f1, \"AUC\": lgb_auc_score})","metadata":{"_uuid":"27b03d74-a54c-45e9-a0c5-b0e9a2231a52","_cell_guid":"55d17844-b063-48b6-926c-7e67bb8908dd","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:11.216202Z","iopub.status.idle":"2024-05-01T03:17:11.216622Z","shell.execute_reply.started":"2024-05-01T03:17:11.216402Z","shell.execute_reply":"2024-05-01T03:17:11.216418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Gradient Boosting","metadata":{}},{"cell_type":"code","source":"# Manually select parameters from the parameter grid\nparams = {\n 'n_estimators': 200,\n 'learning_rate': 0.1,\n 'max_depth': 5,\n 'min_samples_split': 4,\n 'min_samples_leaf': 2,\n 'max_features': 'sqrt'\n}\n\n# Initialize Gradient Boosting Classifier with selected parameters\ngb_model = GradientBoostingClassifier(**params)\n\n# Perform cross-validation\ncv_scores = cross_val_score(gb_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\ngb_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions on the test set\ngb_y_pred = gb_model.predict(test)\ny_pred_proba = gb_model.predict_proba(test)[:, 1]\n\n# Calculate F1-score\nf1 = f1_score(test_labels, gb_y_pred)\ngb_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\n\n# Print the F1-score (Note: No need for a probability array for F1-score)\nprint(f\"F1 Score: {f1}\")\nprint(f\"AUC Score: {auc_score}\")\n\nprint(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\nprint(f\"Average Cross-Validation ROC-AUC Score: {cv_scores.mean()}\")\n\nevaluate()","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.217909Z","iopub.status.idle":"2024-05-01T03:17:11.218207Z","shell.execute_reply.started":"2024-05-01T03:17:11.218058Z","shell.execute_reply":"2024-05-01T03:17:11.218071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'Gradient Boosting Tree', \"F1 Score\": f1, \"AUC\": gb_auc_score})","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.219939Z","iopub.status.idle":"2024-05-01T03:17:11.220258Z","shell.execute_reply.started":"2024-05-01T03:17:11.220098Z","shell.execute_reply":"2024-05-01T03:17:11.220112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model - CatBoost","metadata":{"_uuid":"4b8b000b-9a0f-480a-98ad-4f870fb348e5","_cell_guid":"dbe174d3-8c90-4b21-ae35-5d0fba6dae5a","trusted":true}},{"cell_type":"code","source":"from catboost import CatBoostClassifier\n\n# Manually select parameters from the parameter grid\nparams = {\n 'depth': 10,\n 'learning_rate': 0.2,\n 'iterations': 500,\n 'l2_leaf_reg': 3,\n 'border_count': 50,\n 'bagging_temperature': 0.04019881483063329,\n 'loss_function': 'Logloss',\n 'od_type': 'Iter',\n 'od_wait': 50,\n 'verbose': False\n}\n\n# Initialize CatBoost Classifier with selected parameters\ncat_model = CatBoostClassifier(**params)\n\n# Perform cross-validation\ncv_scores = cross_val_score(cat_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\ncat_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions on the test set\ncat_y_pred = cat_model.predict(test)\ny_pred_proba = cat_model.predict_proba(test)[:, 1]\n\n# Calculate F1-score\nf1 = f1_score(test_labels, cat_y_pred)\ncat_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\n\n# Print the F1-score (Note: No need for a probability array for F1-score)\nprint(f\"F1 Score: {f1}\")\nprint(f\"AUC Score: {cat_auc_score}\")\nprint(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\nprint(f\"Average Cross-Validation ROC-AUC Score: {np.mean(cv_scores)}\")\n\nevaluate()","metadata":{"_uuid":"6f7d6d84-a0ab-4c3f-8619-73ad04802bfa","_cell_guid":"6d050bd9-e069-4349-9e91-7bbf35c3582f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:11.221623Z","iopub.status.idle":"2024-05-01T03:17:11.222005Z","shell.execute_reply.started":"2024-05-01T03:17:11.221813Z","shell.execute_reply":"2024-05-01T03:17:11.221828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'CatBoost', \"F1 Score\": f1, \"AUC\": cat_auc_score})","metadata":{"_uuid":"8af66b70-758f-4dd7-bf2e-304969f76723","_cell_guid":"46ea66d9-d925-45a3-8200-e8b765e0f60a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:11.223158Z","iopub.status.idle":"2024-05-01T03:17:11.223479Z","shell.execute_reply.started":"2024-05-01T03:17:11.223320Z","shell.execute_reply":"2024-05-01T03:17:11.223334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model - Random Forest","metadata":{"_uuid":"50f59850-c645-4ea8-a736-2feda46cb80c","_cell_guid":"7edc1ec3-5c38-4f24-a96e-36b7734502dc","trusted":true}},{"cell_type":"code","source":"# Initialize Random Forest Classifier\nrf_model = RandomForestClassifier(\n n_estimators=100,\n max_depth=20,\n min_samples_split=2,\n min_samples_leaf=1,\n max_features='sqrt',\n bootstrap=False,\n criterion='gini'\n)\n\n# Perform cross-validation\ncv_scores = cross_val_score(rf_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\nrf_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions on the test set\nrf_y_pred = rf_model.predict(test)\ny_pred_proba = rf_model.predict_proba(test)[:, 1]\n\n# Calculate accuracy and AUC Score on the test set\nf1 = f1_score(test_labels, cat_y_pred)\nrf_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\nprint(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\nprint(f\"Average Cross-Validation ROC-AUC Score: {np.mean(cv_scores)}\")\n\nevaluate()","metadata":{"_uuid":"94cc51dc-b29c-48ce-8b1f-6d2f6afb4464","_cell_guid":"a3777945-2215-4c76-ba88-c78d392d901f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:11.225101Z","iopub.status.idle":"2024-05-01T03:17:11.225425Z","shell.execute_reply.started":"2024-05-01T03:17:11.225274Z","shell.execute_reply":"2024-05-01T03:17:11.225286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'Random Forest', \"F1 Score\": f1, \"AUC\": rf_auc_score})","metadata":{"_uuid":"a421381e-be68-4a6a-bf59-6adce1aecbd1","_cell_guid":"0f7265e8-3773-433f-86bf-f2cb9ccf0cf0","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:11.226699Z","iopub.status.idle":"2024-05-01T03:17:11.227028Z","shell.execute_reply.started":"2024-05-01T03:17:11.226866Z","shell.execute_reply":"2024-05-01T03:17:11.226880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model - AdaBoost","metadata":{"_uuid":"96e54bf8-348b-49f8-8f3c-88c6a31ad616","_cell_guid":"2841ff6a-d89b-41bd-8d15-6b0d26841ec1","trusted":true}},{"cell_type":"code","source":"# model performance on test data with chosen hyperparameters\n\n# base estimator\ntree = DecisionTreeClassifier(max_depth=2)\n\n# adaboost with the tree as base estimator\n# learning rate is arbitrarily set, we'll discuss learning_rate below\nABC_model = AdaBoostClassifier(\n base_estimator=tree,\n learning_rate=1.0,\n n_estimators=300,\n algorithm=\"SAMME\")\n\n# Perform cross-validation\ncv_scores = cross_val_score(ABC_model, train_balanced, train_labels_balanced, cv=5, scoring='roc_auc')\n\n# Train the model on the entire training set\nABC_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions on the test set\nABC_y_pred = ABC_model.predict(test)\ny_pred_proba = ABC_model.predict_proba(test)[:, 1]\n\n# Calculate accuracy and AUC Score on the test set\nf1 = f1_score(test_labels, ABC_y_pred)\n\nABC_auc_score = roc_auc_score(test_labels, y_pred_proba)\n\nprint(f\"Cross-Validation ROC-AUC Scores: {cv_scores}\")\nprint(f\"Average Cross-Validation ROC-AUC Score: {np.mean(cv_scores)}\")\n\nevaluate()","metadata":{"_uuid":"4394718d-3321-4748-bf3a-bde0d5ee76f1","_cell_guid":"7ac318b1-ff04-4bd1-b12c-08dcaa0c6a41","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:11.228380Z","iopub.status.idle":"2024-05-01T03:17:11.228827Z","shell.execute_reply.started":"2024-05-01T03:17:11.228604Z","shell.execute_reply":"2024-05-01T03:17:11.228624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'AdaBoost', \"F1 Score\": f1, \"AUC\": ABC_auc_score})","metadata":{"_uuid":"5b95e0b1-5ba3-4bc9-8736-d3d5ce2f3dee","_cell_guid":"c65d81e6-f1a2-4cea-a018-b279ccee75ea","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:11.230226Z","iopub.status.idle":"2024-05-01T03:17:11.230669Z","shell.execute_reply.started":"2024-05-01T03:17:11.230430Z","shell.execute_reply":"2024-05-01T03:17:11.230448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Stacking Generalization Ensembling","metadata":{"_uuid":"5c181af6-0ece-4167-868e-6a2af17e790d","_cell_guid":"617c1db0-6400-4dc8-98f3-147af0058a55","trusted":true}},{"cell_type":"code","source":"from sklearn.ensemble import StackingClassifier\n\n# Define base models\nbase_models = [\n ('Logistic Regression', lr_model),\n ('XGBoost', xgb_model),\n ('Gradient Boosting', gb_model),\n ('LightGBM', lgb_model),\n ('CatBoost', cat_model),\n ('RF', rf_model),\n ('AdaBoost', ABC_model)\n]\n\n# Define meta-model\nmeta_model = LogisticRegression()\n\n# Initialize Stacking Classifier\nstacked_model = StackingClassifier(estimators=base_models, final_estimator=meta_model, cv=5)\n\n# Train the stacked model (Note: The pre-trained model won't be re-trained)\nstacked_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions and evaluation\ny_pred = stacked_model.predict(test)\ny_pred_proba = stacked_model.predict_proba(test)[:, 1] # Probabilities for the positive class\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, y_pred)\nauc_score = roc_auc_score(test_labels, y_pred_proba)\n\nprint(f\"AUC Score: {auc_score}\")\nevaluate()","metadata":{"_uuid":"e7d307d8-6c73-4616-84cc-db48b0fbd784","_cell_guid":"441f7c2b-7ef4-42a0-8bad-07629ebdfd38","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:11.232074Z","iopub.status.idle":"2024-05-01T03:17:11.232499Z","shell.execute_reply.started":"2024-05-01T03:17:11.232277Z","shell.execute_reply":"2024-05-01T03:17:11.232295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'Stacked Model', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{"_uuid":"7f9eebaa-4217-4882-a544-35f081a09ab6","_cell_guid":"1617ee7b-232c-4f5d-94db-8454a514be33","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:11.233773Z","iopub.status.idle":"2024-05-01T03:17:11.234108Z","shell.execute_reply.started":"2024-05-01T03:17:11.233947Z","shell.execute_reply":"2024-05-01T03:17:11.233961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Majority Vote Ensembling","metadata":{"_uuid":"4c4d5b41-4876-4d4d-a11c-966c0e4bcc95","_cell_guid":"3f77fab7-ffb9-4e65-87e5-3295531e4e08","trusted":true}},{"cell_type":"code","source":"from scipy.stats import mode\n\n# Stack predictions for ease of calculation\nstacked_predictions = np.column_stack((gb_y_pred, xgb_y_pred, lgb_y_pred, cat_y_pred, ABC_y_pred))\n\n# Perform majority voting\nmajority_votes = mode(stacked_predictions, axis=1)[0]\n\n# Flatten to get a 1D array of final predictions\nfinal_predictions = np.ravel(majority_votes)\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, final_predictions)\nauc_score = roc_auc_score(test_labels, final_predictions)\nprint(f\"AUC Score: {auc_score}\")\nevaluate()","metadata":{"_uuid":"14578ead-d54f-43e3-85f1-095f51fc2ed4","_cell_guid":"6cbb3c3b-7d47-4ce0-8b57-955af15866a9","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:11.235240Z","iopub.status.idle":"2024-05-01T03:17:11.235535Z","shell.execute_reply.started":"2024-05-01T03:17:11.235385Z","shell.execute_reply":"2024-05-01T03:17:11.235397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'Majority Vote', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{"_uuid":"cc86f48e-e19d-4598-8ccd-98041d7b1912","_cell_guid":"721ce3e9-4725-4287-ab04-603e4c10e076","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-01T03:17:11.237067Z","iopub.status.idle":"2024-05-01T03:17:11.237380Z","shell.execute_reply.started":"2024-05-01T03:17:11.237220Z","shell.execute_reply":"2024-05-01T03:17:11.237232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# XGBoost + Random Forest\n","metadata":{}},{"cell_type":"code","source":"# Define base models\nbase_models = [\n ('XGBoost', xgb_model),\n ('Random Forest', rf_model),\n]\n\n# Define meta-model\nmeta_model = LogisticRegression()\n\n# Initialize Stacking Classifier\nstacked_model = StackingClassifier(estimators=base_models, final_estimator=meta_model, cv=5)\n\n# Train the stacked model (Note: The pre-trained model won't be re-trained)\nstacked_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions and evaluation\ny_pred = stacked_model.predict(test)\ny_pred_proba = stacked_model.predict_proba(test)[:, 1] # Probabilities for the positive class\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, y_pred)\nauc_score = roc_auc_score(test_labels, y_pred_proba)\n\nprint(f\"AUC Score: {auc_score}\")\nresults.append({\"Model\": 'XGBoost + Random Forest', \"F1 Score\": f1, \"AUC\": auc_score})\nevaluate()","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.238627Z","iopub.status.idle":"2024-05-01T03:17:11.238960Z","shell.execute_reply.started":"2024-05-01T03:17:11.238778Z","shell.execute_reply":"2024-05-01T03:17:11.238791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CatBoost + AdaBoosting","metadata":{}},{"cell_type":"code","source":"\n# Define base models\nbase_models = [\n ('CatBoost', cat_model),\n ('AdaBoosting', ABC_model),\n]\n\n# Define meta-model\nmeta_model = LogisticRegression()\n\n# Initialize Stacking Classifier\nstacked_model = StackingClassifier(estimators=base_models, final_estimator=meta_model, cv=5)\n\n# Train the stacked model (Note: The pre-trained model won't be re-trained)\nstacked_model.fit(train_balanced, train_labels_balanced)\n\n# Predictions and evaluation\ny_pred = stacked_model.predict(test)\ny_pred_proba = stacked_model.predict_proba(test)[:, 1] # Probabilities for the positive class\n\n# Calculate accuracy and AUC Score\nf1 = f1_score(test_labels, y_pred)\nauc_score = roc_auc_score(test_labels, y_pred_proba)\n\nprint(f\"AUC Score: {auc_score}\")\nresults.append({\"Model\": 'CatBoost + AdaBoosting', \"F1 Score\": f1, \"AUC\": auc_score})\nevaluate()","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.240411Z","iopub.status.idle":"2024-05-01T03:17:11.240768Z","shell.execute_reply.started":"2024-05-01T03:17:11.240587Z","shell.execute_reply":"2024-05-01T03:17:11.240601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cascade Forest Reimplementation","metadata":{}},{"cell_type":"code","source":"import itertools\n\n\nclass gcForest(object):\n\n def __init__(self, shape_1X=None, n_mgsRFtree=30, window=None, stride=1,\n cascade_test_size=0.2, n_cascadeRF=2, n_cascadeRFtree=101, cascade_layer=np.inf,\n min_samples_mgs=0.1, min_samples_cascade=0.05, tolerance=0.0, n_jobs=1):\n \"\"\" gcForest Classifier.\n\n :param shape_1X: int or tuple list or np.array (default=None)\n Shape of a single sample element [n_lines, n_cols]. Required when calling mg_scanning!\n For sequence data a single int can be given.\n\n :param n_mgsRFtree: int (default=30)\n Number of trees in a Random Forest during Multi Grain Scanning.\n\n :param window: int (default=None)\n List of window sizes to use during Multi Grain Scanning.\n If 'None' no slicing will be done.\n\n :param stride: int (default=1)\n Step used when slicing the data.\n\n :param cascade_test_size: float or int (default=0.2)\n Split fraction or absolute number for cascade training set splitting.\n\n :param n_cascadeRF: int (default=2)\n Number of Random Forests in a cascade layer.\n For each pseudo Random Forest a complete Random Forest is created, hence\n the total numbe of Random Forests in a layer will be 2*n_cascadeRF.\n\n :param n_cascadeRFtree: int (default=101)\n Number of trees in a single Random Forest in a cascade layer.\n\n :param min_samples_mgs: float or int (default=0.1)\n Minimum number of samples in a node to perform a split\n during the training of Multi-Grain Scanning Random Forest.\n If int number_of_samples = int.\n If float, min_samples represents the fraction of the initial n_samples to consider.\n\n :param min_samples_cascade: float or int (default=0.1)\n Minimum number of samples in a node to perform a split\n during the training of Cascade Random Forest.\n If int number_of_samples = int.\n If float, min_samples represents the fraction of the initial n_samples to consider.\n\n :param cascade_layer: int (default=np.inf)\n mMximum number of cascade layers allowed.\n Useful to limit the contruction of the cascade.\n\n :param tolerance: float (default=0.0)\n Accuracy tolerance for the casacade growth.\n If the improvement in accuracy is not better than the tolerance the construction is\n stopped.\n\n :param n_jobs: int (default=1)\n The number of jobs to run in parallel for any Random Forest fit and predict.\n If -1, then the number of jobs is set to the number of cores.\n \"\"\"\n setattr(self, 'shape_1X', shape_1X)\n setattr(self, 'n_layer', 0)\n setattr(self, '_n_samples', 0)\n setattr(self, 'n_cascadeRF', int(n_cascadeRF))\n if isinstance(window, int):\n setattr(self, 'window', [window])\n elif isinstance(window, list):\n setattr(self, 'window', window)\n setattr(self, 'stride', stride)\n setattr(self, 'cascade_test_size', cascade_test_size)\n setattr(self, 'n_mgsRFtree', int(n_mgsRFtree))\n setattr(self, 'n_cascadeRFtree', int(n_cascadeRFtree))\n setattr(self, 'cascade_layer', cascade_layer)\n setattr(self, 'min_samples_mgs', min_samples_mgs)\n setattr(self, 'min_samples_cascade', min_samples_cascade)\n setattr(self, 'tolerance', tolerance)\n setattr(self, 'n_jobs', n_jobs)\n\n def fit(self, X, y):\n \"\"\" Training the gcForest on input data X and associated target y.\n\n :param X: np.array\n Array containing the input samples.\n Must be of shape [n_samples, data] where data is a 1D array.\n\n :param y: np.array\n 1D array containing the target values.\n Must be of shape [n_samples]\n \"\"\"\n if np.shape(X)[0] != len(y):\n raise ValueError('Sizes of y and X do not match.')\n\n mgs_X = self.mg_scanning(X, y)\n _ = self.cascade_forest(mgs_X, y)\n\n def predict_proba(self, X):\n \"\"\" Predict the class probabilities of unknown samples X.\n\n :param X: np.array\n Array containing the input samples.\n Must be of the same shape [n_samples, data] as the training inputs.\n\n :return: np.array\n 1D array containing the predicted class probabilities for each input sample.\n \"\"\"\n mgs_X = self.mg_scanning(X)\n cascade_all_pred_prob = self.cascade_forest(mgs_X)\n predict_proba = np.mean(cascade_all_pred_prob, axis=0)\n\n return predict_proba\n\n def predict(self, X):\n \"\"\" Predict the class of unknown samples X.\n\n :param X: np.array\n Array containing the input samples.\n Must be of the same shape [n_samples, data] as the training inputs.\n\n :return: np.array\n 1D array containing the predicted class for each input sample.\n \"\"\"\n pred_proba = self.predict_proba(X=X)\n predictions = np.argmax(pred_proba, axis=1)\n\n return predictions\n\n def mg_scanning(self, X, y=None):\n \"\"\" Performs a Multi Grain Scanning on input data.\n\n :param X: np.array\n Array containing the input samples.\n Must be of shape [n_samples, data] where data is a 1D array.\n\n :param y: np.array (default=None)\n\n :return: np.array\n Array of shape [n_samples, .. ] containing Multi Grain Scanning sliced data.\n \"\"\"\n setattr(self, '_n_samples', np.shape(X)[0])\n shape_1X = getattr(self, 'shape_1X')\n if isinstance(shape_1X, int):\n shape_1X = [1,shape_1X]\n if not getattr(self, 'window'):\n setattr(self, 'window', [shape_1X[1]])\n\n mgs_pred_prob = []\n\n for wdw_size in getattr(self, 'window'):\n wdw_pred_prob = self.window_slicing_pred_prob(X, wdw_size, shape_1X, y=y)\n mgs_pred_prob.append(wdw_pred_prob)\n\n return np.concatenate(mgs_pred_prob, axis=1)\n\n def window_slicing_pred_prob(self, X, window, shape_1X, y=None):\n \"\"\" Performs a window slicing of the input data and send them through Random Forests.\n If target values 'y' are provided sliced data are then used to train the Random Forests.\n\n :param X: np.array\n Array containing the input samples.\n Must be of shape [n_samples, data] where data is a 1D array.\n\n :param window: int\n Size of the window to use for slicing.\n\n :param shape_1X: list or np.array\n Shape of a single sample.\n\n :param y: np.array (default=None)\n Target values. If 'None' no training is done.\n\n :return: np.array\n Array of size [n_samples, ..] containing the Random Forest.\n prediction probability for each input sample.\n \"\"\"\n n_tree = getattr(self, 'n_mgsRFtree')\n min_samples = getattr(self, 'min_samples_mgs')\n stride = getattr(self, 'stride')\n\n if shape_1X[0] > 1:\n print('Slicing Images...')\n sliced_X, sliced_y = self._window_slicing_img(X, window, shape_1X, y=y, stride=stride)\n else:\n print('Slicing Sequence...')\n sliced_X, sliced_y = self._window_slicing_sequence(X, window, shape_1X, y=y, stride=stride)\n\n if y is not None:\n n_jobs = getattr(self, 'n_jobs')\n prf = RandomForestClassifier(n_estimators=n_tree, max_features='sqrt',\n min_samples_split=min_samples, oob_score=True, n_jobs=n_jobs)\n crf = RandomForestClassifier(n_estimators=n_tree, max_features=1,\n min_samples_split=min_samples, oob_score=True, n_jobs=n_jobs)\n print('Training MGS Random Forests...')\n prf.fit(sliced_X, sliced_y)\n crf.fit(sliced_X, sliced_y)\n setattr(self, '_mgsprf_{}'.format(window), prf)\n setattr(self, '_mgscrf_{}'.format(window), crf)\n pred_prob_prf = prf.oob_decision_function_\n pred_prob_crf = crf.oob_decision_function_\n\n if hasattr(self, '_mgsprf_{}'.format(window)) and y is None:\n prf = getattr(self, '_mgsprf_{}'.format(window))\n crf = getattr(self, '_mgscrf_{}'.format(window))\n pred_prob_prf = prf.predict_proba(sliced_X)\n pred_prob_crf = crf.predict_proba(sliced_X)\n\n pred_prob = np.c_[pred_prob_prf, pred_prob_crf]\n\n return pred_prob.reshape([getattr(self, '_n_samples'), -1])\n\n def _window_slicing_img(self, X, window, shape_1X, y=None, stride=1):\n \"\"\" Slicing procedure for images\n\n :param X: np.array\n Array containing the input samples.\n Must be of shape [n_samples, data] where data is a 1D array.\n\n :param window: int\n Size of the window to use for slicing.\n\n :param shape_1X: list or np.array\n Shape of a single sample [n_lines, n_cols].\n\n :param y: np.array (default=None)\n Target values.\n\n :param stride: int (default=1)\n Step used when slicing the data.\n\n :return: np.array and np.array\n Arrays containing the sliced images and target values (empty if 'y' is None).\n \"\"\"\n if any(s < window for s in shape_1X):\n raise ValueError('window must be smaller than both dimensions for an image')\n\n len_iter_x = np.floor_divide((shape_1X[1] - window), stride) + 1\n len_iter_y = np.floor_divide((shape_1X[0] - window), stride) + 1\n iterx_array = np.arange(0, stride*len_iter_x, stride)\n itery_array = np.arange(0, stride*len_iter_y, stride)\n\n ref_row = np.arange(0, window)\n ref_ind = np.ravel([ref_row + shape_1X[1] * i for i in range(window)])\n inds_to_take = [ref_ind + ix + shape_1X[1] * iy\n for ix, iy in itertools.product(iterx_array, itery_array)]\n\n sliced_imgs = np.take(X, inds_to_take, axis=1).reshape(-1, window**2)\n\n if y is not None:\n sliced_target = np.repeat(y, len_iter_x * len_iter_y)\n elif y is None:\n sliced_target = None\n\n return sliced_imgs, sliced_target\n\n def _window_slicing_sequence(self, X, window, shape_1X, y=None, stride=1):\n \"\"\" Slicing procedure for sequences (aka shape_1X = [.., 1]).\n\n :param X: np.array\n Array containing the input samples.\n Must be of shape [n_samples, data] where data is a 1D array.\n\n :param window: int\n Size of the window to use for slicing.\n\n :param shape_1X: list or np.array\n Shape of a single sample [n_lines, n_col].\n\n :param y: np.array (default=None)\n Target values.\n\n :param stride: int (default=1)\n Step used when slicing the data.\n\n :return: np.array and np.array\n Arrays containing the sliced sequences and target values (empty if 'y' is None).\n \"\"\"\n if shape_1X[1] < window:\n raise ValueError('window must be smaller than the sequence dimension')\n\n len_iter = np.floor_divide((shape_1X[1] - window), stride) + 1\n iter_array = np.arange(0, stride*len_iter, stride)\n\n ind_1X = np.arange(np.prod(shape_1X))\n inds_to_take = [ind_1X[i:i+window] for i in iter_array]\n sliced_sqce = np.take(X, inds_to_take, axis=1).reshape(-1, window)\n\n if y is not None:\n sliced_target = np.repeat(y, len_iter)\n elif y is None:\n sliced_target = None\n\n return sliced_sqce, sliced_target\n\n def cascade_forest(self, X, y=None):\n \"\"\" Perform (or train if 'y' is not None) a cascade forest estimator.\n\n :param X: np.array\n Array containing the input samples.\n Must be of shape [n_samples, data] where data is a 1D array.\n\n :param y: np.array (default=None)\n Target values. If 'None' perform training.\n\n :return: np.array\n 1D array containing the predicted class for each input sample.\n \"\"\"\n if y is not None:\n setattr(self, 'n_layer', 0)\n test_size = getattr(self, 'cascade_test_size')\n max_layers = getattr(self, 'cascade_layer')\n tol = getattr(self, 'tolerance')\n\n X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size)\n\n self.n_layer += 1\n prf_crf_pred_ref = self._cascade_layer(X_train, y_train)\n accuracy_ref = self._cascade_evaluation(X_test, y_test)\n feat_arr = self._create_feat_arr(X_train, prf_crf_pred_ref)\n\n self.n_layer += 1\n prf_crf_pred_layer = self._cascade_layer(feat_arr, y_train)\n accuracy_layer = self._cascade_evaluation(X_test, y_test)\n\n while accuracy_layer > (accuracy_ref + tol) and self.n_layer <= max_layers:\n accuracy_ref = accuracy_layer\n prf_crf_pred_ref = prf_crf_pred_layer\n feat_arr = self._create_feat_arr(X_train, prf_crf_pred_ref)\n self.n_layer += 1\n prf_crf_pred_layer = self._cascade_layer(feat_arr, y_train)\n accuracy_layer = self._cascade_evaluation(X_test, y_test)\n\n if accuracy_layer < accuracy_ref :\n n_cascadeRF = getattr(self, 'n_cascadeRF')\n for irf in range(n_cascadeRF):\n delattr(self, '_casprf{}_{}'.format(self.n_layer, irf))\n delattr(self, '_cascrf{}_{}'.format(self.n_layer, irf))\n self.n_layer -= 1\n\n elif y is None:\n at_layer = 1\n prf_crf_pred_ref = self._cascade_layer(X, layer=at_layer)\n while at_layer < getattr(self, 'n_layer'):\n at_layer += 1\n feat_arr = self._create_feat_arr(X, prf_crf_pred_ref)\n prf_crf_pred_ref = self._cascade_layer(feat_arr, layer=at_layer)\n\n return prf_crf_pred_ref\n\n def _cascade_layer(self, X, y=None, layer=0):\n \"\"\" Cascade layer containing Random Forest estimators.\n If y is not None the layer is trained.\n\n :param X: np.array\n Array containing the input samples.\n Must be of shape [n_samples, data] where data is a 1D array.\n\n :param y: np.array (default=None)\n Target values. If 'None' perform training.\n\n :param layer: int (default=0)\n Layer indice. Used to call the previously trained layer.\n\n :return: list\n List containing the prediction probabilities for all samples.\n \"\"\"\n n_tree = getattr(self, 'n_cascadeRFtree')\n n_cascadeRF = getattr(self, 'n_cascadeRF')\n min_samples = getattr(self, 'min_samples_cascade')\n\n n_jobs = getattr(self, 'n_jobs')\n prf = RandomForestClassifier(n_estimators=n_tree, max_features='sqrt',\n min_samples_split=min_samples, oob_score=True, n_jobs=n_jobs)\n crf = RandomForestClassifier(n_estimators=n_tree, max_features=1,\n min_samples_split=min_samples, oob_score=True, n_jobs=n_jobs)\n\n prf_crf_pred = []\n if y is not None:\n print('Adding/Training Layer, n_layer={}'.format(self.n_layer))\n for irf in range(n_cascadeRF):\n prf.fit(X, y)\n crf.fit(X, y)\n setattr(self, '_casprf{}_{}'.format(self.n_layer, irf), prf)\n setattr(self, '_cascrf{}_{}'.format(self.n_layer, irf), crf)\n prf_crf_pred.append(prf.oob_decision_function_)\n prf_crf_pred.append(crf.oob_decision_function_)\n elif y is None:\n for irf in range(n_cascadeRF):\n prf = getattr(self, '_casprf{}_{}'.format(layer, irf))\n crf = getattr(self, '_cascrf{}_{}'.format(layer, irf))\n prf_crf_pred.append(prf.predict_proba(X))\n prf_crf_pred.append(crf.predict_proba(X))\n\n return prf_crf_pred\n\n def _cascade_evaluation(self, X_test, y_test):\n \"\"\" Evaluate the accuracy of the cascade using X and y.\n\n :param X_test: np.array\n Array containing the test input samples.\n Must be of the same shape as training data.\n\n :param y_test: np.array\n Test target values.\n\n :return: float\n the cascade accuracy.\n \"\"\"\n casc_pred_prob = np.mean(self.cascade_forest(X_test), axis=0)\n casc_pred = np.argmax(casc_pred_prob, axis=1)\n casc_accuracy = accuracy_score(y_true=y_test, y_pred=casc_pred)\n print('Layer validation accuracy = {}'.format(casc_accuracy))\n\n return casc_accuracy\n\n def _create_feat_arr(self, X, prf_crf_pred):\n \"\"\" Concatenate the original feature vector with the predicition probabilities\n of a cascade layer.\n\n :param X: np.array\n Array containing the input samples.\n Must be of shape [n_samples, data] where data is a 1D array.\n\n :param prf_crf_pred: list\n Prediction probabilities by a cascade layer for X.\n\n :return: np.array\n Concatenation of X and the predicted probabilities.\n To be used for the next layer in a cascade forest.\n \"\"\"\n swap_pred = np.swapaxes(prf_crf_pred, 0, 1)\n add_feat = swap_pred.reshape([np.shape(X)[0], -1])\n feat_arr = np.concatenate([add_feat, X], axis=1)\n\n return feat_arr\n \nrows, cols = train_balanced.shape\ntemp = train_balanced.tolist() # Convert data to a list of lists\ntarget_list = train_labels_balanced.tolist()\n\nX_train = temp\ny_train = target_list\n\nmodel = gcForest(shape_1X=cols, window=2, tolerance=0.0)\nmodel.fit(X_train, y_train)\n\n# Predictions\ntemp_test = test.tolist() # Convert data to a list of lists\ny_pred = model.predict(temp_test)\ny_pred_proba = model.predict_proba(temp_test)[:, 1]\n\n# Evaluate the model\nf1 = f1_score(test_labels, y_pred)\nauc_score = roc_auc_score(test_labels, y_pred_proba)\n\nprint(f\"AUC Score: {auc_score}\")\nevaluate()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.242943Z","iopub.status.idle":"2024-05-01T03:17:11.243369Z","shell.execute_reply.started":"2024-05-01T03:17:11.243171Z","shell.execute_reply":"2024-05-01T03:17:11.243189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.append({\"Model\": 'Cascade Random Forest', \"F1 Score\": f1, \"AUC\": auc_score})","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.244685Z","iopub.status.idle":"2024-05-01T03:17:11.245025Z","shell.execute_reply.started":"2024-05-01T03:17:11.244852Z","shell.execute_reply":"2024-05-01T03:17:11.244876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Final Results","metadata":{}},{"cell_type":"code","source":"final_results = pd.DataFrame(results)\nprint(final_results)","metadata":{"execution":{"iopub.status.busy":"2024-05-01T03:17:11.245921Z","iopub.status.idle":"2024-05-01T03:17:11.246241Z","shell.execute_reply.started":"2024-05-01T03:17:11.246077Z","shell.execute_reply":"2024-05-01T03:17:11.246091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Light GBM both achieved good results on Accuracy and AUC. Our method \"Stacking\" also achieve good results. ","metadata":{}}]}