{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<h1><center> Foursquare Location Matching </center></h1>\n<h3><center> Model Selection </center></h3>\n<h3><center> Tao Shan </center></h3>\n\nThis notebook compares different models and choose the best one for future use.\n\n### Other Relevant notebooks and links\n\nCompetition: [Foursquare - Location Matching](https://www.kaggle.com/competitions/foursquare-location-matching)\n\nTrain data generation notebook: [Foursquare - train data generation](https://www.kaggle.com/taos2000/foursquare-train-data-generation)\n\nTrain data preprocessing notebook: [Foursquare - train data preprocess](https://www.kaggle.com/taos2000/foursquare-train-data-preprocess)\n\nModel Selection: [Foursquare - model selection](https://www.kaggle.com/taos2000/foursquare-model-selection)\n\nModel Training: [Foursquare - model training](https://www.kaggle.com/taos2000/foursquare-model-training)","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.preprocessing import RobustScaler\nfrom sklearn.model_selection import cross_val_score\nfrom xgboost import XGBClassifier\nfrom pickle import dump\nimport gc\nimport time\nfrom sklearn.metrics import roc_auc_score, accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nimport pandas as pd\nimport numpy as np\nfrom sklearn.neighbors import NearestNeighbors\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nfrom geopy.geocoders import Nominatim\n\nimport re\nimport string\nfrom nltk.tokenize import word_tokenize\nfrom nltk.corpus import stopwords, wordnet as wn\nfrom nltk.stem import PorterStemmer, WordNetLemmatizer\n\nimport Levenshtein as lev\nimport math\nfrom collections import Counter\n\nfrom pickle import dump, load\nimport time\nfrom sklearn.neighbors import BallTree\n\n\nimport itertools\nfrom tqdm.auto import tqdm\ntqdm.pandas()\nimport gc\n\nfrom fuzzywuzzy import fuzz\nfrom xgboost import XGBClassifier\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.feature_selection import SelectFromModel","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-25T20:31:33.831723Z","iopub.execute_input":"2022-07-25T20:31:33.832299Z","iopub.status.idle":"2022-07-25T20:31:36.236998Z","shell.execute_reply.started":"2022-07-25T20:31:33.832177Z","shell.execute_reply":"2022-07-25T20:31:36.235559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pairs = pd.read_pickle('../input/last-day-four-points/train_pairs (1).pkl').sample(100000,random_state = 8888)\ntest_pairs = pd.read_pickle('../input/last-day-four-points/train_pairs_test.pkl').sample(100000,random_state = 8888)\nsample_pairs = pd.read_pickle('../input/last-day-four-points/train_pairs_sample.pkl')","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:31:36.239528Z","iopub.execute_input":"2022-07-25T20:31:36.240885Z","iopub.status.idle":"2022-07-25T20:32:20.920055Z","shell.execute_reply.started":"2022-07-25T20:31:36.240820Z","shell.execute_reply":"2022-07-25T20:32:20.919108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = ['address_1_missing',\n        'country_same',\n 'address_2_missing',\n 'categories_1_count',\n 'categories_1_count_diff',\n 'categories_2_count',\n 'city_1_count',\n 'city_1_count_diff',\n 'city_1_missing',\n 'city_2_count',\n 'city_2_missing',\n 'country_1_count',\n 'country_1_count_diff',\n 'distance',\n 'latitude_1',\n 'latitude_1_count',\n 'latitude_2_count',\n 'latitude_2_count_diff',\n 'longitude_1',\n 'longitude_1_count',\n 'longitude_2_count',\n 'longitude_2_count_diff',\n 'phone_1_missing',\n 'phone_2_missing',\n 'state_1_count',\n 'state_1_count_diff',\n 'state_2_count',\n 'url_1_missing',\n 'url_2_missing',\n 'zip_1_missing',\n 'zip_2_missing',\n 'name_1_fuzzy',\n 'name_1_fuzzy_partial',\n 'address_1_fuzzy',\n 'categories_1_fuzzy',\n 'categories_1_fuzzy_partial',\n 'url_1_fuzzy']\nids = ['id_1','id_2']\ncols_new = ['address_1_missing',\n 'address_2_missing',\n 'city_1_missing',\n 'city_2_missing',\n 'distance',\n 'latitude_1',\n 'longitude_1',\n 'phone_1_missing',\n 'phone_2_missing',\n 'url_1_missing',\n 'url_2_missing',\n 'zip_1_missing',\n 'zip_2_missing',\n 'name_1_fuzzy',\n 'name_1_fuzzy_partial',\n 'address_1_fuzzy',\n 'categories_1_fuzzy',\n 'categories_1_fuzzy_partial',\n 'url_1_fuzzy']","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:32:20.921839Z","iopub.execute_input":"2022-07-25T20:32:20.923149Z","iopub.status.idle":"2022-07-25T20:32:20.932783Z","shell.execute_reply.started":"2022-07-25T20:32:20.923101Z","shell.execute_reply":"2022-07-25T20:32:20.931496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import GradientBoostingClassifier\nX_train = train_pairs[cols]; y_train = train_pairs['match']\nX_test = test_pairs[cols]; y_test = test_pairs['match']\nmodel = GradientBoostingClassifier()\nselector = SelectFromModel(model).fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:32:20.934917Z","iopub.execute_input":"2022-07-25T20:32:20.935232Z","iopub.status.idle":"2022-07-25T20:33:13.633051Z","shell.execute_reply.started":"2022-07-25T20:32:20.935205Z","shell.execute_reply":"2022-07-25T20:33:13.630819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.columns[selector.get_support()]","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:33:13.635772Z","iopub.execute_input":"2022-07-25T20:33:13.636577Z","iopub.status.idle":"2022-07-25T20:33:13.651995Z","shell.execute_reply.started":"2022-07-25T20:33:13.636532Z","shell.execute_reply":"2022-07-25T20:33:13.651133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"XGboost: ['country_same', 'address_2_missing', 'distance', 'name_1_fuzzy',\n       'name_1_fuzzy_partial', 'categories_1_fuzzy_partial'],\n\nrandom forest: ['country_same', 'distance', 'name_1_fuzzy', 'name_1_fuzzy_partial',\n       'categories_1_fuzzy', 'categories_1_fuzzy_partial']\n       \ngradient boost: ['country_same', 'distance', 'name_1_fuzzy', 'name_1_fuzzy_partial',\n       'categories_1_fuzzy_partial']","metadata":{}},{"cell_type":"code","source":"select_col = ['country_same', 'address_2_missing', 'distance', 'name_1_fuzzy', 'name_1_fuzzy_partial', 'categories_1_fuzzy_partial','categories_1_fuzzy']","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:33:13.653642Z","iopub.execute_input":"2022-07-25T20:33:13.654247Z","iopub.status.idle":"2022-07-25T20:33:13.659019Z","shell.execute_reply.started":"2022-07-25T20:33:13.654215Z","shell.execute_reply":"2022-07-25T20:33:13.658046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = train_pairs[select_col]; y_train = train_pairs['match']\nX_test = test_pairs[select_col]; y_test = test_pairs['match']\n\nxgb = XGBClassifier(nthread = -1)\nxgb.fit(X_train,y_train)\ntest_pred = xgb.predict_proba(X_test)\nprint(roc_auc_score(y_test, test_pred[:, 1]))","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:33:13.660738Z","iopub.execute_input":"2022-07-25T20:33:13.661355Z","iopub.status.idle":"2022-07-25T20:33:19.886806Z","shell.execute_reply.started":"2022-07-25T20:33:13.661321Z","shell.execute_reply":"2022-07-25T20:33:19.885324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"X_train","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:33:19.889607Z","iopub.execute_input":"2022-07-25T20:33:19.890033Z","iopub.status.idle":"2022-07-25T20:33:19.923281Z","shell.execute_reply.started":"2022-07-25T20:33:19.889992Z","shell.execute_reply":"2022-07-25T20:33:19.921875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:33:19.925523Z","iopub.execute_input":"2022-07-25T20:33:19.926114Z","iopub.status.idle":"2022-07-25T20:33:19.948749Z","shell.execute_reply.started":"2022-07-25T20:33:19.926060Z","shell.execute_reply":"2022-07-25T20:33:19.947223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = train_pairs[cols]; y_train = train_pairs['match']\nX_test = test_pairs[cols]; y_test = test_pairs['match']\n\nxgb = XGBClassifier(nthread = -1)\nxgb.fit(X_train,y_train)\ntest_pred = xgb.predict_proba(X_test)\nprint(roc_auc_score(y_test, test_pred[:, 1]))","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:33:19.954685Z","iopub.execute_input":"2022-07-25T20:33:19.955548Z","iopub.status.idle":"2022-07-25T20:33:37.976060Z","shell.execute_reply.started":"2022-07-25T20:33:19.955496Z","shell.execute_reply":"2022-07-25T20:33:37.974580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = MinMaxScaler()\nscaler.fit(X_train,y_train)\nX_train_temp = scaler.transform(X_train)\nX_test_temp = scaler.transform(X_test)\nxgb = XGBClassifier(nthread = -1)\nxgb.fit(X_train_temp,y_train)\ntest_pred = xgb.predict_proba(X_test_temp)\nprint(roc_auc_score(y_test, test_pred[:, 1]))","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:33:37.977828Z","iopub.execute_input":"2022-07-25T20:33:37.979112Z","iopub.status.idle":"2022-07-25T20:33:53.433468Z","shell.execute_reply.started":"2022-07-25T20:33:37.979059Z","shell.execute_reply":"2022-07-25T20:33:53.432258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = train_pairs[cols]; y_train = train_pairs['match']\nX_test = test_pairs[cols]; y_test = test_pairs['match']\nscaler = RobustScaler()\nscaler.fit(X_train,y_train)\nX_train_temp = scaler.transform(X_train)\nX_test_temp = scaler.transform(X_test)\nxgb = XGBClassifier(nthread = -1)\nxgb.fit(X_train_temp,y_train)\ntest_pred = xgb.predict_proba(X_test_temp)\nprint(roc_auc_score(y_test, test_pred[:, 1]))","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:33:53.435121Z","iopub.execute_input":"2022-07-25T20:33:53.436138Z","iopub.status.idle":"2022-07-25T20:34:08.120068Z","shell.execute_reply.started":"2022-07-25T20:33:53.436094Z","shell.execute_reply":"2022-07-25T20:34:08.118638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = train_pairs[cols_new]; y_train = train_pairs['match']\nX_test = test_pairs[cols_new]; y_test = test_pairs['match']\n\nxgb = XGBClassifier(nthread = -1)\nxgb.fit(X_train,y_train)\ntest_pred = xgb.predict_proba(X_test)\nprint(roc_auc_score(y_test, test_pred[:, 1]))","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:34:08.121856Z","iopub.execute_input":"2022-07-25T20:34:08.122349Z","iopub.status.idle":"2022-07-25T20:34:17.143282Z","shell.execute_reply.started":"2022-07-25T20:34:08.122303Z","shell.execute_reply":"2022-07-25T20:34:17.141684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = MinMaxScaler()\nscaler.fit(X_train,y_train)\nX_train_temp = scaler.transform(X_train)\nX_test_temp = scaler.transform(X_test)\nxgb = XGBClassifier(nthread = -1)\nxgb.fit(X_train_temp,y_train)\ntest_pred = xgb.predict_proba(X_test_temp)\nprint(roc_auc_score(y_test, test_pred[:, 1]))","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:34:17.145424Z","iopub.execute_input":"2022-07-25T20:34:17.145958Z","iopub.status.idle":"2022-07-25T20:34:27.168504Z","shell.execute_reply.started":"2022-07-25T20:34:17.145911Z","shell.execute_reply":"2022-07-25T20:34:27.167150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = RobustScaler()\nscaler.fit(X_train,y_train)\nX_train_temp = scaler.transform(X_train)\nX_test_temp = scaler.transform(X_test)\nxgb = XGBClassifier(nthread = -1)\nxgb.fit(X_train_temp,y_train)\ntest_pred = xgb.predict_proba(X_test_temp)\nprint(roc_auc_score(y_test, test_pred[:, 1]))","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:36:10.730424Z","iopub.execute_input":"2022-07-25T20:36:10.730963Z","iopub.status.idle":"2022-07-25T20:36:20.144962Z","shell.execute_reply.started":"2022-07-25T20:36:10.730915Z","shell.execute_reply":"2022-07-25T20:36:20.143497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = test_pairs[cols]; y_train = test_pairs['match']\nX_test = sample_pairs[cols]; y_test = sample_pairs['match']\n\nxgb = XGBClassifier(nthread = -1)\nxgb.fit(X_train,y_train)\ntest_pred = xgb.predict_proba(X_test)\nprint(roc_auc_score(y_test, test_pred[:, 1]))","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:36:20.147160Z","iopub.execute_input":"2022-07-25T20:36:20.147982Z","iopub.status.idle":"2022-07-25T20:36:37.373238Z","shell.execute_reply.started":"2022-07-25T20:36:20.147935Z","shell.execute_reply":"2022-07-25T20:36:37.371831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = test_pairs[cols_new]; y_train = test_pairs['match']\nX_test = sample_pairs[cols_new]; y_test = sample_pairs['match']\n\nxgb = XGBClassifier(nthread = -1)\nxgb.fit(X_train,y_train)\ntest_pred = xgb.predict_proba(X_test)\nprint(roc_auc_score(y_test, test_pred[:, 1]))","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:36:37.374696Z","iopub.execute_input":"2022-07-25T20:36:37.375070Z","iopub.status.idle":"2022-07-25T20:36:46.409187Z","shell.execute_reply.started":"2022-07-25T20:36:37.375040Z","shell.execute_reply":"2022-07-25T20:36:46.407825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_pairs.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:36:46.413110Z","iopub.execute_input":"2022-07-25T20:36:46.413877Z","iopub.status.idle":"2022-07-25T20:36:46.422015Z","shell.execute_reply.started":"2022-07-25T20:36:46.413822Z","shell.execute_reply":"2022-07-25T20:36:46.420784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = train_pairs[cols]; y_train = train_pairs['match']\nX_test = test_pairs[cols]; y_test = test_pairs['match']\nRobust_col = X_train.drop(cols_new, axis=1).columns\n\nX_train_temp = X_train.copy()\nX_test_temp = X_test.copy()\n#for col in tqdm(Robust_col):\n#    X_test_temp[col] = X_test_temp[col] / 600000*4871594\n    \nxgb = XGBClassifier(nthread = -1)\nxgb.fit(X_train_temp,y_train)\ntest_pred = xgb.predict_proba(X_test_temp)\nprint(roc_auc_score(y_test, test_pred[:, 1]))","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:36:46.424157Z","iopub.execute_input":"2022-07-25T20:36:46.425298Z","iopub.status.idle":"2022-07-25T20:37:01.144775Z","shell.execute_reply.started":"2022-07-25T20:36:46.425044Z","shell.execute_reply":"2022-07-25T20:37:01.143381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom catboost import CatBoostClassifier\nimport lightgbm as lgb\n#clf = lgb.LGBMClassifier()","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:37:01.146324Z","iopub.execute_input":"2022-07-25T20:37:01.146762Z","iopub.status.idle":"2022-07-25T20:37:02.703864Z","shell.execute_reply.started":"2022-07-25T20:37:01.146722Z","shell.execute_reply":"2022-07-25T20:37:02.702524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = train_pairs[cols]; y_train = train_pairs['match']\nX_test = test_pairs[cols]; y_test = test_pairs['match']\nscaler = RobustScaler()\nscaler.fit(X_train,y_train)\nX_train_temp = scaler.transform(X_train)\nX_test_temp = scaler.transform(X_test)\nxgb = lgb.LGBMClassifier()\nxgb.fit(X_train_temp,y_train)\ntest_pred = xgb.predict_proba(X_test_temp)\nprint(roc_auc_score(y_test, test_pred[:, 1]))","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:37:02.705795Z","iopub.execute_input":"2022-07-25T20:37:02.706312Z","iopub.status.idle":"2022-07-25T20:37:05.760728Z","shell.execute_reply.started":"2022-07-25T20:37:02.706261Z","shell.execute_reply":"2022-07-25T20:37:05.759259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"xgboost: 0.77\n\nada boost: 0.69\n\ngradient boost: 0.715\n\nrandom forest: 0.74\n\nlogistic: 0.62\n\ncatboost: 0.785\n\nlgb: 0.76 (fast)","metadata":{}},{"cell_type":"code","source":"xgb_parameters = {'learning_rate': [0.01, 0.1],\n        'max_depth': [3, 5, 7, 10],\n        'min_child_weight': [1, 3, 5],\n        'subsample': [0.5, 0.7],\n        'colsample_bytree': [0.5, 0.7],\n        'n_estimators' : [100, 200, 500],\n        'objective': ['reg:squarederror','binary:logistic']}","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:37:05.762059Z","iopub.execute_input":"2022-07-25T20:37:05.762635Z","iopub.status.idle":"2022-07-25T20:37:05.768764Z","shell.execute_reply.started":"2022-07-25T20:37:05.762599Z","shell.execute_reply":"2022-07-25T20:37:05.767384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = train_pairs[cols]; y_train = train_pairs['match']\nX_test = test_pairs[cols]; y_test = test_pairs['match']\nscaler = RobustScaler()\nscaler.fit(X_train,y_train)\nX_train_temp = scaler.transform(X_train)\nX_test_temp = scaler.transform(X_test)\nxgb = XGBClassifier(nthread = -1, objective = 'binary:logistic', n_estimators = 800, learning_rate = 0.1)\nxgb.fit(X_train_temp,y_train)\ntest_pred = xgb.predict_proba(X_test_temp)\nprint(roc_auc_score(y_test, test_pred[:, 1]))","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:37:05.770540Z","iopub.execute_input":"2022-07-25T20:37:05.771481Z","iopub.status.idle":"2022-07-25T20:39:02.493898Z","shell.execute_reply.started":"2022-07-25T20:37:05.771426Z","shell.execute_reply":"2022-07-25T20:39:02.492512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"nthread = -1, objective = 'binary:logistic'","metadata":{}},{"cell_type":"code","source":"X_train = train_pairs[cols]; y_train = train_pairs['match']\nX_test = test_pairs[cols]; y_test = test_pairs['match']\nscaler = RobustScaler()\nscaler.fit(X_train,y_train)\nX_train_temp = scaler.transform(X_train)\nX_test_temp = scaler.transform(X_test)\n\nfor i in [400,800,1600]:\n    xgb = XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n                  colsample_bynode=1, colsample_bytree=0.5, gamma=0, gpu_id=-1,\n                  importance_type='gain', objective = 'binary:logistic',\n                  learning_rate=0.1, max_delta_step=0, max_depth=3,\n                  min_child_weight=2, n_estimators=1600, n_jobs=-1, nthread=-1, num_parallel_tree=1,\n                  random_state=0, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n                  subsample=0.9, tree_method='exact', validate_parameters=1,\n                  verbosity=0)\n    xgb.fit(X_train_temp,y_train)\n    test_pred = xgb.predict_proba(X_test_temp)\n    print(roc_auc_score(y_test, test_pred[:, 1]))","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:39:02.497627Z","iopub.execute_input":"2022-07-25T20:39:02.498146Z","iopub.status.idle":"2022-07-25T20:42:50.044163Z","shell.execute_reply.started":"2022-07-25T20:39:02.498106Z","shell.execute_reply":"2022-07-25T20:42:50.042754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"xgb = XGBClassifier(nthread = -1, objective = 'binary:logistic', n_estimators = 800, learning_rate = 0.1)\n\nxgb = XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n                  colsample_bynode=1, colsample_bytree=0.5, gamma=0, gpu_id=-1,\n                  importance_type='gain', objective = 'binary:logistic',\n                  learning_rate=0.1, max_delta_step=0, max_depth=3,\n                  min_child_weight=2, n_estimators=1600, n_jobs=-1, nthread=-1, num_parallel_tree=1,\n                  random_state=0, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n                  subsample=0.9, tree_method='exact', validate_parameters=1,\n                  verbosity=0)","metadata":{}},{"cell_type":"code","source":"X_train = train_pairs[cols]; y_train = train_pairs['match']\nX_test = test_pairs[cols]; y_test = test_pairs['match']","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:43:27.115253Z","iopub.execute_input":"2022-07-25T20:43:27.115752Z","iopub.status.idle":"2022-07-25T20:43:27.139339Z","shell.execute_reply.started":"2022-07-25T20:43:27.115710Z","shell.execute_reply":"2022-07-25T20:43:27.138021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = RobustScaler()\nscaler.fit(X_train,y_train)\nX_train_temp = scaler.transform(X_train)\nX_test_temp = scaler.transform(X_test)\nxgb = CatBoostClassifier(eval_metric = 'Accuracy', depth = 5,verbose=False)\nxgb.fit(X_train_temp, y_train,\n       eval_set=(X_test_temp, y_test))\ntest_pred = xgb.predict_proba(X_test_temp)\nprint(roc_auc_score(y_test, test_pred[:, 1]))","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:43:28.244878Z","iopub.execute_input":"2022-07-25T20:43:28.245522Z","iopub.status.idle":"2022-07-25T20:43:52.364871Z","shell.execute_reply.started":"2022-07-25T20:43:28.245468Z","shell.execute_reply":"2022-07-25T20:43:52.363497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = RobustScaler()\nscaler.fit(X_train,y_train)\nX_train_temp = scaler.transform(X_train)\nX_test_temp = scaler.transform(X_test)\n\nxgb = XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n                  colsample_bynode=1, colsample_bytree=0.5, gamma=0, gpu_id=-1,\n                  importance_type='gain', objective = 'binary:logistic',\n                  learning_rate=0.1, max_delta_step=0, max_depth=3,\n                  min_child_weight=2, n_estimators=4000, n_jobs=-1, nthread=-1, num_parallel_tree=1,\n                  random_state=0, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n                  subsample=0.9, tree_method='exact', validate_parameters=1,eval_metric = 'auc',\n                  verbosity=0)\n\nxgb.fit(X_train_temp, y_train, eval_set=[(X_train_temp, y_train),(X_test_temp, y_test)],early_stopping_rounds=100)\ntest_pred = xgb.predict_proba(X_test_temp)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:49:04.448743Z","iopub.execute_input":"2022-07-25T20:49:04.449646Z","iopub.status.idle":"2022-07-25T20:51:08.982119Z","shell.execute_reply.started":"2022-07-25T20:49:04.449580Z","shell.execute_reply":"2022-07-25T20:51:08.980930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred[:,1]","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:54:23.792966Z","iopub.execute_input":"2022-07-25T20:54:23.793490Z","iopub.status.idle":"2022-07-25T20:54:23.802902Z","shell.execute_reply.started":"2022-07-25T20:54:23.793455Z","shell.execute_reply":"2022-07-25T20:54:23.801640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred_temp = [ 0 if x <= 0.8 else 1 for x in test_pred[:,1].tolist()]\nconfusion_matrix(y_test,test_pred_temp)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T20:54:25.278050Z","iopub.execute_input":"2022-07-25T20:54:25.278506Z","iopub.status.idle":"2022-07-25T20:54:25.347652Z","shell.execute_reply.started":"2022-07-25T20:54:25.278470Z","shell.execute_reply":"2022-07-25T20:54:25.346384Z"},"trusted":true},"execution_count":null,"outputs":[]}]}