{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<div class=\"alert alert-success\">  \n    <h1 align=\"center\" style=\"color:darkgreen;\">Tabular Playground Series - Jul 2022</h1>  \n</div>\n\n<div>\n    <h1 align=\"center\" style=\"color:darkgreen;\">C l u s t e r i n g - 1/3</h1>\n</div>\n\n<div class=\"alert alert-success\">  \n</div>","metadata":{}},{"cell_type":"markdown","source":"#### - This notebook is a fork of the excellent notebook below:\n>#### https://www.kaggle.com/code/hiro5299834/tps-jul-2022-unsupervised-and-supervised-learning\n>#### Thanks to: @hiro5299834\n\n#### - In the \"Supervised Learning\" section, we have only used \"KNeighborsClassifier\" and got better results.\n\n#### - Then we developed our own notebook using \"iteration\", but at the same time we were introduced to \"sklego\", which is very fast. (Thanks to: @karlcini) That's why our second notebook, using this library Excellent and also the results of this notebook will be.","metadata":{}},{"cell_type":"markdown","source":"<div class=\"alert alert-success\">  \n</div>","metadata":{}},{"cell_type":"markdown","source":"# Librairies","metadata":{"papermill":{"duration":0.004566,"end_time":"2022-07-17T01:28:31.044567","exception":false,"start_time":"2022-07-17T01:28:31.040001","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import warnings # suppress warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-07-24T09:52:05.496883Z","iopub.execute_input":"2022-07-24T09:52:05.497505Z","iopub.status.idle":"2022-07-24T09:52:05.522717Z","shell.execute_reply.started":"2022-07-24T09:52:05.497403Z","shell.execute_reply":"2022-07-24T09:52:05.521913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport random\n\nimport numpy as np \nimport pandas as pd\nimport seaborn as sns\n\nfrom tqdm import tqdm\nfrom scipy import stats\nfrom pathlib import Path\n\nimport matplotlib.pyplot as plt\nimport plotly.figure_factory as ff\nimport plotly.express as px\n%matplotlib inline\n!ls ../input/*","metadata":{"execution":{"iopub.status.busy":"2022-07-24T09:52:23.692406Z","iopub.execute_input":"2022-07-24T09:52:23.692851Z","iopub.status.idle":"2022-07-24T09:52:28.043364Z","shell.execute_reply.started":"2022-07-24T09:52:23.692812Z","shell.execute_reply":"2022-07-24T09:52:28.042211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import PowerTransformer\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import balanced_accuracy_score, roc_auc_score\nfrom sklearn.metrics import silhouette_score, calinski_harabasz_score, davies_bouldin_score\n\nfrom sklearn.mixture import BayesianGaussianMixture\nfrom cuml.neighbors import KNeighborsClassifier\n\npd.options.display.max_columns = 100\npd.options.display.max_rows = 100","metadata":{"papermill":{"duration":4.622558,"end_time":"2022-07-17T01:28:35.671943","exception":false,"start_time":"2022-07-17T01:28:31.049385","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-24T09:53:01.463057Z","iopub.execute_input":"2022-07-24T09:53:01.463688Z","iopub.status.idle":"2022-07-24T09:53:03.611603Z","shell.execute_reply.started":"2022-07-24T09:53:01.463652Z","shell.execute_reply":"2022-07-24T09:53:03.610615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parameters","metadata":{"papermill":{"duration":0.004769,"end_time":"2022-07-17T01:28:35.682617","exception":false,"start_time":"2022-07-17T01:28:35.677848","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CFG:\n    seed_bgm = 1\n    seed = 42\n    n_splits = 9\n    n_clusters = 7\n    threshold = 0.65","metadata":{"papermill":{"duration":0.013624,"end_time":"2022-07-17T01:28:35.701009","exception":false,"start_time":"2022-07-17T01:28:35.687385","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T04:00:52.23305Z","iopub.execute_input":"2022-07-19T04:00:52.235586Z","iopub.status.idle":"2022-07-19T04:00:52.241711Z","shell.execute_reply.started":"2022-07-19T04:00:52.235557Z","shell.execute_reply":"2022-07-19T04:00:52.240707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)","metadata":{"papermill":{"duration":0.013081,"end_time":"2022-07-17T01:28:35.718972","exception":false,"start_time":"2022-07-17T01:28:35.705891","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T04:00:52.243222Z","iopub.execute_input":"2022-07-19T04:00:52.24356Z","iopub.status.idle":"2022-07-19T04:00:52.259496Z","shell.execute_reply.started":"2022-07-19T04:00:52.243525Z","shell.execute_reply":"2022-07-19T04:00:52.258334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{"papermill":{"duration":0.005286,"end_time":"2022-07-17T01:28:35.729548","exception":false,"start_time":"2022-07-17T01:28:35.724262","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df = pd.read_csv(\"../input/tabular-playground-series-jul-2022/data.csv\").drop('id', axis=1)\ndf","metadata":{"papermill":{"duration":1.082404,"end_time":"2022-07-17T01:28:36.816805","exception":false,"start_time":"2022-07-17T01:28:35.734401","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T04:01:09.055937Z","iopub.execute_input":"2022-07-19T04:01:09.056403Z","iopub.status.idle":"2022-07-19T04:01:09.619424Z","shell.execute_reply.started":"2022-07-19T04:01:09.05636Z","shell.execute_reply":"2022-07-19T04:01:09.618433Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_scores = []\nbest_features = [f\"f_{i:02d}\" for i in list(range(7, 14)) + list(range(22, 29))]\nfeatures = df.columns\n\ndef scores(preds, lib, df=df[best_features], verbose=True, compute_silhouette=True): \n    # Silhouette is very slow\n    sil = 0\n    if compute_silhouette:\n        sil = silhouette_score(df, preds, metric='euclidean')\n    \n    s = (lib,\n         sil, \n         calinski_harabasz_score(df, preds), \n         davies_bouldin_score(df, preds))\n    \n    if verbose:\n        print(f\"{s[0]} : Silhouette : {s[1]:.1%} | Calinski Harabasz : {s[2]:.1f} | Davis Bouldin : {s[3]:.3f}\")\n        \n    return s","metadata":{"papermill":{"duration":0.017779,"end_time":"2022-07-17T01:28:36.840036","exception":false,"start_time":"2022-07-17T01:28:36.822257","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-18T08:17:39.422225Z","iopub.execute_input":"2022-07-18T08:17:39.422959Z","iopub.status.idle":"2022-07-18T08:17:39.434523Z","shell.execute_reply.started":"2022-07-18T08:17:39.422907Z","shell.execute_reply":"2022-07-18T08:17:39.433127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bayesian Gaussian Mixture","metadata":{"papermill":{"duration":0.005041,"end_time":"2022-07-17T01:28:36.850182","exception":false,"start_time":"2022-07-17T01:28:36.845141","status":"completed"},"tags":[]}},{"cell_type":"code","source":"seed_everything(CFG.seed_bgm)\ndf_scaled = pd.DataFrame(PowerTransformer().fit_transform(df[features]), columns=features)\n\nBGM = BayesianGaussianMixture(n_components=CFG.n_clusters, covariance_type='full', random_state=CFG.seed_bgm, n_init=1, max_iter=200, tol=1e-3)\nBGM.fit(df_scaled[best_features])\n\nBGM_predict_proba = BGM.predict_proba(df_scaled[best_features])\nBGM_predict = np.argmax(BGM_predict_proba, axis=1)\n\nall_scores.append(scores(BGM_predict, lib=\"BayesianGaussianMixture after powertransformer\"))","metadata":{"papermill":{"duration":139.612692,"end_time":"2022-07-17T01:30:56.468036","exception":false,"start_time":"2022-07-17T01:28:36.855344","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2022-07-18T08:17:39.436128Z","iopub.execute_input":"2022-07-18T08:17:39.436772Z","iopub.status.idle":"2022-07-18T08:21:39.194473Z","shell.execute_reply.started":"2022-07-18T08:17:39.436738Z","shell.execute_reply":"2022-07-18T08:21:39.190984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Trusted Data","metadata":{"papermill":{"duration":0.019955,"end_time":"2022-07-17T01:30:56.511106","exception":false,"start_time":"2022-07-17T01:30:56.491151","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# get trusted data to train LGB model.\nproba_threshold = CFG.threshold\n\ndf_scaled['predict'] = BGM_predict\ndf_scaled['predict_proba'] = 0\nfor n in range(CFG.n_clusters):\n    df_scaled[f'predict_proba_{n}'] = BGM_predict_proba[:, n]\n    df_scaled.loc[df_scaled['predict']==n, 'predict_proba'] = df_scaled[f'predict_proba_{n}']\n    \n    \nidxs = np.array([])\nfor n in range(CFG.n_clusters):\n    median = df_scaled[df_scaled.predict==n]['predict_proba'].median()\n    idx = df_scaled[(df_scaled.predict==n) & (df_scaled.predict_proba > proba_threshold)].index\n    idxs = np.concatenate((idxs, idx))\n    print(f'Class n{n}  |  Median : {median:.4f}  |  Training data : {len(idx)/len(df_scaled[(df_scaled.predict==n)]):.1%}')\n    \nX = df_scaled.loc[idxs][best_features].reset_index(drop=True)\ny = df_scaled.loc[idxs]['predict'].reset_index(drop=True)","metadata":{"papermill":{"duration":0.213867,"end_time":"2022-07-17T01:30:56.743299","exception":false,"start_time":"2022-07-17T01:30:56.529432","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-18T08:21:39.195842Z","iopub.execute_input":"2022-07-18T08:21:39.196439Z","iopub.status.idle":"2022-07-18T08:21:39.392378Z","shell.execute_reply.started":"2022-07-18T08:21:39.196401Z","shell.execute_reply":"2022-07-18T08:21:39.390345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(X.shape)\ndisplay(X)","metadata":{"papermill":{"duration":0.04887,"end_time":"2022-07-17T01:30:56.801587","exception":false,"start_time":"2022-07-17T01:30:56.752717","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-18T08:21:39.393776Z","iopub.execute_input":"2022-07-18T08:21:39.3941Z","iopub.status.idle":"2022-07-18T08:21:39.420425Z","shell.execute_reply.started":"2022-07-18T08:21:39.394074Z","shell.execute_reply":"2022-07-18T08:21:39.419415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Supervised Learning\n### KNeighborsClassifier","metadata":{"papermill":{"duration":0.009837,"end_time":"2022-07-17T01:30:56.821233","exception":false,"start_time":"2022-07-17T01:30:56.811396","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_score(labels, preds, probas):\n    s = (balanced_accuracy_score(labels, preds),\n        roc_auc_score(labels, probas, average=\"weighted\", multi_class=\"ovo\"))\n    return s","metadata":{"papermill":{"duration":0.016814,"end_time":"2022-07-17T01:30:56.845643","exception":false,"start_time":"2022-07-17T01:30:56.828829","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-18T08:21:39.422346Z","iopub.execute_input":"2022-07-18T08:21:39.422817Z","iopub.status.idle":"2022-07-18T08:21:39.428366Z","shell.execute_reply.started":"2022-07-18T08:21:39.422774Z","shell.execute_reply":"2022-07-18T08:21:39.427168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything(CFG.seed)\nknc_predict_proba = 0\nclassif_scores = []\n\nskf = StratifiedKFold(CFG.n_splits, shuffle=True, random_state=CFG.seed)\n\nfor fold, (trn_idx, val_idx) in enumerate(skf.split(X, y)):\n    print(f\"===== fold{fold} =====\")\n    X_train, y_train = X.iloc[trn_idx], y.iloc[trn_idx]\n    X_valid, y_valid = X.iloc[val_idx], y.iloc[val_idx]\n\n    # KNeighborsClassifier\n    model = KNeighborsClassifier(n_neighbors=15)\n    model.fit(X_train, y_train) # on trusted data only\n    \n    y_pred = model.predict(X_valid)\n    y_pred_proba = model.predict_proba(X_valid)\n    \n    s = get_score(y_valid, y_pred, y_pred_proba)\n    print(f\"KNeighbors AUC : {s[1]:.3f} | Accuracy : {s[0]:.1%}\")\n    classif_scores.append(s)\n\n    knc_predict_proba += model.predict_proba(df_scaled[best_features]) / CFG.n_splits\n\n    del model, s, y_pred, y_pred_proba\n    gc.collect()\n\nall_scores.append(scores(np.argmax(knc_predict_proba, axis=1), lib=\"KNeighbors after BayesianGaussianMixture\"))\n\npd.DataFrame(classif_scores, columns = [\"balanced_accuracy_score\", \"roc_auc_score\"]).mean(0)","metadata":{"papermill":{"duration":1465.264169,"end_time":"2022-07-17T01:55:22.135252","exception":false,"start_time":"2022-07-17T01:30:56.871083","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-18T08:21:39.432545Z","iopub.execute_input":"2022-07-18T08:21:39.432946Z","iopub.status.idle":"2022-07-18T08:23:30.940433Z","shell.execute_reply.started":"2022-07-18T08:21:39.432899Z","shell.execute_reply":"2022-07-18T08:23:30.939327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Soft voting","metadata":{"papermill":{"duration":0.017768,"end_time":"2022-07-17T01:55:22.17324","exception":false,"start_time":"2022-07-17T01:55:22.155472","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def soft_voting(preds_probas, weights):\n    pred_test = np.zeros((df.shape[0], CFG.n_clusters))\n    \n    for i, (p, w) in enumerate(zip(preds_probas, weights)):\n        preds = np.argmax(p, axis=1)\n        pred_idx = pd.Series(preds).value_counts().index.tolist()\n        pred_test += p[:, pred_idx] * w\n    \n    return np.argmax(pred_test, axis=1)","metadata":{"papermill":{"duration":0.03384,"end_time":"2022-07-17T01:55:22.225915","exception":false,"start_time":"2022-07-17T01:55:22.192075","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-18T08:23:30.945677Z","iopub.execute_input":"2022-07-18T08:23:30.94654Z","iopub.status.idle":"2022-07-18T08:23:30.959969Z","shell.execute_reply.started":"2022-07-18T08:23:30.946492Z","shell.execute_reply":"2022-07-18T08:23:30.957908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sv_predict = soft_voting([knc_predict_proba], [2.0])\n\nall_scores.append(scores(sv_predict, lib=\"Soft voting: KNeighbors\"))","metadata":{"papermill":{"duration":94.701929,"end_time":"2022-07-17T01:56:56.938326","exception":false,"start_time":"2022-07-17T01:55:22.236397","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-18T08:23:30.961363Z","iopub.execute_input":"2022-07-18T08:23:30.961735Z","iopub.status.idle":"2022-07-18T08:25:05.616048Z","shell.execute_reply.started":"2022-07-18T08:23:30.961697Z","shell.execute_reply":"2022-07-18T08:25:05.61284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(all_scores, columns=[\"Model\", \"silhouette\", \"Calinski_Harabasz\", \"Davis_Bouldin\"])","metadata":{"papermill":{"duration":0.039703,"end_time":"2022-07-17T01:56:56.99712","exception":false,"start_time":"2022-07-17T01:56:56.957417","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-18T08:25:05.617501Z","iopub.execute_input":"2022-07-18T08:25:05.618136Z","iopub.status.idle":"2022-07-18T08:25:05.63863Z","shell.execute_reply.started":"2022-07-18T08:25:05.6181Z","shell.execute_reply":"2022-07-18T08:25:05.637171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submissions","metadata":{"papermill":{"duration":0.019794,"end_time":"2022-07-17T01:56:57.036763","exception":false,"start_time":"2022-07-17T01:56:57.016969","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sub = pd.read_csv(\"../input/tabular-playground-series-jul-2022/sample_submission.csv\")\nsub['Predicted'] = sv_predict\nsub","metadata":{"papermill":{"duration":0.191221,"end_time":"2022-07-17T01:56:57.241094","exception":false,"start_time":"2022-07-17T01:56:57.049873","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-18T08:25:05.639673Z","iopub.execute_input":"2022-07-18T08:25:05.640214Z","iopub.status.idle":"2022-07-18T08:25:05.847498Z","shell.execute_reply.started":"2022-07-18T08:25:05.640178Z","shell.execute_reply":"2022-07-18T08:25:05.846287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv(\"submission.csv\", index=False)\n!ls","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div class=\"alert alert-success\">  \n    <h1 align=\"center\" style=\"color:darkgreen;\">Good Luck</h1>  \n</div>","metadata":{}}]}