{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\nimport matplotlib.pyplot as plt\nplt.style.use('ggplot')\nimport seaborn as sns\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom pandarallel import pandarallel\n# Initialization\npandarallel.initialize()\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-28T03:01:41.976962Z","iopub.execute_input":"2022-07-28T03:01:41.977569Z","iopub.status.idle":"2022-07-28T03:01:43.238926Z","shell.execute_reply.started":"2022-07-28T03:01:41.977485Z","shell.execute_reply":"2022-07-28T03:01:43.237591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# load data","metadata":{}},{"cell_type":"code","source":"# from: https://www.kaggle.com/code/digvijaysinhgohil/auto-eda-with-dataprep-and-bgm-model\nimp_cols = ['f_07','f_08','f_09','f_10','f_11','f_12','f_13','f_22','f_23','f_24','f_25','f_26','f_27','f_28']\npath = '/kaggle/input/tabular-playground-series-jul-2022/data.csv'\npath_sub = '/kaggle/input/tabular-playground-series-jul-2022/sample_submission.csv'\n\ntrain_df = pd.read_csv(path)\nsub_df = pd.read_csv(path_sub)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T03:01:43.658156Z","iopub.execute_input":"2022-07-28T03:01:43.658856Z","iopub.status.idle":"2022-07-28T03:01:45.029690Z","shell.execute_reply.started":"2022-07-28T03:01:43.658832Z","shell.execute_reply":"2022-07-28T03:01:45.028252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-28T03:01:45.048898Z","iopub.execute_input":"2022-07-28T03:01:45.049276Z","iopub.status.idle":"2022-07-28T03:01:45.058827Z","shell.execute_reply.started":"2022-07-28T03:01:45.049247Z","shell.execute_reply":"2022-07-28T03:01:45.057864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[imp_cols].head()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T03:01:46.187174Z","iopub.execute_input":"2022-07-28T03:01:46.187949Z","iopub.status.idle":"2022-07-28T03:01:46.221683Z","shell.execute_reply.started":"2022-07-28T03:01:46.187910Z","shell.execute_reply":"2022-07-28T03:01:46.220941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cluster","metadata":{}},{"cell_type":"code","source":"# https://www.kaggle.com/code/cabaxiom/tps-jul-22-bgmm-semi-supervised\ndef best_class(df):\n    new_df = df.copy()\n    new_df[\"highest_prob\"] = df.max(axis=1)\n    new_df[\"best_class\"] = df.idxmax(axis=1)\n    new_df[\"second_highest_prob\"] = df.parallel_apply(lambda x: x.nlargest(2).values[-1], axis=1)\n    new_df[\"second_best_class\"] = df.parallel_apply(lambda x: np.where(x == x.nlargest(2).values[-1])[0][0], axis=1)\n    return new_df","metadata":{"execution":{"iopub.status.busy":"2022-07-28T03:01:47.879746Z","iopub.execute_input":"2022-07-28T03:01:47.881675Z","iopub.status.idle":"2022-07-28T03:01:47.889649Z","shell.execute_reply.started":"2022-07-28T03:01:47.881615Z","shell.execute_reply":"2022-07-28T03:01:47.888358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import cluster\nfrom sklearn.cluster import DBSCAN, AgglomerativeClustering, KMeans\nfrom sklearn.mixture import GaussianMixture, BayesianGaussianMixture\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import PowerTransformer, RobustScaler\n# pt = PowerTransformer()\n# df_x = pt.fit_transform(train_df[imp_cols])\n\ntransformer = Pipeline(\n    steps = [\n        (\"robust\", RobustScaler()),\n        (\"power\", PowerTransformer()),\n    ]\n)\n\ndf_x = transformer.fit_transform(train_df[imp_cols])","metadata":{"execution":{"iopub.status.busy":"2022-07-28T03:01:48.819167Z","iopub.execute_input":"2022-07-28T03:01:48.819749Z","iopub.status.idle":"2022-07-28T03:01:52.548960Z","shell.execute_reply.started":"2022-07-28T03:01:48.819711Z","shell.execute_reply":"2022-07-28T03:01:52.547545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Methods')\n# ------------------------------------------------------------------------------\n# 0.26\n# pt = PowerTransformer()\n# df_x = pt.fit_transform(train_df[imp_cols])\n# kms = KMeans(7)\n# pred_ = kms.fit_predict(df_x) \n\n# ------------------------------------------------------------------------------\n# 0.41897\n# pt = PowerTransformer()\n# df_x = pt.fit_transform(train_df[imp_cols])\n# pred_list =[]\n# for i in tqdm(range(n_iter)):\n#     seed = i * 100 // 42\n#     model = BayesianGaussianMixture(n_components=7, \n#                                     covariance_type='full',\n#                                     max_iter=256,\n#                                     n_init=1,\n#                                     random_state=seed)\n\n#     model.fit(df_x)\n#     pred = model.predict(df_x)\n#     pred_list.append(pred)\n# # vote \n# aa = np.stack(pred_list)\n# f_pred = []\n# for i in range(aa.shape[1]):\n#     f_pred.append(\n#         np.argmax(np.bincount(aa[:, i]))\n#     )\n\n# ------------------------------------------------------------------------------\n# 0.32660\n# transformer = Pipeline(\n#     steps = [\n#         (\"robust\", RobustScaler()),\n#         (\"power\", PowerTransformer()),\n#     ]\n# )\n# df_x = transformer.fit_transform(train_df[imp_cols])\n# n_iter = 5\n# pred_list =[]\n# for i in tqdm(range(n_iter)):\n#     seed = i * 100 // 42\n#     model = BayesianGaussianMixture(n_components=7, \n#                                     covariance_type='full',\n#                                     max_iter=256,\n#                                     n_init=1,\n#                                     random_state=seed)\n\n#     model.fit(df_x)\n#     pred = model.predict(df_x)\n#     pred_list.append(pred)\n## vote \n# aa = np.stack(pred_list)\n# f_pred = []\n# for i in range(aa.shape[1]):\n#     f_pred.append(\n#         np.argmax(np.bincount(aa[:, i]))\n#     )\n\n# **********************************************\n# kms 0.58859\n# aa = np.stack(pred_list)\n# kms = KMeans(7)\n# f_pred = kms.fit_predict(aa.T) \n\n# np.unique(f_pred, return_counts=True)\n\n# ------------------------------------------------------------------------------\n#  0.29\n# transformer = Pipeline(\n#     steps = [\n#         (\"robust\", RobustScaler()),\n#         (\"power\", PowerTransformer()),\n#     ]\n# )\n# df_x = transformer.fit_transform(train_df[imp_cols])\n# n_iter = 5\n# pred_list =[]\n# for i in tqdm(range(n_iter)):\n#     print('--'*35)\n#     seed = i * 100 // 42\n#     model = BayesianGaussianMixture(n_components=7, \n#                                     covariance_type='full',\n#                                     max_iter=256,\n#                                     n_init=2,\n#                                     random_state=seed,\n#                                     init_params=\"kmeans\"\n#                                    )\n#     print(f'[{i}] fit ....')\n#     model.fit(df_x)\n#     print(f'[{i}] pred ....')\n#     df = pd.DataFrame(model.predict_proba(df_x))\n#     print(f'[{i}] pred_processing ....')\n#     pred = best_class(df)[[\"best_class\", 'second_best_class']].values\n#     pred_list.append(pred)\n# kms\n# aa = np.concatenate([i.T for i in pred_list])\n# kms = KMeans(7)\n# f_pred = kms.fit_predict(aa.T) \n\n# np.unique(f_pred, return_counts=True)\n\n# ------------------------------------------------------------------------------\n# 0.34052\n# transformer = Pipeline(\n#     steps = [\n#         (\"robust\", RobustScaler()),\n#         (\"power\", PowerTransformer()),\n#     ]\n# )\n# df_x = transformer.fit_transform(train_df[imp_cols])\n\n# n_iter = 5\n# pred_df = pd.DataFrame()\n# for i in tqdm(range(n_iter)):\n#     print('--'*35)\n#     seed = i * 100 // 42\n#     model = BayesianGaussianMixture(n_components=7, \n#                                     covariance_type='full',\n#                                     max_iter=256,\n#                                     n_init=2,\n#                                     random_state=seed,\n#                                     init_params=\"kmeans\"\n#                                    )\n#     print(f'[{i}] fit ....')\n#     model.fit(df_x)\n#     print(f'[{i}] pred ....')\n#     if i == 0:\n#         pred_df = pd.DataFrame(model.predict_proba(df_x))\n#         continue\n#     pred_df += pd.DataFrame(model.predict_proba(df_x))\n# soft vote\n# pred_df /= n_iter\n# f_pred = np.argmax(pred_df.values, axis=1)\n\n# ------------------------------------------------------------------------------\n#  \n# transformer = Pipeline(\n#     steps = [\n#         (\"robust\", RobustScaler()),\n#         (\"power\", PowerTransformer()),\n#     ]\n# )\n# df_x = transformer.fit_transform(train_df[imp_cols])\n# n_iter = 5\n# pred_df = pd.DataFrame()\n# for i in tqdm(range(n_iter)):\n#     print('--'*35)\n#     seed = i * 100 // 42\n#     model = BayesianGaussianMixture(n_components=7, \n#                                     covariance_type='full',\n#                                     max_iter=256,\n#                                     n_init=1,\n#                                     random_state=seed\n#                                    )\n#     print(f'[{i}] fit ....')\n#     model.fit(df_x)\n#     print(f'[{i}] pred ....')\n#     if i == 0:\n#         pred_df = pd.DataFrame(model.predict_proba(df_x))\n#         continue\n#     pred_df += pd.DataFrame(model.predict_proba(df_x))\n# vote 0.279\n# pred_df /= n_iter\n# f_pred = np.argmax(pred_df.values, axis=1)\n# **********************************************\n# kms 0.55937\n# kms = KMeans(7)\n# f_pred = kms.fit_predict(pred_df) \n# np.unique(f_pred, return_counts=True)\n# (array([0, 1, 2, 3, 4, 5, 6], dtype=int32),\n#  array([13740, 15126, 13843, 15342, 10859, 17723, 11367]))\n","metadata":{"execution":{"iopub.status.busy":"2022-07-28T03:01:52.550807Z","iopub.execute_input":"2022-07-28T03:01:52.551076Z","iopub.status.idle":"2022-07-28T03:01:52.560820Z","shell.execute_reply.started":"2022-07-28T03:01:52.551052Z","shell.execute_reply":"2022-07-28T03:01:52.559610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.notebook import tqdm\nn_iter = 10\npred_df = pd.DataFrame(np.zeros((len(df_x),7)), columns=range(7))\nfor i in tqdm(range(n_iter)):\n    print('--'*15, f'[ iter: {i+1} ]' ,'--'*15)\n    seed = i * 100 // 42\n    model = BayesianGaussianMixture(n_components=7, \n                                    covariance_type='full',\n                                    max_iter=256,\n                                    n_init=3,\n                                    init_params=\"kmeans\",\n                                    random_state=seed\n                                   )\n    print(f'[{i}] fit ....')\n    model.fit(df_x)\n    print(f'[{i}] pred ....')\n    if i == 0:\n        initial_centers = model.means_\n        pred_df = pd.DataFrame(model.predict_proba(df_x))\n        continue\n\n    # same center cumsum\n    new_classes = []\n    for mean2 in model.means_:\n        distances = [np.linalg.norm(mean1 - mean2) for mean1 in initial_centers]\n        new_class = np.argmin(distances)\n        new_classes.append(new_class)\n    \n    print(list(range(7)),new_classes)\n    if len(new_classes) != len(set(new_classes)):\n        print(\"iteration\", i, \"could not determine the cluster label mapping, skipping\")\n        continue\n    \n    pred_probs = pd.DataFrame(model.predict_proba(df_x))\n    pred_probs = pred_probs.rename(columns=dict(zip(range(7), new_classes)))\n    display(pred_probs.head(2))\n    display(pred_df.head(2))\n    pred_df += pred_probs\n    a = pd.Series(np.argmax(pred_df.values, axis=1)).value_counts().sort_index()\n    sns.barplot(x=a.index, y=a.values)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T03:01:52.561975Z","iopub.execute_input":"2022-07-28T03:01:52.563196Z","iopub.status.idle":"2022-07-28T03:19:14.473245Z","shell.execute_reply.started":"2022-07-28T03:01:52.563073Z","shell.execute_reply":"2022-07-28T03:19:14.472201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# vote 0.59696\npred_df = pred_df.div(pred_probs.sum(axis=1), axis=0)\nf_pred = np.argmax(pred_df.values, axis=1)\n\npd.Series(f_pred).value_counts().sort_values().plot.bar(alpha=0.9)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T03:19:56.780146Z","iopub.execute_input":"2022-07-28T03:19:56.780796Z","iopub.status.idle":"2022-07-28T03:19:56.966846Z","shell.execute_reply.started":"2022-07-28T03:19:56.780741Z","shell.execute_reply":"2022-07-28T03:19:56.965524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# kms\n# (array([0, 1, 2, 3, 4, 5, 6], dtype=int32),\n#  array([13740, 15126, 13843, 15342, 10859, 17723, 11367]))\n\n# kms = KMeans(7)\n# f_pred = kms.fit_predict(pred_df) \n\n# pd.Series(f_pred).value_counts().sort_values().plot.bar(alpha=0.9)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T03:20:50.569510Z","iopub.execute_input":"2022-07-28T03:20:50.569910Z","iopub.status.idle":"2022-07-28T03:20:52.619942Z","shell.execute_reply.started":"2022-07-28T03:20:50.569880Z","shell.execute_reply":"2022-07-28T03:20:52.618829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 0.55\na = pd.Series(sorted([13740, 15126, 13843, 15342, 10859, 17723, 11367]))\na.plot.bar(alpha=0.9)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T02:54:18.767158Z","iopub.execute_input":"2022-07-28T02:54:18.767596Z","iopub.status.idle":"2022-07-28T02:54:18.954714Z","shell.execute_reply.started":"2022-07-28T02:54:18.767560Z","shell.execute_reply":"2022-07-28T02:54:18.953758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df['Predicted'] = f_pred\nsub_df.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T03:20:10.121938Z","iopub.execute_input":"2022-07-28T03:20:10.122311Z","iopub.status.idle":"2022-07-28T03:20:10.283805Z","shell.execute_reply.started":"2022-07-28T03:20:10.122282Z","shell.execute_reply":"2022-07-28T03:20:10.282061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}