{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Inspiration from these notebooks:\n- Clustering from: https://www.kaggle.com/code/nitishraj/pseudo-label-soft-voting-with-sklearn\n- Clustering from: www.kaggle.com/competitions/tabular-playground-series-jul-2022/discussion/334808\n- Ideas from: https://www.kaggle.com/code/adaubas/tps-jul22-lgbm-extratree-qda-soft-voting\n- Bayesian GMM classifier from: https://www.kaggle.com/code/karlcini/bayesiangmmclassifier","metadata":{}},{"cell_type":"code","source":"pip install scikit-lego","metadata":{"execution":{"iopub.status.busy":"2022-07-23T15:54:17.562493Z","iopub.execute_input":"2022-07-23T15:54:17.562910Z","iopub.status.idle":"2022-07-23T15:54:32.047825Z","shell.execute_reply.started":"2022-07-23T15:54:17.562881Z","shell.execute_reply":"2022-07-23T15:54:32.046242Z"},"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split, StratifiedKFold, GridSearchCV, TimeSeriesSplit\nfrom sklearn.preprocessing import RobustScaler, PowerTransformer\nfrom sklearn.mixture import GaussianMixture, BayesianGaussianMixture\nfrom sklego.mixture import BayesianGMMClassifier, GMMClassifier\nfrom sklearn.metrics import accuracy_score\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-23T15:54:46.816261Z","iopub.execute_input":"2022-07-23T15:54:46.816733Z","iopub.status.idle":"2022-07-23T15:54:46.831309Z","shell.execute_reply.started":"2022-07-23T15:54:46.816694Z","shell.execute_reply":"2022-07-23T15:54:46.830359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=pd.read_csv(\"../input/tabular-playground-series-jul-2022/data.csv\")\nss=pd.read_csv(\"../input/tabular-playground-series-jul-2022/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-23T15:55:07.559603Z","iopub.execute_input":"2022-07-23T15:55:07.559958Z","iopub.status.idle":"2022-07-23T15:55:08.973597Z","shell.execute_reply.started":"2022-07-23T15:55:07.559930Z","shell.execute_reply":"2022-07-23T15:55:08.972241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clustering from: https://www.kaggle.com/code/nitishraj/pseudo-label-soft-voting-with-sklearn\n\n# Using PowerTransformer before Scaling to adjust for Outliers\ndata_scaled = pd.DataFrame(PowerTransformer().fit_transform(data), columns=data.columns)\ndata_scaled = pd.DataFrame(\n    RobustScaler().fit_transform(data_scaled), columns=data_scaled.columns\n)\n\n# www.kaggle.com/competitions/tabular-playground-series-jul-2022/discussion/334808\nuseful_cols = [\n    \"f_07\",\n    \"f_08\",\n    \"f_09\",\n    \"f_10\",\n    \"f_11\",\n    \"f_12\",\n    \"f_13\",\n    \"f_22\",\n    \"f_23\",\n    \"f_24\",\n    \"f_25\",\n    \"f_26\",\n    \"f_27\",\n    \"f_28\",\n]\n\n# Test Data for predictions later\ntest_data = data_scaled[useful_cols].copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-23T15:56:08.162995Z","iopub.execute_input":"2022-07-23T15:56:08.163780Z","iopub.status.idle":"2022-07-23T15:56:12.302973Z","shell.execute_reply.started":"2022-07-23T15:56:08.163735Z","shell.execute_reply":"2022-07-23T15:56:12.301696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Fit Bayesian Gaussian Mixture\nprint(\"Fitting Bayesian Gaussian Mixture..\")\nbgm = BayesianGaussianMixture(\n    n_components=7,\n    max_iter=300,\n    n_init=10,\n    random_state=2,\n    verbose_interval=100,\n)\n\nbgm_labels = bgm.fit_predict(data_scaled[useful_cols])\nbgm_proba = bgm.predict_proba(data_scaled[useful_cols])","metadata":{"execution":{"iopub.status.busy":"2022-07-23T15:58:40.096439Z","iopub.execute_input":"2022-07-23T15:58:40.096839Z","iopub.status.idle":"2022-07-23T16:05:34.984721Z","shell.execute_reply.started":"2022-07-23T15:58:40.096806Z","shell.execute_reply":"2022-07-23T16:05:34.983344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Using idea from: https://www.kaggle.com/code/adaubas/tps-jul22-lgbm-extratree-qda-soft-voting\n\n# Creating Best data based on predicted probability of BGM model\nn_components = 7\ndata_scaled[\"predict\"] = bgm_labels\ndata_scaled[\"predict_proba\"] = 0\n\nfor n in range(n_components):\n    data_scaled[f\"bgm_proba_{n}\"] = bgm_proba[:, n]\n    data_scaled.loc[data_scaled.predict == n, \"bgm_proba\"] = data_scaled[\n        f\"bgm_proba_{n}\"\n    ]\n\ntrain_index = np.array([])\nfor n in range(n_components):\n    median = data_scaled[data_scaled.predict == n][\"bgm_proba\"].median()\n\n    # Experiment with different thresholds\n    # Higher thereshold might overfit\n    n_inx = data_scaled[\n        (data_scaled.predict == n) & (data_scaled.bgm_proba > 0.675)\n    ].index\n\n    train_index = np.concatenate((train_index, n_inx))\n    print(\n        f\"class:{n}\",\n        f\"median: {round(median,4)}\",\n        \"Training data:\"\n        + str(round(len(n_inx) / len(data_scaled[(data_scaled.predict == n)]), 2) * 100)\n        + \"%\",\n    )\n\n\nprint(f\"\\nSize of Training data : {len(train_index)}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-23T16:05:34.987449Z","iopub.execute_input":"2022-07-23T16:05:34.988411Z","iopub.status.idle":"2022-07-23T16:05:35.168557Z","shell.execute_reply.started":"2022-07-23T16:05:34.988319Z","shell.execute_reply":"2022-07-23T16:05:35.167341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = data_scaled.loc[train_index][useful_cols]\ny = data_scaled.loc[train_index][\"predict\"]","metadata":{"execution":{"iopub.status.busy":"2022-07-23T16:05:35.170029Z","iopub.execute_input":"2022-07-23T16:05:35.170421Z","iopub.status.idle":"2022-07-23T16:05:35.229821Z","shell.execute_reply.started":"2022-07-23T16:05:35.170389Z","shell.execute_reply":"2022-07-23T16:05:35.228773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# https://www.kaggle.com/code/karlcini/bayesiangmmclassifier\n\nbgm = BayesianGMMClassifier(\n    n_components=7,\n    random_state=42,\n    # tol =1e-3,\n    covariance_type=\"full\",\n    max_iter=500,\n    n_init=7,\n    init_params=\"kmeans\", # you can use k-means++\n)\nbgm.fit(X, y)","metadata":{"execution":{"iopub.status.busy":"2022-07-23T16:21:07.659522Z","iopub.execute_input":"2022-07-23T16:21:07.659939Z","iopub.status.idle":"2022-07-23T16:27:36.233559Z","shell.execute_reply.started":"2022-07-23T16:21:07.659896Z","shell.execute_reply":"2022-07-23T16:27:36.232186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = bgm.predict(X)\naccuracy_score(y, y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-07-23T16:27:36.235880Z","iopub.execute_input":"2022-07-23T16:27:36.236671Z","iopub.status.idle":"2022-07-23T16:27:37.397791Z","shell.execute_reply.started":"2022-07-23T16:27:36.236619Z","shell.execute_reply":"2022-07-23T16:27:37.396384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = bgm.predict(test_data)\nss[\"Predicted\"] = predictions\nss.to_csv(\n    \"submission.csv\",\n    index=False,\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-23T16:28:37.491432Z","iopub.execute_input":"2022-07-23T16:28:37.491815Z","iopub.status.idle":"2022-07-23T16:28:39.444692Z","shell.execute_reply.started":"2022-07-23T16:28:37.491785Z","shell.execute_reply":"2022-07-23T16:28:39.443512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}