{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports\n---","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom sklearn.preprocessing import PowerTransformer\nfrom sklearn.feature_selection import VarianceThreshold\nfrom yellowbrick.cluster import KElbowVisualizer\nfrom sklearn.decomposition import PCA\nfrom sklearn.mixture import BayesianGaussianMixture\nfrom sklearn.cluster import KMeans\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:32:25.734606Z","iopub.execute_input":"2022-07-13T21:32:25.735486Z","iopub.status.idle":"2022-07-13T21:32:26.898301Z","shell.execute_reply.started":"2022-07-13T21:32:25.735406Z","shell.execute_reply":"2022-07-13T21:32:26.896938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loads\n---","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('../input/tabular-playground-series-jul-2022/data.csv', index_col='id')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:32:26.905017Z","iopub.execute_input":"2022-07-13T21:32:26.905668Z","iopub.status.idle":"2022-07-13T21:32:28.173527Z","shell.execute_reply.started":"2022-07-13T21:32:26.905619Z","shell.execute_reply":"2022-07-13T21:32:28.172306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PreProcessing\n---","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:53:01.517433Z","iopub.execute_input":"2022-07-13T15:53:01.517906Z","iopub.status.idle":"2022-07-13T15:53:01.525139Z","shell.execute_reply.started":"2022-07-13T15:53:01.517869Z","shell.execute_reply":"2022-07-13T15:53:01.523846Z"}}},{"cell_type":"code","source":"df.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:32:28.175184Z","iopub.execute_input":"2022-07-13T21:32:28.175959Z","iopub.status.idle":"2022-07-13T21:32:28.379608Z","shell.execute_reply.started":"2022-07-13T21:32:28.175911Z","shell.execute_reply":"2022-07-13T21:32:28.378363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isna().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:32:28.382066Z","iopub.execute_input":"2022-07-13T21:32:28.382531Z","iopub.status.idle":"2022-07-13T21:32:28.395574Z","shell.execute_reply.started":"2022-07-13T21:32:28.382499Z","shell.execute_reply":"2022-07-13T21:32:28.394353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:32:28.396698Z","iopub.execute_input":"2022-07-13T21:32:28.397472Z","iopub.status.idle":"2022-07-13T21:32:28.420907Z","shell.execute_reply.started":"2022-07-13T21:32:28.397421Z","shell.execute_reply":"2022-07-13T21:32:28.419770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(19,8)) \nsns.boxplot(x=\"variable\", y=\"value\", data=pd.melt(df)).set_title('Boxplot of each feature',size=15)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:32:28.422100Z","iopub.execute_input":"2022-07-13T21:32:28.422432Z","iopub.status.idle":"2022-07-13T21:32:30.427339Z","shell.execute_reply.started":"2022-07-13T21:32:28.422402Z","shell.execute_reply":"2022-07-13T21:32:30.426458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.pairplot(df, kind='scatter', corner=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:32:30.428597Z","iopub.execute_input":"2022-07-13T21:32:30.429161Z","iopub.status.idle":"2022-07-13T21:35:52.435487Z","shell.execute_reply.started":"2022-07-13T21:32:30.429107Z","shell.execute_reply":"2022-07-13T21:35:52.433661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selector = VarianceThreshold(threshold=1.5)\nselector.fit_transform(df)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:35:52.436842Z","iopub.execute_input":"2022-07-13T21:35:52.437340Z","iopub.status.idle":"2022-07-13T21:35:52.504038Z","shell.execute_reply.started":"2022-07-13T21:35:52.437309Z","shell.execute_reply":"2022-07-13T21:35:52.503191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,10))\nsns.barplot(x=selector.variances_, y=df.columns,orient='h' ).set_title('Feature selection with VarianceThreshold',size=15);\nplt.xlabel('Variance');\nplt.axvline(x=1.5, color='r', linestyle='--', label='Threshold')\nplt.legend();","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:35:52.505032Z","iopub.execute_input":"2022-07-13T21:35:52.505984Z","iopub.status.idle":"2022-07-13T21:35:52.935089Z","shell.execute_reply.started":"2022-07-13T21:35:52.505920Z","shell.execute_reply":"2022-07-13T21:35:52.934139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = ['f_07', 'f_08', 'f_09', 'f_10', 'f_11',\n            'f_12', 'f_13', 'f_22', 'f_23', 'f_24',\n            'f_25', 'f_26', 'f_27', 'f_28']","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:35:52.936297Z","iopub.execute_input":"2022-07-13T21:35:52.936871Z","iopub.status.idle":"2022-07-13T21:35:52.941715Z","shell.execute_reply.started":"2022-07-13T21:35:52.936838Z","shell.execute_reply":"2022-07-13T21:35:52.940644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = PowerTransformer()\ndf_scaled = pd.DataFrame(scaler.fit_transform(df[features]))\ndf_scaled.columns = features\ndf_scaled.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:35:52.943185Z","iopub.execute_input":"2022-07-13T21:35:52.943733Z","iopub.status.idle":"2022-07-13T21:35:54.627561Z","shell.execute_reply.started":"2022-07-13T21:35:52.943696Z","shell.execute_reply":"2022-07-13T21:35:54.626392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_scaled.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:35:54.628994Z","iopub.execute_input":"2022-07-13T21:35:54.629433Z","iopub.status.idle":"2022-07-13T21:35:54.741545Z","shell.execute_reply.started":"2022-07-13T21:35:54.629402Z","shell.execute_reply":"2022-07-13T21:35:54.740460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(19,8)) \nsns.boxplot(x=\"variable\", y=\"value\", data=pd.melt(df_scaled)).set_title('Boxplot of each feature',size=15)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:35:54.744771Z","iopub.execute_input":"2022-07-13T21:35:54.745100Z","iopub.status.idle":"2022-07-13T21:35:55.762534Z","shell.execute_reply.started":"2022-07-13T21:35:54.745072Z","shell.execute_reply":"2022-07-13T21:35:55.760357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = KMeans()\nvlz = KElbowVisualizer(model, k=(4,13))\n\nvlz.fit(df_scaled)\nvlz.show()     ","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:35:55.763785Z","iopub.execute_input":"2022-07-13T21:35:55.764103Z","iopub.status.idle":"2022-07-13T21:36:46.898912Z","shell.execute_reply.started":"2022-07-13T21:35:55.764073Z","shell.execute_reply":"2022-07-13T21:36:46.898165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction\n---","metadata":{}},{"cell_type":"code","source":"# BGM\npreds1 = BayesianGaussianMixture(n_components=7,\n                                covariance_type='full',\n                                max_iter=1000,\n                                n_init=5,\n                                random_state=42).fit_predict(df_scaled)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:45:46.429048Z","iopub.execute_input":"2022-07-13T21:45:46.429488Z","iopub.status.idle":"2022-07-13T21:49:25.518685Z","shell.execute_reply.started":"2022-07-13T21:45:46.429455Z","shell.execute_reply":"2022-07-13T21:49:25.517194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# KMeans\nmodel_km = KMeans(n_clusters=7, random_state=42)\npreds2 = model_km.fit_predict(df_scaled)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:40:27.730841Z","iopub.execute_input":"2022-07-13T21:40:27.731773Z","iopub.status.idle":"2022-07-13T21:40:32.218490Z","shell.execute_reply.started":"2022-07-13T21:40:27.731718Z","shell.execute_reply":"2022-07-13T21:40:32.217484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca = PCA(n_components=3, random_state=42)\n\nPCA_DF = pd.DataFrame(pca.fit_transform(df_scaled))\nPCA_DF[\"Cluster\"]=preds1\n\n\nfig = plt.figure()\nax = fig.add_subplot(111, projection='3d')\nx = np.array(PCA_DF.iloc[:, 0])\ny = np.array(PCA_DF.iloc[:, 1])\nz = np.array(PCA_DF.iloc[:, 2])\n\nax.scatter(x,y,z, marker=\"s\", c=PCA_DF[\"Cluster\"], s=40, cmap=\"RdBu\")\nplt.title('PCA Cluster BGM')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:49:25.521003Z","iopub.execute_input":"2022-07-13T21:49:25.521698Z","iopub.status.idle":"2022-07-13T21:49:26.978047Z","shell.execute_reply.started":"2022-07-13T21:49:25.521653Z","shell.execute_reply":"2022-07-13T21:49:26.976861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca = PCA(n_components=3, random_state=42)\n\nPCA_DF2 = pd.DataFrame(pca.fit_transform(df_scaled))\nPCA_DF2[\"Cluster\"]=preds2\n\nfig = plt.figure()\nax = fig.add_subplot(111, projection='3d')\nx = np.array(PCA_DF2.iloc[:, 0])\ny = np.array(PCA_DF2.iloc[:, 1])\nz = np.array(PCA_DF2.iloc[:, 2])\n\nax.scatter(x,y,z, marker=\"s\", c=PCA_DF2[\"Cluster\"], s=40, cmap=\"RdBu\")\nplt.title('PCA Cluster KMeans')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:41:26.724404Z","iopub.execute_input":"2022-07-13T21:41:26.724826Z","iopub.status.idle":"2022-07-13T21:41:28.124839Z","shell.execute_reply.started":"2022-07-13T21:41:26.724787Z","shell.execute_reply":"2022-07-13T21:41:28.123627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,4))\nsns.countplot(x=preds1)\nplt.title('Predicted Clusters BGM')","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:49:26.979246Z","iopub.execute_input":"2022-07-13T21:49:26.979556Z","iopub.status.idle":"2022-07-13T21:49:27.204139Z","shell.execute_reply.started":"2022-07-13T21:49:26.979527Z","shell.execute_reply":"2022-07-13T21:49:27.202893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,4))\nsns.countplot(x=preds2)\nplt.title('Predicted Clusters KMeans')","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:40:35.426980Z","iopub.execute_input":"2022-07-13T21:40:35.427501Z","iopub.status.idle":"2022-07-13T21:40:35.664317Z","shell.execute_reply.started":"2022-07-13T21:40:35.427453Z","shell.execute_reply":"2022-07-13T21:40:35.663208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\n---","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv('../input/tabular-playground-series-jul-2022/sample_submission.csv')\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:42:06.895933Z","iopub.execute_input":"2022-07-13T21:42:06.896717Z","iopub.status.idle":"2022-07-13T21:42:06.920214Z","shell.execute_reply.started":"2022-07-13T21:42:06.896674Z","shell.execute_reply":"2022-07-13T21:42:06.919018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_BGM = sub.copy()\nsub_BGM['Predicted']=preds1\nsub_BGM.to_csv('submission_bgm.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:49:27.206171Z","iopub.execute_input":"2022-07-13T21:49:27.206598Z","iopub.status.idle":"2022-07-13T21:49:27.304986Z","shell.execute_reply.started":"2022-07-13T21:49:27.206565Z","shell.execute_reply":"2022-07-13T21:49:27.303992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_kmeans = sub.copy()\nsub_kmeans['Predicted']=preds2\nsub_kmeans.to_csv('submission_kmeans.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:40:35.801862Z","iopub.execute_input":"2022-07-13T21:40:35.802227Z","iopub.status.idle":"2022-07-13T21:40:35.901006Z","shell.execute_reply.started":"2022-07-13T21:40:35.802194Z","shell.execute_reply":"2022-07-13T21:40:35.899838Z"},"trusted":true},"execution_count":null,"outputs":[]}]}