{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Clustering of tabular data\n## Table of Contents\n* [EDA](#1)\n* [Principle Component Analysis](#2)\n* [k-Means Algorithm](#3)\n* [Gaussian Mixture Model](#4)","metadata":{}},{"cell_type":"code","source":"# look for available files\ncat(system('ls -l \"../input/tabular-playground-series-jul-2022\"', intern=TRUE), sep='\\n')","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:52:16.170184Z","iopub.execute_input":"2022-07-13T15:52:16.177611Z","iopub.status.idle":"2022-07-13T15:52:16.323995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# installations\ninstall.packages('factoextra')","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-13T15:52:16.328518Z","iopub.execute_input":"2022-07-13T15:52:16.367909Z","iopub.status.idle":"2022-07-13T15:57:02.367651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# packages\nsuppressPackageStartupMessages(library(tidyverse))\nsuppressPackageStartupMessages(library(DataExplorer))\nsuppressPackageStartupMessages(library(cluster))\nsuppressPackageStartupMessages(library(factoextra))\nsuppressPackageStartupMessages(library(mclust))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-13T15:57:02.371274Z","iopub.execute_input":"2022-07-13T15:57:02.373063Z","iopub.status.idle":"2022-07-13T15:57:03.526010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# configure default plot size\nplot_w <- 8\nplot_h <- 5\noptions(repr.plot.width = plot_w, repr.plot.height = plot_h)\n\n# configure data frame display\noptions(repr.matrix.max.cols=500)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:57:03.529803Z","iopub.execute_input":"2022-07-13T15:57:03.532046Z","iopub.status.idle":"2022-07-13T15:57:03.555275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load data\nt1 <- Sys.time()\ndf <- read.csv('../input/tabular-playground-series-jul-2022/data.csv')\nt2 <- Sys.time()\nprint(t2-t1)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:57:03.557997Z","iopub.execute_input":"2022-07-13T15:57:03.559647Z","iopub.status.idle":"2022-07-13T15:57:12.908663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='1'></a>\n# EDA","metadata":{}},{"cell_type":"code","source":"# id conversion + first overview\ndf$id <- as.factor(df$id)\nstr(df)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:57:12.912705Z","iopub.execute_input":"2022-07-13T15:57:12.914528Z","iopub.status.idle":"2022-07-13T15:57:13.080492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# general info about data\nDataExplorer::introduce(df)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:57:13.083138Z","iopub.execute_input":"2022-07-13T15:57:13.084647Z","iopub.status.idle":"2022-07-13T15:57:14.066354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Plots:","metadata":{}},{"cell_type":"code","source":"# plot features (here all are numerical)\noptions(repr.plot.width = 12, repr.plot.height = 8)\nDataExplorer::plot_histogram(df)\noptions(repr.plot.width = plot_w, repr.plot.height = plot_h)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:57:14.069616Z","iopub.execute_input":"2022-07-13T15:57:14.071372Z","iopub.status.idle":"2022-07-13T15:57:23.981658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Correlations:","metadata":{}},{"cell_type":"code","source":"# plot correlations\noptions(repr.plot.width = 8, repr.plot.height = 8)\nsuppressWarnings(\n    DataExplorer::plot_correlation(df)\n)\noptions(repr.plot.width = plot_w, repr.plot.height = plot_h)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:57:23.984577Z","iopub.execute_input":"2022-07-13T15:57:23.986566Z","iopub.status.idle":"2022-07-13T15:57:25.024823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Basic stats:","metadata":{}},{"cell_type":"code","source":"# basic stats\nfeatures <- setdiff(colnames(df),'id')\nsummary(df[features])","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:57:25.029375Z","iopub.execute_input":"2022-07-13T15:57:25.031432Z","iopub.status.idle":"2022-07-13T15:57:25.237665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Look closer at integer features again:","metadata":{}},{"cell_type":"code","source":"# integer features\nfeatures_int <- c('f_07','f_08','f_09','f_10','f_11','f_12','f_13')","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:57:25.240863Z","iopub.execute_input":"2022-07-13T15:57:25.242472Z","iopub.status.idle":"2022-07-13T15:57:25.258598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot all integer distributions as categorical variables\noptions(repr.plot.width = 14, repr.plot.height = 5)\nfor (f in features_int) {\n    plot(as.factor(df[,f]), main=f)\n    grid()\n}\noptions(repr.plot.width = plot_w, repr.plot.height = plot_h)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:57:25.261305Z","iopub.execute_input":"2022-07-13T15:57:25.262862Z","iopub.status.idle":"2022-07-13T15:57:25.935770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot correlations (integers only)\noptions(repr.plot.width = 8, repr.plot.height = 8)\nsuppressWarnings(\n    DataExplorer::plot_correlation(df[features_int])\n)\noptions(repr.plot.width = plot_w, repr.plot.height = plot_h)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:57:25.938842Z","iopub.execute_input":"2022-07-13T15:57:25.940647Z","iopub.status.idle":"2022-07-13T15:57:26.293427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### QQ-Plots for non-integers:","metadata":{}},{"cell_type":"code","source":"# check for normality using QQ plots\noptions(repr.plot.width = 8, repr.plot.height = 8)\nDataExplorer::plot_qq(df[setdiff(features,features_int)])\noptions(repr.plot.width = plot_w, repr.plot.height = plot_h)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:57:26.297677Z","iopub.execute_input":"2022-07-13T15:57:26.299968Z","iopub.status.idle":"2022-07-13T15:59:12.617297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='2'></a>\n# Principle Component Analysis","metadata":{}},{"cell_type":"code","source":"options(repr.plot.width = 14, repr.plot.height = 12)\nDataExplorer::plot_prcomp(df[features], variance_cap = 1, nrow = 2 , ncol = 3)\noptions(repr.plot.width = plot_w, repr.plot.height = plot_h)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:59:12.621129Z","iopub.execute_input":"2022-07-13T15:59:12.622693Z","iopub.status.idle":"2022-07-13T15:59:18.120140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='3'></a>\n# k-Means Algorithm","metadata":{}},{"cell_type":"code","source":"# standardize data first\ndf_norm <- as.data.frame(scale(df[features]))","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:59:18.124443Z","iopub.execute_input":"2022-07-13T15:59:18.127121Z","iopub.status.idle":"2022-07-13T15:59:18.380400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set.seed(1234321) # make reproducible","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:59:18.383239Z","iopub.execute_input":"2022-07-13T15:59:18.384912Z","iopub.status.idle":"2022-07-13T15:59:18.401552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use \"elbow method\" to get idea about number of clusters\nt1 <- Sys.time()\ndf_sub <- dplyr::sample_n(df_norm, 5000) # use subset => performance issues\nfactoextra::fviz_nbclust(df_sub, kmeans, method = 'wss')\nt2 <- Sys.time()\nprint(t2-t1)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:59:18.406083Z","iopub.execute_input":"2022-07-13T15:59:18.407964Z","iopub.status.idle":"2022-07-13T15:59:42.142030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# run k-means algorithm\nset.seed(1234321) # make reproducible\nn_clust <- 6\nt1 <- Sys.time()\nsuppressWarnings(\nclust <- kmeans(df_norm, \n                centers=n_clust, # number of clusters\n                iter.max=100, # maximum iterations\n                nstart=25  # number of initial (random) configurations tried by algo\n               )\n)\nt2 <- Sys.time()\nprint(t2-t1)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:59:42.146332Z","iopub.execute_input":"2022-07-13T15:59:42.148313Z","iopub.status.idle":"2022-07-13T16:00:49.127461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show (high-dimensional) cluster centers\nclust$centers","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:00:49.130582Z","iopub.execute_input":"2022-07-13T16:00:49.132327Z","iopub.status.idle":"2022-07-13T16:00:49.169180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show cluster distribution\nplot(as.factor(clust$cluster), main='k-means Cluster distribution (count)')\ngrid()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:00:49.172735Z","iopub.execute_input":"2022-07-13T16:00:49.174448Z","iopub.status.idle":"2022-07-13T16:00:49.259446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 2D plot of clusters\noptions(repr.plot.width = 9, repr.plot.height = 9)\ncluster::clusplot(df[features], clus=clust$cluster, color=FALSE,\n                  col.p=clust$cluster)\ngrid()\noptions(repr.plot.width = plot_w, repr.plot.height = plot_h)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:00:49.262172Z","iopub.execute_input":"2022-07-13T16:00:49.263733Z","iopub.status.idle":"2022-07-13T16:00:53.318397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Show some statistics:","metadata":{}},{"cell_type":"code","source":"# some statistics\ncat('Total sum of squares :', clust$totss, '\\n')\ncat('> within Clusters    :', clust$tot.withinss, '\\n')\ncat('> between Clusters   :', clust$betweenss, '\\n')","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:00:53.322389Z","iopub.execute_input":"2022-07-13T16:00:53.324698Z","iopub.status.idle":"2022-07-13T16:00:53.361075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split of sum of squares within clusters\nprint(clust$withinss)\nbarplot(clust$withinss, main='Within-cluster sum of squares')\ngrid()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:00:53.364906Z","iopub.execute_input":"2022-07-13T16:00:53.367537Z","iopub.status.idle":"2022-07-13T16:00:53.452415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='4'></a>\n# Gaussian Mixture Model","metadata":{}},{"cell_type":"code","source":"set.seed(1234321) # make reproducible\nn_clust <- 7\nt1 <- Sys.time()\nclust_gmm <- Mclust(df_norm, G=n_clust)\nt2 <- Sys.time()\nprint(t2-t1)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:00:53.455206Z","iopub.execute_input":"2022-07-13T16:00:53.456788Z","iopub.status.idle":"2022-07-13T16:09:00.675669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary(as.factor(clust_gmm$classification))\nplot(as.factor(clust_gmm$classification), main='GMM Cluster distribution (count)')\ngrid()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:09:00.678311Z","iopub.execute_input":"2022-07-13T16:09:00.679799Z","iopub.status.idle":"2022-07-13T16:09:00.933789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 2D plot of clusters\noptions(repr.plot.width = 9, repr.plot.height = 9)\ncluster::clusplot(df[features], clus=clust_gmm$classification, color=FALSE,\n                  col.p=clust_gmm$classification)\ngrid()\noptions(repr.plot.width = plot_w, repr.plot.height = plot_h)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:21:04.205677Z","iopub.execute_input":"2022-07-13T16:21:04.207690Z","iopub.status.idle":"2022-07-13T16:21:08.548938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Try to measure feature impact by interpreting clustering as multiclass problem:","metadata":{}},{"cell_type":"code","source":"# add cluster variable => our target for the following\ndf$cluster_gmm <- clust_gmm$classification\ndf$cluster_gmm <- as.factor(df$cluster_gmm)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:13:41.292083Z","iopub.execute_input":"2022-07-13T16:13:41.294754Z","iopub.status.idle":"2022-07-13T16:13:41.417793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# start H2O\nlibrary(h2o)\nh2o.init()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:11:50.006599Z","iopub.execute_input":"2022-07-13T16:11:50.008446Z","iopub.status.idle":"2022-07-13T16:11:57.730712Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# upload data\ndf_hex <- as.h2o(df)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:13:48.355784Z","iopub.execute_input":"2022-07-13T16:13:48.357951Z","iopub.status.idle":"2022-07-13T16:13:51.768621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train model\nt1 <- Sys.time()\nfit <- h2o.glm(x=features, y='cluster_gmm',\n               training_frame = df_hex,\n               nfolds=5)\nt2 <- Sys.time()\nprint(t2-t1)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:22:48.210020Z","iopub.execute_input":"2022-07-13T16:22:48.213409Z","iopub.status.idle":"2022-07-13T16:24:58.314312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fit","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:25:06.885483Z","iopub.execute_input":"2022-07-13T16:25:06.887559Z","iopub.status.idle":"2022-07-13T16:25:06.932983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# variable importance of model\nh2o.varimp_plot(fit,30)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:25:17.750510Z","iopub.execute_input":"2022-07-13T16:25:17.752661Z","iopub.status.idle":"2022-07-13T16:25:17.835065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h2o.varimp(fit)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:25:20.111731Z","iopub.execute_input":"2022-07-13T16:25:20.113677Z","iopub.status.idle":"2022-07-13T16:25:20.156785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submissions","metadata":{}},{"cell_type":"code","source":"df_sub <- read.csv('../input/tabular-playground-series-jul-2022/sample_submission.csv')\nstr(df_sub)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:09:00.936153Z","iopub.execute_input":"2022-07-13T16:09:00.937599Z","iopub.status.idle":"2022-07-13T16:09:01.034797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub_kmeans <- df_sub\ndf_sub_kmeans$Predicted <- clust$cluster\nsummary(df_sub_kmeans)\nwrite_delim(df_sub_kmeans, file='submission_kmeans.csv', delim=',')","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:09:01.037125Z","iopub.execute_input":"2022-07-13T16:09:01.038679Z","iopub.status.idle":"2022-07-13T16:09:01.178661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub_gmm <- df_sub\ndf_sub_gmm$Predicted <- clust_gmm$classification\nsummary(df_sub_gmm)\nwrite_delim(df_sub_gmm, file='submission_gmm.csv', delim=',')","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:09:01.181418Z","iopub.execute_input":"2022-07-13T16:09:01.182903Z","iopub.status.idle":"2022-07-13T16:09:01.248131Z"},"trusted":true},"execution_count":null,"outputs":[]}]}