{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

\n
\n

1. Introduction

\n
\n\n
\n

1.1 | Goal

\n
\n\nIn this challenge, we are given a dataset where each row belongs to a particular cluster. Our job is to predict the cluster each row belongs to.
\n","metadata":{}},{"cell_type":"markdown","source":"
\n

1.2 | Load data and preprocessing

\n
","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport seaborn as sns\nfrom sklearn.metrics import silhouette_samples\nfrom sklearn.cluster import KMeans, AgglomerativeClustering\nfrom matplotlib import cm\nimport seaborn as sns\nimport missingno as no","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:35:14.405640Z","iopub.execute_input":"2022-07-28T14:35:14.406563Z","iopub.status.idle":"2022-07-28T14:35:19.440423Z","shell.execute_reply.started":"2022-07-28T14:35:14.406456Z","shell.execute_reply":"2022-07-28T14:35:19.439049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data= pd.read_csv(\"../input/tabular-playground-series-jul-2022/data.csv\")\npd.set_option('display.max_columns', None)\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:37:17.275906Z","iopub.execute_input":"2022-07-28T14:37:17.276508Z","iopub.status.idle":"2022-07-28T14:37:18.246550Z","shell.execute_reply.started":"2022-07-28T14:37:17.276453Z","shell.execute_reply":"2022-07-28T14:37:18.245109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.drop('id', axis=1).describe().T.style.bar(subset=['mean'], color=px.colors.qualitative.Pastel[4])\\\n .background_gradient(subset=['std'], cmap='Greens')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:37:18.649304Z","iopub.execute_input":"2022-07-28T14:37:18.650360Z","iopub.status.idle":"2022-07-28T14:37:18.935571Z","shell.execute_reply.started":"2022-07-28T14:37:18.650315Z","shell.execute_reply":"2022-07-28T14:37:18.934161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# any missing values\nprint(\"Data Shape: There are {:,.0f} rows and {:,.0f} columns.\\nMissing values = {}, Duplicates = {}.\\n\".\n format(data.shape[0], data.shape[1],data.isna().sum().sum(), data.duplicated().sum()))","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:37:19.450765Z","iopub.execute_input":"2022-07-28T14:37:19.451248Z","iopub.status.idle":"2022-07-28T14:37:19.681109Z","shell.execute_reply.started":"2022-07-28T14:37:19.451216Z","shell.execute_reply":"2022-07-28T14:37:19.679777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(28,16))\ncorr_mat = data.drop([\"id\"], axis=1).corr()\nmask = np.zeros_like(corr_mat, dtype=np.bool)\nmask[np.triu_indices_from(mask)] = True\nsns.heatmap(corr_mat, mask=mask, square=True, ax=ax, linewidths=0.1,center=0, cmap='coolwarm_r');","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:37:20.290972Z","iopub.execute_input":"2022-07-28T14:37:20.291561Z","iopub.status.idle":"2022-07-28T14:37:21.463408Z","shell.execute_reply.started":"2022-07-28T14:37:20.291512Z","shell.execute_reply":"2022-07-28T14:37:21.462056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(20,40))\n\nfor i, col in enumerate(data.columns):\n plt.subplot(25,3, i+1)\n sns.kdeplot(data[col], fill=True, color='red')\n \nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:37:21.466341Z","iopub.execute_input":"2022-07-28T14:37:21.467312Z","iopub.status.idle":"2022-07-28T14:37:39.285270Z","shell.execute_reply.started":"2022-07-28T14:37:21.467252Z","shell.execute_reply":"2022-07-28T14:37:39.284053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# scaling all the columns except ID\nfrom sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\ndata_scaled = scaler.fit_transform(data.drop(columns=\"id\"))\ndata_scaled_df=pd.DataFrame(data_scaled, index=data.index, columns=data.drop(columns=\"id\").columns)\ndata_scaled_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:37:39.287254Z","iopub.execute_input":"2022-07-28T14:37:39.287744Z","iopub.status.idle":"2022-07-28T14:37:39.394570Z","shell.execute_reply.started":"2022-07-28T14:37:39.287697Z","shell.execute_reply":"2022-07-28T14:37:39.393302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### All the Kagglers almost agreed that there are seven clusters, we will keep it same.","metadata":{}},{"cell_type":"code","source":"top_features = [f\"f_{i:02d}\" for i in list(range(7, 14)) + list(range(22, 29))]\ntop_features","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:37:39.396294Z","iopub.execute_input":"2022-07-28T14:37:39.397600Z","iopub.status.idle":"2022-07-28T14:37:39.407621Z","shell.execute_reply.started":"2022-07-28T14:37:39.397558Z","shell.execute_reply":"2022-07-28T14:37:39.406059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

3.0| Clustering the data - Unsupervised

\n
\n\nClustering of the data can be performed using either all the columns or reduced dimentions. In our case we can use output of the PCA or Autoencoder.","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import PowerTransformer\nr=PowerTransformer()\nX=r.fit_transform(data_scaled_df[top_features])\ndf_trans=pd.DataFrame(X,columns=top_features)\ndf_trans.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:37:39.411732Z","iopub.execute_input":"2022-07-28T14:37:39.412230Z","iopub.status.idle":"2022-07-28T14:37:41.130356Z","shell.execute_reply.started":"2022-07-28T14:37:39.412190Z","shell.execute_reply":"2022-07-28T14:37:41.129049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install sklego\nfrom sklego.mixture import BayesianGMMClassifier","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:37:41.132571Z","iopub.execute_input":"2022-07-28T14:37:41.133057Z","iopub.status.idle":"2022-07-28T14:37:56.898991Z","shell.execute_reply.started":"2022-07-28T14:37:41.132993Z","shell.execute_reply":"2022-07-28T14:37:56.897612Z"},"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://www.kaggle.com/code/mehrankazeminia/1-3-tps22jul-pseudo-supervised-clustering\n#https://www.kaggle.com/code/thedevastator/the-fine-art-of-fine-tuning\nfrom tqdm import tqdm\nsub_prime = pd.read_csv('../input/the-fine-art-of-fine-tuning/submission.csv', index_col=[0])\nX = np.array(df_trans)\ny = np.ravel(sub_prime)\n\nbgm = BayesianGMMClassifier(n_components=7, random_state=123, tol=0.001, max_iter=200, n_init=3, verbose=0)\n\nbgm.fit(X,y)\npred = bgm.predict(X)\ny = pred","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:37:56.902399Z","iopub.execute_input":"2022-07-28T14:37:56.903379Z","iopub.status.idle":"2022-07-28T14:40:16.050250Z","shell.execute_reply.started":"2022-07-28T14:37:56.903329Z","shell.execute_reply":"2022-07-28T14:40:16.048668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

4.0| Supervised clustering - DAE

\n
\n","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.compose import make_column_transformer\nfrom sklearn.model_selection import StratifiedKFold\nfrom tensorflow.keras.models import Model, Sequential\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler, MinMaxScaler\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\nimport gc\nRUNS = 1\nFOLDS = 15 # 10 # For debugging, change this when you use this notebook!\nSEED = 2021\nEPOCHS = 100 #0 # 90 # For debugging, change this when you use this notebook!\nVERBOSE = 10\nLR = 0.00012\nBATCH_SIZE =64 \ntrain_df=df_trans.copy()\ntrain_df['target'] = pred\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:40:16.054099Z","iopub.execute_input":"2022-07-28T14:40:16.054920Z","iopub.status.idle":"2022-07-28T14:40:22.105113Z","shell.execute_reply.started":"2022-07-28T14:40:16.054876Z","shell.execute_reply":"2022-07-28T14:40:22.103946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SwapRowNoise:\n def __init__(self, proba):\n self.proba = proba\n \n def apply(self, X):\n random_idx = np.random.randint(low=0, high=X.shape[0], size=1)[0]\n swap_matrix = K.random_bernoulli(shape=X.shape, p=self.proba) * tf.ones(shape=X.shape) \n corrupted = tf.where(swap_matrix==1, X.iloc[random_idx], X)\n return corrupted.numpy()\n \n# create autoencoder\nclass EncodingLayer(layers.Layer):\n def __init__(self, encoding_dim, activation='relu'):\n super().__init__()\n self.enc1 = layers.Dense(encoding_dim, activation)\n self.enc2 = layers.Dense(encoding_dim, activation)\n self.enc3 = layers.Dense(encoding_dim, activation)\n self.concat = layers.Concatenate()\n \n def call(self, inputs):\n enc1 = self.enc1(inputs)\n enc2 = self.enc2(enc1)\n enc3 = self.enc3(enc2)\n merge = self.concat([enc1, enc2, enc3])\n return merge\n\nclass DecodingLayer(layers.Layer):\n def __init__(self, num_outputs, activation='linear'):\n super().__init__()\n self.dec = layers.Dense(num_outputs, activation)\n \n def call(self, inputs):\n return self.dec(inputs) \n\n# create custom layer\nclass DenseBlock(layers.Layer):\n def __init__(self, units, activation='relu', dropout_rate=0, l2=0):\n super().__init__()\n self.dense = layers.Dense(\n units, activation,\n kernel_regularizer=keras.regularizers.l2(l2)\n )\n self.batchn = layers.BatchNormalization()\n self.dropout = layers.Dropout(dropout_rate)\n \n def call(self, inputs):\n x = self.dense(inputs)\n x = self.batchn(x)\n x = self.dropout(x)\n return x","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:40:22.109424Z","iopub.execute_input":"2022-07-28T14:40:22.110362Z","iopub.status.idle":"2022-07-28T14:40:22.133019Z","shell.execute_reply.started":"2022-07-28T14:40:22.110331Z","shell.execute_reply":"2022-07-28T14:40:22.130628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_oof_list = []\ntest_pred_list = []\n#TPU = True\nTPU = False\nfrom tensorflow.keras import backend as K\ntry:\n tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n tf.config.experimental_connect_to_cluster(tpu)\n tf.tpu.experimental.initialize_tpu_system(tpu)\n strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n strategy = tf.distribute.get_strategy()\n \nprint('Replicas:', strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:40:22.135208Z","iopub.execute_input":"2022-07-28T14:40:22.135971Z","iopub.status.idle":"2022-07-28T14:40:22.165737Z","shell.execute_reply.started":"2022-07-28T14:40:22.135927Z","shell.execute_reply":"2022-07-28T14:40:22.164247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features=top_features\nle = LabelEncoder()\ntrain_df[\"target\"] = le.fit_transform(train_df[\"target\"])\ntarget = le.fit_transform(train_df.target)\n\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:40:22.167340Z","iopub.execute_input":"2022-07-28T14:40:22.167622Z","iopub.status.idle":"2022-07-28T14:40:22.205465Z","shell.execute_reply.started":"2022-07-28T14:40:22.167597Z","shell.execute_reply":"2022-07-28T14:40:22.203900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder, StandardScaler, MinMaxScaler\n\ndata_pipe_transformer = make_pipeline(\n StandardScaler()\n)\n\npreprocessor = make_column_transformer(\n (data_pipe_transformer, features)\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:40:22.207276Z","iopub.execute_input":"2022-07-28T14:40:22.207898Z","iopub.status.idle":"2022-07-28T14:40:22.216036Z","shell.execute_reply.started":"2022-07-28T14:40:22.207858Z","shell.execute_reply":"2022-07-28T14:40:22.214078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create Auto-Encoder NN\nclass AutoEncoder(keras.Model):\n def __init__(self, encoding_dim, num_outputs, activation='relu'):\n super().__init__()\n self.encoder = EncodingLayer(encoding_dim, activation,)\n self.decoder = DecodingLayer(num_outputs)\n \n def call(self, inputs):\n encoder = self.encoder(inputs)\n decoder = self.decoder(encoder)\n return decoder\n \n def get_encoder(self):\n return self.encoder\n \n# Create Fully-Connected NN\nclass MLP(keras.Model):\n def __init__(self, hidden_layers, autoencoder, activation='relu', dropout_rate=0, l2=0):\n super().__init__()\n self.encoder = autoencoder.get_encoder()\n self.hidden_layers = [DenseBlock(units, activation, l2) for units in hidden_layers]\n self.softmax = layers.Dense(units=len(le.classes_), activation='softmax')\n self.concat = layers.Concatenate()\n \n def call(self, inputs):\n x = self.encoder(inputs)\n for layer in self.hidden_layers:\n x = layer(x)\n x = self.softmax(x)\n return x ","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:40:22.218110Z","iopub.execute_input":"2022-07-28T14:40:22.219639Z","iopub.status.idle":"2022-07-28T14:40:22.233539Z","shell.execute_reply.started":"2022-07-28T14:40:22.219598Z","shell.execute_reply":"2022-07-28T14:40:22.232080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(2021)\ntf.random.set_seed(2021)\n\nscore_list, history_list = [], []\noof_list = [np.full((len(train_df), len(le.classes_)), -1.0, dtype='float32') for run in range(RUNS)]\n# oof_list = [np.full((len(train_df), len(le.classes_)), -1.0, dtype='float32') for run in range(RUNS)]\nfor run in range(RUNS):\n kf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=1)\n for fold, (train_idx, val_idx) in enumerate(kf.split(train_df, y=train_df.target)):\n print(f\"Fold {run}.{fold}\")\n K.clear_session()\n \n X_tr = train_df.iloc[train_idx]\n X_va = train_df.iloc[val_idx]\n y_tr = target[train_idx]\n y_va = target[val_idx]\n X_tr = X_tr[features]\n X_va = X_va[features]\n\n# X_tr[features] = preprocessor.fit_transform(X_tr)\n# X_va[features] = preprocessor.transform(X_va)\n\n noise_maker = SwapRowNoise(0.20)\n X_noise_train = noise_maker.apply(X_tr)\n X_noise_valid = noise_maker.apply(X_va)\n\n # TPU model\n if TPU:\n with strategy.scope():\n ae = AutoEncoder(encoding_dim=128,num_outputs=X_tr.shape[-1],activation='relu')\n ae.compile(optimizer=keras.optimizers.Adam(learning_rate=1e-3),loss=keras.losses.MeanSquaredError())\n lr = ReduceLROnPlateau(monitor = \"val_loss\", factor = 0.5, patience = 5, verbose = VERBOSE)\n es = EarlyStopping(monitor = \"val_loss\", patience = 10, verbose = VERBOSE, mode = \"min\", restore_best_weights = True)\n history_ae = ae.fit(X_noise_train, X_tr,validation_data=(X_noise_valid, X_va),epochs=EPOCHS,batch_size=BATCH_SIZE,validation_batch_size=BATCH_SIZE,shuffle=True,verbose=False,callbacks=[lr,es])\n\n with strategy.scope():\n model = MLP(hidden_layers=[32,32,32],autoencoder=ae,activation='relu')\n model.compile(optimizer=keras.optimizers.Adam(learning_rate=1e-3),loss=\"sparse_categorical_crossentropy\",metrics=['acc'])\n\n else:\n # GPU model\n ae = AutoEncoder(encoding_dim=128,num_outputs=X_tr.shape[-1],activation='relu')\n ae.compile(optimizer=keras.optimizers.Adam(learning_rate=1e-3),loss=keras.losses.MeanSquaredError())\n lr = ReduceLROnPlateau(monitor = \"val_loss\", factor = 0.5, patience = 5, verbose = VERBOSE)\n es = EarlyStopping(monitor = \"val_loss\", patience = 10, verbose = VERBOSE, mode = \"min\", restore_best_weights = True)\n history_ae = ae.fit(X_noise_train, X_tr,validation_data=(X_noise_valid, X_va),epochs=EPOCHS,batch_size=BATCH_SIZE,validation_batch_size=BATCH_SIZE,shuffle=True,verbose=False,callbacks=[lr,es])\n\n model = MLP(hidden_layers=[32,32],autoencoder=ae,activation='relu')\n model.compile(optimizer=keras.optimizers.Adam(learning_rate=1e-3),loss=\"sparse_categorical_crossentropy\",metrics=['acc'])\n\n lr = ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5,\n patience=5, verbose=VERBOSE)\n\n es = EarlyStopping(monitor=\"val_acc\", patience=10,\n verbose=VERBOSE, mode=\"max\",\n restore_best_weights=True)\n\n history = model.fit(X_tr, y_tr,\n validation_data=(X_va, y_va),\n epochs=EPOCHS,\n verbose=VERBOSE,\n batch_size=BATCH_SIZE,\n validation_batch_size=BATCH_SIZE,\n shuffle=True,\n callbacks=[lr, es])\n history_list.append(history.history)\n\n y_va_pred = model.predict(X_va, batch_size=len(X_va))\n oof_list[run][val_idx] = y_va_pred\n y_va_pred = le.inverse_transform(np.argmax(y_va_pred, axis=1))\n test_pred_list.append(model.predict(df_trans[features], batch_size=BATCH_SIZE))\n del model, y_va_pred\n gc.collect()\ntotal_oof_list += oof_list","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:42:44.942797Z","iopub.execute_input":"2022-07-28T14:42:44.943329Z","iopub.status.idle":"2022-07-28T16:02:44.232399Z","shell.execute_reply.started":"2022-07-28T14:42:44.943301Z","shell.execute_reply":"2022-07-28T16:02:44.231037Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub=pd.read_csv(\"../input/tabular-playground-series-jul-2022/sample_submission.csv\")\nsub['Predicted']= le.inverse_transform(np.argmax(sum(test_pred_list), axis=1)) \nsub.to_csv(\"submission_highnoise.csv\",index=False)\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T16:08:39.912295Z","iopub.execute_input":"2022-07-28T16:08:39.912769Z","iopub.status.idle":"2022-07-28T16:08:40.114393Z","shell.execute_reply.started":"2022-07-28T16:08:39.912738Z","shell.execute_reply":"2022-07-28T16:08:40.112960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = plt.figure(figsize=(12, 6))\ncluster=sub['Predicted'].value_counts().sort_index(ascending = False )\nsns.barplot(x=cluster.index,y=cluster,color=\"salmon\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T16:08:58.149476Z","iopub.execute_input":"2022-07-28T16:08:58.149878Z","iopub.status.idle":"2022-07-28T16:08:58.391383Z","shell.execute_reply.started":"2022-07-28T16:08:58.149825Z","shell.execute_reply":"2022-07-28T16:08:58.390045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}