{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport os\nimport sys\nimport warnings\n\n\nfrom sklearn.decomposition import PCA\nfrom sklearn.cluster import KMeans\nfrom sklearn.metrics import silhouette_score\nfrom IPython.display import display, HTML\nfrom tqdm import tqdm\n\nwarnings.filterwarnings('ignore')\nplt.style.use('fivethirtyeight')","metadata":{"execution":{"iopub.status.busy":"2022-07-12T14:46:26.929392Z","iopub.execute_input":"2022-07-12T14:46:26.930108Z","iopub.status.idle":"2022-07-12T14:46:28.661439Z","shell.execute_reply.started":"2022-07-12T14:46:26.929946Z","shell.execute_reply":"2022-07-12T14:46:28.660443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/tabular-playground-series-jul-2022/data.csv\", index_col=\"id\")\nprint(f'Number of samples in the data : {df.shape[0]}')\nprint(f'Number of attributes in the data : {df.shape[1]}')","metadata":{"execution":{"iopub.status.busy":"2022-07-12T14:46:28.663647Z","iopub.execute_input":"2022-07-12T14:46:28.664348Z","iopub.status.idle":"2022-07-12T14:46:30.088527Z","shell.execute_reply.started":"2022-07-12T14:46:28.664305Z","shell.execute_reply":"2022-07-12T14:46:30.087427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HTML(df.head().to_html())","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-12T14:46:30.089979Z","iopub.execute_input":"2022-07-12T14:46:30.090322Z","iopub.status.idle":"2022-07-12T14:46:30.113919Z","shell.execute_reply.started":"2022-07-12T14:46:30.090293Z","shell.execute_reply":"2022-07-12T14:46:30.112744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Gather Information on Missing Data","metadata":{}},{"cell_type":"code","source":"miss = pd.DataFrame((df.isna().sum() / df.isna().count()) * 100).rename(columns={0:'percent_missing'}).sort_values(by='percent_missing')\nmiss[miss['percent_missing'] > 0]","metadata":{"execution":{"iopub.status.busy":"2022-07-12T14:46:30.116655Z","iopub.execute_input":"2022-07-12T14:46:30.117010Z","iopub.status.idle":"2022-07-12T14:46:30.153005Z","shell.execute_reply.started":"2022-07-12T14:46:30.116980Z","shell.execute_reply":"2022-07-12T14:46:30.151441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So the dataset is a clean one.","metadata":{}},{"cell_type":"markdown","source":"# Dataset Statistic","metadata":{}},{"cell_type":"code","source":"df.info()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-12T14:46:30.154820Z","iopub.execute_input":"2022-07-12T14:46:30.155214Z","iopub.status.idle":"2022-07-12T14:46:30.178997Z","shell.execute_reply.started":"2022-07-12T14:46:30.155183Z","shell.execute_reply":"2022-07-12T14:46:30.177634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are $7$ categorical features in this dataset, and $22$ continuous features.","metadata":{}},{"cell_type":"code","source":"HTML(df[df.select_dtypes('float').columns].describe().style\\\n     .set_table_styles([{\"selector\":\"th\", \"props\":\"background-color:darkslateblue; font-size:1.5em; font-weight:bold;\"},\n                       {\"selector\":\"td:hover\", \"props\":\"background-color:blueviolet\"},\n                       {\"selector\":\"td\", \"props\":\"border:1px solid darkorchid; text-align:center;\"}])\n     .to_html())","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-12T14:46:30.180735Z","iopub.execute_input":"2022-07-12T14:46:30.181247Z","iopub.status.idle":"2022-07-12T14:46:30.415928Z","shell.execute_reply.started":"2022-07-12T14:46:30.181216Z","shell.execute_reply":"2022-07-12T14:46:30.414640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since our data size is large, therefore by the *Central Limit Theorem* it can be derived that the above continuous features will have a normal distribution with a mean of $\\mu$ and a variance $\\sigma^2$ respectively.","metadata":{}},{"cell_type":"code","source":"HTML(df[df.select_dtypes('int').columns].describe().style\\\n     .set_table_styles([{\"selector\":\"th\", \"props\":\"background-color:crimson; font-size:1.5em; font-weight:bold;\"},\n                       {\"selector\":\"td:hover\", \"props\":\"background-color:tomato;\"},\n                        {\"selector\":\"td\", \"props\":\"border:1px solid orangered; text-align:center;\"}])\n    .to_html())","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-12T14:46:30.417506Z","iopub.execute_input":"2022-07-12T14:46:30.417952Z","iopub.status.idle":"2022-07-12T14:46:30.473593Z","shell.execute_reply.started":"2022-07-12T14:46:30.417908Z","shell.execute_reply":"2022-07-12T14:46:30.472317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratorty Data Analysis","metadata":{}},{"cell_type":"markdown","source":"## - Continuous Features","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(2, 11, figsize=(12, 8))\nplt.subplots_adjust(top=1., right=2.5, wspace=.5, hspace=.5)\n\ncolors = np.repeat([\"blueviolet\", \"crimson\", \"goldenrod\", \"forestgreen\", \"darkorchid\", \"orangered\", \"dodgerblue\", \n          \"hotpink\", \"indigo\", \"firebrick\", \"gold\"], 2)\n\nfor i, col in enumerate(df.select_dtypes('float').columns):\n    sns.kdeplot(x=col, data=df, ax=ax[i%2, i//2], color=colors[i], fill=True)\n    \nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-12T14:46:30.476806Z","iopub.execute_input":"2022-07-12T14:46:30.477254Z","iopub.status.idle":"2022-07-12T14:46:42.646221Z","shell.execute_reply.started":"2022-07-12T14:46:30.477209Z","shell.execute_reply":"2022-07-12T14:46:42.645141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"All of the features have a symmetric distribution, with a $\\mu \\approx 0$","metadata":{}},{"cell_type":"markdown","source":"## - Categorical Features","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(12, 8))\n\nax1 = plt.subplot2grid((3, 3), (0, 0), colspan=2)\nax2 = plt.subplot2grid((3, 3), (0, 2), colspan=2)\nax3 = plt.subplot2grid((3, 3), (1, 0), colspan=1)\nax4 = plt.subplot2grid((3, 3), (1, 1), colspan=1)\nax5 = plt.subplot2grid((3, 3), (1, 2), colspan=1)\nax6 = plt.subplot2grid((3, 3), (2, 0), colspan=2)\nax7 = plt.subplot2grid((3, 3), (2, 2), colspan=1)\n\nplt.subplots_adjust(top=1., right=2.5, wspace=.5, hspace=.5)\n\naxes = [ax1, ax2, ax3, ax4, ax5, ax6, ax7]\ncolors = [\"blueviolet\", \"crimson\", \"goldenrod\", \"forestgreen\", \"darkorchid\", \"orangered\", \"dodgerblue\"]\n\nfor i, c in enumerate(df.select_dtypes('int').columns):\n    sns.countplot(x=c, data=df, ax=axes[i], color=colors[i])\n    axes[i].tick_params(labelrotation=45, axis='x')\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-12T14:46:42.647961Z","iopub.execute_input":"2022-07-12T14:46:42.648552Z","iopub.status.idle":"2022-07-12T14:46:44.672569Z","shell.execute_reply.started":"2022-07-12T14:46:42.648518Z","shell.execute_reply":"2022-07-12T14:46:44.671682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The Frequency distributions represents right skewed data, with heavy tail at the right side.","metadata":{}},{"cell_type":"markdown","source":"## Correlation Map","metadata":{}},{"cell_type":"code","source":"corr = df[df.select_dtypes('float').columns].corr()\nplt.figure(figsize=(15, 10))\nmask = np.tril(np.ones_like(corr))\nsns.heatmap(corr, mask=mask, vmin=-1, vmax=1., linewidth=2, cmap=\"coolwarm\")\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-12T14:46:44.674183Z","iopub.execute_input":"2022-07-12T14:46:44.674556Z","iopub.status.idle":"2022-07-12T14:46:45.302789Z","shell.execute_reply.started":"2022-07-12T14:46:44.674523Z","shell.execute_reply":"2022-07-12T14:46:45.301654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So most of the attributes have *zero* **correlation** or a *weak* **correlation**","metadata":{}},{"cell_type":"markdown","source":"## Outlier Analysis \n\nSince the KMeans algorithm uses the mean $\\mu$ to define the centroids of the cluster it is sensitive to the presence of outliers in the data.","metadata":{}},{"cell_type":"markdown","source":"### Univariate Outlier Detection  (Inspired from notebook : [Outliers+EDA+Clustering Tutorial](https://www.kaggle.com/code/javigallego/outliers-eda-clustering-tutorial/notebook))\n\n**Grubbs Test**\n\n\nThe Grubbs Test is used to detect if there are any outliers in a dataset. The hypothesis for the dataset are:\n$$\n\\begin{align*}\nH_0 &: \\text{There are no outliers in the dataset} \\\\\nH_A &: \\text{There is exactly one outlier in the dataset}\n\\end{align*}\n$$\n\nThe Grubbs Test statistic is defined as follow:\n$$\n\\begin{equation}\nG_{calculated} = \\frac{max|X_i  - bar{X}|}{SD}\n\\end{equation}\n$$\n\nwhere $\\bar{X}$ represents the *mean* of the sample and $SD$ represents the *standard deviation* of the sample.\n\nThe rejection region of the test is defined as:\n$$\n\\begin{equation}\nG_{critical} = \\frac{(N - 1)}{\\sqrt{N}}\\sqrt{\\frac{t_{\\alpha/2N, N-2} ^ 2}{N - 2 + t_{\\alpha/2N, N-2}^2}}\n\\end{equation}\n$$\n\nWe reject the null hypothesis with strong evidence if $G_{calculated} > G_{critical}$","metadata":{}},{"cell_type":"code","source":"import scipy.stats as stats\nfrom termcolor import colored\n\n\ndef grubbs_test(data, attr):\n    \"\"\"\n    Function to perform the Grubbs's Test on the data\n    \n    Parameters:\n    ----------\n    data : pandas.DataFrame\n        Represents the data with which we are working.\n        \n    attr : str\n        Represents the attribute name\n        \n        \n    Returns:\n    --------\n    is_rejected : bool\n        Represents whether the null hypothesis was accepted or rejected.\n        \n    \"\"\"\n    \n    n = data[col].shape[0]\n    mu = data[col].mean()\n    std = data[col].std()\n    \n    \n    test_stat = max(np.abs(data[col] - mu)) / std\n    t_val = stats.t.ppf(1 - 0.05/(2 * n), n - 2)\n    \n    critical_val = (n - 1)/np.sqrt(n) * np.sqrt((t_val**2)/(n - 2 + t_val**2))\n    \n    is_rejected = True if test_stat >= critical_val else False\n    \n    return is_rejected","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-12T14:46:45.305678Z","iopub.execute_input":"2022-07-12T14:46:45.306565Z","iopub.status.idle":"2022-07-12T14:46:45.319693Z","shell.execute_reply.started":"2022-07-12T14:46:45.306515Z","shell.execute_reply":"2022-07-12T14:46:45.318505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out_attr = list()\nfor col in df.columns:\n    grub_res = grubbs_test(df, col)\n    if grub_res:\n        print(f'{col} has {colored(\"Rejected\", \"red\")} the null hypothesis')\n        out_attr.append(col)\n    else:\n        print(f'{col} has {colored(\"Accepted\", \"green\")} the null hypothesis')\n        ","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-12T14:46:45.321422Z","iopub.execute_input":"2022-07-12T14:46:45.321820Z","iopub.status.idle":"2022-07-12T14:46:45.817129Z","shell.execute_reply.started":"2022-07-12T14:46:45.321791Z","shell.execute_reply":"2022-07-12T14:46:45.815927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Z-score based outlier detection**\n\nThe *Z-score* is a method to detect outliers in any univariate data. The **idea** of this approach is that points which are far away from the distribution *mean* $\\mu$ are considered to be an outlier.\n\nThe **algorithm** of Z-score based outlier detection can be summarized as follow:\n\n* Calculate the mean $\\mu$ of the data\n* Calculate the standard deviation $\\sigma$ of the data\n* Calculate the z-score as follow $$z_i = \\frac{x_i - \\mu}{\\sigma}$$ \n* Select a threshold (say 4 standard deviation from the mean)\n* Consider a point to be an outlier if $z_i > \\text{threshold}$\n\n<div style=\"text-align:center\">\n    <img src=\"https://cdn.scribbr.com/wp-content/uploads/2020/10/normal-distribution.png\"/>\n    <figcaption style=\"font-style:italic\">Source : Scribbr</figcaption>\n</div>\n\nSo, from the above image it is clear that outliers are points with very little probability of occuring. Because any point which is $>3\\sigma$ has an extremely low probability of occuring.","metadata":{}},{"cell_type":"code","source":"def z_score_outlier_detect(data, attr, thresh=3): \n    \"\"\"\n    Function to determine outliers based on Z-Score\n    \n    Parameters:\n    ----------\n    data : pandas.DataFrame\n        Represents the data with which we are working.\n        \n    attr : str\n        Represents the attribute name\n        \n    thresh : int\n        Represents the threshold values (in this case the number of stnd deviations from the mean)\n        \n    Returns:\n    -------\n    out : array-like\n        Represents the indices of outlier points in the data.\n    \"\"\"\n    \n    mu = data[attr].mean()\n    std = data[attr].std()\n    \n    z_score = (data[attr] - mu) / std\n    out = np.where(np.abs(z_score) > thresh)[0]\n    \n    return out","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-12T14:46:45.818365Z","iopub.execute_input":"2022-07-12T14:46:45.818696Z","iopub.status.idle":"2022-07-12T14:46:45.825993Z","shell.execute_reply.started":"2022-07-12T14:46:45.818668Z","shell.execute_reply":"2022-07-12T14:46:45.824761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outlier_stats_z_score = list()\nfor col in out_attr:\n    _out = z_score_outlier_detect(df, col)\n    \n    outlier_stats_z_score.append((col,len(_out) / df[col].shape[0] * 100))\n    \npd.DataFrame.from_records(outlier_stats_z_score, columns=[\"attributes\", \"percent_outliers\"])\\\n.set_index(\"attributes\")\\\n.sort_values(by=\"percent_outliers\").plot(kind=\"bar\", figsize=(12, 8), title=\"Percentage of Outliers in each attributes\", ylabel=\"percentage of outliers\", legend=False)\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-12T14:46:45.827793Z","iopub.execute_input":"2022-07-12T14:46:45.828523Z","iopub.status.idle":"2022-07-12T14:46:46.060886Z","shell.execute_reply.started":"2022-07-12T14:46:45.828458Z","shell.execute_reply":"2022-07-12T14:46:46.059733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Isolation Forest**\n\nIsolation Forest is another algorithm to detect outliers in any univariate data. The algorithm is based on random trees. \n\n The main idea of this algorithm is *outliers* will be quickly isolated compared to normal examples.\n\nThe overall algorithm can be summarized as follow:\n\n* Grow a tree where each decision stump represents a random split of a random feature.\n* Stop when each example is *isolated* (each leaf of the tree has only one example).\n* The *isolation score* is the *depth* before an example gets isolated.\n\n\n<div style=\"text-align:center\">\n    <img src=\"https://upload.wikimedia.org/wikipedia/commons/thumb/c/ce/Isolating_a_Non-Anomalous_Point.png/600px-Isolating_a_Non-Anomalous_Point.png\"/>\n    <figcaption style=\"font-style:italic\">Source : Wikipedia</figcaption>\n</div>","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import IsolationForest\n\noutliers_iso_forest = list()\n\nfor col in out_attr:\n    model = IsolationForest(bootstrap=True, random_state=42)\n    X = df[col].values.reshape(-1, 1)\n    model.fit(X)\n    \n    predictions = model.predict(X)\n    percent_outliers = np.where(predictions == -1)[0].shape[0] / X.shape[0] * 100 \n    \n    outliers_iso_forest.append((col, percent_outliers))\n    \npd.DataFrame.from_records(outliers_iso_forest, columns=[\"attribute\", \"percent_outliers\"])\\\n.set_index(\"attribute\").sort_values(by=\"percent_outliers\")\\\n.plot(kind=\"bar\", figsize=(12, 8), color=\"crimson\", ylabel=\"percentage of outliers\", legend=False)\n\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-12T14:46:46.062289Z","iopub.execute_input":"2022-07-12T14:46:46.062658Z","iopub.status.idle":"2022-07-12T14:47:11.095732Z","shell.execute_reply.started":"2022-07-12T14:46:46.062626Z","shell.execute_reply":"2022-07-12T14:47:11.094559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now the question is can we remove these outliers? Well the answer to this question is not straightforward. We can remove the outliers but then again we are working with a sample of a population and if we zoom out from this population then it can be seen that these points which are considered to be outliers are not actually outliers with respect to the population, rather they might be part of some cluster which might be extremely valuable to the population. \n\nTherefore, a wise decision would be to not delete the outliers from the data.","metadata":{}},{"cell_type":"markdown","source":"# Clustering\n\nFor clustering the data in this notebook, I will use the KMeans algorithm to cluster the data. But before we can cluster the data, let's first look at the data. \n\nThe data is mixed with *numeric* and *categorical* data. So a direct application of KMeans algorithm on this data won't result in fruitful results. Why? To answer that we need to look into the KMeans algorithm first.\n\n\n## KMeans Algorithm\n\nGiven $k$, the *k-means* algorithm implements the following four steps:\n\n1. Initial guess of the centroid (the $\\mu$) for each cluster.\n2. Assigns a point $x_i$ to a cluster which has the **minimum** distance with the centroid.\n3. Updates the mean based on the assingment.\n4. Goto step-2 and repeat until convergence.\n\n\nNow it is obvious from the above algorithm that the KMeans utilizes the mean ($\\mu$) of the data to form the cluster centroids, for a categorical data there is no $\\mu$ as there is no point of origin for such data. Therefore, a direct application of KMeans on the data won't be a fair choice.\n\n\n## The Solution\n\nOne of the solution to the above problem would be to **one-hot encode** the categorical data and then apply KMeans on that data.","metadata":{}},{"cell_type":"markdown","source":"## Transform categorical data","metadata":{}},{"cell_type":"code","source":"df_new = pd.get_dummies(df, prefix=df.select_dtypes('int').columns, columns=df.select_dtypes('int').columns, drop_first=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T14:47:11.098728Z","iopub.execute_input":"2022-07-12T14:47:11.099641Z","iopub.status.idle":"2022-07-12T14:47:11.294947Z","shell.execute_reply.started":"2022-07-12T14:47:11.099582Z","shell.execute_reply":"2022-07-12T14:47:11.293885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Number of attributes in the data: {df_new.shape[-1]}')","metadata":{"execution":{"iopub.status.busy":"2022-07-12T14:47:11.296940Z","iopub.execute_input":"2022-07-12T14:47:11.297387Z","iopub.status.idle":"2022-07-12T14:47:11.303039Z","shell.execute_reply.started":"2022-07-12T14:47:11.297343Z","shell.execute_reply":"2022-07-12T14:47:11.301749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Problem\n\nThe problem with this approach is the data dimension is **huge** when using the one-hot encoded matrix as there are $7$ categorical attribute/feature which has almost $30$ categories within each. Therefore, roughly in total $7 \\times 30 = 210$ dimensional one-hot encoding generated from the data. And as we all know KMeans suffer from the curse of dimensionality when the data dimension is huge, therefore this solution needs to be modified in order to work with KMeans with this data.\n\n\n## Solution 2 \nWe can apply *PCA* on the data and reduce the dimension of the data. \n\n\n## Problem with Solution 2\nPCA on one-hot encoded matrix does not yield frutiful results as PCA maximizes the projected inertia of all attributes into a direction. For a table with one-hot encoded modalities, the inertia given to a categorical variable would inherently depend on the number of modalities available to the variable, and on the probabilities of these modalities. As a result, it would be **impossible to give a similar weight to all the initial variables over the calculated components**.\n\n\n## Solution 3 (FAMD algorithm)\nWe can use the FAMD algorithm in this case. \n\n>The FAMD algorithm wants to give the exact same weight to all the variables, numerical or categorical, when searching for the $k$ principal components.\n\nThe FAMD algorithm can be described in the following steps:\n\nFor the $P$ numerical attributes/features:\n\n* Scale them to 1\n* Center them\n\nFor the $M$ categorical attributes/features:\n\n* We divide each column by the probability $p_m$ of the modality (number of ones in the column divided by $N$)\n* Center them\n","metadata":{}},{"cell_type":"markdown","source":"## Reduce Data Dimension","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\n\nnum_attr = df_new.select_dtypes('float').columns.tolist()\n\nscaler = MinMaxScaler()\nX = df_new[num_attr]\nX = scaler.fit_transform(X)\n\ndf_new[num_attr] = X","metadata":{"execution":{"iopub.status.busy":"2022-07-12T14:53:16.524412Z","iopub.execute_input":"2022-07-12T14:53:16.524798Z","iopub.status.idle":"2022-07-12T14:53:16.568299Z","shell.execute_reply.started":"2022-07-12T14:53:16.524767Z","shell.execute_reply":"2022-07-12T14:53:16.566944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_hot_cols = list(filter(lambda x: x not in df_new.select_dtypes('float').columns.tolist(), df_new.columns))\n\nfor col in tqdm(one_hot_cols):\n    \n    freq_1 = df_new.groupby(col).size().to_dict()[1]\n    prob_1 = freq_1 / df_new.shape[0]\n    \n    df_new[col] = df_new[col] / np.sqrt(prob_1)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T14:54:53.229807Z","iopub.execute_input":"2022-07-12T14:54:53.230202Z","iopub.status.idle":"2022-07-12T14:54:55.725142Z","shell.execute_reply.started":"2022-07-12T14:54:53.230171Z","shell.execute_reply":"2022-07-12T14:54:55.723932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca = PCA(n_components=0.95, random_state=42)\npca.fit(df_new)\n\ndata_red = pca.transform(df_new)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T14:55:00.561288Z","iopub.execute_input":"2022-07-12T14:55:00.562380Z","iopub.status.idle":"2022-07-12T14:55:04.991192Z","shell.execute_reply.started":"2022-07-12T14:55:00.562331Z","shell.execute_reply":"2022-07-12T14:55:04.989549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Number of samples in reduced data : {data_red.shape[0]}')\nprint(f'Number of attributes in reduced data : {data_red.shape[-1]}')","metadata":{"execution":{"iopub.status.busy":"2022-07-12T14:56:29.899808Z","iopub.execute_input":"2022-07-12T14:56:29.901673Z","iopub.status.idle":"2022-07-12T14:56:29.909241Z","shell.execute_reply.started":"2022-07-12T14:56:29.901612Z","shell.execute_reply":"2022-07-12T14:56:29.908201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clustering\n\nWith the data dimension reduced, now we are good to go for the KMeans algorithm application. To know how many clusters would be appropriate for the KMeans algorithm, I used the Elbow Method technique and Silehoutte's method to determine the value of $k$ in KMeans.\n\n\n## [The Elbow Method](https://towardsdatascience.com/silhouette-method-better-than-elbow-method-to-find-optimal-clusters-378d62ff6891)\n\nIn the Elbow method we are trying to find the best value of $k$ for the KMeans algorithm by varying the values of $k$. For each value of $k$, the algorithm is trying to find the value of $k$ for which the *WCSS*(Within Cluster Sum of Square) distance is the lowest. When this value is plotted on a graph the point which looks an *elbow* in the graph is the best value of $k$ for the KMeans algorithm.\n\n<div style=\"text-align:center\">\n    <img src=\"https://editor.analyticsvidhya.com/uploads/62725cluster0.PNG\"/>\n    <figcaption style=\"font-style:italic\">Source : Analytics Vidya </figcaption>\n</div>\n\n\n## [Silhouette Method](https://towardsdatascience.com/silhouette-method-better-than-elbow-method-to-find-optimal-clusters-378d62ff6891)\n\nThe silhouette method is also a method to find the optimal number of clusters and interpretation and validation consistency within clusters of data. \n\n> The silhouette method computes the silhouette coefficients of each point that measure how much a point is similar to its own cluster compared to other clusters.\n\nThe objective of this algorithm is to compute the silhouette coefficients for each point, and average it out for all the samples to get the silhouette score.\n\nThe silhouette value is a measure of how similar an object is to its own cluster(**cohesion**) compared to other clusters(**separation**), and the value of the silhouette ranges between $(-1, 1)$ where a high value indicates that the object is well matched to its own cluster and poorly matched to its neighbouring cluster.\n\n","metadata":{}},{"cell_type":"code","source":"class OptimalK:\n    \n    def __init__(self):\n        pass\n        \n    def find_k(self, data, k_range):\n        \n        # Using Elbow Method\n        cost = list()\n        \n        # Using Silehoutte's Method\n        sil = list()\n        \n        for i in tqdm(k_range):\n            model = KMeans(n_clusters=i, random_state=42)\n            model.fit(data)\n            \n            cost.append(model.inertia_)\n            \n            labels = model.labels_\n            sil.append(silhouette_score(data, labels, metric='euclidean', sample_size=1000))\n            \n        print(\"=\"*20)\n        print(\"Elbow Method Results\")\n        print(\"=\"*20)\n        \n        plt.figure(figsize=(15, 10))\n        plt.plot(k_range, cost, 'bx-', label=\"Elbow Method\", color=\"royalblue\")\n        plt.show()\n        \n        print(\"=\"*20)\n        print(\"Silhouette Method Results\")\n        print(\"=\"*20)\n        \n        plt.figure(figsize=(15, 10))\n        plt.plot(k_range, sil, 'bx-', label=\"Silhouette Method\", color=\"orangered\")\n        plt.show()\n        ","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:21:04.681283Z","iopub.execute_input":"2022-07-12T15:21:04.682647Z","iopub.status.idle":"2022-07-12T15:21:04.692469Z","shell.execute_reply.started":"2022-07-12T15:21:04.682602Z","shell.execute_reply":"2022-07-12T15:21:04.691568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"op = OptimalK()\nop.find_k(data_red, range(4, 20))","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:23:16.673610Z","iopub.execute_input":"2022-07-12T15:23:16.674047Z","iopub.status.idle":"2022-07-12T15:27:22.089746Z","shell.execute_reply.started":"2022-07-12T15:23:16.674014Z","shell.execute_reply":"2022-07-12T15:27:22.088546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The above plots suggest that with $k=5$ we will obtain the best results of KMeans. Therefore, let's fit KMeans with the decided value on the reduced data. ","metadata":{}},{"cell_type":"code","source":"model = KMeans(n_clusters=5, random_state=42)\nmodel.fit(data_red)\n\npred = model.predict(data_red)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T16:17:12.584403Z","iopub.execute_input":"2022-07-12T16:17:12.584906Z","iopub.status.idle":"2022-07-12T16:17:24.329572Z","shell.execute_reply.started":"2022-07-12T16:17:12.584867Z","shell.execute_reply":"2022-07-12T16:17:24.327946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualization of predictions","metadata":{"execution":{"iopub.status.busy":"2022-07-12T16:17:41.640463Z","iopub.execute_input":"2022-07-12T16:17:41.641673Z","iopub.status.idle":"2022-07-12T16:17:41.651463Z","shell.execute_reply.started":"2022-07-12T16:17:41.641628Z","shell.execute_reply":"2022-07-12T16:17:41.650655Z"}}},{"cell_type":"code","source":"pca = PCA(n_components=2, random_state=42)\n\ndf_res = pd.DataFrame(pca.fit_transform(data_red), columns=[\"X1\", \"X2\"])\ndf_res.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T16:27:11.049759Z","iopub.execute_input":"2022-07-12T16:27:11.050234Z","iopub.status.idle":"2022-07-12T16:27:13.490775Z","shell.execute_reply.started":"2022-07-12T16:27:11.050198Z","shell.execute_reply":"2022-07-12T16:27:13.489675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_res[\"predictions\"] = pred.astype(int)\n\nplt.figure(figsize=(12, 8))\nsns.countplot(x=\"predictions\", data=df_res, color=\"blueviolet\")\nplt.yscale(\"log\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T16:27:13.493058Z","iopub.execute_input":"2022-07-12T16:27:13.493665Z","iopub.status.idle":"2022-07-12T16:27:14.020442Z","shell.execute_reply.started":"2022-07-12T16:27:13.493619Z","shell.execute_reply":"2022-07-12T16:27:14.019200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So, most of the data belongs to *cluster $2$* and there is only **one** instance of *cluster $4$*","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15, 10))\nsns.scatterplot(x=\"X1\", y=\"X2\", hue=\"predictions\", data=df_res, palette=\"tab10\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T16:27:14.021876Z","iopub.execute_input":"2022-07-12T16:27:14.022197Z","iopub.status.idle":"2022-07-12T16:27:17.318803Z","shell.execute_reply.started":"2022-07-12T16:27:14.022169Z","shell.execute_reply":"2022-07-12T16:27:17.317679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission of the results","metadata":{}},{"cell_type":"code","source":"df_sub = pd.read_csv(\"../input/tabular-playground-series-jul-2022/sample_submission.csv\")\ndf_sub.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T16:27:22.947600Z","iopub.execute_input":"2022-07-12T16:27:22.948405Z","iopub.status.idle":"2022-07-12T16:27:23.002452Z","shell.execute_reply.started":"2022-07-12T16:27:22.948357Z","shell.execute_reply":"2022-07-12T16:27:23.001236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub[\"Predicted\"] = pred","metadata":{"execution":{"iopub.status.busy":"2022-07-12T16:27:24.662346Z","iopub.execute_input":"2022-07-12T16:27:24.663838Z","iopub.status.idle":"2022-07-12T16:27:24.669893Z","shell.execute_reply.started":"2022-07-12T16:27:24.663797Z","shell.execute_reply":"2022-07-12T16:27:24.668676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T16:27:25.611331Z","iopub.execute_input":"2022-07-12T16:27:25.612692Z","iopub.status.idle":"2022-07-12T16:27:25.778189Z","shell.execute_reply.started":"2022-07-12T16:27:25.612635Z","shell.execute_reply":"2022-07-12T16:27:25.776519Z"},"trusted":true},"execution_count":null,"outputs":[]}]}