{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#Libraries Required\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\nimport numpy as np \nimport pandas as pd \n\npd.set_option(\"display.max_rows\", 2000)\npd.set_option(\"display.max_columns\", 100)\nimport matplotlib.pyplot as plt\ncolor_pal = plt.rcParams['axes.prop_cycle'].by_key()['color']\nimport seaborn as sns\nimport plotly.express as px\nimport pickle \n\nimport datetime\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom scipy.stats import shapiro, stats, probplot\n\nfrom sklearn.preprocessing import StandardScaler, PowerTransformer\nfrom sklearn.decomposition import PCA\nfrom sklearn.manifold import TSNE\nfrom umap import UMAP\n\nfrom sklearn.mixture import BayesianGaussianMixture, GaussianMixture\nfrom sklearn.cluster import KMeans, DBSCAN\nfrom yellowbrick.cluster import KElbowVisualizer\nfrom sklearn.neighbors import NearestNeighbors\n\nimport lightgbm as lgb\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis\nfrom sklearn.naive_bayes import GaussianNB","metadata":{"execution":{"iopub.status.busy":"2022-07-18T04:39:17.402457Z","iopub.execute_input":"2022-07-18T04:39:17.402940Z","iopub.status.idle":"2022-07-18T04:39:46.693359Z","shell.execute_reply.started":"2022-07-18T04:39:17.402846Z","shell.execute_reply":"2022-07-18T04:39:46.692098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Table of Contents\n\n* [Sneak Peek into Data](#section-one)\n - [Numerical Variables](#subsection-one)\n - [All about Normality](#subsection-two)\n - [Categorical Variables](#subsection-three)\n \n* [Dimensionality Reduction](#section-two)\n - [PCA](#subsection-four)\n - [t-SNE](#subsection-five)\n - [UMAP](#subsection-six)\n\n* [Clustering](#section-three)\n - [KMeans](#subsection-four)\n - [Variables Plotting based on Clusters](#subsection-five)\n - [Clusters Mean](#subsection-six)\n - [Gaussian Mixture Model (GMM)](#subsection-seven)\n - [Bayesian Gaussian Mixture Model (BGMM)](#subsection-eight)\n - [DBSCAN](#subsection-nine)\n - [Model Results Comparison](#subsection-ten)\n - [LGBM (BAM!!!)](#subsection-eleven)\n* [Submission](#section-four)","metadata":{}},{"cell_type":"code","source":"data = pd.read_csv('/kaggle/input/tabular-playground-series-jul-2022/data.csv')\nsubmission = pd.read_csv('/kaggle/input/tabular-playground-series-jul-2022/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-18T04:39:46.695243Z","iopub.execute_input":"2022-07-18T04:39:46.696683Z","iopub.status.idle":"2022-07-18T04:39:48.005526Z","shell.execute_reply.started":"2022-07-18T04:39:46.696645Z","shell.execute_reply":"2022-07-18T04:39:48.004063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Sneak Peek into Data","metadata":{}},{"cell_type":"code","source":"## To create the DF summary\ndef summarytable(df):\n print(f\"Dataset Shape: {df.shape}\")\n summary = pd.DataFrame(df.dtypes,columns=['dtypes'])\n summary = summary.reset_index()\n summary['Name'] = summary['index']\n summary = summary[['Name','dtypes']]\n summary['Missing'] = df.isnull().sum().values \n summary['Missing_pct'] = df.isnull().mean().round(2).values\n summary['Uniques'] = df.nunique().values\n summary['First Value'] = df.loc[0].values\n summary['Second Value'] = df.loc[1].values\n summary['Third Value'] = df.loc[2].values\n return summary\n\nprint(\"Displaying data ------>\")\ndisplay(summarytable(data))","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:25:07.227032Z","iopub.execute_input":"2022-07-17T15:25:07.228138Z","iopub.status.idle":"2022-07-17T15:25:07.399653Z","shell.execute_reply.started":"2022-07-17T15:25:07.228098Z","shell.execute_reply":"2022-07-17T15:25:07.398833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## Numerical Variables","metadata":{}},{"cell_type":"code","source":"# reference - https://www.kdnuggets.com/2019/07/annotated-heatmaps-correlation-matrix.html\n\ncorr_mat = data.drop([\"id\"], axis=1).corr()\nmask = np.zeros_like(corr_mat, dtype=np.bool)\nmask[np.triu_indices_from(mask)]= True\n\nf, ax = plt.subplots(figsize=(30, 20))\n\nheatmap = sns.heatmap(corr_mat.round(2),\n mask = mask,\n square = True,\n linewidths = .5,\n cmap = 'coolwarm',\n vmin = -1,\n vmax = 1,\n annot = True,\n annot_kws = {'size': 12})\n\n#add the column names as labels\nax.set_yticklabels(corr_mat.columns, rotation = 0)\nax.set_xticklabels(corr_mat.columns)\n\nsns.set_style({'xtick.bottom': True}, {'ytick.left': True})","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:25:07.401941Z","iopub.execute_input":"2022-07-17T15:25:07.402499Z","iopub.status.idle":"2022-07-17T15:25:09.634430Z","shell.execute_reply.started":"2022-07-17T15:25:07.402452Z","shell.execute_reply":"2022-07-17T15:25:09.633244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" 👉 **Observations**\n\nOnly INT variables showed some correlation amongst them, rest all FLOAT features have very weak correlation.","metadata":{}},{"cell_type":"markdown","source":"\n## All about Normality","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(30, 30))\ncols = list(data.columns)\ncols.remove('id')\n\nfor e, c in enumerate(cols, start=1):\n plt.subplot(6,5,e)\n p = sns.histplot(data, x=c, kde=True)\n plt.xlabel(c)\n plt.title(f\"{c}'s Distribution\")","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:25:09.636312Z","iopub.execute_input":"2022-07-17T15:25:09.637692Z","iopub.status.idle":"2022-07-17T15:25:35.700186Z","shell.execute_reply.started":"2022-07-17T15:25:09.637638Z","shell.execute_reply":"2022-07-17T15:25:35.699005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" 👉 **Observations**\n\nAll of the FLOAT features are distributed normally, but INT variables are positively (rightly) skewed.","metadata":{}},{"cell_type":"markdown","source":"### Shapiro-Wilk Test for Normality\n\nShapiro-Wilk test is a test of normality, it determines whether the given sample comes from the normal distribution or not. Shapiro-Wilk’s test or Shapiro test is a normality test in frequentist statistics. The null hypothesis of Shapiro’s test is that the population is distributed normally.\n\nThis is a hypotheses test and the two hypotheses are as follows:\n\n- Ho(Accepted): Sample is from the normal distributions.(Po>0.05)\n- Ha(Rejected): Sample is not from the normal distributions.\n\nLets apply this test on our dataset - ","metadata":{}},{"cell_type":"code","source":"# conduct the Shapiro-Wilk Test\nf_cols = data.drop(columns='id').columns.tolist()\n\nshapiro_output = []\n\nfor col in f_cols:\n x = shapiro(data[col])\n \n if x.pvalue > 0.05:\n result = \"Normality Approved\"\n shapiro_output.append((col, result))\n \n else:\n result = \"Normality Rejected\"\n shapiro_output.append((col, result))\n \ndef highlight_rows(row):\n value = row.loc['Test Status']\n if value == 'Normality Rejected':\n color = '#FFB3BA' # Red\n elif value == 'Normality Approved':\n color = '#BAFFC9' # Green\n else:\n color = '#BAE1FF' # Blue\n return ['background-color: {}'.format(color) for r in row]\n\nshapiro_output = pd.DataFrame(shapiro_output, columns=['Column', 'Test Status'])\nshapiro_output.style.apply(highlight_rows, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:25:35.702293Z","iopub.execute_input":"2022-07-17T15:25:35.702642Z","iopub.status.idle":"2022-07-17T15:25:36.052067Z","shell.execute_reply.started":"2022-07-17T15:25:35.702614Z","shell.execute_reply":"2022-07-17T15:25:36.050505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Q-Q Plots for Normality","metadata":{}},{"cell_type":"markdown","source":"When the quantiles of two variables are plotted against each other, then the plot obtained is known as quantile – quantile plot or qqplot. This plot provides a summary of whether the distributions of two variables are similar or not with respect to the locations.\n\nAll point of quantiles lie on or close to straight line at an angle of 45 degree from x – axis. It indicates that two samples have similar distributions.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(30, 20))\nplt.suptitle('Q-Q Plots for Normality', fontsize=18) \n\ni = 1\nfor col in f_cols:\n plt.subplot(5, 6, i)\n probplot(data[col], dist=\"norm\", plot=plt)\n plt.title(col)\n i+=1\n \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:25:36.053737Z","iopub.execute_input":"2022-07-17T15:25:36.054093Z","iopub.status.idle":"2022-07-17T15:25:44.683838Z","shell.execute_reply.started":"2022-07-17T15:25:36.054065Z","shell.execute_reply":"2022-07-17T15:25:44.683017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Categorical Variables","metadata":{}},{"cell_type":"code","source":"categorical_cols = [col for col in data.columns if data[col].dtype == 'int']\ncategorical_cols.remove('id')\n\nplt.figure(figsize=(30, 20))\nplt.suptitle('Categorical Columns Distribution', fontsize=18)\n\nfor e, c in enumerate(categorical_cols, start=1):\n plt.subplot(3,3,e)\n sns.countplot(data=data, x=c)\n plt.title(f\"{c}'s Count Distribution\")","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:25:44.685119Z","iopub.execute_input":"2022-07-17T15:25:44.686262Z","iopub.status.idle":"2022-07-17T15:25:47.057471Z","shell.execute_reply.started":"2022-07-17T15:25:44.686226Z","shell.execute_reply":"2022-07-17T15:25:47.056241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# PCA & Scree Plot","metadata":{}},{"cell_type":"code","source":"def run_pca(df, components, drop_cols=False, transformation=False):\n \n if drop_cols:\n df = df.drop(columns=drop_cols)\n else:\n df\n\n if transformation:\n df = transformation.fit_transform(df)\n \n pca = PCA(n_components=components)\n pca_out = pca.fit_transform(df)\n explained_variance = np.round(pca.explained_variance_ratio_*100, 1)\n labels = [f\"PC_{i}\" for i in range(1,components+1)]\n \n plt.figure(figsize=(30, 10))\n plt.suptitle(f\"Explained Variance & Scree Plot of {transformation}\", fontsize=18)\n \n plt.subplot(1,2,1)\n plt.plot(labels, explained_variance, 'go--', linewidth=2, markersize=10)\n plt.title(f\"Explained Variance vis-a-vis PCs\")\n \n plt.subplot(1,2,2)\n plt.bar(x=labels, height= explained_variance)\n plt.xlabel('Principal Component')\n plt.ylabel('Percentage of explained variance')\n plt.title('Scree Plot')\n \n plt.show()\n \n return df, pca_out","metadata":{"execution":{"iopub.status.busy":"2022-07-18T04:42:10.032630Z","iopub.execute_input":"2022-07-18T04:42:10.033084Z","iopub.status.idle":"2022-07-18T04:42:10.045013Z","shell.execute_reply.started":"2022-07-18T04:42:10.033047Z","shell.execute_reply":"2022-07-18T04:42:10.044088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SS_scaled_output, SS_PCA_output = run_pca(df=data, components=15, drop_cols='id', transformation=StandardScaler())","metadata":{"execution":{"iopub.status.busy":"2022-07-18T04:42:10.452001Z","iopub.execute_input":"2022-07-18T04:42:10.452603Z","iopub.status.idle":"2022-07-18T04:42:12.187012Z","shell.execute_reply.started":"2022-07-18T04:42:10.452569Z","shell.execute_reply":"2022-07-18T04:42:12.185771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PT_scaled_output, PT_PCA_output = run_pca(df=data, components=15, drop_cols='id', transformation=PowerTransformer())","metadata":{"execution":{"iopub.status.busy":"2022-07-18T04:42:12.189138Z","iopub.execute_input":"2022-07-18T04:42:12.190123Z","iopub.status.idle":"2022-07-18T04:42:17.564345Z","shell.execute_reply.started":"2022-07-18T04:42:12.190066Z","shell.execute_reply":"2022-07-18T04:42:17.562901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"👉 **Observations**\n\n**From the above PCA charts, we can see that the explained variance ratio is very less (*in single digits*) for PC1, PC2, .... Hence, they have very less informative power for the entire dataset.**","metadata":{}},{"cell_type":"markdown","source":"\n# Clustering\n\n## What is Clustering\nClustering is an unsupervised machine learning technique, which partitions the entire dataset into a fixed number of clusters or groups in such a way that observations in each group exhibit similar behavior or characteristics and are homogeneous. The observations in the same group are more similar than the observations in other groups.\n\n## Clustering Methods:\nBased on the type of clustering algorithm used, there are different types of clustering methods:","metadata":{}},{"cell_type":"markdown","source":"\n\n1. **Hierarchical clustering**: It is a tree based clustering method where the observations are divided into a tree like structure using distance as a measure.\n2. **Centroid based clustering**: In this method, the observations are partitioned into a predefined number of clusters (k) such that within cluster variances is minimum. k-means clustering is the most commonly used centroid based clustering algorithm\n3. **Density-based clustering**: In this method, the observations are grouped together based on density of points that are closely packed. DBSCAN and OPTICS are two popular density-based algorithms\n4. **Distribution-based clustering**: In this method, the observations are grouped based on the distribution models and observations in the same cluster are most likely coming from the same distribution. Gaussian mixture algorithm is an example of Distribution-based clustering\n\nThe clustering types 2,3, and 4 described in the above list are also categorized as Non-Hierarchical Clustering.","metadata":{},"attachments":{"56cb8646-a852-4786-8d98-7f2b27f6e177.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAlwAAAI0CAIAAABH2WDQAABb7ElEQVR42uy9fXRUZ37n+dy3epMKFcICIVQCYXAZbLfsbhBt2oPabiUNK6bpZJIe7VlPYpqGiTdhTjZnV7NuelthZmk2zMkfPfQOCW4vdMJkSPokDh4IpE23jeOQQXYa09jQZTAvkizAQkJSSVWlqvuyh3rEw9WtF1XdurdUV/X9HI6tert16/e79/d9fr/nTdQ0jQAAAACAEB4mAAAAACCKAAAAAEQRAAAAgCgCAAAAEEUAAAAAoggAAABAFAEAAACIIgAAAABRBAAAACCKAAAAAEQRAAAAgCgCAAAAEEUAAAAAoggAAABAFAEAAACIIgAAAABRBAAAACCKAAAAAEQRAAAAgCgCAAAAEEUAAAAAoggAAABAFAEAAACIIgAAAABRBAAAACCKAAAAAEQRAAAAgCgCAAAAEEUAAAAAoggAAABAFAEAAIC5iggTAKBHVVVCiKZpMIXdcA+AKQBEEYAyVcTDhw9DFEsmioSQl156iedRsgIQRQDKCS3FoUOHdv+H/7i8uZk+KSvq7NyWQqWIxM2bN27cuNHd3S0IAi5CAFEEoIxyRFVVr127tv65f/Evv/WH+cuhmjGl1DImRnNWTc21HjiO+/jtP7t+/bqiKBzHIV8EEEUAyihNVFVVURSXwE/E5c9G4qpaigqqS8pXCSYTahZpySA2eb7z/pOZ3qnY/9tFga+d56LNEUVRRFHUNA39iwCiCEC5ZIpKCkIIz3GxhDIynlTzS4BMC4iTFIArqJExwzE4ngtUuxakRJGaXVVVVFABRBGAMsoUaWimzwg8p3Gc21d9d/jebJ0VX0TaNHOay5mWP870i3T4ks/rdYmcoTlCXYBMEUAUASgjXWSieF+TeK4mUDN/gf/DSzeKzO9yjGUthQyU00DaukcCTUsa+vt7DWk6rkAAUQSg7ETRoF5ul6u5OVBXS06/cz2HvnBzRrJmPtfs6p6HGR5dGvjKl5/+4Bef6RsEmg5chwCiCEBZ4/V4mlvqr/bK167ftELWLOma48owo9Rm+viC2tpv/dYz13sJxz0cWxRPKrjGAEQRgLJGNzXgvvzMD3hqAzU3BVGWk7MrPDOIZRnDcXzz0qA+L8dlBsr09ocJAJiR6qoqnuf0KQ4oRBE5SUL7G0AUAZhT6SOPoZGmZTHjswrG1wCIIgCOQ50puIM8SciwAYAoAuBQ0O0FAEQRAABsjzs8Fq8BEEUAnMbUpPJUyogBk6ZhRefUInpE06YK0iosCiCKADgCBQEbAIgiAACUktnasRIAiCIA5olNTsIIFoLJLQCiCIBj0E2f01K7VUx7CCyTRpgAQBQBAJVMIgkbAIgiAA5Hw7gbK8AuwgCiCAAAAEAUAXB0aohZiXYZFjYAEEUAHItLItgaHgCIIgDARo50tR3YuX52z2HftrXHuttXNwVK9o2G/aN4HuNPQdmBTc4AsFd4Qo017OErh96/1DsCswAAUQTAUTeGwE/tGTUtmSkssznW3R6JJbfsPk0fbl4X3Lt1jVW6SOWWHdwEXa+9V3rDog4NIIoAOJcpFayu9hX6SVodfXHfGfbM8XN9x8/1VawpvR4X+zs1jgm1UwBRBMDZ5Dt0cnVToKHW1xMezOfNuzpbWkN1+pzvSFdbJJZ8ef9Zpq8NtfdVORJLvrjvzOqmwN6ta1gySgjpCQ/uOXqBELJjU6ijNUhfCvePslxw87rg9o2hV0+FO9uW+73SwHD05f1n6ffSzJUe80RP34qGebTeS7/LkPgazryYrFcQIIoAoghA2ZOQVf3inLypsWjPPbGIEHLh+nDx57Nv21q/V6KSuWNTaFdny56jF7bsPp1ePqUi9+qpMM1Hj3W379u2Vl8j3b4xdKKn7+DJcLbv6mgNhvtH6TGPdbcf2LmeCfOx7namske62gxJMABzBow+BSCP+6TAcZJ1NR5CyLVbkeK/2u+TBoai9O+DJ8M0I8xIa6iuJzzIKrQnevpCjTX6waUDw9EcimhILnvCgzQ9pWJMCDn85hX68J2Lt/1eqdBhq1yqfeH14moCEEUAgFn6BydCjTX7tq3N/bbN64KG3HRg+L6ULl/sZ8+cvzqU+yCjEwn29+BonNaBbfpdKJ6C8gTlUwAsSxDtYM/RC7QuSseyZita0qxu+8bQ9o0hy8/h4MlwR2vw19YvpT2IG56qHxiOFjOGFgsGAYgiABXEhevDraG6555YZMnsC1oypYNl9P186bAORWuhaSgVZjoGJ8c5ZG9hoC4FIIoAOJx4fCq3KehTx8/1dbYt3/BUfe4+PAorVFIFXd0U8HulSCyZfswNT9Y3LJjq59OXOgkh7350p6M1yHoBraWluTZHkjpzXqj7++H2wqifgrIEbTcAbOF7Ry/4vRIdqElZ3RTIuKzaux/dIYS89Csr6UM23YKinwgRaqxhYkmllOZwhJBLvSPh/tGO1iA7/uZ1Qf23F8PrZ2/6vdKx7nb2z6ojA4BMEQAHYOjzMrEg+KXekS27Tx/patOrWsaJfZd6R149Fd6+MUTf+cqh93duWc1effVUmB2Bzi+kfx88GX5mxQLaiUgnWnS99t6uzhamqcXkdgboDBP9yR/pajvS1Zb/8bP1IaZWDgIAoghA2cOlKn3T1bDgkl822TA8b1jsRt9jl2MdnPSOvWwTNjIeRP9mKuH6Vw+eDLPa74an6sP9o3o5HxiK6td0zYfx8SghPoyyAWUOmmkATE8KEbLTiMSSeglc3RQINdaE+0eLOaaswM4AmSIAwIG8vP/sgZ3r9XVgtrBc3kACAUQRgDmDxld4VDcxB8OYgmODDOAEUD4FAJQOVSWoTwOIIgBOBDPpbDQmhtsAiCIADslmEK8BgCgCADKiKLABABBFAAAhPD+VNSJ5tApaOEVtGkAUAXASbJVOFXpoTfOChxYCiCIAAOjWAZ+ORxJgHABRBKB8ybbSCoccpzQ6CQBEEYCyYvpsAQ6TzgGAKAIAgNWtDZgAQBQBcCgKMkR7ELBdFIAoAgAqFg5pI4AoAjBn8LgehHENuaMZVE3F3ETgFLBLBgBGWH2P042QlGUle2DXCsqRKhBZVrBLBoAoAjBH+PDSL1VVyT6JwDbxK6jGmP8kh+IXdy34F2vhj6/cG5lHiAeXE4AoAuA8WNj/0z87P3RvhGWNXIGCoGVXNm5GNbRJbWdjjmA0Pvnqn//8V5//PC4tAFEEwJFomvaTn34wODQq8nwps6oSqT5noaHyeltkLHHyJx8sX9qISwtAFAFwHteu904mJj2uXEuRqUUsjSrw3ExZpmkh4swlr4aeVMtRleS1G73VPglXF4AoAuAAqj1TN4WmKh6BeLzSzPo0Hb6EXXulTp2LzE112shxWPUUQBQBKFcM6ZHAcQ213lk5k0RSzTPxc2dZTTt9U/ts6psxZUx/s8BbnzvKiorN9wFEEYCy1kUqjY/McweqXAV9tpgiqkucnenCcnF7YhUkaRlLshxHriuq3QVbACCKAJiRQ47jmpqa/vpvXn/nH74Mm9jUAkhtrPhQAm/euPGNb3yD53nWIgFglqMBKhgAaJqWTCZlWR4fH798+fInn3wSi8WUFOV/8oP3xv/y9AVaYP3GVz63sLa6nBsfqqpyHMfzvCRJLperqqqqvb3d4/F4vV4+Ba5GgEwRgNkP1jQiS5K0cuXKxsbGeDzORLHMG443b9879VGUimLrs+uX1s8v57OVZdnlcmma5na7XS6Xz+eTJEkURaSJAKIIQBnB87wgCG63m+M4URQ9Ho+qqlqKMtfF0TgR3T4qijU1gdra2jJvf3AcpyiK2+0WRdGVgooi0kQAUQSgjESR/SFJkqIoTBHLHP+4zESxrm5BIBBwRGpOEVOgcAogigCUqS7SlEUUxfLPESkul5vnRSqKGrmf7DpCFOl/BUHAEBsAUQSgfHWR53mn5IgPRNHFC1Oi6HK5HCGKhpQRFx6AKAKASG1dgnv/bDkm6vAgAOZvKJgAAAAAgCgCAAAAEEUAAAAAoggAAABAFAEAAACIIgAAAABRBAAAACCKAAAAAEQRAAAAgCgCAAAAEEUAAAAAoggAAABAFAEAAACIIgAAAGA72DrKRlRVdcQutXMAtpM7TAEAgCiWI9euXfvRj37EFBHSaKsiEkKWLVv20ksvYTdBAABEsbyg+7a/8MILS4JLg01Nlh9fVmZBX0Wh3JOwwz/6s6ampueff14QBFyEAACIYrmgptA07aXf+QNx4WqOkISsZklxMic9Gd6Z38f5sqwf8jxXhDHzagH4POLtT7deu3Ztw4YNHMchXwQAQBTLKE2kvYmCwN2LJMZjSVlRZ/GUXGIBCjGZVDOpL2dap7M9r6iW5btuSXi0wS8rmizLiqKIoqhpGvoXAQAQxXLJFBVFoZ2IPE9Go/J4NKmq6FOcmYKEjPbSihK/uNYzJbQpVFVFBRUAAFEso0yRimIiqRIPEQTi81ePT8Sj0dhcbgrkHEnEmXyNcDlfXrJ4YSQyyh7STJG6AJkiAACiWEa6aAj7y5qCt+7cvj14r4C0iXAmvjp7Ema7SGikpNnwui+sGo9MUlGkP5wVrgEAAKJYvqJICJEkcdNXH3/rzOVPbt4rQGUsPa25ZOR//bVVGl//y1/e1Iu9pgPXIQAAoliOKAohqR6uKq/nhbZVd//2w5GRUXu/krNAK+1KLK1QqyefCD3TsvjnF3FxAQAgik5m3jxPzTz/yOiYptlZ5dPmcmLp8/mWBhuz5ei4xgAAxYDpXCWH4wSBxzAQs8bjeY5PjeshyYRqLjkGAACI4iySnr4ghJsWRZgOAABRBCCrTMIGAACIonPTRvR72ZB+c0i+AQAQRWdFb5cLxrC2eQEbAAAgis5BRdgGAACIIsillAqWXEG+CACAKAJ0e1l44fLTdpXCcBsAAETRGYgCjAwAABBFAAAAAKIIclBdXQUjFInXM+2hwKN4CgCAKDoEFrB5GNs6YzIw7xMAAFF0oKE5mNrqfNGL7fUBABBFh4NanwX5Im9IH7nUgCYYFgAAUSxjZAWVPQAAgCiCTMTjFhxk37a1x7rbVzcFZvGH7NgUOtbdvnldMMerOzaFivyNM+eLyAwBABDFOczmdcF0Ocn4ZKVftQ8uW1VVYA0AgIVg5/1SIAgPh4RoxS1s0/Xae2X+Yw+eDB88GS7pVyJlBAAgU3QKmDIAAADIFIERVSXFV/x2dba0hupeOfT+pd4R/TP0757w4J6jFwzv3Lt1DXtpdVOAPqToj3Osu70nPFhT5Qo11hBCtuw+TYu32zdOVW4jseSL+87oT+bAzvUNtT5CSLh/lKWw9CMnevpYvrhv21p6TMMZHulq83sl+rf+/QAAAFEEZqB6Q7WNCt6uzhamOoSQvVvXMOWjb6BqRyVN/5AQ0hqqC/ePsmeovDEZ29XZoj84Vb6XT56lbzN8b8YzpEK4uilwqXfkSFfb0TPXjp/ro0fuaA0ODEfpwxmZmFAzFjlik/LoxKRMRCHF3C9CqJoo8JFoAjcCABBFJ8EVMlCyozXY0RrM552rmwKhxpoTPX1Uby71jvSEB1nWyDIzlgte6h3RS+D5q0MNrb7N64J6KdL3WVKhYlJn0DyW2x0/19fRGmysq5rxDAkhLNfUJ52vn73ZGqpraa7NUxQNsKkvJ969dCX+95LLw3E8VxkLCK1atuh8uB+3GAAQRQdJYmGdioZCor6AaeC5JxYRQt796A57ZnA0TqWIidDrZ29m+6KB4ajhmXD/qP5hQ63P8Ey2j0eiyYYFvmxnmP5FBtjZFslnIxN8311BchOO5ypjCaHwzUHcYQBAFMF96mo8tECa/0dySGxGRicsKM1duxXJ+Hw+kxEzoigqIbyK3ZoBABDFOYGVA1L1g2Vys2NTqKM1yDLRQgXSQmjvpn54jmmBTFlT41OL5/2v/+rZb37zm9XV1RXSp0j5uHfwW3t+jJsKAEvAlAwbCVS5bD3+hevDhJDli/15vn9Fwzw6jzDP9w8MRzP2FOYPLe3SIqoe+sw7H94utnGB+S4AAIiig7C1xHf8XN/AcFSf7e3YFDqwc32299NaKF2ebXVTYMY08fzVoYZaH1tMh44+LegML/WODAxH9eOG6OhT2svY0lzLnpzFVBsAABgon5awAcITWbb4mC/vP6tfKXRgOPry/rPZ3rzn6IV929Zu3xiicvjqqXBuXaQ5JRsNmz5PMc8zPLBzPTtDOhr2Uu9IQ62vozVInz/R07fhqfqCfzwWsgEAWA2HApTVqaE6OTmZSCSefPLJPX984K7YfHskRsSqhXWL1qy5nxid+unNK59cVxQFljeBwAter3fDl77w5Weln18kH330ydDQ3cUB96qlNf/vd7Zu3ry5gvsU77cRfrjrNx5rqsN1AoD57AUmsBVteqEPOmhRzo0kEQAAUQRgOqKI+j8AAKLoNBRFJhgdYk3mPcW0EUzIGwEAEEVnBPHpQvgglCOKAwAARBEAazBWTdnypwAAAFF0XgIJE1gEcm4AAETRcRqoaTwPU6NZAQCAKFY2kjDNwlU+mKTo3JBDdggAgCgCAAAAEMW5mN8g1ykWTWWGhTEAABBFx4ihQRphkWJRsY8iAACiOKe0EcNEAAAAogiA3c0NAACAKDohdmMNa6ssiRo0AACiODfyGCzzZhXS9DVt0OQAAEAUHZTcTHuoaSo2UzSHpqmGx7AJAMBCsPOOjchp4yQVlYxFxhVFyZ5M5lIEmFTTtPhkPEeDAwAAIIrlHMQfCGRSJoT0fzoycOsOx5lL0O0L/5o9p1G8imf4LkVR/+HsP25Y9yVcXQAAiKKDufXZyF+9fj7/cSK53qhZq4c2ya1dKh4ZT+z9/j999YVncVEBACCKjmQsMt7zxjVJcHyxT5uNTDad6MTk3/392UeXLsGlBQCAKDqMZFK+2XvT6xKKP1SRy7kIOYdp5so/ZxrSos2giZqZhHgGlBu9/V4PrmEAAETRMXmVpigkGY/7XNNM7ZL4PKUn2zhVLb+Pq+ocH57Da1j1DQAAUSxj9L2GqkoW1rhJjbuszjAhFyAk7kz6nUWSNdPifT+LNZszslG+mNcPAIAolq8uJhVtvt9FCGmqqyqZhlnywSJTzISsmP5sdFIxpNqZLTy9WisrGjU7dBEAAFEsLzmkPPvss7/z4tdhkFKyc+dOnuchjQAAiGJ56aIgCD/4wQ9+93d/NxaLRaNRRVFkWc7RQVgm/Je/PR+JJYlG5vlcL3/96TI3sqqqfAqXy+V2ux9//HG/309FERchAACiWC7BmkZqSZJWrlwZi8Xi8biSovxFcV7dXXU8fl8U/d7HH3+8zE0ty7LL5dI0ze12u1wuj8cjSZIoihBFAABEsYzgeV4QBLfbzXGcKIoej0dVVS1Fmeui21styRLRiMvrqa2tLf/2B8dxiqK43W5RFF0pqCjyPBb1BQBAFMtGFNkfkiQpisIUscyRvNVSUkiJoi8QCDglNaeIKWiajosQAABRLDtdpCmLKIrlnyNSBEHiBJlo9/9wu91OEUXWj4shNgAAiGL56iLP807JEafOWZT4lCjyomNE0ZAy4sIDAEAUEaktPlt0ywEAKjSfgQkAAAAAiCIAjsdZ9XkAIIoAABuZqnhzJd65CwCIIqio/AMmcJ7D4DQAIIrApvwDJgAAQBQBAAAAiCIAAAAAIIogCxjSCACAKAIwBRaHAQBAFAEAAACIIgAAAABRhAkAAAAAiCIAAAAAUQQAAAAyMftbR+k34MXSxjbBNoRi/y0SVVXhr5L5DltFAlApong4hV4dga262NbW1t3dXeReiW+99dbu3bvhspI5bunSpd3d3cuXL4c1AJjLoqgoyh/+4R9+Yd2XvtD6pbzOVTDdWC6qlR1PyLPgGMH6yvadW/0/+tGPvvOd72iaJgiCubRe07TDhw/LivY/ff1f2+y1ohw3K16zyXGn3virw4cPd3d3m/MaAMABoqiqqqIohJANX/1X0sJVEgslmcLgZFLNeBCey5wPpQfzHMmTHmHuFqk0oj2yKPy3P/6vsiyLomhub301haIo69Y/V/fEV6bZlbPca5kdl/HNc9hxgsAtu3BOSWHOawCAchdFmnAoikL/mIgrkWg8qajlaSOXWEAYyqgE6RGby5IDZYztGZ9U1AJKl5LAB/xSdcryyWSS4zhBEDRNK6inijpLVVVaNU3I6u3huDx3vZa/47JZMf35grxGCPFIQnChj5ZVFEURRbFQrwEAHCOKsizT8CoIJBKXY5OKminCzhhHiggRnOnDlmNYynJOPM9XeYUAkcgDUaSKaGLQDU0TaYpPXTMykVTkzIqS23E2eS33kctUTLKcligK0rwpaadmV1UVFVQA5nimyJ6sqvaPRqIT0WiFuyT34JVCtaTK56uZV6UmJ5jlZVmm2V6hisi8RoeeTuWgLrfb577z2SC8Zm3TbVnTkrGRYfp3Qp6qWlMXIFMEYK6JIouw0xMabsXyZf23+2/fHrG7DW4uupUiFlk6nLO+PtDUEBwc/GxaAje9LWLCa4aPNzbUP/KI74NfXC9B8lQhjnuh7ZnhoRgTRWp5fVsEADDXRJEN6JcVjXX+eNyur21adeqnl6/duJdPkKycCQFc4dF4+bL5v/G1Z949e9dgczpYxpwupisiZc3Ty+bXaG+euZ7PmVfUNA4TjvuNr62qrgoMD8XoQ1qE1nQgbAEwN0VxeoCYmrxfXeV5tnXV3aFfjI5FZiuSclZkCJw9eUiedqiZ5+/46jOpGQICS8RtNdqXvtj8y6vx3v6BWZTA4h3H2ZNA5m+Ex1Y0P/v04otXUifDo0wKQKWKop7a+R6f1zsWicxWo1grm4OY1AaOq50/3+OaeqgoCmdDip/+cNHCuk9v3WbDcJzouNnNwiRJWrK4Ph+bAwBsouzmPLkkQkc5CrzAEbSUTWRLnMALsyVIPC9gGIjppgyfcpysTBNoyCEAlSiKGW98BANzqjjt0YO53iVsYUAUzYkiZuUDAFEE9gVZ+9PFQqeig0KTfhgBAIgijbYYfe4YUN8DAEAUbWweA0swLExtd08fR0gsPgmzW9HIgA0AqHhRlAQUcgGYIpGYJowqdBKAys0UgdWJB0wAAACOEUV5evchHSzp83pLtDpXJZF7a8OCspFEpm0l0LloCYbLHlNcAKjITJHD/W8x4xPTtEpWNBuSUKigPdm9hhsBgAoXxaJzF5CR/FeRLqZB4nLB0gAAiKJ9p4XtxWcnXS8qQVEVTKSxABEbJgIAUQSWY1iCNJ8OP5RDywrqMqwKDkAliiK78eNxOGUWo3ChOQ0/1ROMwG0pWLsCAGSKGUqmiLRW5IuqKOa7FwrHWTCDA14DAEAUwVyhCE2rrvbBfkXi8UgwAgAQxYeFO33hyNoOrgM71x/panOch3Z1thzrbt+8LlgqT5gRUH2SX7Juyc3rgse623d1tuR+2+qmwLHu9n3b1hb5nllBSPUo8piqBECpEMvztFRVXdVU83//1jOEkFcOvX+pdwSuKlnrpEwi8L5ta0ONNfpnesKDe45eqAQvpNauAABUpChWe6QETTumB+MtXwwODEf9XunX1i+tHFHc1dnSGqrTtwOKkYFEctrD6OTMe+JzfF65nn41TuY4OybSbNl9miVze7euOdbdzoxz/Fzf8XN9Mx7hUu8IO0gx7ykNnG6wKVYIAqD0lG+f4uPBmv7B6MBQdFVTAH4y41rDdDeHV+CYbn17pnrpXALjegGouExxqlE8PT1pe2K+3yu9fvbmc08sDDXWbF4XNOQEx7rbDUdgCcSBnesbaqeN+DjR03fwZDj9S/XvHBiOvrz/rD4pOdHTt6JhHq3gRWLJF/edoc8bvo6yeV1w+8ZQjkM9s2JBQ62PHocQcqSrze+VDKfHfhT9lnD/aNdr7+3YFOpoDb56Knz8XN+x7nb6pP4n+L0SPabhsO/fI+PyXLteT/T0dbQG6fVAbUtrqrTWqs/2ctiNWUnvKfae3Icy1HWzXVoAAIiilXyh2R+JJS/1jlzqvdfRGtzwZL1eFPUxjo6dYapARYJGNBq8snVJ0g+y2Hesu/3AzvUsRBJCOlqDPeHBLbtPs8Ide/+Bnev3bl3DPkuDJouPR7raMh6KPXOkq+3omWv0F+3qbOloDQ4MR4+f69uy+3R6+VRPuH/U0M3WUOvrCQ+ynzAwHP03/+kdgRd2//baNcGq82nHUDUtEk2o/KRMREEQDAVPTZtqnKiaNjph3BxRVdXE5GQimUjfAjoeJ2ReKS6Mdz+609EabGmuNTSS3vnwdqixZsemEJOoFQ3zIrFken31wM71rGmyq7NF/5F8DkUbUtT1rCVkoS7SLdQwsAYAiOK0Al/zIt/Hn46xxEuvBDs23Q9Dh9+8MhXCLt7uaA2ubgpc6h1Z3RRoqPWd6JmKg4ffvLJ365rnnliULjC7Olv8XumVQ+9nTEGYAtH+vEu9IwPDURYKCSHnrw41tPrYmzc8VR/uH2WR8eiZa9s3hvSHisSS+q5BJuGEkNfP3mwN1aVH+YzQeM2OTIekXrg+TH8RIeTl/WfpVPof/nTgP/3WykeryJvTjxCJJvb9tx7J7ZVcHo4XDP24o+OJqT8m4t/54bvGbF7TFDmhKvLFa3cblj42K5cIdWVNlXGt1ePn+jrblq9oeKjMDQt8A0PR9CP4vdLlnP21OQ61eV2wodb36qkwe+eGJ+s3PFVvY7IIdQSgtJRjn2LIn8oJPvyMKRDTQqtorKuimag+BSGEtDTXPlSIiYROS5KR2MNRKwPDUZql0QKp3ytdHRhjr167FWGvUi5nHyhU0Bii4+f6IrHkhifr6cMNT9azDKaxroqeFSOpEYk3RldZUa/fGr02MHL105Er/fcM/+QHi5fKspb+6pX+e1c/Hfnk05HxB6ZIDVUtl7B9uXeENZ42rwv6vdI7H95Of9vAULQ1VJd7Ike2Q9HLQ998GZ1IsHq1tagqRtkAUKmZon6VTp4n81NpwIn3+vUVM9ZyP3gy3NEaZENSNzxVPzAcpX/TlO6ZFQvoO1/6lZU5Slt6kStUnPQsX+ynBdKO1gKmEqb3iebJwFC0YYGPZTBMbv1eye+VDIeNKSbTDa3Az9B6qqoqdl8qq1OjrvTtFcaF68OtoTqaRrc012asnRJCaK9ha6iOVpv1Ve4ZD0UzVNO+AwBAFM1QlTqpv/7Ol6elj7qWOyGEBjWqbfq4RvMzFrb0BVIDhgb+6uLGuOY/4IL2UOrHfRQUZN/58DatzdKf8PrZm/r8lZZPBUEILlnyLzetTJ9csWCe98h3Oqqrq6uqqtL7FGdSPnVycjKRSGwbeGO2ro3nnljEKsbpaXRn23La/byqKZAjO6eWpz3B+7at1Q9cyudQtk7eGL+v9yICEwAQxSmaU9OW/+ofb/3lW2H5fgqpsfhFxz7Qlru+W45BpSKfyf79gxOtoTraEzljtM3N8XN92zeG6mo8BYX1jJU9QsjgaHzGr+tsW07reCxFNmSQ9/06R/cfooWBbP2vl3tHVjUFaEFb31zIyMGT4WdWLPD7pPwPdXVgLNRYo79s7IYWtF0ilmMEoESU3c02XyIJlfzs4pAhfhFCaF309bM3aZ2Q/WOLt9EuNzpSlP3LmALSERY7t6xm2VtHazDcP5rPaJd0esKDtNrGjpYj+aM9f6zz0rDyHH2VCmc2aLxe1RSgva2Uw29e8Xsl/UJlzz5CQg8GiwhW7z9kxz7+M3Ksu93vlTIWPCn02ti5ZbW+uWBI05nB6bCsSDSZ/6EOngxHYkn9RMldnS22LQ6HPkUAKjtTVBR5yXzRw5PbGcYMTs1GWN0UoIKhTwePdLUd6Wqj8wj9Xklfydy3be3erWsy5o5bdp+mqsmEzfTaMXuOXtixKbR9Y4hNVcxRYTt+rq+h1tfRGqRffaKnb8NT9fpXNzxZT3sos50SHbBq6C6lc9uPdbf/7Xe/MpUNR0l4bFp4VW1YIWX6OBvODiHUNz5y+4iNE2YjkNPf8M7F2+yYhkmf+RzqxX1nDuxcn88RzN4FKraOAmAW4WZlKSklxfj4eEtLy7//j/85Vr3y9kiMl6qWNCxpXVszNEJOnPpw8O5dVj7Vc6SrbWAoqo9EbLb1vm1rGxb49JVVw7TrivBoqk9x+bKlG9ubiUb++ed3Bwc/45Ro/XyPOPLxH31n57lz54rtU9y2bUHDo5/71Zc+vRuLynxjY9PqVbUN9eT4T+5+cPGSLMuahqBeeNGGFyRJWrfm6a+3V//iY3Lxw/47twdq/VLzoqq3j+yprZ3/3e9+t7q6Wkih/+DHvYPf2vNj2iL54a7feKypDsYEwPydWA4nYVjOZmoMY5asIxJL6qctrm4KhBprwv2jbHy8fkOJDU/VZxuFWAl4PSyDK9XECSzXCQBwMs4b5/by/rP6+pW+qrbn6IVdnS36Mma2MffAcvmTFZjESpuidQFApYti/oXcHDpXIVsLzYjAl6wGwBn2UwQW2BQL2QAwS5RXMONnZbPaORxbeQRZh3sQdwEAFSiKojDtNKqr4RdHoE1P9GEQq9uFAIAKzxQBAAAAiOIUHMp8FjERjc1O0ggPWnxLwAQAVJ4oGm58FJAsx44FaGi9VHmwmjvmm1vVtKC+YpMRebQzAKjYTBFYhZpp+ryF4bXai0WrkRkCAFG0JYl5GL45HjEBVLAi4vIHAKIISpI4YmyoY9AgjQBAFNFMtkUL7VxlRhS4B467/4fPC3sXfwvgHgAAogiszzX04RVx1kmyCBMAAFEsaX4DAAAAlLUosg6vZEI2pjyg7BMQNTUbA7tFWXAfpO4E9P8CgEwxa6gFJiKrqqmiIM5KN+39L4c8mrzgNQ3XPACzTZnONhuLjKUGTKKLxWR4HYtESv+990ZGZVmB10yjqEpkYoIQH21gTHkTmSMAlSqKU6NDfvLW5WgsjpF4xTB8b+T989f/xRebp2IrZ6PPqKd6+0Y+uX4TXiuyNRO+cuXaTYmQADoPAKhgUdRtkPP3P7t89ZPbQqZZ/CZazCZCdI5vyXa0bB+ZXYHoef9Gagmbh3uOuES7quV37o688XfnrfKaCdOZ8Fq2T82u12LRyR+/cfnLX3oGsQkAZIrkyrWbd4eGRUNsnSFIcSaDZQXw/j/fWFy/UBLs/ZaxyPh7P78mpiuiacdVttei45M/O/PB0sbFCE8AVLQoTkxEY7G4z126U1JV88FXyLkinWbytaJez1i6HL037PN5PZJdNpRlua+3d254rRjHFePVDI7T5P7+Tz1uAREKgAoVRYFwgqZU5x283RKfZ+DRMqUdWt75iTAXOslU0YZfoWpEUTWvQITivJbRHVqWZDFPx80Jr5GkoqKPFoDKFEVu0XyPg6yWkNW8ZUDIEty1fCQ5mxCohcuA8GBHrmLiLP0s/a+mkYU17jnptWyO0zL5In/5VgtvdSWVqcGnUEcAKkUU7w0ONNYuzjhh0nShrGGBr4jQaX5BnUTS/DwzuYiqYDyRxzmrZGToFveAInXx5s2bj3/xnrtsvFaM48raazzh4hMagS4CMKdFkcZlQRC+8Y1v/IdX/h3cUDJ+/dd/XRAEnudNRFgmqOvXr/+DP/iDv/7L/wp7loYlS5Z8//vfp16DNAJgrzxpszHST00Rj8ej0eh4iomJCVmWk8kkXGJ5+4MQ4nK5JEny+Xzz5s2rqqrypOA4jucLmKShaVoymZRleXx8PBKJTKSQU+ToBQQmXKaqKp/C7XZ7PJ558+b5/X6Px+P1eunz+vd/3Dv4rT0/pqN5f7jrNx5rqoMNAXBqpihJktfrvX8eoijLsqIoiK12mFpM4U4hiqIgCCZyDiqiPM8zrwmCoKSAKFqLLMsul0vTNLfb7XK5PB6PJEmiKCJNBGDOiiLP8zRM0z88Hg9NH+ESOxBSiKJIU0ZBEMz1UfE8LwiC2+2mQku9pmlsJWvoopVNGUVRaCPGlYKKYkHJPQDAGaLIdJEGaEmSaGyFKNoE7Y6ivYn0v+ZiK/sU9R3N7KGFtkojS/RNew0A4ABRZBGWqqP2ALjE1vBK1bGY2Kr3miiKyBFt9Rr9r7lyNwDAYaJI73bE0xIHWUtaM6wdA6uWrE0DOwAwx0XR2kgNEKkBAKDYRj9MAAAAAEAUAQAAgGmIMAFgqJo2MDgWm0xm7CoUBV5WMDx41qip9sQn5cmkbHi+984I24fr/t9pNW04rjwdR/G6pSV183j0RJQNHMZKAMbuH55+/3Lf6Hg846vNSxZc/3QIVppFzLkAjitnx82rcresbNjz8kZYCaIIyo723zsoy6qKS8Kpd/PM+zqCcoPnOFHkT/9gB0xRJqB8Ch6SSCqEkBXBR0JL62r9Xvb87eFIfa0f9ikTItFJv2/apl0//ukv4gnZI4m/+ZXP6Z+H48rZccORWPjm4NW+u/S+AxBFUKaEmh752nOrHlu6kD6ciCWqvC6YpXwYG48LAq93yps9V24PRQJ+7ze/1sqehOPK3HEf3/zsDU272ncXloEognJHEHiBn+r5n1flhkHKivnzvFkdxz8crwHHlbnjBAGD/8sReAUAAACAKAIAAAAQRQAAAACiCAAAZQHmwkEUAQAATIHl9CGKwDH4MWoRgFIki8gXyxFMybD7unfUpoOpUx0bj6spHNj0tqDtTX+4s6KVpqn0n6IojnNZ8V5jO10zrznCfYosT8Qn6XnLslzmnrLca+X7Y9FUsYlDhw7t3r3bWed8eyhCCPF5JK9bkkTBcTb/6U9/2tzczPPm6x9vvfXW7t27b9y44awfPjgyoSiqIPB1gSpnnXlbW9vhw4eLibBUC1944QXHeS0pK7HJZDSeJITUL3DYwkNvvfVWc3MzMkVQQLaxe/fu3+x88ctf/rKJj/cOTpj+apfIl/iDqb34udKf8H39dk9dwPGE8sf79rz99tvLli0z33JXlEOHDgWblv77b/9feX6kb3C8iB9ustnhkorwVBHyI5n1FHOTgcmk8r/93r/92c9+9vzzz5trymiapqrq4cOHr1+/8f3/ctAtCfCa3V6LJeQ//qM9b7311tKlS4tpgEIUK0sRFUXRNG3V063XE0v0u/bke2PMpyFDzVTHyJTvk4fPJnK+M+PzijKr1YJiyrQxkloZhHukxu0SeUVRVFXlOM7EvaqqqizLqqouCS7rVRrjk3JeLgvoQnxCzWJwLqML4vk6N81f6iz5y3RpNpop9Ah8TbVLVTXqtVTTijchikqKYNPSMc/ySDRJb7f8vZbNcZZ7bdYcZ6nX7t9r8zwc4WiI0zRt7tVRIYp2iSL9W1bVOyNxWXbebnYFNWyz6DeXT6TIGla4jMmclnaX8lUeoa7GIyuanILneVVVC4qwtA+V3ueph9rgWCKRUOasvxJ5tbeyxbviZUAU+CqfWFPtogm6oijmYiv1GrvdBscmEwnVQZtH2uG1bI4r3muCwPnc4iPzPOyWmZOdixBFi6H1HBZe719hmhZNksnJRBEqm7OBmf2i5Oz8meVgbUkSOV7xeYQHkqnIsixJkrkbVR9eNU2LxJVkpo1hcx96hoDMETuc5ay4JAiCJHJVvqngQxN0c1cUHaLCRqnwHLkXTagKSR9wVIZec5bjBEHgeOJ1C/qbRRCEuRfDIYp26aKmaYmkSu43qojb43F7qwduf1bUYUuufLNjvRnjyIM/PB73vEBgPDKqV7Uiw6t+zK3kEqvmBfo/vQ1nFekpPR6Pu37hwpHhQUNbRBAE0/n99KGbXN2ihVev9cJr5hyX7ccGlzwyHhnRG197wBxLFiGK1itixmkYDfWLFi7yvvfz6yU7ExMXai41KfBoHLH029OYH/A8uerxe8OjE+NjLHOlUdL0NJj0Dy6qe6Smxv3BxesOclYJApTpMkGgxvPYoysjY6P56mp+LtM3ZTiOuCRp06+ue+PkPznrFrPbccUUdzY8+8y9e2MT46PsOA6bbAZRLAdp1DWECRGIqqlfXLPM49bO/OP1Sm6lWkJVlefXN68ajQTuDY/mVrVCmzLpH/9Cy7LagHb6zHV4qkiqqz1f37zqzm3/fVHUKYDp5F5flTE873Z5/s/fffZ7+8/iFiue7b/1+bFI4N69sXxa/xBFMIMiGljz+eZLH0fufDZoVa2jAuF5/unPrXp0WeDnFwuwfDG0fak5/Mnkzb5Py8dTBcXxUopBtu8SBaH1C6uWLw3cuV2i4UvVNZ4vPbv2H872lM8t5jivEUKe++LnH10WOF+qew2iWInUzq8ZvDukqgpMYS6weD2emnnzS9K6efhn/aKF/QO3yme9GK1sDpKn10RRnB+YX+IcbZ6/2u12T07G4TVzVPl8NTWBimt2I8qW+F6IxSd5nsdiwGbTxKkh4KpGkvoR6vaYU99ZxfMCvGbaa3QEjWr/ap+cYU2ylOPgAlNeeximkkm1ZE0ZiCIAhQa96Y+5Un43RNGU3WajMaGi48Gie63SLAlRBHPj9oVcOTWJtOpQosClVxd5ASHOwsZNJSSKEMWSgxXYLbIjsdWQKhINO8INN82wPFoyDrnXkCkCiwIr3dHmwUOXCyYBBbC6KXCsu33ftrVO/yH/z9Yv/Leu9SVMaAoW283rgse623d1thT/7Ts2hY51t+/YFKIPj3S1Hdhp2W+38DwLbMpX0H2H0acAzNxyjE9O7v+dtYtrvfrnT/T0HTwZdsRPOLBzfUOtT//MK4fev9Q7UiEeTP/5DvIdgCjO9fRRUWEEC4SKn9aGtXwkR8aWcSSWfHHfGdZm374x1NEa3LL7tE2/8VLviP7gq5sCe7eu6QkP7jl6wcTR9Cd/YOf6vVvXzKYu2pZ6PDywVoDvjp/rO36ub8aDH+tuD/ePdr32Xo73HDwZtkpu6Xnq9TvP87TVtvxcH4ON8mkJQQ+Ktc2LWe32O36u75VD71OBcZzpTvTcD6zPPbGolF/q9Ux7KMxej6KjfTcrDdCK6mJHpmi/iQVehhUcTsbhUZd6R3rCg62hutVNAZZy7du2NtRYw7SHtfH3bVvbsMD3vaMX9m5dk/4q7XzyeyVCyMBw9OUH65Ox1IQdtjVUd6y7ndY/0xPHI11tkVjy5enLm+XDrs6W1lAd/Vt/Avraoz7ZYnlMxo/QM6d/9ISnrd/kSf3GWS/MGHxnyMLpQ3b+e45eYPYJNdYw+1/qHTnW3d4THqypclHvbNl9Oj29M9jq1VNhlu1RVWam058GM3tHa7CjNUiNn14t0B/ZcPAZLzkAUQSgIBWcOZW5cH24NVT33BOLqCjSQEaLcixasTDk90qsaLmrs6WjNTgwHKUh7MDO9UxydnW27NgUMgSvrtfeSw+IA8PRVU0BfeT1e6V3Ls68rQcNsuwrdnW21FS56GnTb9m3bS0tEu7bttbvlehLOzaFdnW20G/fsSnU0RpkQZYOJ9FrOZPJ9P68MimCGXynR2/nAzvXr24K0L8zlk9bQ3Xh/tEcVfSGWl9n23L6hl2dLfSqmLEK+vL+sxn1VY/BC+kHz3HJAdNXDrCY6uoqGMFBhbjcY4av3YoQQupqPFSTGmp9r5562P0T7h/d8FS9/v2sG48G2ZbmWha8+gcn6N97jl7Iszl//uqQ3yttXhekD+nRsn3W75WOdbfTfw21vss6Jdhz9AIL9Jd6RwaGow0LpmTM75MGhqa2YD94Msz0eMNT9eH+UfZdR89ca6j10TOhYyP3H7vEgnskliSEcNODjR0+o2lhnqvx6X2nZ3WqnXHh+jA7/xl7XnP3MhJCWIZNDbjhyXpLfm9Ha1DvhT1HL0RiyY7WYD6XnDmECphGg0zRduhFRBe6UjHIxgpjTkmjV7Bj3mdCVvOvwumh4UbfDB+dSLBSKpMc9ncklqypmpLcgaFoa6iO5WF5cvBkuKM1uOHJevqlq5oC4f7RbG/WFz9phsHSQeM7o0n/gyJn/+BEa6jO8M7VTQG/V7o6MGYQGJoR1lS5IrGk/pfeGo75l0wdkOe0Ml/491LvSCSWzDOfI4TksDlL6A0PWZujGGgTRO8FeiHleckx9AX/dP78vKF8QiCKAICsLF/sJ4QMjsapGOj70gqC9hrS/sL0/rncEZlGWFo7NYTIHGq6omFeqLGG9YYaIiPN7Wh6QbvTjnW3M1mlv5r2dWVISX1SmXgnFp/M03fpid2RrrbtG0PbN4ZMD/ctDQbFHZ1ImLj2siSFgtfr2fiVZ1kDtEKAKJYaAct4FJ8vGgtxHFvlyyokkY8lNTJTmY9mh+9+dIc9Y3qGBo1NuXO4dN758Pb2jaHN64K5a6fp6KOnviuUDdBgr1JJoP1b+o7DQkdtGDbV5zjiEi3vvtF0N1pemb3edwZdpKZoDdWld/EWCWtzFI+hvzY9ESz/e63cQJ+ijcgKlgqb49BBFjTZolna6qaitto5eDI8MBzNmGxl7Nk6fq4vEktueLI+d+00HRo96TEban0zfpZ2kdKyKi0qpvfGUfoHJ/xeSW8Hw6IHdsOqfLnLfXrf5W6psF9qTswMutVQ64tEp46jr1SnT5KhRekcHiGErGiYN+3gC3yG3BFAFMuX+Gxv6zbjsmEWLiK1q7PlWHc7GwNiS75YwtZqdbUvozEjsSRL6Q6eDEdiyW/rrLersyWfRdpWNwWOdLWxv/VBMz3DaKwzDtSi3Uh+r/TOh7fz/Dmb1wVDjTVMCCOxJEsNd3W26Ouo+mpwqLGGqQKdz8D8S61B/3797E1CyEu/spI+ZFNNyod03+nZsSnEvEZXa2MlVr2hCoL5lx758JtX6MOrA2N+r0S/ZfM6YzmaCrZB9vT0hAdDjTXMC3SoMBvi5NB7bdZB+bQUCIKgq+/kur7SO70rajmufJtyD9pyU0M2SnXH0gGc7GF6/fDFfWcO7FzP3jPj6ics9r1z8XY+nzp65tr2jSF99x6toFK5yj0qJMfJv7jvDB2VSvuoesKDbKbHq6fC7FP6zs49Ry/s2BSivW70GVZ9vdQ7QudQ0g+e6OlbGPCufeyRkuSI5n2nT9b3bVvL3qx/5/5jl9jvyr9OHu4fvTowxg6ov6MPngzX1XjYTMQtu08b+qRP9PR1tAaz9TTTHt+MXrDhXlNJZaxAAlEsR9iVTZfj0k/ILQjDtKoZlw2zcBGpch6bYC4o7PyT92RZ1rQZhqFmGyOTrnP6ifDZ1gYzxLiMDqJFtss5W04zjtzJFkxzXBI51jMzXGmCILhd7o6N05aPKUHnAhszPOPPN5xwtkaJ4W0Z7WawGPNyNlsZ7hTDAQ1GTj+BHDda7ksOZL3lYYJZbL3OCL2ZrZrVBIqQxvK9U35t/VJWtAQUN3akAcgUyx9afih0klYklmTDLujQRPY8a/fRnO9ET98zKxY01PoiseTl3pFsq1JlWzZsy+7T6bmjfr0rQymJjtTXF3z0H6SnynJcuhoW/bqMLeL0nifT+bF9WsiV5dZyraG6geGoU2rspZ+qi9HeAKI4d6CzpOmSIjs2hZ5ZsYBpybHudv34eDpvrCc8qH8mx6L+GZcNM0BH4bM3MEnW13OOdbdT9aKv5hi8TmM3Pf8jXW1HutqYqNMVO+lDw/SAjExMqPoiR4WP8rVvmw6L5VCZ8hodFKpit20nMD79XqsEUD4tSVvV7OAtOpSRjlU7eDKsF7xw/6hhqHcklrS2J6+zbfnAcJQdk84WMKxbxvI5OvYyx0g5/UiBdy7eZqP26axzum8D28DB1mGrM8TuVLBmDtOPGUYUL/c2PpucmPJfdbUXNrGWSmiAIlMsgSQWfBmxgiQdkJbxPelLV1y2tIBGk1TDMemiX/pNIfRTqfSV3nT00wzoVKrli/3min5KKufAmnkgn3iNpgyAKDqebEKo39+nNBgWwcq4JlaRHD/X19m2nC3gScemF9qbyFu9ThBqezagPUziqNfsmf7GcehGtAxVUQwNUJ6HKAK7okNh0KExbJJTaQTSsGpJtkVMik9J2YCggjrJaO+UZk8ywOlWuCrzNaydCGeLy6YFbp5H08byyIXyKSiCQJXrrnUlPrqAU0H1xtyrUuU+FN0rwLB+SmNdlWEDhOKhS1uV7WgRfRuZQy3OkqAjzp2wk2Ms2xz7dq1iiicQxdLB80QuYg/+SCzJRtbs2BTKJ02ccVWqjMuGMd65eFs/oHRXZ4t+v0CroPsfGRbymEkj58L9aZjukvGH6yfhUNJHC+tNl77+kWGP33TD0o+nT4ChlQnDbvszjljOdf0LiAEAogis4+X9Z490tdEQFokl6fpPuT8y46pUGZcN08uVYYcgO9acowNN9UGZrpQ2c+5oZ89RyVRXLzB0dbH0tj8zDtVItu0ilSj2/s3rgt/ubDHMXmXTYOjHDYbdsSkUiSXpkuIZ+3H9XqnQXR5zMG33X/T8AYhiBVJoCT53JcSgW2xGYPriT9mez71sWPr7cyzllR4o9TNGDB/M8b0drcGB4aj+NPoHJxpqffoxrjOIo00KxulrR6WI4F2vvUe7irNN9zx4MrzhqXqW3NO1bNjq0gZvfruzJRJL5nAKIeSZFQsGhqKjE4mMhQeqlyUe2+XMtk2lwGkQRWDtPYqbND3yRpP63W7zWaKFn7t7Uu45euFIV9szKxbk/5GMM1sMsz8zQnfkeLUnTG2ersR+r0RrCRYmi6UI3BxX4h1p9FVuQzVFv1STfkEoffHcUI7W7wpgWLjc8EWIHhBFp6IoMpqtOdKjfdvW6nvX8h87MDVkY25JJN0KKlvwpUJFH75+9mZrqG77xtC1WxGDLubeQVefaNLksrNteUYlPn6ub8OT9YbJqWabhEKGHNz5hBpr/D6J1kLolcx08UhX29Ez16iFd3W26Isi+q7ZAzvXM/PqV3SiS0qxmhBVRCqTGTuk7YB/sJ/wXBoeBVFEaugAXczXng/+sG/mvqwYx9nJpZ2OQZdl0IuQfm8g/STOS70jr54Kb98YovFR3y+r30M4G6uaAmxDWqrEGZXv8JtX9m5d89KvrCx+lKNW2tutNMs76GvU1Fa/tn4pNaO+v4O2YFqaa4+f66NrOV24PkxfYh/fvC6oH8tGWyQbnqqnorjhqfqB4Sj9m7neRmNmTsPnfjjCMm8l5cFdiiEGjmnMlcNU5VdPhbfsPk3/GfYFPH6ub8vu07RMun1j6MDO9Xkek9ZXz18dog/pBsU0dzRwqXck3D9KJdNiLwqWGXcWV1KlSxMzW2Ub0a1vbdC3bd8YMixnSPN7fcfw6ESCVV/9Xql/cELv+tJGLQQXAJyDHUsyFrkwimE2Rf6bfuRO8mh+YOjkoyNoaF9UjjXZ9dD9yFh99fi5vu0bQ9nm53S99t6x7vadW1ab3tVdv892RcVxw1wjxov7zhzpaqM1AFZHpa7P+BHLWyT5u64E9xpEEaCoapUFOXsOPs1B5pKQHAN3c0NXacj2qn6xWQO0dkdXHbo6MBZqrNm8LphNiekEVkO/VI5Bvz3hwdZQHV1pwbxh7deq6U2Z2ZkaSZf5NUyYSVc7Wlzdt22tYZSTU7Y9maugfFqSCK5pPA9Tg7zyy9yjRpcv9uezCC2bY5rxVTY2lVVlt+w+TbuyMlZQ6bDYSCxp2COlCOmaO81N/foYdM1C2itMGxC0Lp0DKpmsKZMtKUwvzJZsJxkNoggsRJrea1Llg0msTQXmFHSAYk94MEehtbNtOdO8AzvX70ptLkahG42xsuqrp8INtT59L+PmdVMrB9HaqSGRPX6uLxJLrspepqMbfjnLayUI6H6vxIy8c8tq5gKa7tNuQjoSVd/02bdtLfubtXLo/mvf1vl0V2cLe+fl3pGGWh/VQjYwFVgOyqezE9hhgvLO7KeFVcXO0aetoTp9VS1j6Uw/+nRgOMrGNL68/+yx7nb2cf1LbC6//g3s+KHGmnD/aPoXXe4daQ3VZSu6HjwZfmbFgoZanx2bpVhN6bbG7QkP6kujzIPHz/U11PrYEoYnevpYnk07gNlH9JMRX9x3hq7oRB/qq697jl7Y1dnCLoYtu0/rhRYNUIgi8pvK1i2VGdaur1AUe6diZFuHyKBDuTsmZzxCxjdk+5R+8E7GCRj69XGKvB9saMpo+gOXZjhqbvsb3Kf/O8f8lhxGNiyhkL40o81BrCJiC8qnJc0JoYwWpADYXNiJgSbVp+5yPci+gYPuuApzF0RxNrQRQQEAezUYNrAApSIboLh2wJxqblh88FRq7/XAzA7wmgZddPK9BlGsvOsptYY1Kn9WCZU0vTfcwhXCqz2iKKDM7bRA9rBngqvAOD5LpoYoAsvaVpyGRVEtyQdKa0b4zDSTiUS6BcfjSVim/PXPVWG7Q0MUS5PcTHuoqgpsYlKWUuSwrbXQQT1TX3j/m5HpmzQjV/KWDPsWeM2kAafXtSqnHY8pGTYip1VLFZV8OnBbVTXUdUxHuslEYnTsHiHz7czvjd6JRqNJOTlD/o/cMntTRmc9eiNo9qlg6u+p5n5STj6I5vBagfYkWnxysmQNUIhiBQXxKYFMyoSQn/zscmR8otLmLHKF3YozoCjKLz68/NzazxNSogEwI6PxDy9/zHG8zT+d2BO1SykGWb9LlpV/vnD580+tKuWF9/7586qqFe44Z3nNRscpivoP/3T2y1/8UkXFK4hi6fjZO5c/+uVtoSS7xpuQXQdVRyYmJv/0z84/u+aZEnxXNB5/9c/P5mPPXG0dzbJYptkYte29Mm/cvPXX/500LSmRLv7JkfODd0dmdFzhXjPjOM1e89rouPHxye99/+zGF9ZDFIHFXLvZN3xvRMq0gZw2i4GqxHmzdT8zNpH4x/9xvnHxQkW1MdIkEsm/+pv/IQm8E0XItPDad1oXP7ozPMxVeV32fQXtBn773Q+G740613Fl5bXoROLET84+urQRoggsubjvX96xWDwWm/Q6axRX9vuSnynZ1ay93acbU/dYuXP7M0ni8mr1F/5FqqZ9fOWqRzLvtWJ2vhXKxMjTwq4F5h26OzhZ7bNDrJjFevs+TSQSph0Hr2UyinL9Zp/PI0IUgQXwHFftItWuDB1gCbmANMed5SZPvx+0Qop16SMeBEd1eSYV1cI+WnYoTdPkRNLnvn+DuMTMMTx9PF5Bw0cyukl17Bi/3Ceud5GaTEj2NRBVVeSImN1xGUdR5u+4ivXafbXgiOUNUIhiRcClYC3HxfPnwoIoCVkrRL/zqhJrhah3xvfSSfbJB1uBczqKUURCiMclLFtUVR6WL7bllDGNyGjPjB7J6I7iW078gxUSigyv+o/zHLd0Ibxmp9d4DqIITIpiMBj86Pw5b+BmtrfJyqxNnwpUuWy922NFh4lCGf2Y3Lhx07QcGqJJ8JGq5oXV+bxZLvliyW5pdorwSh6Xa/624HT/Y14z4Tv6WbraeF2NZ1GNpzy9NluOs8Nrc14XsbSK5cUbVZblZDL52muv/cVf/IWSIn3KOeX6rVFNIzxPav3eap/EYxONPHVL03ieZyblU4ii+IMf/CAUCnm9XvpMQV5TVTUajR46dOj3f//3YeFScv78+ccee8zlcomiaOJei8fjv/zlL9etWwdLlpI33nijvb3d5XLxPD/H1BGiaH28pkIYjUZjKeLxuKIoqqqmm/rf/eefKarqdYlff27l5x9f6HNJMGD+yLIsSZKmaW632+Vy+VK4UxSaMmqapqpqPB6PxWIff/xxNBqdnJyUU8DOltdReJ4XBMHtdkuSFAqFqqurqSKaEEVFURKJRCwWC4fD0Wg0kSJbGxSYg+d5VVV5npdSeDyexx9/3Ov1Ug/yc27ldZRPrb/n6W3vSu0dR29+moikv1nyVPGK5nKL1f558wPzfW6IYsHWVlXV5XKxICsIgulCnCAILpdr5cqVk5OTiUSCNmVgZJt0UZIkl8vl8XhEURQEwURspcGauv6xxx6Lx+PJZJK2YyCKdjRAqddou1MURX6ObkQCUbSlYcVxnCRJHMeJopijfCq6fZyiim7JV1Xtn+evcrtgPXOtEI7jqDSaC69UREVRZNKoT+4RYa31F71HqKckSaLh1VyEpWVzprI0d4TLbHKcpmm03UnvNdNegyhWaJim7V/tARnfKUgewquSy+X1VlVVVVd7IIpF6SIbc2E6fWEp44OlwBFbbUwWWSnVtNfYB5nX4DK7bzTmsrk60AaiaGO+qKoqbV5lvch4ntMIJ3C8wNMsB3YrUheLT/HpEB7kiCXIF4tsx+i9xo4Dl5Ugy7dkmDdEsUJ1caaLjOc4IvACP6WJEMWyEFfYwXFew70DLIvbMMGso6iaguYtAABAFAFF4OEIAACAKAIqigJKdgAAAFEEKRRZ1VRUUAEAAKIIKBjfAQAAEEVAEQX4AgAAIIoVDx15qmBFMQAAgCgCAAAAEEUwBToTAQAAogimENCbCAAAEEUAAAAAogimgwmKAAAAUQQUbGULAAAQRaDLFbEgOAAAQBQBBdsVAQBAOYD9FMujbVK4KObY0B9Y3mRBqwUAiCIoHQXtkvHWW2+98MILMFrJWLp06Xe/+92XXnqJxw5fAEAUQVmhqurhw4f/5//l33Ru+/2MeSJnc4bKuDsWN/1Zl2ReXVyiUMRnC/5ejZCzb75+5syZ3/7t38blBwBEEZQELV9FVFI0NTXfjldNxGVFUa0Voclk5gOm1w85ImV6G8ny8emPFaIomiMyeJ9H5FKWV1WV4zgkiwBAFIHt3Nc2LV9RnJrBoZHBscnJSWUu9XZxhSS6CVnJR7xTCTHJM6k2SJ7XLVZ5JGp5WZZ5nldVFboIAEQR2J8rzjRqRtO0aaKYivWTCjc+MXMZk+fNK6c64+7HnIlX8pPIIsxVPF6P5HFrD1otiizLkiRhxA0AEEVQRsJJ63gPdYPnli9f9uEvr8I4lqlxSgdXNAdH7w2zQ1HLY7gvABBFUKowPlNRTtOhf97tdv/m17949G/+yeF6n90yJc/NfvX5Z4aG4lOiOJUuq1QUoYsAQBRBKchzq4yMQdnr8fzvv/vsH+0/O0cT5JJ+27/97c8ramBo6HY+lgcAQBSBXblinoqYMToHajyfe3LVhYuX5rqVNNPWy4eWp1Y/ujTw8XWsMQQARBE4GZWQ6iqfS5ISyURFGsCCNE4UxWqfzzAnZeZxRgCAOQQGl5cLihVbZXAcx3Pwqak7gecN2SGmXgAAUQQOzBENaoqyn9VknBAJAIAogrIXSAwGKd6G0xUQFgUAogigf5UOxpoCAFEEDkPBhv0AAABRBKAUoHcWAIgicBy00McTEotPkpJPdYc0AgAgiqBcBBEmsINEAjYAAKIIAAAAQBTBHEkbMWDSkltier0U1VMAIIrAKSoIG9hoWMghABBF4EhcLtjARlRMfQEAogicF7sxb9EKJCySDwBEETgLrMZZujsEe0gBAFEEjkBkGxPr4zZiePE5dyrlRpctABBF4DzYhkfV1b4iD3Vg5/ojXW3s4eqmwLHu9gM718PIAACIIqh0vt3ZQgh5ef/ZyvnJHo+kfyjwyLsBgCgCx+SImZ61qOh3YOd6v1d65dD7sDMAoHLASLs50bThibU7xe/qbGmo9b16KnypdyTbe1Y3BfZuXXOip29Fw7xQYw0hJBJLvrjvDH2evueVQ+/rj7B5XXD7xhD9e2A4qs9B9Z8yfPBIV9vAUPTqwFhHazD9Vf0He8KDe45eKP7ne724pgBApghAih2bQq2hup7w4PFzfTO+uaM1ODqR2LL79CuH3vd7pWPd7Xu3rtmy+/SW3acHhqN6nduxKbR9Y+hETx991e+VWG8lFTb6fPoHCSGhxpoNT9VnfHXv1jU94UH6UmNd1eqmQFHJt65hoWkqh2FLAEAUQYnRCIkllLHxydGJGf6NTUyORSeT8rT5iJylcwb8XqmjNRjuH80z5WLvvNQ7MjAcJYRs2X2avnT+6hDNDunDDU/Vh/tHD54M04dHz1xrqPXRVy/1jrBPpX+Q5aD07xM9fexVKoEXrg/Tl17efzZHagsAADOC8unsM5lQjr175ac/vyHkrH9qmqbICVWRP7px99HH7DqZSCw5MBQNNdZsXhfMJ1McnXi4o0Qkmox4k+wh1ciGWh9VL79Xujowxl69divCXjVAP2g4q4wfvNQ7EoklaUk2n7OdkYwja3g0HQGAKIKSoajqZyPRz2bKcDRNU5WkpsjjOoWgxOMs57SArtfeO9LVtn1jSC8zB3auZwKmz9vyZPliP621sn5BA/ruxoJ4cd8ZerbbN4as6lPEfH0AIIpgFvBIwoSi3I/BBcqZavO88u8dvbB365oDO9ezsTCWTMw40dPHyqd6dmwKdbQG2auFCiRV6H3b1raG6nZsCmX8CrPyiIsUAIgiKBXVPtdEPPlUc93WTU+uaAy4RCGXEKrq5ORkIpHYNvDGg9zRrhO71DtyoqevozW4q7PFktzr+Lm+7RtDdTWejK+uaJhHCClSzLpee+9Yd3u2rygU7MMFQGWC3hJnox9lo6qKhcnNwZPhcP9oa6hOP+ClGHrCg/qj0eVy6N+0Y5KNnck/TdyxKbRv21r2NyFkcDRezEmOT8RxUQGATBE4G5v2NsrYuWiaPUcv0FkZTPPYiNM9Ry/s27aWvfTqqXCeunjwZHjftrVMXLOVZwEAAKIICia94zDHgBrDPAoqovqHx8/1GdT04MlwNtFK/2y2czB8r+GD1iIrKKICUFmgfOpoNHjTWhTsSQkARBE4F+OSKxgeYgXYah8AiCIAIC0TRyMDAIgicHTmCCwn91QZAABEEcwysqIachhZgVUAAACiCLA4pw2gcAoARBE4ECzTaadRYVwAIIrAIamMMbOBSay4HwTcEQBAFAEAAAAAUZxzaSMKfpYCcwIAUQQO0cFpBVMVo08tEcEHZhUETMMAAKIInIaiyDACAABAFAEAAACIIgC4JQAAiABAj8hPDQKh+wx7PDBJsXCZJn1iYQQAIIqg3Kn2SNle0lTMWDQDlrABAEAU5xqqqmgEwd2kKKrYMgqAigc778+RFEfTyMdXrqX2yLVrbl3+x3WoLMuy8umt2xqpwUUFAEQROJu3/uH63eF7XM6lULlSaViZTHk3oeI3+z5984xraVMzrigAIIrAqVy7efudszdEgbMhDS1cigpcozxHZx5XeDJa/ALpb71743NPcJLgxXUFAEQROI+R0bHzF/slsytZa7bJpuawxPIhH12+UV/3CC4tACCKwAHQVIyKkaaqd+/c8bulrOJSrv17xZwXP0M+qJn/3gcvT4yNSi6BECz2BgBEETiHgFcg3oIDd0LOV5LcEp+ntGSrgmr5pZdCWS69LWC7SgAgiqCcc0TWYycK/KIAZuw/RBL5fNQ3m35n7aRMJeCcDpgaAIgiKCNR5DjuH999JzpZon0xZGV2Zu95XEIR52xl1fj9c+82NtRDDgGAKIKyE0We57u6uvbu3Xvz6kfJZFJ7AIxjCZqmCYKgqiqVQCHFms8//fzzzwuCAF0EAKIIyksRRVF89NFH9+/fH4vF4vG4oiiqqkIULU6OZVmSJE3T3G63y+XypRBFked56CIAEEVQLqJIddHlctEkxu12qylgHDusraqqy+WidpYkiW47DFEEAKIIygWaqUiSxHGcKIqKoqB2ancrhOM4Ko2CIPDYMgMAiCIotzAtiqIgCOhNLKUu0hwdBgEAogjKMV+kI0GgiKXURZgCAIgiKF9dhBEAAMD66AoTAAAAABBFAAAAAKIIAAAAQBQBAAAAiCIAAAAAUQQAAAAgigAAAABEEQAAAIAoAgAAABBFAAAAAKIIAAAAQBQBAAAAiCIAAAAAUQQAAAAgigAAAECpwH6KjuTGjRvYdr9kcBzX3NwMOwAAUQTlyKFDh775zW/CDqXku9/9bnd3N/Z2BgCiCMoLVVXffvvtrld2vfCvXqbPaCRTvqhlznhmuBoEblZ+1Hgsaf4KFnhbP6gR8vZf/wlNzXH5AQBRBOWliEoKjnA374yPRZOyohZzQJdUgKJMJjJ/V7rUZlTfbIqc/rSilov8iAI/r0qilldVleM4JIsAQBRBeYmiqk6J0/BEYiKarJDfXlAOm1QyyCrP5Zs6s6f9VRITRVmWeZ5XVRW6CABEEcw+mqYZRJHniEKE+GQykZQt+Qo+Z301tywpZquLXJFv4XJZrIgcWvK4BXZ8RVFkWZYkacYSNAAAoghKqotMFO9HbJ4LNgWv3+yPxuKwj1VU+TzLly/7tL/PYHl0KwIAUQRlpIgM/fMet7vjq0+f+un5kdH4HP7tBaeKnMnctDbg2dLxxV9cvK1/krZFMA0GAIgicIA8eD2eb734zP7Xfh6NWqiLmilNmQWTFHb6OfF6Pf/H7z378fVChRkAAFEEs6SIGaNzoMazYnnzLz68bH/sLlttKFbFOY5b+Wgz+xvXGwAQReBUVEKqq3ySKCaSCVjDnIoLglDt8yWnTzxRVSSIAFQQGFw+p+A4jufgU1N3As8bskNMvQAAoggcmCOq01MblP2sJiErMAIAEEXgQIHEYJDibZhSQNY1C4sCAFEE0D8AAIAoAkdR3LqnAAAAIIpzDsyfswv0zgIAUQTOdWQsPknKeRohpBEAAFEEtmaJMIEdJDDVEwCIIgAg4w5TAACIInBg2ojORRuARAIAUQROUUHYwEbDQg4BgCgCR+JykdSsc0zRsAUVdgUAoghAZSJNXySfx3YZAEAUQZmD1TgBAACiCKYhCg/cp09mHJXYbF4XPNbdvmNTKJ83H9i5/lh3e7ZXd3W2HOtuX90UKP6saL0UXbYAQBSB82AbHlVX+/L8SLp+HOlqy6E3AABQKckGTAD2bVvr90qvngqX+Xm+vP+s3V/h8Uj6hwImLQKATBE4J0d84EW9Gwss+u3qbAk11pzo6Tt+rg8mBQAgUwRzpYFT+E7xm9cFW0N1PeHBgydzpYlHutoGhqJXB8Y6WoP0mVcOvX+pd+RIV5vfez+1OtHTZziCvhi7ZfdpQ2IaaqwhhERiyaNnrqWf0vaND7sYXz0VZmpNP6g/GjuBcP/o6MS0xdl2bAqxs9UfZIZ2RsqGXi+uJgAgiqDCWL7Yv31jaGA4uufohRnfHGqs8fskKkhHutr2bl3DxGZXZ0tHa3BgOEqFZ3VTYO/WNeH+0a7X3qNKdqy7nSkZFTaqqTs2hfT6x5SMSeyuzhb6hoySdqSrjSnuvm1rW0N1emXtaA0yLTzS1WYiD9Y0lSMCrhMAKiu7gAlmnXhC7h+MXO0f+bjvXu5/V/pHrn46Mh6blhJxZifSbd8YisSSeXbU6d95uXdEn369fvYmIaSluZa++tKvrIzEklQRCSH0j12dLVQvQ401PeHBS6kjHDwZ7gkP6r+lozUY7h9lSeeeoxcisSRL+Azy6fdKLNHseu29geEoe7Wh1qeX0hf3ncFlBgBApugM7o7G3v6gz+918TmHdWiaKieTipz89O74aiu+tyc82Bqq29XZkk+mGIkl2d+Do3FCyLVbEfqQKlxNlWtKkBb4Boaihs/SV5cv9hNCLlwfZi9duD7MMrzN6+6L39WBMf1nB4aitNZqoK7GY8gg+wcnqBYSQt796E5Ha1CfoeZJxpE1PJqOAEAUQQngCHGJ/L1I/F4kPuObNU1TlaSmyIMjMWOuOfXpwsbYvH72Zk2VqzVUt3ldkKnLrs4WfR2S1jkLOqzfK4UaazJO8GCilQN9wkcIMfQUMpgGZ+RS78grh97fu3UNPY38+xQfJN86DwEAIIqgNKx/asn5j+9cvzVa6AdVi+aVd7323pGutu0bQ0wz8skaZ4R1KJrAIJy5xS+3LrIe0O0bQ9duRQpVdwAARBGUlB2bP1eAEKrq5ORkIpHYNvDGg9zRgnP43tELe7euObBzvVWzAAeGo36flPElWtVsaa5lGsx6ImktdPvG0IqGedM0coHPkDtSrg6MhRpr9DluY11Vxi89euba9o2h5Yv9BYki9uECoDJBb4mzoaNs6LJkqppaEJUrOKM60dPXUOujY2GKJ/1oR7raaH/hpd6RgeEoK88aSrW0m5NKHX1IVxXYf+xS+rfQwThsDM6Bnev1Kea+bWvZ0nEbnqzX94DmZnwijosKAGSKoKI5eDK8omGeoXPRNMfP9V27FWH9eYYuvZf3n2VLmA4MR2nPH/vsnqMX6DQMNlUjx0iZLbtPH+tup4fqCQ+evzrENJKWhQ2zKuFoAMDMmQbKRE7hYfl027bG5sce//Jv9Q/FxuJaQ0Pj6lULVzaT//7mvZ5/vqgosooNAE3UTHheEITWLzz99a/WXLlBPrhw69ZAf6BKbKrz/fKtH/Vdv/Knf/qn1dXVQgqYC4A5GwpgAiejGb2JJk5xKNioGQCIInBwpo9JA7Yk5bABABBFAIAhE0fmDQBEETgkXmfOHIHluER0IgIAUQRljJzW9SUrsAoAAEAUK5zUPEUszmm5UVE4BQCiCJwEYnYp1BEmAACiCBypkRBJS+4HAXcEABBFAAAAAEAU50qeyP5Cwc9SYE4AIIrAITo4rWCqYvSpJSL4wKxYyw0AiCJwHooiwwgAAABRBAAAACCKIBMeD2xgyy2BOaAAQBSBAxD5qUEgdJ9hUDywJAAAouhIqj1StpewFIs5YDcAAERxLsV0ktrzSEFwNy2KioIhvABUOiJMMGdSnMvhq6qiWrXDYkHSmn/d0RLJtqnKqSrqJzduauRzuKgAgCgCZ/P/HTk/MjrGTRsWohWjKTYJT5l32302OPRXb1x++qlVuKIAgCgCp/J3Pzn/2dBI2rKd9uuPZkIUOdOpcAHSXUR74IOLtyMRLjBvPq4r8P+zd3erTTRxAIc7O/tBw/uCHulJwbtQr633VqxeREGwYj3yXJLY3Z0RdjEEbG0haZJNn+c4tNt/6P52JpsEUWRiQgifv3xdLJbVkMR/Ryo+EKS81fytV23Tv3LHU735/uPni4VbUUEUmUYIV6Vpf92WOf+/djPqjl/e26/8ZCFtl4uiCCcnPuwNRJHpNOHVy22+af+2TY9/cFPFew4qP6a++Z6i/f3gWOxtzbbHXw2IIg+vEVevydVV8eb1f7s/hq5PB3E9kDdt251VvuvHhtXk1+cPiCIHEcWzs7PLyw83N98md/xdv58d2zJuVLJPHy/fv3srh/AsTrPe6z0VKaWu69q2vbq6uri4uL6+bts2Df6scjyVmy9Ac4wxpTRef8QYy7Jsmub8/Hw2m52enhYDgwJRZP/n634wn88Xg+Vy2fd9SsmTuOUVbddVVZVzbpqmruvZoBnYQYXjZvt0Otcvg6Io6roevwW3aZr1lSLbnXZKqa7rcc5VVY1fO6yIIIociqIoQghVVYUQyrLs+z4PTObprkJCCGMaY4w2TuH4//GdUidn3C9dMZAddHFcoxsIiCIH2kV31uy4i0YBoggAz4gdIQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQAQRQA4Wr8DAAD//7ewsUqY0epQAAAAAElFTkSuQmCC"}}},{"cell_type":"markdown","source":"\n## K-Means Clustering (Model 1)\n\nK-means is a very popular clustering technique and widely used. K-means algorithm performs partitioning of the data in such a way that within cluster variance is minimized. You have to specify the number of clusters — k as an input to the algorithm.","metadata":{}},{"cell_type":"markdown","source":"### Finding the optimal value of K\n\nThe optimal value of k can be chosen using the elbow method. In this method the cost function (variance) is plotted against different values of k. As the value of k increases, the number of clusters decreases and the average distortion also decreases. The optimal value of k is where we see the sharp change in the rate of change of error. For example in the following illustration, the optimal value of k is 7.\n\n### How does it work?\n1. Randomly choose the K Centroids (for K Clusters).\n2. Compute Euclidean distance of each observation with these Centroids.\n3. Assign the objects to clusters with shortest distance\n4. Compute the new centroid (mean) of each cluster based on the observations assigned to each cluster. The K number of means obtained will become the new centroids for each cluster\n5. Repeat step 2 to 4 till there is convergence. i.e. there is no movement of the observed points from one cluster to another Or threshold number of iterations have occurred.\n\nThe cost function is given by the following equation:\n\n\n\nwhere,\nk = number of clusters,\nj = cluster number,\nm = number of observations,\nj = Centroid of cluster j\n\nThe objective of the k-means algorithm is to minimize the cost function.","metadata":{},"attachments":{"1532862e-b45e-476f-8c69-cf3efe1c8741.JPG":{"image/jpeg":"/9j/4AAQSkZJRgABAQEAYABgAAD/4RDyRXhpZgAATU0AKgAAAAgABAE7AAIAAAANAAAISodpAAQAAAABAAAIWJydAAEAAAAaAAAQ0OocAAcAAAgMAAAAPgAAAAAc6gAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEFrYXNoIE1hdGh1cgAAAAWQAwACAAAAFAAAEKaQBAACAAAAFAAAELqSkQACAAAAAzEwAACSkgACAAAAAzEwAADqHAAHAAAIDAAACJoAAAAAHOoAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAyMDIyOjA3OjE1IDE4OjIyOjEzADIwMjI6MDc6MTUgMTg6MjI6MTMAAABBAGsAYQBzAGgAIABNAGEAdABoAHUAcgAAAP/hCx9odHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvADw/eHBhY2tldCBiZWdpbj0n77u/JyBpZD0nVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkJz8+DQo8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIj48cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPjxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSJ1dWlkOmZhZjViZGQ1LWJhM2QtMTFkYS1hZDMxLWQzM2Q3NTE4MmYxYiIgeG1sbnM6ZGM9Imh0dHA6Ly9wdXJsLm9yZy9kYy9lbGVtZW50cy8xLjEvIi8+PHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9InV1aWQ6ZmFmNWJkZDUtYmEzZC0xMWRhLWFkMzEtZDMzZDc1MTgyZjFiIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iPjx4bXA6Q3JlYXRlRGF0ZT4yMDIyLTA3LTE1VDE4OjIyOjEzLjA5OTwveG1wOkNyZWF0ZURhdGU+PC9yZGY6RGVzY3JpcHRpb24+PHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9InV1aWQ6ZmFmNWJkZDUtYmEzZC0xMWRhLWFkMzEtZDMzZDc1MTgyZjFiIiB4bWxuczpkYz0iaHR0cDovL3B1cmwub3JnL2RjL2VsZW1lbnRzLzEuMS8iPjxkYzpjcmVhdG9yPjxyZGY6U2VxIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+PHJkZjpsaT5Ba2FzaCBNYXRodXI8L3JkZjpsaT48L3JkZjpTZXE+DQoJCQk8L2RjOmNyZWF0b3I+PC9yZGY6RGVzY3JpcHRpb24+PC9yZGY6UkRGPjwveDp4bXBtZXRhPg0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICA8P3hwYWNrZXQgZW5kPSd3Jz8+/9sAQwAHBQUGBQQHBgUGCAcHCAoRCwoJCQoVDxAMERgVGhkYFRgXGx4nIRsdJR0XGCIuIiUoKSssKxogLzMvKjInKisq/9sAQwEHCAgKCQoUCwsUKhwYHCoqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioq/8AAEQgAPwDAAwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkqNDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/EAB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMRBAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A+kaKKKACiiigAooooAKKz9f1JdH8OajqTzRwi0tZJvMlQuq7VJyVBBI46ZGfUVF4a1W41nQLa9vLR7WWRBncUKy8D94mx3Gxuq5OcdaN7g9LeZq0VR1jVYtG0/7ZcRSyp5scZWEAsN7hc4JGcZzgZPoCeKq6B4ltvESSNa211AI0Rz9oVRnduGBhjnBUgnoeoJHNAGxRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFZWh+JNN8RLctpZumFrKYpftFlNb4cdVHmIuSMc4zjvVXxD400bw5MbO+uwNSkt2mtbIKxkujnaEjAHzsWwNoye+Mc1N4W0ufSfClnaXm03nlmW6KnAaaQl5Dkf7bNQtU3/AF/WgeX9f1sbNFedxeFNeW6R20vChwSf+E91VsDP90xYP0PFeiUdA62MTxFrfhnT4lsvFGsabYLcAOsV7epAZArA8ZYEjIGe3Y1D4d8Q+ELjZpXhjXdJu3UM621pfpM4Gck4DE4GfoBgdMUnjPQV1jSUu7e6j0/U9LY3djfucLA6g53n/nmy5Vx3UnuAauaDdS674f0rVNU042V5JEs5t5DkwOykHB+hPXBweQDxQuv9egPpYZr+ueG9La3j8S6xp2nlm82Fb28SAsV4yAzDOM/gcHris/w3q/gltRktfC2u6Xc3NwiD7Laams3yxoFGyIOQoCgZ2gdOas+MdATXNGEkV0un6jp7farG/PH2aVR1b/YIyrjupIqzoF3Nrvh7S9S1XT1tLxkE3kltwjcqVyp9CCcZwcNyAaI9f69AZr0VwfjTxPplnqt3Ya5qk9la2VgL0WllcSRXV8SXzsMZEmxBHk7D3JYhVOd3wtdiLwfpbXF7Lqcr2kcsl3GGnEzPjJDKCCNxPA6D0Ao6X/rr/kD0/r0/zNKbWLGBLx5Z9q2LKtwdjHYSAw7c8MOmetXa4nVv+PPxj/18Qf8AomKoNSvLCXxpGI9cuhpd8Wtbtbe+l2m8jIMcUbK/7tsCTekeCSq56kEWrSB6He0VBLNKbe4+xRCS4iBVEnLRI74yBv2n5eR8wDd+pGKyLO88XPeRLf6HosFsW/eyQazLK6j1Cm1UE+24fWgOlzeorD8bXE1p4B8QXFrLJDPDptw8csbFWRhExBBHIIPeuf0Hxrpeo+LIY7XWlvYZtHjaJYJDMssqs5k27chnA27gMkZXPUUr62/rZv8AQbVo839dP8zt7q6hsrOa6un2QwRtJI2CdqgZJwOegqrdXkMyyWcN8bS4ktjMswQExx5wXG4FcjPfPuCOKg8TsH8GauwzhrCYjIIP+rPY9KzmsrXUfEUNpqFtDdW02jbZIZ4w6OPMXgqeCKNW7f1s3+gN2Sl/W6/zNLw5JdSaMv269+3SLI4W4KqryR7iY2cIAoYoVJwAOc4HStWqum6Xp+jWK2ekWNtYWqElYLWFYkUk5OFUAcmrVUxBRRRSAKKKKACiiigAooprtsjZtrNtBOFGSfpRsBzPiqDXtQv7O00ezsbmyiInu4ru8ltvOIPyJuWCRSuRllyCcKD8pIZbbWfEQ8Q2On6zpumWMdyJHD2t9Pclwq/dybZEU5IPLgkA4B5xmSeFLTxxa6jealbyWUV9dRSwifTYDMY44tq+ZHcxOAdzSdVDYxjjroad4Wn8PtolloyW0un2U00k8koitnXepA2RQQrG3Lc/d6dzQtNGD1V0P8XW+uahPY2WkWdjdWbEy3cV1eS2/mhSNsZZIZBtJOSpwWAxyN1EWteI49e06x1rTNLsYrx3Aktb6e6LFULbebZEUnGfmcZAbAPaHxH4Ih8V6pOdUitY7byYVhuFgguJiVMhKsk8DoF+cEEZPXpzl9n4Vm0G30Kw0Rbaazsbx553mEVqwVkdfkjghWMnMmei9OSSc0R8xyt0OqridQ06XUPHrW82jxNE09pdG+8piDFCsjKCxQKHExAADFtr5wAK7aihaO4ujRA9layLMJLaFhOQZg0YPmEAAFvXgAc+gqeiigDM8P2EunaY8Vzb2tvNJczTOtq5ZCXkZs5Kryc88de56nToooAqatpyavo15p0s81ul3C0LS27BZEDDBKkggHn0rI8XL9v09PD62M13/ae2N2aAvCsQdPMLvgqp2FioPUjA5roqKAGSwxzQPDNGkkTqUeN1BVlIwQR3FILeFZhKsMYkVPLDhRkL1259PapKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA//Z"}}},{"cell_type":"code","source":"# Instantiate the clustering model and visualizer\nmodel = KMeans()\nvisualizer = KElbowVisualizer(model, k=(4,12))\n\nvisualizer.fit(SS_scaled_output) # Fit the data to the visualizer\nvisualizer.show() # Finalize and render the figure","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:25:53.869121Z","iopub.execute_input":"2022-07-17T15:25:53.869943Z","iopub.status.idle":"2022-07-17T15:26:58.627604Z","shell.execute_reply.started":"2022-07-17T15:25:53.869895Z","shell.execute_reply":"2022-07-17T15:26:58.626630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"👉 **Observations**\n\n- The optimal K value signifying the number of clusters is 7.\n- By default, the scoring parameter metric is set to distortion, which computes the sum of squared distances from each point to its assigned center. \n- Next, we will calculate the calinski_harabasz score computes the ratio of dispersion between and within clusters.","metadata":{}},{"cell_type":"code","source":"# metric='calinski_harabasz'\n\nvisualizer1 = KElbowVisualizer(\n model, k=(4,12), metric='calinski_harabasz', timings=False\n)\n\nvisualizer1.fit(SS_scaled_output) # Fit the data to the visualizer\nvisualizer1.show() # Finalize and render the figure","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:26:58.629324Z","iopub.execute_input":"2022-07-17T15:26:58.629949Z","iopub.status.idle":"2022-07-17T15:28:02.067394Z","shell.execute_reply.started":"2022-07-17T15:26:58.629914Z","shell.execute_reply":"2022-07-17T15:28:02.066328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Baseline Scenerio","metadata":{}},{"cell_type":"code","source":"kmeans_model = KMeans(n_clusters = 7)\npreds_kmeans = kmeans_model.fit_predict(SS_scaled_output)\n\nsubmission[\"Predicted\"] = preds_kmeans\n## Lets improve our baseline model by trying many other different methods ","metadata":{"execution":{"iopub.status.busy":"2022-07-18T04:43:08.194076Z","iopub.execute_input":"2022-07-18T04:43:08.194506Z","iopub.status.idle":"2022-07-18T04:43:14.529749Z","shell.execute_reply.started":"2022-07-18T04:43:08.194471Z","shell.execute_reply":"2022-07-18T04:43:14.528802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n## PCA Plots\n#### From the above PCA charts, we can see that the explained variance ratio is very less (*in single digits*) *for PC1, PC2,etc. We will take TOP 2 componenets and try to visualize the clusters*","metadata":{}},{"cell_type":"code","source":"df = pd.DataFrame(SS_PCA_output, columns=[f\"PC_{i}\" for i in range(1,16)])\ndf = pd.concat([df, pd.Series(preds_kmeans)], axis=1).rename(columns={0:'clusters'})\nsns.scatterplot(df['PC_1'], df['PC_2'], hue=df['clusters'])\nplt.title('K means Clusters Visualization for PC1 and PC2')","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:30:22.489723Z","iopub.execute_input":"2022-07-17T15:30:22.490115Z","iopub.status.idle":"2022-07-17T15:30:28.846237Z","shell.execute_reply.started":"2022-07-17T15:30:22.490085Z","shell.execute_reply":"2022-07-17T15:30:28.845441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(30,10))\nplt.suptitle('K means Clusters Visualization for PC1 and PC2')\n \ni=1\ncolours = ['turbo', 'Oranges_r', 'coolwarm', 'cividis', 'magma', 'winter', 'viridis']\n\nfor c in set(df['clusters']):\n x = df[df['clusters']==c]\n plt.subplot(2,4,i)\n sns.scatterplot(x['PC_1'], x['PC_2'], hue=x['clusters'], palette=colours[i-1])\n plt.title(f\"Clusters {c}\")\n i+=1\n \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:30:28.847772Z","iopub.execute_input":"2022-07-17T15:30:28.848265Z","iopub.status.idle":"2022-07-17T15:30:33.524778Z","shell.execute_reply.started":"2022-07-17T15:30:28.848236Z","shell.execute_reply":"2022-07-17T15:30:33.523412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = pd.DataFrame(SS_scaled_output, columns=f_cols)\ndf1 = pd.concat([df1, pd.Series(preds_kmeans)], axis=1).rename(columns={0:'clusters'})\n\nplt.figure(figsize=(30, 20))\nplt.suptitle('Features Density plots based on Clusters', fontsize=18) \n\nj=1\nfor col in f_cols:\n plt.subplot(5, 6, j)\n sns.kdeplot(data=df1, x=col, hue=\"clusters\", multiple=\"stack\")\n plt.title(col)\n j+=1\n \nplt.tight_layout()\nplt.show","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:30:33.526529Z","iopub.execute_input":"2022-07-17T15:30:33.526899Z","iopub.status.idle":"2022-07-17T15:31:01.533871Z","shell.execute_reply.started":"2022-07-17T15:30:33.526840Z","shell.execute_reply":"2022-07-17T15:31:01.532995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"👉 **Observations**\n- We have plotted cluster-wise histograms of each of the float features. These histograms show that the float features f_00 through f_21 do not help to distinguish the clusters: All clusters have mean 0 and standard deviation 1. We can (or even should?) drop these features from the data. Only the last seven features, f_22 through f_28, are useful.\n\n- All seven int features help to distinguish the clusters, these distributions are not Gaussian.","metadata":{}},{"cell_type":"markdown","source":"\n### Finding the cluster mean","metadata":{}},{"cell_type":"code","source":"cluster_mean_df = pd.DataFrame()\n\nfor col in f_cols:\n for e in range(7):\n cluster_mean = df1[df1['clusters']== e][col].mean()\n cluster_mean_df.loc[e, col] = cluster_mean\n \ncluster_mean_plot = pd.melt(cluster_mean_df.reset_index().rename(columns={'index':'cluster'}),\n id_vars='cluster', var_name=\"f_column\", value_name='cluster_mean')\ncluster_mean_plot['cluster'] = cluster_mean_plot['cluster'].apply(lambda x : x+1)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:31:01.536208Z","iopub.execute_input":"2022-07-17T15:31:01.536755Z","iopub.status.idle":"2022-07-17T15:31:02.309664Z","shell.execute_reply.started":"2022-07-17T15:31:01.536719Z","shell.execute_reply":"2022-07-17T15:31:02.308329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.scatter(cluster_mean_plot, x=\"cluster\", y=\"cluster_mean\", animation_frame=\"f_column\",\n size=\"cluster\", color=\"cluster\", hover_name=\"cluster_mean\", facet_col=\"cluster\", range_y=[-1,2])\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:31:02.311134Z","iopub.execute_input":"2022-07-17T15:31:02.311476Z","iopub.status.idle":"2022-07-17T15:31:05.332314Z","shell.execute_reply.started":"2022-07-17T15:31:02.311446Z","shell.execute_reply":"2022-07-17T15:31:05.331018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.scatter(cluster_mean_plot, x=\"f_column\", y=\"cluster_mean\", color=\"cluster\", title='Summarized View of F_COLS by Cluster Mean')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:31:05.333842Z","iopub.execute_input":"2022-07-17T15:31:05.334224Z","iopub.status.idle":"2022-07-17T15:31:05.408125Z","shell.execute_reply.started":"2022-07-17T15:31:05.334180Z","shell.execute_reply":"2022-07-17T15:31:05.406939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"👉 **Observations**\n- The above diagram shows the seven cluster centers for every feature. \n- f_00-f_06, f_14-f_21 clusters have mean 0 and cannot be distinguished.\n- Rest has significant mean variance and will be considered to improve our model ","metadata":{}},{"cell_type":"markdown","source":"### Getting rid of zero mean clusters - ","metadata":{}},{"cell_type":"code","source":"zero_mean_cols1 = [f\"f_0{i}\" for i in range(0,8)]\nzero_mean_cols2 = [f\"f_{i}\" for i in range(14,22)]\n\nkmeans_df_2 = df1.drop(columns= zero_mean_cols1+zero_mean_cols2)\nbest_data = kmeans_df_2.columns.tolist()\n\nmodel2 = KMeans(n_clusters = 7)\npreds2 = model2.fit_predict(kmeans_df_2)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T04:45:22.469406Z","iopub.execute_input":"2022-07-18T04:45:22.469795Z","iopub.status.idle":"2022-07-18T04:45:26.117214Z","shell.execute_reply.started":"2022-07-18T04:45:22.469764Z","shell.execute_reply":"2022-07-18T04:45:26.115958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaled_new, pca_out_new = run_pca(df=kmeans_df_2, components=10, drop_cols=None, transformation=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T04:45:26.121756Z","iopub.execute_input":"2022-07-18T04:45:26.122129Z","iopub.status.idle":"2022-07-18T04:45:27.173786Z","shell.execute_reply.started":"2022-07-18T04:45:26.122099Z","shell.execute_reply":"2022-07-18T04:45:27.172194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x1 = pd.DataFrame(pca_out_new).loc[:, 0:1].rename(columns={0:'PC_1',1:'PC_2'})\nx2 = pd.concat([x1, pd.Series(preds2)], axis=1).rename(columns={0:'clusters'})\n\nsns.scatterplot(x2['PC_1'], x2['PC_2'], hue=x2['clusters'])\nplt.title('K means Clusters Visualization for PC1 and PC2')","metadata":{"execution":{"iopub.status.busy":"2022-07-18T04:45:27.175342Z","iopub.execute_input":"2022-07-18T04:45:27.175683Z","iopub.status.idle":"2022-07-18T04:45:33.707652Z","shell.execute_reply.started":"2022-07-18T04:45:27.175653Z","shell.execute_reply":"2022-07-18T04:45:33.706137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"👉 **Observations**\n\n- Clusters look far better than baseline iteration.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(30,10))\nplt.suptitle('K means Clusters Visualization for PC1 and PC2')\n \ni=1\ncolours = ['turbo', 'Oranges_r', 'coolwarm', 'cividis', 'magma', 'winter', 'viridis']\n\nfor c in set(x2['clusters']):\n x = x2[x2['clusters']==c]\n plt.subplot(2,4,i)\n sns.scatterplot(x['PC_1'], x['PC_2'], hue=x['clusters'], palette=colours[i-1])\n plt.title(f\"Clusters {c}\")\n i+=1\n \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T04:45:33.710282Z","iopub.execute_input":"2022-07-18T04:45:33.710639Z","iopub.status.idle":"2022-07-18T04:45:38.577351Z","shell.execute_reply.started":"2022-07-18T04:45:33.710606Z","shell.execute_reply":"2022-07-18T04:45:38.575891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## Gaussian Mixture Model (Model 2)","metadata":{}},{"cell_type":"markdown","source":"**Let’s come to a further unsupervised learning cluster algorithm: The Gaussian Mixture Models. As simple or good as the K-Means algorithm is, it is often difficult to use in real world situations. In particular, the non-probabilistic nature of k-means and its use of simple distance-from-cluster-center to assign cluster membership leads to poor performance for many real-world problems. Therefore we will look at Gaussian mixture models (GMMs), which can be viewed as an extension of the ideas behind k-means, but can also be a powerful tool for estimation beyond simple clustering.**\n\n--- Gaussian mixture models can be used to cluster unlabeled data in much the same way as k-means. There are, however, a couple of advantages to using Gaussian mixture models over k-means.\n\n1.- First and foremost, k-means does not account for variance. By variance, we are referring to the width of the bell shape curve.\n- One way to think about the k-means model is that it places a circle (or, in higher dimensions, a hyper-sphere) at the center of each cluster, with a radius defined by the most distant point in the cluster. This works fine for when your data is circular. However, when your data takes on different shape, you end up with overlapping the clusters.\n\n- In contrast, Gaussian mixture models can handle even very oblong clusters.\n\n2.- The second difference between k-means and Gaussian mixture models is that the former performs hard classification whereas the latter performs soft classification. In other words, k-means tells us what data point belong to which cluster but won’t provide us with the probabilities that a given data point belongs to each of the possible clusters.\n\n- In calling the predict function, the model will assign every data point to one of the clusters.\n- On the other hand, we can call the predict_proba function to return the probabilities that a data point belongs to each of the K clusters.","metadata":{}},{"cell_type":"markdown","source":"### Determine the optimal k for GMM\nWith k-Means, you could use the inertia or silhouette score to select the appropriate number of clusters. But with GMM, it’s not possible to use these metrics because they are not reliable when clusters are not spherical or have different sizes. Instead you can try to find the model that minimizes a theoretical information criterion, such as “AIC” or “BIC”.\n\nLet’s do it -","metadata":{}},{"cell_type":"code","source":"n_components = np.arange(1, 15)\nmodels = [GaussianMixture(n, covariance_type='full', random_state=0).fit(PT_scaled_output) for n in n_components]\nplt.plot(n_components, [m.bic(PT_scaled_output) for m in models], label='BIC')\nplt.plot(n_components, [m.aic(PT_scaled_output) for m in models], label='AIC')\nplt.legend(loc='best')\nplt.xlabel('n_components')","metadata":{"execution":{"iopub.status.busy":"2022-07-18T04:45:38.579167Z","iopub.execute_input":"2022-07-18T04:45:38.579581Z","iopub.status.idle":"2022-07-18T05:02:48.522228Z","shell.execute_reply.started":"2022-07-18T04:45:38.579547Z","shell.execute_reply":"2022-07-18T05:02:48.520772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The optimal number of clusters (K) is the value that minimizes the Akaike information criterion (AIC) or the Bayesian information criterion (BIC).**\n- BIC and AIC both minimizes around 7-8 clusters and then become *almost* a staright line.\n- Now, We will train our model using the optimal number of clusters (in this case, 7).","metadata":{}},{"cell_type":"code","source":"PT_scaled_new = pd.DataFrame(PT_scaled_output, columns=f_cols).drop(columns= zero_mean_cols1+zero_mean_cols2)\n\nmodel_gm = GaussianMixture(n_components=7, covariance_type='full', random_state=0)\npred_gm = model_gm.fit_predict(PT_scaled_new)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T05:02:48.524536Z","iopub.execute_input":"2022-07-18T05:02:48.524943Z","iopub.status.idle":"2022-07-18T05:03:32.053291Z","shell.execute_reply.started":"2022-07-18T05:02:48.524905Z","shell.execute_reply":"2022-07-18T05:03:32.052133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Advantages of Gaussian Distribution model\n1. They are more generic than the partitioning models\n2. They are simple and easy to implement\n\n### Limitations of Gaussian Distribution Model\n1. The number of clusters has to be specified as an input to the model.","metadata":{}},{"cell_type":"markdown","source":"\n## Bayesian Gaussian Mixture Model (Model 3)\n\nNow we come to an kind of extension of GMM the Bayesian Gaussian Mixture Models. As we have seen at “GMM”, we could either only infer the number of clusters by eye or by comparing the theoretical information criterions “AIC” and “BIC” for different k.\n\nRather than manually search for the optimal number of k, you can use the Bayesian Gaussian Mixture Model, which is capable of giving weights equal (or close) to zero to unnecessary cluster. Set the parameter n_components to a value that you have good reason to believe is greater than the optimal number of clusters, and the algorithm will eliminate the unnecessary cluster automatically.","metadata":{}},{"cell_type":"code","source":"bgm = BayesianGaussianMixture(n_components=7, n_init=5, random_state=42)\npred_bgm = bgm.fit_predict(PT_scaled_new)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T05:03:32.054894Z","iopub.execute_input":"2022-07-18T05:03:32.055311Z","iopub.status.idle":"2022-07-18T05:11:32.730288Z","shell.execute_reply.started":"2022-07-18T05:03:32.055279Z","shell.execute_reply":"2022-07-18T05:11:32.728980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bgm_weights = bgm.weights_\nnp.round(bgm_weights, 2)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T05:11:32.731961Z","iopub.execute_input":"2022-07-18T05:11:32.732458Z","iopub.status.idle":"2022-07-18T05:11:32.741046Z","shell.execute_reply.started":"2022-07-18T05:11:32.732417Z","shell.execute_reply":"2022-07-18T05:11:32.739753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see three clusters have been identified. This makes sense after looking at the scatter plot, even though we specified to generate 7 clusters when creating the data. But two clusters are so close together that they were correctly identified as one cluster.","metadata":{}},{"cell_type":"code","source":"n_clusters_ = (np.round(bgm_weights, 2) > 0).sum()\nprint('Estimated number of clusters: ' + str(n_clusters_))","metadata":{"execution":{"iopub.status.busy":"2022-07-18T05:11:32.742810Z","iopub.execute_input":"2022-07-18T05:11:32.743284Z","iopub.status.idle":"2022-07-18T05:11:32.754800Z","shell.execute_reply.started":"2022-07-18T05:11:32.743237Z","shell.execute_reply":"2022-07-18T05:11:32.753480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As with “Gaussian Mixture Models”, we have a ‘predict_proba’ function here-","metadata":{}},{"cell_type":"code","source":"predict_proba = bgm.predict_proba(PT_scaled_new)\npredict_proba = predict_proba.round(3)\npredict_proba","metadata":{"execution":{"iopub.status.busy":"2022-07-18T05:11:32.759432Z","iopub.execute_input":"2022-07-18T05:11:32.759975Z","iopub.status.idle":"2022-07-18T05:11:33.294192Z","shell.execute_reply.started":"2022-07-18T05:11:32.759938Z","shell.execute_reply":"2022-07-18T05:11:33.292471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## DBSCAN (Model 4)\n\nIt’s is a non parametric clustering method which groups the points which are closely located (high density of points) and the points that lie in low density areas are treated as outliers.\n\nThe algorithm takes the following inputs:\n\n•\tEpsilon Parameter — ε: Degree of closeness of points in the same cluster. It is also referred as radius of the circle created around each point to check the density\n\n•\tMinPts: Minimum number of points required to inside the circle to be classified as a core point\n\nHow does it work:\n1.\tFind the points which are in the neighborhood of the selected point within the radial distance equal to ε\n2.\tClassify the points into “Core Point”,”Border Point” and “Noise”.\n\n•\t**Core Point** is a point that has more than the specified MinPts in its vicinity within the distance ε. The core points form the interior points of the cluster. Core Point always belongs in a dense region. For example, let’s consider ‘p’ is set to be a core point if ‘p’ has ≥ minPoints in an eps radius around it.\n\n•\t**Border Point** is the point which is still reachable within the distance ε but does not meet the MinPts criteria. This point can be a part of the cluster formed by the Core Points. For example, p is set to be a border point if ‘p’ is not a core point. i.e ‘p’ has < minPoints in eps radius. But ‘p’ should belong to the neighborhood ‘q’. Where ‘q’ is a core point.\n\n•\t**Noise**: This point has no other point within the threshold distance ε. This point is neither a core point nor a border point.","metadata":{}},{"cell_type":"code","source":"neigh = NearestNeighbors(n_neighbors=2)\nnbrs = neigh.fit(kmeans_df_2)\ndistances, indices = nbrs.kneighbors(kmeans_df_2)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T05:11:33.296052Z","iopub.execute_input":"2022-07-18T05:11:33.296518Z","iopub.status.idle":"2022-07-18T05:13:18.901777Z","shell.execute_reply.started":"2022-07-18T05:11:33.296472Z","shell.execute_reply":"2022-07-18T05:13:18.900538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Parameter selection for DBSCAN:**\nDBSCAN is sensitive to the choice of hyper parameters epsilon ε and MinPts.\n\nWe can calculate the distance from each point to its closest neighbour using the NearestNeighbors. The point itself is included in n_neighbors. The kneighbors method returns two arrays, one which contains the distance to the closest n_neighbors points and the other which contains the index for each of those points.","metadata":{}},{"cell_type":"code","source":"distances = np.sort(distances, axis=0)\ndistances = distances[:,1]\nplt.plot(distances)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T05:13:18.903107Z","iopub.execute_input":"2022-07-18T05:13:18.903449Z","iopub.status.idle":"2022-07-18T05:13:19.179774Z","shell.execute_reply.started":"2022-07-18T05:13:18.903419Z","shell.execute_reply":"2022-07-18T05:13:19.178691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The optimal value for epsilon will be found at the point of maximum curvature. We train our model, selecting 2 for eps and setting min_samples to 5.**","metadata":{}},{"cell_type":"code","source":"# dbs = DBSCAN(eps=2, min_samples=5)\n# pred_dbs = dbs.fit_predict(kmeans_df_2)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T05:13:19.180963Z","iopub.execute_input":"2022-07-18T05:13:19.181258Z","iopub.status.idle":"2022-07-18T05:13:19.185938Z","shell.execute_reply.started":"2022-07-18T05:13:19.181231Z","shell.execute_reply":"2022-07-18T05:13:19.184781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### NOT better than BGM, didn't improve public score.","metadata":{}},{"cell_type":"markdown","source":"\n## Model Results Comparison","metadata":{}},{"cell_type":"code","source":"s1 = submission.copy()\n\ns1 = pd.concat([s1, pd.Series(preds_kmeans), pd.Series(pred_gm), pd.Series(pred_bgm)], axis=1).rename(columns={0:'kmeans',1:'gm',2:'bgm'}).drop(columns='Predicted')\ns1['kmeans_vs_gm'] = np.where(s1['kmeans']==s1['gm'], \"Match\", \"No Match\")\ns1['kmeans_vs_bgm'] = np.where(s1['kmeans']==s1['bgm'], \"Match\", \"No Match\")","metadata":{"execution":{"iopub.status.busy":"2022-07-18T05:13:19.187397Z","iopub.execute_input":"2022-07-18T05:13:19.187760Z","iopub.status.idle":"2022-07-18T05:13:19.237030Z","shell.execute_reply.started":"2022-07-18T05:13:19.187730Z","shell.execute_reply":"2022-07-18T05:13:19.235815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.suptitle('Model Results Comparison', fontsize=18) \n\nplt.subplot(1, 2, 1)\ns1['kmeans_vs_gm'].value_counts(normalize=True).round(2).plot(kind='barh', title='KMeans vs GMM', figsize=(30,10))\n\nplt.subplot(1, 2, 2)\ns1['kmeans_vs_bgm'].value_counts(normalize=True).round(2).plot(kind='barh', title='KMeans vs BGMM', figsize=(30,10),sharey=True)\n \nplt.tight_layout()\nplt.show","metadata":{"execution":{"iopub.status.busy":"2022-07-18T05:13:19.238916Z","iopub.execute_input":"2022-07-18T05:13:19.239388Z","iopub.status.idle":"2022-07-18T05:13:19.839642Z","shell.execute_reply.started":"2022-07-18T05:13:19.239339Z","shell.execute_reply":"2022-07-18T05:13:19.838336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It looks like BGMM results are very different from K means and GMM, while DBSCAN doesnt stand a chance","metadata":{}},{"cell_type":"markdown","source":"\n## LightGBM Model","metadata":{}},{"cell_type":"code","source":"PT_scaled_new['predict']=pred_bgm\nPT_scaled_new['predict_proba']=0\nfor n in range(7):\n PT_scaled_new[f'predict_proba_{n}']=predict_proba[:,n]\n PT_scaled_new.loc[PT_scaled_new.predict == n,'predict_proba']=PT_scaled_new[f'predict_proba_{n}']\n \n \ntrain_index=np.array([])\nfor n in range(7):\n median=PT_scaled_new[PT_scaled_new.predict==n]['predict_proba'].median()\n n_inx=PT_scaled_new[(PT_scaled_new.predict==n) & (PT_scaled_new.predict_proba > 0.7)].index\n train_index = np.concatenate((train_index, n_inx))\n print(f'class:{n}',f'median: {round(median,4)}','Training data:'+str(round(len(n_inx)/len(PT_scaled_new[(PT_scaled_new.predict==n)]),2)*100)+'%')","metadata":{"execution":{"iopub.status.busy":"2022-07-18T05:13:19.841262Z","iopub.execute_input":"2022-07-18T05:13:19.841598Z","iopub.status.idle":"2022-07-18T05:13:19.955399Z","shell.execute_reply.started":"2022-07-18T05:13:19.841568Z","shell.execute_reply":"2022-07-18T05:13:19.954192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_data.remove('clusters')\nX=PT_scaled_new.loc[train_index][best_data]\ny=PT_scaled_new.loc[train_index]['predict']\n\nparams_lgb = {'learning_rate': 0.07,'objective': 'multiclass','boosting': 'gbdt','verbosity': -1,'n_jobs': -1, 'num_classes':7} \nN_FOLDS = 10\nmodel_list=[]\n\ngkf = StratifiedKFold(N_FOLDS)\nfor fold, (train_idx, valid_idx) in enumerate(gkf.split(X,y)): \n\n tr_dataset = lgb.Dataset(X.iloc[train_idx],y.iloc[train_idx],feature_name = best_data)\n vl_dataset = lgb.Dataset(X.iloc[valid_idx],y.iloc[valid_idx],feature_name = best_data)\n \n model = lgb.train(params = params_lgb, \n train_set = tr_dataset, \n valid_sets = vl_dataset, \n num_boost_round = 5000, \n callbacks=[ lgb.early_stopping(stopping_rounds=300, verbose=True), lgb.log_evaluation(period=200)]) \n \n model_list.append(model) ","metadata":{"execution":{"iopub.status.busy":"2022-07-18T05:15:16.292962Z","iopub.execute_input":"2022-07-18T05:15:16.293322Z","iopub.status.idle":"2022-07-18T05:15:30.337685Z","shell.execute_reply.started":"2022-07-18T05:15:16.293294Z","shell.execute_reply":"2022-07-18T05:15:30.335750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_preds=0\nfor model in model_list:\n lgb_preds+=model.predict(PT_scaled_new[best_data])","metadata":{"execution":{"iopub.status.busy":"2022-07-18T05:13:20.086254Z","iopub.status.idle":"2022-07-18T05:13:20.087382Z","shell.execute_reply.started":"2022-07-18T05:13:20.087077Z","shell.execute_reply":"2022-07-18T05:13:20.087107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Submission","metadata":{}},{"cell_type":"code","source":"submission_final = pd.read_csv('/kaggle/input/tabular-playground-series-jul-2022/sample_submission.csv')\nsubmission_final['Predicted'] = np.argmax(lgb_preds, axis=1)\nsubmission_final.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T05:13:20.088931Z","iopub.status.idle":"2022-07-18T05:13:20.089861Z","shell.execute_reply.started":"2022-07-18T05:13:20.089532Z","shell.execute_reply":"2022-07-18T05:13:20.089563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### If you like my work, kindly press the **upvote** button to show your support🤓💖","metadata":{}}]}