{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.stats import ttest_ind\nfrom sklearn import metrics\nfrom scipy import stats\nfrom statistics import mode\n\nfrom sklearn.mixture import GaussianMixture\nfrom sklearn.mixture import BayesianGaussianMixture\nfrom sklearn.cluster import MiniBatchKMeans\nfrom sklearn.ensemble import VotingClassifier \n\nfrom sklearn import preprocessing\nfrom sklearn.decomposition import PCA","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading Data","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('../input/tabular-playground-series-jul-2022/data.csv', index_col=False)\ndf = df.fillna(0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Standardization","metadata":{}},{"cell_type":"code","source":"df_copy = df.drop(columns = ['id'])\n\nscaler = preprocessing.PowerTransformer(method = 'yeo-johnson', standardize=True).fit(df_copy.values)\nscaled_df = pd.DataFrame(scaler.transform(df_copy.values), index = df_copy.index, columns = df_copy.columns)\np_vals = []\nfor col in df_copy.columns:\n    pre_transform = stats.shapiro(df[col]).pvalue\n    post_transform = stats.shapiro(scaled_df[col]).pvalue\n    p_vals.append([col, pre_transform, post_transform])\n\np_val_df = pd.DataFrame(p_vals, columns = ['Variable', 'Pre-Transform', 'Post-Transform'])\nprint(p_val_df.sort_values(by=['Pre-Transform']))\n\n\nmelted_df_pre = df_copy.melt(value_vars = df_copy.columns,\n                    value_name = 'Value', var_name = 'Variable')\nmelted_df_post = scaled_df.melt(value_vars = df_copy.columns,\n                    value_name = 'Value', var_name = 'Variable')\nmelted_df_pre['Transform'] = 'No Transform'\nmelted_df_post['Transform'] = 'yeo-johnson'\nmelted_df = pd.concat([melted_df_pre, melted_df_post], ignore_index = True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Selection","metadata":{}},{"cell_type":"code","source":"best_data =['f_07','f_08', 'f_09', 'f_10','f_11', 'f_12', 'f_13', 'f_22','f_23', 'f_24', 'f_25','f_26','f_27', 'f_28']\nscaled_df = scaled_df[best_data]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hyperparameters Tuning","metadata":{}},{"cell_type":"code","source":"sample_df = scaled_df.sample(n = 5000)\nclusters = range(2,15)\nscores = []\n\nfor i in clusters:\n    gm = GaussianMixture(n_components=i, n_init=5, init_params='kmeans',\n                        verbose = 0)\n    gm_prediction = gm.fit_predict(sample_df)\n    # Calculate Silhoutte Score and append to a list\n    score = metrics.silhouette_score(sample_df, gm_prediction, metric='euclidean')\n    scores.append(score)\n    print('Number of Clusters: ', i, ' Score: ', score)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Models Training","metadata":{}},{"cell_type":"code","source":"model_bayesian = BayesianGaussianMixture(n_components=7, n_init=5, verbose = 0.5,tol = 0.0001, max_iter = 200).fit(scaled_df)\nmodel_gaussian = GaussianMixture(n_components=7, n_init=5, verbose = 0.5,tol = 0.0001, max_iter = 200).fit(scaled_df)\nmodel_minibatch = MiniBatchKMeans(n_clusters=7).fit(scaled_df)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_bayesian = metrics.silhouette_score(scaled_df, model_bayesian.predict(scaled_df), metric='euclidean')\nscore_gaussian = metrics.silhouette_score(scaled_df, model_gaussian.predict(scaled_df), metric='euclidean')\nscore_minibatch = metrics.silhouette_score(scaled_df, model_minibatch.predict(scaled_df), metric='euclidean')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensemble","metadata":{}},{"cell_type":"code","source":"preds_df = pd.DataFrame(np.array([model_bayesian.predict(scaled_df),\n                           model_gaussian.predict(scaled_df),\n                           model_minibatch.predict(scaled_df)]).T, columns = ['BGM', 'GM', 'MiniBatchK'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_predictions = []\nfor index, row in preds_df.iterrows():\n    all_predictions.append(mode([row['BGM']]* int(round((score_bayesian**-1)*100,0)) + \n                               [row['GM']] * int(round((score_gaussian**-1)*100, 0))+\n                               [row['MiniBatchK']] * int(round((score_minibatch**-1)*100, 0))))\npreds_df['Weighted_Pred'] = all_predictions","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame(np.array([df['id'], preds_df['Weighted_Pred']]).T,\n                                 columns = ['Id', 'Predicted'])\nsubmission.to_csv('ensemble.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}]}