{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Bayesian Gaussian Mixture Models - Semi-Supervised","metadata":{}},{"cell_type":"markdown","source":"### Contents\n\n- Determine the optimal number of clusters using GMM and BIC scores\n- Estimate Feature importance\n- Cluster the data using soft-voting\n- Classification using the confident predictions from clustering as training labels\n- Iterative Classification with BGMM Classifier","metadata":{}},{"cell_type":"markdown","source":"### References\nInspiration was taken from these great notebooks:\n\n- https://www.kaggle.com/code/ambrosm/tpsjul22-gaussian-mixture-cluster-analysis\n- https://www.kaggle.com/code/ricopue/tps-jul22-clusters-and-lgb\n- https://www.kaggle.com/code/pourchot/simple-soft-voting\n- https://www.kaggle.com/code/hiro5299834/tps-jul-2022-unsupervised-and-supervised-learning\n- https://www.kaggle.com/code/karlcini/bayesiangmmclassifier\n\nBaseline Model: \n\n- https://www.kaggle.com/code/cabaxiom/tps-jul-22-gmm-baseline","metadata":{}},{"cell_type":"markdown","source":"# Preliminaries","metadata":{}},{"cell_type":"code","source":"!pip install sklego","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-24T13:01:40.213768Z","iopub.execute_input":"2022-07-24T13:01:40.214351Z","iopub.status.idle":"2022-07-24T13:01:54.773606Z","shell.execute_reply.started":"2022-07-24T13:01:40.214232Z","shell.execute_reply":"2022-07-24T13:01:54.771948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set_style('darkgrid')\n\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, MinMaxScaler, PowerTransformer\nfrom sklearn.mixture import GaussianMixture, BayesianGaussianMixture\nfrom sklearn.decomposition import PCA\nfrom sklearn.metrics import silhouette_score, calinski_harabasz_score, davies_bouldin_score\n\nfrom lightgbm import LGBMClassifier\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis, LinearDiscriminantAnalysis\nfrom sklego.mixture import BayesianGMMClassifier","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-24T13:01:54.781207Z","iopub.execute_input":"2022-07-24T13:01:54.782696Z","iopub.status.idle":"2022-07-24T13:01:56.984905Z","shell.execute_reply.started":"2022-07-24T13:01:54.782632Z","shell.execute_reply":"2022-07-24T13:01:56.983606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/tabular-playground-series-jul-2022/data.csv\")\ndf = df.drop(columns=\"id\")","metadata":{"execution":{"iopub.status.busy":"2022-07-24T13:01:56.986303Z","iopub.execute_input":"2022-07-24T13:01:56.987188Z","iopub.status.idle":"2022-07-24T13:01:58.299522Z","shell.execute_reply.started":"2022-07-24T13:01:56.987149Z","shell.execute_reply":"2022-07-24T13:01:58.298238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"int_cols = [i for i in df.columns if df[i].dtype == int]\nfloat_cols = [i for i in df.columns if df[i].dtype == float]","metadata":{"execution":{"iopub.status.busy":"2022-07-24T13:01:58.303113Z","iopub.execute_input":"2022-07-24T13:01:58.303469Z","iopub.status.idle":"2022-07-24T13:01:58.313363Z","shell.execute_reply.started":"2022-07-24T13:01:58.303437Z","shell.execute_reply":"2022-07-24T13:01:58.312023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"transformer = PowerTransformer()\nX_scaled = transformer.fit_transform(df)\nX_scaled = pd.DataFrame(X_scaled, columns = df.columns)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T13:01:58.315325Z","iopub.execute_input":"2022-07-24T13:01:58.315849Z","iopub.status.idle":"2022-07-24T13:02:02.148644Z","shell.execute_reply.started":"2022-07-24T13:01:58.315802Z","shell.execute_reply":"2022-07-24T13:02:02.147355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Optimal Number of Clusters","metadata":{}},{"cell_type":"markdown","source":"We attempt to find the optimal number of clusters using a Gaussian Mixture Model. To do this we use the Bayesian Information Criterion (BIC), where our goal is to minimise the BIC score.","metadata":{}},{"cell_type":"code","source":"def components_number_multiple(max_n, n_seeds):\n    bic_scores = []\n    for n in range(2,max_n):\n        bic_scores_n = []\n        for seed in range(n_seeds):\n            gmm = GaussianMixture(n_components=n, covariance_type = 'full', n_init=3, random_state=seed)\n            gmm.fit(X_scaled)\n            bic_scores_n.append(gmm.bic(X_scaled))\n        bic_scores.append(bic_scores_n)\n    return bic_scores","metadata":{"execution":{"iopub.status.busy":"2022-07-24T13:02:02.149915Z","iopub.execute_input":"2022-07-24T13:02:02.150222Z","iopub.status.idle":"2022-07-24T13:02:02.158558Z","shell.execute_reply.started":"2022-07-24T13:02:02.150194Z","shell.execute_reply":"2022-07-24T13:02:02.157243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_components_number_multiple(max_n, n_seeds):\n    bic_scores = components_number_multiple(max_n + 1, n_seeds)\n    bic_df = pd.DataFrame(data = bic_scores).T\n    bic_df.columns = range(2,max_n+1)\n    \n    f,ax = plt.subplots(figsize=(20,7))\n    for i in range(n_seeds):\n        sns.lineplot(x=bic_df.columns, y=bic_df.loc[i].values)\n    ax.set_xticks(range(2,max_n+1))\n    \n    return bic_df","metadata":{"execution":{"iopub.status.busy":"2022-07-24T13:02:02.160142Z","iopub.execute_input":"2022-07-24T13:02:02.160469Z","iopub.status.idle":"2022-07-24T13:02:02.187775Z","shell.execute_reply.started":"2022-07-24T13:02:02.160440Z","shell.execute_reply":"2022-07-24T13:02:02.186801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = plot_components_number_multiple(max_n = 15, n_seeds = 10)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T13:02:02.188892Z","iopub.execute_input":"2022-07-24T13:02:02.189246Z","iopub.status.idle":"2022-07-24T14:45:15.141081Z","shell.execute_reply.started":"2022-07-24T13:02:02.189214Z","shell.execute_reply":"2022-07-24T14:45:15.132699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks like 6 or 7 is the best choice for the number of clusters. We use 7.","metadata":{}},{"cell_type":"markdown","source":"# Feature Importance","metadata":{}},{"cell_type":"markdown","source":"Lets fit a BGMM and take a look at the results:","metadata":{}},{"cell_type":"code","source":"bgmm = BayesianGaussianMixture(n_components=7, covariance_type = 'full', n_init=3, random_state=2)\npredicted_class = bgmm.fit_predict(X_scaled)\ndf[\"class\"] = predicted_class","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:45:15.143011Z","iopub.status.idle":"2022-07-24T14:45:15.143983Z","shell.execute_reply.started":"2022-07-24T14:45:15.143674Z","shell.execute_reply":"2022-07-24T14:45:15.143707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualising Feature Importance","metadata":{}},{"cell_type":"markdown","source":"We plot a graph showing the distributions of each feature - with each graph showing the distribution of each class. The more distinct the different classes distributions are, the more useful that feature is.","metadata":{}},{"cell_type":"markdown","source":"### Float features","metadata":{}},{"cell_type":"code","source":"f,ax = plt.subplots(figsize=(25,50))\nfor n,feature in enumerate(float_cols):\n    plt.subplot(8,3,n+1)\n    sns.kdeplot(data=df, x=feature, hue=\"class\", palette=sns.color_palette(\"hls\", 7));","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:45:15.145495Z","iopub.status.idle":"2022-07-24T14:45:15.146754Z","shell.execute_reply.started":"2022-07-24T14:45:15.146334Z","shell.execute_reply":"2022-07-24T14:45:15.146367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Integer features","metadata":{}},{"cell_type":"code","source":"f,ax = plt.subplots(figsize=(25,20))\nfor n,feature in enumerate(int_cols):\n    ax = plt.subplot(3,3,n+1)\n    sns.kdeplot(data=df, x=feature, hue=\"class\", bw_adjust=2, palette=sns.color_palette(\"hls\", 7));\n    ax.set_xlim([-2,30])","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:45:15.149087Z","iopub.status.idle":"2022-07-24T14:45:15.149784Z","shell.execute_reply.started":"2022-07-24T14:45:15.149393Z","shell.execute_reply":"2022-07-24T14:45:15.149419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Select Best Columns","metadata":{}},{"cell_type":"markdown","source":"Based on the graphs above:\n\n- All the integer features seem important \\[`f_07`, `f_08`, ..., `f_13`\\]\n- The float features  \\[`f_22`, `f_23`, ... , `f_28`\\] seem important, but the other float features do not.\n\nWhen we re-train our model, we'll only use these important features.","metadata":{}},{"cell_type":"code","source":"best_cols = ['f_07','f_08', 'f_09', 'f_10','f_11', 'f_12', 'f_13', 'f_22','f_23', 'f_24', 'f_25','f_26','f_27', 'f_28']","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:45:33.330234Z","iopub.execute_input":"2022-07-24T14:45:33.330635Z","iopub.status.idle":"2022-07-24T14:45:33.336675Z","shell.execute_reply.started":"2022-07-24T14:45:33.330602Z","shell.execute_reply":"2022-07-24T14:45:33.335283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clustering","metadata":{}},{"cell_type":"markdown","source":"Before clustering its useful to be able to assess performance of our clustering predictions.\n\nTo evaluate clustering performance without ground truth labels we can use:\n\n**Silhoutte Coefficient**\n\nThe score is higher when clusters are dense and well separated, which relates to a standard concept of a cluster.\n\nThe score is bounded between -1 for incorrect clustering and +1 for highly dense clustering. Scores around zero indicate overlapping clusters.\n\nhttps://scikit-learn.org/stable/modules/clustering.html#silhouette-coefficient\n\n**Calinski-Harabasz Index**\n\nThe score is higher when clusters are dense and well separated\n\nhttps://scikit-learn.org/stable/modules/clustering.html#calinski-harabasz-index\n\n**Davies-Bouldin Index**\n\nLower Davies-Bouldin index relates to a model with better separation between the clusters\n\nhttps://scikit-learn.org/stable/modules/clustering.html#davies-bouldin-index\n","metadata":{}},{"cell_type":"code","source":"def score_clusters(X, predictions, silhouette = True, verbose=False):\n    \"\"\"Evaluate how good our cluster label predictions are\"\"\"\n    \n    db_score = davies_bouldin_score(X=X, labels=predictions)\n\n    ch_score = calinski_harabasz_score(X=X, labels=predictions)\n    #the silhouette score is the slowest to compute ~90 secs\n    s_score = silhouette_score(X=X, labels=predictions, metric='euclidean')\n    \n    if verbose:\n        print(\"David Bouldin score: {0:0.4f}\".format(db_score))\n        print(\"Calinski Harabasz score: {0:0.3f}\".format(ch_score))\n        print(\"Silhouette score: {0:0.4f}\".format(s_score))\n        \n    return db_score, ch_score, s_score","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:45:33.884409Z","iopub.execute_input":"2022-07-24T14:45:33.885376Z","iopub.status.idle":"2022-07-24T14:45:33.892913Z","shell.execute_reply.started":"2022-07-24T14:45:33.885325Z","shell.execute_reply":"2022-07-24T14:45:33.891710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"(Bayesian) Gausian Mixture Models can output probabilities, so we fit the model multiple times with different random seeds and sum the predicted probabilities. The class with the highest predicted probability will then be use for the final class assignment (soft voting).\n\nThe main challenge in implementing this is each time that we fit the model, the model will output different cluster labels. To get around this we calculate the distance between between the cluster centroids from the first model and the cluster centroids of each subsequently fitted model. The cluster centroid of a class will be in a similar locations each time the model is fit, even though the randomly assigned label for that cluster may be different. We can use this info to ensure we are consistent with the cluster label assignments from the first fitted model. This is not a perfect solution, as it relies on the cluster centroids being in similar locations for each (B)GMM fit, but it seems sufficient.\n","metadata":{}},{"cell_type":"code","source":"def soft_voting(predict_number, best_cols = best_cols):\n    #initialise dataframe with 0's\n    predicted_probabilities = pd.DataFrame(np.zeros((len(df),7)), columns=range(1,8))\n    scores = []\n    # loop with a different random seeds\n    for i in range(predict_number):\n        #print(\"=========\", i, \"==========\")\n        X_scaled_sample = X_scaled.sample(50000)\n        gmm = BayesianGaussianMixture(n_components=7, covariance_type = 'full', max_iter=300, init_params=\"kmeans\", n_init=3, random_state=i)\n        gmm.fit(X_scaled_sample[best_cols])\n        pred_probs = gmm.predict_proba(X_scaled[best_cols])\n        pred_probs = pd.DataFrame(pred_probs, columns=range(1,8))\n        \n        # ensuring clusters are labeled the same value at each fit\n        if i == 0:\n            initial_centers = gmm.means_\n        new_classes = []\n        for mean2 in gmm.means_:\n            #for the current center of the current gmm, find the distances to every center in the initial gmm\n            distances = [np.linalg.norm(mean1-mean2) for mean1 in initial_centers]\n            # select the class with the minimum distance\n            new_class = np.argmin(distances) + 1 #add 1 as our labels are 1-7 but index is 0-6\n            new_classes.append(new_class)\n        # if the mapping from old cluster labels to new cluster labels isn't 1 to 1\n        if len(new_classes) != len(set(new_classes)):\n            print(\"iteration\", i, \"could not determine the cluster label mapping, skipping\")\n            continue\n        #apply the mapping by renaming the dataframe columns representing the original labels to the new labels    \n        pred_probs = pred_probs.rename(columns=dict(zip(range(1,8),new_classes)))\n        \n        #add the current prediction probabilities to the overall prediction probabilities\n        predicted_probabilities = predicted_probabilities + pred_probs\n        # lets score the cluster labels each iteration to see if soft voting is helpful\n        db, ch, s = score_clusters(X_scaled[best_cols], predicted_probabilities.idxmax(axis=1), verbose=False)\n        scores.append((db,ch,s))\n    \n    #normalise dataframe so each row sums to 1\n    predicted_probabilities = predicted_probabilities.div(predicted_probabilities.sum(axis=1), axis=0)\n    display(pd.DataFrame(scores, columns=[\"Davies-Bouldin Index\",\"Calinski-Harabasz Index\",\"Silhouette Coefficient\"]))\n    return predicted_probabilities","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:45:34.179108Z","iopub.execute_input":"2022-07-24T14:45:34.180179Z","iopub.status.idle":"2022-07-24T14:45:34.195742Z","shell.execute_reply.started":"2022-07-24T14:45:34.180109Z","shell.execute_reply":"2022-07-24T14:45:34.194341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_probs = soft_voting(10)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:45:34.284775Z","iopub.execute_input":"2022-07-24T14:45:34.285252Z","iopub.status.idle":"2022-07-24T14:51:19.155112Z","shell.execute_reply.started":"2022-07-24T14:45:34.285217Z","shell.execute_reply":"2022-07-24T14:51:19.153772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def best_class(df):\n    new_df = df.copy()\n    new_df[\"highest_prob\"] = df.max(axis=1)\n    new_df[\"best_class\"] = df.idxmax(axis=1)\n    new_df[\"second_highest_prob\"] = df.apply(lambda x: x.nlargest(2).values[-1], axis=1)\n    new_df[\"second_best_class\"] = df.apply(lambda x: np.where(x == x.nlargest(2).values[-1])[0][0]+1, axis=1)\n    return new_df","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:51:19.157804Z","iopub.execute_input":"2022-07-24T14:51:19.158214Z","iopub.status.idle":"2022-07-24T14:51:19.167683Z","shell.execute_reply.started":"2022-07-24T14:51:19.158169Z","shell.execute_reply":"2022-07-24T14:51:19.166398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cluster_class_probs = best_class(pred_probs)\ncluster_class_probs.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:51:19.169175Z","iopub.execute_input":"2022-07-24T14:51:19.169780Z","iopub.status.idle":"2022-07-24T14:52:13.587431Z","shell.execute_reply.started":"2022-07-24T14:51:19.169743Z","shell.execute_reply":"2022-07-24T14:52:13.586052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets visualise how confident our predictions are:","metadata":{}},{"cell_type":"code","source":"def plot_prediction_probability_histogram(probabilities):\n    f,ax = plt.subplots(figsize=(20,7))\n    sns.histplot(probabilities, bins=100);","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:52:13.591195Z","iopub.execute_input":"2022-07-24T14:52:13.591741Z","iopub.status.idle":"2022-07-24T14:52:13.602142Z","shell.execute_reply.started":"2022-07-24T14:52:13.591692Z","shell.execute_reply":"2022-07-24T14:52:13.600603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_mean_class_confidence(dataframe):\n    confidence_s = dataframe.groupby(\"best_class\")[\"highest_prob\"].mean()\n    f,ax = plt.subplots(figsize=(8,6))\n    sns.barplot(x=confidence_s.index, y = confidence_s.values, palette=sns.color_palette(\"hls\", 7) );\n    ax.set_ylabel(\"Mean probability of point belonging to target class\");\n    ax.set_ylim([0.65,0.95]);","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:52:13.603827Z","iopub.execute_input":"2022-07-24T14:52:13.604304Z","iopub.status.idle":"2022-07-24T14:52:13.613429Z","shell.execute_reply.started":"2022-07-24T14:52:13.604270Z","shell.execute_reply":"2022-07-24T14:52:13.612408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_second_best_class_probabilities_sum(dataframe):\n    \n    second_highest_probs_sum = dataframe.groupby([\"best_class\",\"second_best_class\"])[\"second_highest_prob\"].sum().reset_index()\n\n    f,ax = plt.subplots(figsize=(25,12))\n    format_dataframe = pd.DataFrame({\"second_best_class\":range(1,8)})\n    for i in range(1,8):\n        second_best_match_for_i = second_highest_probs_sum.loc[second_highest_probs_sum[\"best_class\"] == i, [\"second_best_class\",\"second_highest_prob\"]]\n        #We merge so that all classes 1 to 7 are available, we do this to keep colours consistent throughout plots\n        plot_df = pd.merge(left=format_dataframe, right=second_best_match_for_i, how=\"left\", on=\"second_best_class\")\n        ax = plt.subplot(2,4,i)\n        sns.barplot(data= plot_df, x=\"second_best_class\", y=\"second_highest_prob\",palette=sns.color_palette(\"hls\", 7) );\n        ax.set_ylabel(\"Probability sum\")\n        ax.set_title(\"Assigned Class: \" + str(i))","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:52:13.615252Z","iopub.execute_input":"2022-07-24T14:52:13.615785Z","iopub.status.idle":"2022-07-24T14:52:13.628943Z","shell.execute_reply.started":"2022-07-24T14:52:13.615735Z","shell.execute_reply":"2022-07-24T14:52:13.627465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_prediction_probability_histogram(cluster_class_probs[\"highest_prob\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:52:13.630412Z","iopub.execute_input":"2022-07-24T14:52:13.631202Z","iopub.status.idle":"2022-07-24T14:52:14.257178Z","shell.execute_reply.started":"2022-07-24T14:52:13.631156Z","shell.execute_reply":"2022-07-24T14:52:14.255777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_mean_class_confidence(dataframe=cluster_class_probs)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:52:14.258648Z","iopub.execute_input":"2022-07-24T14:52:14.258994Z","iopub.status.idle":"2022-07-24T14:52:14.504845Z","shell.execute_reply.started":"2022-07-24T14:52:14.258964Z","shell.execute_reply":"2022-07-24T14:52:14.503346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets visualise which clusters are commonly confused with each other","metadata":{}},{"cell_type":"code","source":"plot_second_best_class_probabilities_sum(dataframe=cluster_class_probs)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:52:14.506422Z","iopub.execute_input":"2022-07-24T14:52:14.506812Z","iopub.status.idle":"2022-07-24T14:52:16.022233Z","shell.execute_reply.started":"2022-07-24T14:52:14.506764Z","shell.execute_reply":"2022-07-24T14:52:16.021038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Classification","metadata":{}},{"cell_type":"markdown","source":"We adopt a semi-supervised approach for this clustering challenege.\n\nTo do this we fit our classification model using points the clustering algorithm was confident that it got correct. We can then use the classification model to predict the points it was not confident that it got correct.\n\nThe hope is that the classification algorithm is better than the clustering algorithm. If this is true we can trust the classification algorithms predictions more than the clustering algorithms predictions and improve our score.","metadata":{}},{"cell_type":"code","source":"confident_predictions = cluster_class_probs.loc[cluster_class_probs[\"highest_prob\"] >= 0.8]\nconfident_predictions_class = confident_predictions[\"best_class\"]\nX_scaled[\"class\"] = confident_predictions_class","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:52:16.026841Z","iopub.execute_input":"2022-07-24T14:52:16.027258Z","iopub.status.idle":"2022-07-24T14:52:16.048570Z","shell.execute_reply.started":"2022-07-24T14:52:16.027222Z","shell.execute_reply":"2022-07-24T14:52:16.047519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = X_scaled.loc[X_scaled[\"class\"] == X_scaled[\"class\"]]\ntest_df = X_scaled.loc[X_scaled[\"class\"] != X_scaled[\"class\"]]","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:52:16.049747Z","iopub.execute_input":"2022-07-24T14:52:16.050479Z","iopub.status.idle":"2022-07-24T14:52:16.089105Z","shell.execute_reply.started":"2022-07-24T14:52:16.050440Z","shell.execute_reply":"2022-07-24T14:52:16.087930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_df.drop(columns=\"class\").reset_index(drop=True)\ny = train_df[\"class\"].reset_index(drop=True)\nX_test = test_df.drop(columns=\"class\").reset_index(drop=True)\nX_full = X_scaled.drop(columns=\"class\")","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:52:16.090929Z","iopub.execute_input":"2022-07-24T14:52:16.091383Z","iopub.status.idle":"2022-07-24T14:52:16.130204Z","shell.execute_reply.started":"2022-07-24T14:52:16.091339Z","shell.execute_reply":"2022-07-24T14:52:16.128996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cross Validation","metadata":{}},{"cell_type":"markdown","source":"Lets see how well the classification algorithms perform. The results might be a bit biased as we are evaluating using labels that the clustering algorithm was confident with, it will be much harder for the algorithm to predict the points the clustering algorithm was not confident with.","metadata":{}},{"cell_type":"markdown","source":"### Define models","metadata":{}},{"cell_type":"code","source":"model_et = ExtraTreesClassifier(n_estimators = 2000,\n                                n_jobs = -1,\n                                random_state=42\n                               )","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:52:16.132412Z","iopub.execute_input":"2022-07-24T14:52:16.133418Z","iopub.status.idle":"2022-07-24T14:52:16.139051Z","shell.execute_reply.started":"2022-07-24T14:52:16.133364Z","shell.execute_reply":"2022-07-24T14:52:16.137777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_lgbm = LGBMClassifier(objective = 'multiclass',\n                            n_estimators = 2500,\n                            random_state = 42,\n                            learning_rate = 0.1,\n                            n_jobs = -1)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:52:16.140932Z","iopub.execute_input":"2022-07-24T14:52:16.141293Z","iopub.status.idle":"2022-07-24T14:52:16.151963Z","shell.execute_reply.started":"2022-07-24T14:52:16.141261Z","shell.execute_reply":"2022-07-24T14:52:16.150575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_qda = QuadraticDiscriminantAnalysis()\nmodel_lda = LinearDiscriminantAnalysis()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:52:16.153890Z","iopub.execute_input":"2022-07-24T14:52:16.154252Z","iopub.status.idle":"2022-07-24T14:52:16.168883Z","shell.execute_reply.started":"2022-07-24T14:52:16.154221Z","shell.execute_reply":"2022-07-24T14:52:16.167454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_bgmm = BayesianGMMClassifier(\n            n_components=7,\n            random_state = 1,\n            tol =1e-3,\n            covariance_type = 'full',\n            max_iter = 400,\n            n_init=4,\n            init_params='kmeans')","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:52:16.171014Z","iopub.execute_input":"2022-07-24T14:52:16.171506Z","iopub.status.idle":"2022-07-24T14:52:16.181152Z","shell.execute_reply.started":"2022-07-24T14:52:16.171460Z","shell.execute_reply":"2022-07-24T14:52:16.180263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = {\"ET\":model_et, \"LGBM\":model_lgbm, \"QDA\":model_qda, \"LDA\":model_lda, \"BGMM_C\":model_bgmm}","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:52:16.182786Z","iopub.execute_input":"2022-07-24T14:52:16.183193Z","iopub.status.idle":"2022-07-24T14:52:16.193276Z","shell.execute_reply.started":"2022-07-24T14:52:16.183160Z","shell.execute_reply":"2022-07-24T14:52:16.192120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Compare Classificatin performance","metadata":{}},{"cell_type":"code","source":"def k_fold_cv(model,X,y, verbose=True):\n    kfold = StratifiedKFold(n_splits = 5, shuffle=True, random_state = 0)\n\n    feature_imp, y_pred_list, y_true_list, acc_list  = [],[],[],[]\n    for fold, (train_index, val_index) in enumerate(kfold.split(X, y)):\n        if verbose: print(\"==fold==\", fold)\n        X_train = X.loc[train_index]\n        X_val = X.loc[val_index]\n\n        y_train = y.loc[train_index]\n        y_val = y.loc[val_index]\n\n        model.fit(X_train,y_train)\n\n        y_pred = model.predict(X_val)\n\n        y_pred_list = np.append(y_pred_list, y_pred)\n        y_true_list = np.append(y_true_list, y_val)\n\n        acc_list.append(accuracy_score(y_pred, y_val))\n        if verbose: print('Acc', accuracy_score(y_pred, y_val))\n\n        try:\n            feature_imp.append(model.feature_importances_)\n        except AttributeError: # if model does not have .feature_importances_ attribute\n            pass # returns empty list\n            \n    return feature_imp, y_pred_list, y_true_list, acc_list, X_val, y_val","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:52:16.194576Z","iopub.execute_input":"2022-07-24T14:52:16.195093Z","iopub.status.idle":"2022-07-24T14:52:16.208215Z","shell.execute_reply.started":"2022-07-24T14:52:16.195057Z","shell.execute_reply":"2022-07-24T14:52:16.206920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate_models():\n    for model_name, model in models.items():\n        print(\"===\",model_name,\"===\")\n        feature_imp, y_pred_list, y_true_list, acc_list, X_val, y_val = k_fold_cv(model=model,X=X,y=y, verbose=False)\n        acc_score = accuracy_score(y_pred_list, y_true_list)\n        print(\"{0:0.4f}\".format(acc_score))","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:52:16.209707Z","iopub.execute_input":"2022-07-24T14:52:16.210315Z","iopub.status.idle":"2022-07-24T14:52:16.220092Z","shell.execute_reply.started":"2022-07-24T14:52:16.210269Z","shell.execute_reply":"2022-07-24T14:52:16.219068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_models()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T14:52:16.221583Z","iopub.execute_input":"2022-07-24T14:52:16.222166Z","iopub.status.idle":"2022-07-24T15:27:48.595927Z","shell.execute_reply.started":"2022-07-24T14:52:16.222122Z","shell.execute_reply":"2022-07-24T15:27:48.594585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Just because a classification model performs better than another does not mean the clustering labels will be better. This is because the train/validation data is from a different distribution from the test data (as we select points in the train/val datasets that are BGMM was confident in predicting) so models that generalise well may outperform those with a high accuracy on the validation set.","metadata":{}},{"cell_type":"markdown","source":"### Feature importance","metadata":{}},{"cell_type":"markdown","source":"Lets just reconfirm that the features we dropped in the clustering algorithm are also not important for the classification algorithm.","metadata":{}},{"cell_type":"code","source":"feature_imp, y_pred_list, y_true_list, acc_list, X_val, y_val = k_fold_cv(model=model_lgbm,X=X,y=y)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T15:27:48.598155Z","iopub.execute_input":"2022-07-24T15:27:48.598959Z","iopub.status.idle":"2022-07-24T15:37:09.855013Z","shell.execute_reply.started":"2022-07-24T15:27:48.598905Z","shell.execute_reply":"2022-07-24T15:37:09.854021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fold_feature_importances(model_importances, column_names, model_name, n_folds = 5, ax=None, boxplot=False):\n    importances_df = pd.DataFrame({\"feature_cols\": column_names, \"importances_fold_0\": model_importances[0]})\n    for i in range(1,n_folds):\n        importances_df[\"importances_fold_\"+str(i)] = model_importances[i]\n    importances_df[\"importances_fold_median\"] = importances_df.drop(columns=[\"feature_cols\"]).median(axis=1)\n    importances_df = importances_df.sort_values(by=\"importances_fold_median\", ascending=False)\n    if ax == None:\n        f, ax = plt.subplots(figsize=(15, 25))\n    if boxplot == False:\n        ax = sns.barplot(data = importances_df, x = \"importances_fold_median\", y=\"feature_cols\", color=\"blue\")\n        ax.set_xlabel(\"Median Feature importance across all folds\");\n    elif boxplot == True:\n        importances_df = importances_df.drop(columns=\"importances_fold_median\")\n        importances_df = importances_df.set_index(\"feature_cols\").stack().reset_index().rename(columns={0:\"feature_importance\"})\n        ax = sns.boxplot(data = importances_df, y = \"feature_cols\", x=\"feature_importance\", color=\"blue\", orient=\"h\")\n        ax.set_xlabel(\"Feature importance across all folds\");\n    plt.title(model_name)\n    ax.set_ylabel(\"Feature Columns\")\n    return ax","metadata":{"execution":{"iopub.status.busy":"2022-07-24T15:37:09.858745Z","iopub.execute_input":"2022-07-24T15:37:09.859129Z","iopub.status.idle":"2022-07-24T15:37:09.871081Z","shell.execute_reply.started":"2022-07-24T15:37:09.859096Z","shell.execute_reply":"2022-07-24T15:37:09.870185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(figsize=(15, 15))\nfold_feature_importances(model_importances = feature_imp, column_names = X_val.columns, model_name = \"LGBM\", n_folds = 2, ax=ax, boxplot=False);","metadata":{"execution":{"iopub.status.busy":"2022-07-24T15:37:09.872777Z","iopub.execute_input":"2022-07-24T15:37:09.873495Z","iopub.status.idle":"2022-07-24T15:37:10.340459Z","shell.execute_reply.started":"2022-07-24T15:37:09.873448Z","shell.execute_reply":"2022-07-24T15:37:10.339322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Yep as expected, \\[`f_07`, `f_08`, ..., `f_13`\\] and \\[`f_22`, `f_23`, ... , `f_28`\\] seem important, but the other features do not.","metadata":{}},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"def fit_predict_all():\n    predictions = []\n    model_names = []\n    scores = []\n    for model_name, model in models.items():\n        print(\"===\",model_name,\"===\")\n        model.fit(X[best_cols], y)\n        preds_prob =  model.predict_proba(X_full[best_cols])\n        preds_prob_df = pd.DataFrame(preds_prob, columns=range(1,8), index=X_scaled.index)\n        db, ch, s = score_clusters(X_scaled[best_cols], preds_prob_df.idxmax(axis=1), verbose=True)\n        scores.append((db,ch,s))\n        predictions.append(preds_prob_df)\n        model_names.append(model_name)\n    \n    return predictions, model_names, scores\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-24T15:37:10.342155Z","iopub.execute_input":"2022-07-24T15:37:10.342484Z","iopub.status.idle":"2022-07-24T15:37:10.351700Z","shell.execute_reply.started":"2022-07-24T15:37:10.342456Z","shell.execute_reply":"2022-07-24T15:37:10.350582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions, model_names, scores = fit_predict_all()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T15:37:10.353155Z","iopub.execute_input":"2022-07-24T15:37:10.353453Z","iopub.status.idle":"2022-07-24T15:51:42.092514Z","shell.execute_reply.started":"2022-07-24T15:37:10.353426Z","shell.execute_reply":"2022-07-24T15:51:42.091308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets compare these scores to our clustering scores and also add our clustering predictions to the list of classification predictions","metadata":{}},{"cell_type":"code","source":"cluster_class_probs = cluster_class_probs.loc[:,[1,2,3,4,5,6,7]]","metadata":{"execution":{"iopub.status.busy":"2022-07-24T15:51:42.094145Z","iopub.execute_input":"2022-07-24T15:51:42.094593Z","iopub.status.idle":"2022-07-24T15:51:42.103696Z","shell.execute_reply.started":"2022-07-24T15:51:42.094546Z","shell.execute_reply":"2022-07-24T15:51:42.102578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions.append(cluster_class_probs)\nmodel_names.append(\"BGMM\")","metadata":{"execution":{"iopub.status.busy":"2022-07-24T15:51:42.105148Z","iopub.execute_input":"2022-07-24T15:51:42.105494Z","iopub.status.idle":"2022-07-24T15:51:42.116084Z","shell.execute_reply.started":"2022-07-24T15:51:42.105454Z","shell.execute_reply":"2022-07-24T15:51:42.115072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"db, ch, s = score_clusters(X_scaled[best_cols], cluster_class_probs.idxmax(axis=1), verbose=True)\nscores.append((db,ch,s))","metadata":{"execution":{"iopub.status.busy":"2022-07-24T15:51:42.122414Z","iopub.execute_input":"2022-07-24T15:51:42.122789Z","iopub.status.idle":"2022-07-24T15:53:19.815023Z","shell.execute_reply.started":"2022-07-24T15:51:42.122752Z","shell.execute_reply":"2022-07-24T15:53:19.813877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Our classification algorithm appears to be performing better than our clustering algorithms; although I don't trust the clustering scores too much.","metadata":{}},{"cell_type":"markdown","source":"## Combine Predictions","metadata":{}},{"cell_type":"markdown","source":"Lets combine the probabilities from the different classification/clustering algorithms.","metadata":{}},{"cell_type":"code","source":"#chosen fairly randomly\npredictions_df = 0.5 * predictions[0] + 1.5 * predictions[1] + 0.5 * predictions[2] + 1.5 * predictions[4] + 0.5 * predictions[5]\n\n#normalise so rows sums to 1\npredictions_df = predictions_df.div(predictions_df.sum(axis=1), axis=0)\npredictions_df = best_class(predictions_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T15:53:19.816454Z","iopub.execute_input":"2022-07-24T15:53:19.816920Z","iopub.status.idle":"2022-07-24T15:54:14.847704Z","shell.execute_reply.started":"2022-07-24T15:53:19.816883Z","shell.execute_reply":"2022-07-24T15:54:14.846782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"db, ch, s = score_clusters(X_scaled[best_cols], predictions_df[\"best_class\"], verbose=True)\nscores.append((db,ch,s))\nmodel_names.append(\"combined\")","metadata":{"execution":{"iopub.status.busy":"2022-07-24T15:54:14.849228Z","iopub.execute_input":"2022-07-24T15:54:14.849891Z","iopub.status.idle":"2022-07-24T15:55:51.936708Z","shell.execute_reply.started":"2022-07-24T15:54:14.849856Z","shell.execute_reply":"2022-07-24T15:55:51.935473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(scores, index=model_names, columns=[\"Davies-Bouldin Index\",\"Calinski-Harabasz Index\",\"Silhouette Coefficient\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-24T15:55:51.937932Z","iopub.execute_input":"2022-07-24T15:55:51.938243Z","iopub.status.idle":"2022-07-24T15:55:51.952300Z","shell.execute_reply.started":"2022-07-24T15:55:51.938215Z","shell.execute_reply":"2022-07-24T15:55:51.951156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Just because a model has good evaluation clustering scores, does not necessarily mean it will have a better leaderboard score. LDA for example has the best evaluation scores, but has the worst leaderboard score.","metadata":{}},{"cell_type":"code","source":"f,ax = plt.subplots(figsize=(10,5))\nval_c = predictions_df[\"best_class\"].value_counts()\nsns.barplot(x=val_c.index, y=val_c.values);","metadata":{"execution":{"iopub.status.busy":"2022-07-24T15:55:51.954070Z","iopub.execute_input":"2022-07-24T15:55:51.954556Z","iopub.status.idle":"2022-07-24T15:55:52.188666Z","shell.execute_reply.started":"2022-07-24T15:55:51.954492Z","shell.execute_reply":"2022-07-24T15:55:52.187543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca = PCA(random_state = 10)\nX_pca = pca.fit_transform(X_scaled[best_cols])\nPCA_df = pd.DataFrame({\"PCA_1\" : X_pca[:,0], \"PCA_2\" : X_pca[:,1]})   \nPCA_df[\"class\"] = predictions_df[\"best_class\"]\n    \nf,ax = plt.subplots(figsize=(10, 10))\nsns.scatterplot(data = PCA_df, x = \"PCA_1\", y = \"PCA_2\", hue=\"class\", s=2, palette=sns.color_palette(\"hls\", PCA_df[\"class\"].nunique()));","metadata":{"execution":{"iopub.status.busy":"2022-07-24T15:55:52.190275Z","iopub.execute_input":"2022-07-24T15:55:52.191442Z","iopub.status.idle":"2022-07-24T15:55:54.654805Z","shell.execute_reply.started":"2022-07-24T15:55:52.191405Z","shell.execute_reply":"2022-07-24T15:55:54.653860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_mean_class_confidence(dataframe=predictions_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T15:55:54.656289Z","iopub.execute_input":"2022-07-24T15:55:54.656850Z","iopub.status.idle":"2022-07-24T15:55:54.891667Z","shell.execute_reply.started":"2022-07-24T15:55:54.656814Z","shell.execute_reply":"2022-07-24T15:55:54.890584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_second_best_class_probabilities_sum(dataframe=predictions_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T15:55:54.893064Z","iopub.execute_input":"2022-07-24T15:55:54.893932Z","iopub.status.idle":"2022-07-24T15:55:56.083128Z","shell.execute_reply.started":"2022-07-24T15:55:54.893895Z","shell.execute_reply":"2022-07-24T15:55:56.081886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Iterative Classification - BGMM Classifier","metadata":{}},{"cell_type":"markdown","source":"To improve performance further we can iteratively use the the predicted labels from the previous iteration's model as our training labels for the current iteration's model.","metadata":{}},{"cell_type":"code","source":"def update_predictions(predict_number, y):\n    scores = []\n    for i in range(predict_number):\n        print(\"=========\", i, \"==========\")\n        X_scaled_sample = X_scaled.sample(60000)\n        y_sample = y.loc[X_scaled_sample.index]\n        \n        bgmmC = BayesianGMMClassifier(\n        n_components=7,\n        random_state = i,\n        tol =1e-3,\n        covariance_type = 'full',\n        max_iter = 300,\n        n_init=3,\n        init_params='kmeans')\n        \n        bgmmC.fit(X_scaled_sample[best_cols], y_sample)\n        \n        pred_probs = bgmmC.predict_proba(X_scaled[best_cols])\n        pred_probs = pd.DataFrame(pred_probs, columns=range(1,8))\n        \n        # lets score the cluster labels each iteration\n        db, ch, s  = score_clusters(X_scaled[best_cols], pred_probs.idxmax(axis=1), verbose=False)\n        scores.append((db, ch, s))\n        y = pred_probs.idxmax(axis=1)\n        \n    display(pd.DataFrame(scores, columns=[\"Davies-Bouldin Index\",\"Calinski-Harabasz Index\",\"Silhouette Coefficient\"]))\n    return pred_probs","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:00:15.405450Z","iopub.execute_input":"2022-07-24T16:00:15.405878Z","iopub.status.idle":"2022-07-24T16:00:15.417693Z","shell.execute_reply.started":"2022-07-24T16:00:15.405844Z","shell.execute_reply":"2022-07-24T16:00:15.416479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_probabilities = update_predictions(predict_number=50, y=predictions_df[\"best_class\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:00:24.629242Z","iopub.execute_input":"2022-07-24T16:00:24.630430Z","iopub.status.idle":"2022-07-24T16:03:59.339065Z","shell.execute_reply.started":"2022-07-24T16:00:24.630389Z","shell.execute_reply":"2022-07-24T16:03:59.337894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The clustering scores are getting worse each iteration, but the leaderboard performance improves.","metadata":{}},{"cell_type":"code","source":"predictions_df = best_class(predicted_probabilities)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:03:59.341302Z","iopub.execute_input":"2022-07-24T16:03:59.341679Z","iopub.status.idle":"2022-07-24T16:04:55.437727Z","shell.execute_reply.started":"2022-07-24T16:03:59.341647Z","shell.execute_reply":"2022-07-24T16:04:55.436469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_second_best_class_probabilities_sum(dataframe=predictions_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:04:55.439239Z","iopub.execute_input":"2022-07-24T16:04:55.439677Z","iopub.status.idle":"2022-07-24T16:04:56.863777Z","shell.execute_reply.started":"2022-07-24T16:04:55.439642Z","shell.execute_reply":"2022-07-24T16:04:56.862620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f,ax = plt.subplots(figsize=(10,5))\nval_c = predictions_df[\"best_class\"].value_counts()\nsns.barplot(x=val_c.index, y=val_c.values);","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:04:56.866008Z","iopub.execute_input":"2022-07-24T16:04:56.866337Z","iopub.status.idle":"2022-07-24T16:04:57.102739Z","shell.execute_reply.started":"2022-07-24T16:04:56.866306Z","shell.execute_reply":"2022-07-24T16:04:57.101522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/tabular-playground-series-jul-2022/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-18T18:39:47.183183Z","iopub.execute_input":"2022-07-18T18:39:47.184031Z","iopub.status.idle":"2022-07-18T18:39:47.24063Z","shell.execute_reply.started":"2022-07-18T18:39:47.183991Z","shell.execute_reply":"2022-07-18T18:39:47.239229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission[\"Predicted\"] = predictions_df[\"best_class\"]\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T20:03:04.204076Z","iopub.execute_input":"2022-07-12T20:03:04.205226Z","iopub.status.idle":"2022-07-12T20:03:04.224771Z","shell.execute_reply.started":"2022-07-12T20:03:04.20517Z","shell.execute_reply":"2022-07-12T20:03:04.223319Z"},"trusted":true},"execution_count":null,"outputs":[]}]}