{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<!-- # <p style=\"background-color:#FF7F50;font-family:newtimeroman;color:#FFF9ED;font-size:150%;text-align:center;border-radius:10px 10px;\">Importing Libraries</p> -->\n# Importing Libraries","metadata":{}},{"cell_type":"code","source":"#Importing the Libraries\nimport numpy as np\nimport pandas as pd\nimport matplotlib\nimport matplotlib.pyplot as plt\nfrom matplotlib import colors\nimport seaborn as sns\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import RobustScaler,PowerTransformer\nfrom sklearn.decomposition import PCA\nfrom sklearn.mixture import BayesianGaussianMixture,GaussianMixture\nfrom yellowbrick.cluster import KElbowVisualizer\nfrom sklearn.metrics import silhouette_score\nfrom sklearn.cluster import KMeans\nimport matplotlib.pyplot as plt\nfrom sklearn import metrics\nimport lightgbm as lgb\nfrom mpl_toolkits.mplot3d import Axes3D\nfrom matplotlib.colors import ListedColormap\nfrom sklearn import metrics\nimport warnings\nimport sys\nif not sys.warnoptions:\n    warnings.simplefilter(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T17:16:55.859449Z","iopub.execute_input":"2022-07-15T17:16:55.859902Z","iopub.status.idle":"2022-07-15T17:16:58.023784Z","shell.execute_reply.started":"2022-07-15T17:16:55.859805Z","shell.execute_reply":"2022-07-15T17:16:58.022842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<!-- # <p style=\"background-color:#FF7F50;font-family:newtimeroman;color:#FFF9ED;font-size:150%;text-align:center;border-radius:10px 10px;\">Loading Data</p> -->\n# Loading Data","metadata":{}},{"cell_type":"code","source":"df=pd.read_csv(\"../input/tabular-playground-series-jul-2022/data.csv\")\ndf=df.drop(\"id\",axis=1)\nss=pd.read_csv(\"../input/tabular-playground-series-jul-2022/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T17:17:01.428133Z","iopub.execute_input":"2022-07-15T17:17:01.428800Z","iopub.status.idle":"2022-07-15T17:17:02.812173Z","shell.execute_reply.started":"2022-07-15T17:17:01.428750Z","shell.execute_reply":"2022-07-15T17:17:02.810873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<!-- # <p style=\"background-color:#FF7F50;font-family:newtimeroman;color:#FFF9ED;font-size:150%;text-align:center;border-radius:10px 10px;\">Confusion Matrix</p>\n -->\n# Let's Look at correlation Heat Map","metadata":{}},{"cell_type":"code","source":"mask = np.triu(np.ones_like(df.corr(), dtype=bool))\ncmap = sns.diverging_palette(230, 20, as_cmap=True)\nf, ax = plt.subplots(figsize=(20,20))\nsns.heatmap(df.corr(), mask=mask, vmax=.3, center=0,\n            square=True, linewidths=.5, cbar_kws={\"shrink\": .5},annot=True,fmt='.2f')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T17:17:04.550151Z","iopub.execute_input":"2022-07-15T17:17:04.550743Z","iopub.status.idle":"2022-07-15T17:17:07.163218Z","shell.execute_reply.started":"2022-07-15T17:17:04.550705Z","shell.execute_reply":"2022-07-15T17:17:07.161846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<!-- [AmbrosM](https://www.kaggle.com/code/ambrosm/tpsjul22-gaussian-mixture-cluster-analysis) suggested some features are just not as useful as others.\n> I'll be using all the integer features i.e from **(f_07, ... ,f_13)** & continuos features **(f_22 ,... ,f_28)**. -->","metadata":{}},{"cell_type":"markdown","source":"# Feature Selection\n> This features are most important features, take a look in my other [kernel](https://www.kaggle.com/code/ashaykatrojwar/feature-selection-eda) to know more about `feature selection`.","metadata":{}},{"cell_type":"code","source":"feats= ['f_07', 'f_08', 'f_09', 'f_10', 'f_11', 'f_12', 'f_13', 'f_22', 'f_23', 'f_24', 'f_25', 'f_26', 'f_27', 'f_28']","metadata":{"execution":{"iopub.status.busy":"2022-07-15T17:17:12.098831Z","iopub.execute_input":"2022-07-15T17:17:12.099231Z","iopub.status.idle":"2022-07-15T17:17:12.104405Z","shell.execute_reply.started":"2022-07-15T17:17:12.099196Z","shell.execute_reply":"2022-07-15T17:17:12.103230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<!-- # <p style=\"background-color:#FF7F50;font-family:newtimeroman;color:#FFF9ED;font-size:150%;text-align:center;border-radius:10px 10px;\">Scaling Data Data</p> -->\n# Preprocessing Data \n> `Power Transformer` makes distribution more gaussian and is most appropriate for this competition.","metadata":{}},{"cell_type":"code","source":"transformer = PowerTransformer()\nX=transformer.fit_transform(df[feats])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T17:17:13.482883Z","iopub.execute_input":"2022-07-15T17:17:13.483462Z","iopub.status.idle":"2022-07-15T17:17:15.176216Z","shell.execute_reply.started":"2022-07-15T17:17:13.483430Z","shell.execute_reply":"2022-07-15T17:17:15.175335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dimensionality Reduction\n\nIn this problem, there are many factors on the basis of which the final classification will be done. These factors are basically attributes or features. The higher the number of features, the harder it is to work with it. Many of these features are correlated, and hence redundant. This is why I will be performing dimensionality reduction on the features before putting them through a classifier.\nDimensionality reduction is the process of reducing the number of random variables under consideration, by obtaining a set of principal variables.\n\n`Principal component analysis (PCA)` is a technique for reducing the dimensionality of such datasets, increasing interpretability but at the same time minimizing information loss.\n\nFor this data, I will be reducing the dimensions to **`3`**.","metadata":{}},{"cell_type":"code","source":"pca = PCA(n_components=3,random_state=1)\npca.fit(X)\nPCA_ds = pd.DataFrame(pca.transform(df[feats]), columns=([\"col1\",\"col2\",\"col3\"]))\nPCA_ds.describe().T","metadata":{"execution":{"iopub.status.busy":"2022-07-15T17:17:17.200232Z","iopub.execute_input":"2022-07-15T17:17:17.200858Z","iopub.status.idle":"2022-07-15T17:17:17.709533Z","shell.execute_reply.started":"2022-07-15T17:17:17.200806Z","shell.execute_reply":"2022-07-15T17:17:17.708329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x =PCA_ds[\"col1\"]\ny =PCA_ds[\"col2\"]\nz =PCA_ds[\"col3\"]\n#To plot\nfig = plt.figure(figsize=(10,8))\nax = fig.add_subplot(111, projection=\"3d\")\nax.scatter(x,y,z, c=\"maroon\", marker=\"o\" )\nax.set_title(\"A 3D Projection Of Data In The Reduced Dimension\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T17:17:20.178417Z","iopub.execute_input":"2022-07-15T17:17:20.178840Z","iopub.status.idle":"2022-07-15T17:17:22.112146Z","shell.execute_reply.started":"2022-07-15T17:17:20.178807Z","shell.execute_reply":"2022-07-15T17:17:22.111314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Elbow Method to determine optimum number of clusters to be formed:\n> `7 clusters` gives the best results on leaderboard.","metadata":{}},{"cell_type":"code","source":"print('Elbow Method to determine the number of clusters to be formed:')\nElbow_M = KElbowVisualizer(KMeans(random_state=23), k=(4,12))\nElbow_M.fit(X)\nElbow_M.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T16:39:50.287446Z","iopub.execute_input":"2022-07-15T16:39:50.287879Z","iopub.status.idle":"2022-07-15T16:40:40.619416Z","shell.execute_reply.started":"2022-07-15T16:39:50.287843Z","shell.execute_reply":"2022-07-15T16:40:40.618383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bayesian Gaussian Mixture ","metadata":{}},{"cell_type":"code","source":"BGM = BayesianGaussianMixture(n_components=7,covariance_type='full',random_state=1,n_init=15)\n# fit model and predict clusters\npreds = BGM.fit_predict(X)\nPCA_ds[\"Clusters\"] = preds\n#Adding the Clusters feature to the orignal dataframe.\ndf[\"Clusters\"]= preds\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T17:17:32.523698Z","iopub.execute_input":"2022-07-15T17:17:32.524110Z","iopub.status.idle":"2022-07-15T17:18:10.118080Z","shell.execute_reply.started":"2022-07-15T17:17:32.524078Z","shell.execute_reply":"2022-07-15T17:18:10.116692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating a classifier\n> Training a LGBM model from the clusters predicted from Bayessian Gaussian Mixture","metadata":{}},{"cell_type":"code","source":"pp=BGM.predict_proba(X)# Calcualting the probabilities of each prediction\ndf_new=pd.DataFrame(X,columns=feats) \ndf_new[[f'predict_proba_{i}' for i in range(7)]]=pp # creating new dataframe columns of probabilites \ndf_new['preds']=preds\ndf_new['predict_proba']=np.max(pp,axis=1)\ndf_new['predict']=np.argmax(pp,axis=1)\n    \ntrain_index=np.array([])\nfor n in range(7):\n    n_inx=df_new[(df_new.preds==n) & (df_new.predict_proba > 0.68)].index\n    train_index = np.concatenate((train_index, n_inx))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T17:19:43.021819Z","iopub.execute_input":"2022-07-15T17:19:43.022226Z","iopub.status.idle":"2022-07-15T17:19:43.363398Z","shell.execute_reply.started":"2022-07-15T17:19:43.022195Z","shell.execute_reply":"2022-07-15T17:19:43.362483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ricopue's notebook's code snippet\nfrom sklearn.model_selection import StratifiedKFold\nX_new=df_new.loc[train_index][feats]\ny=df_new.loc[train_index]['preds']\n\nparams_lgb = {'learning_rate': 0.06,'objective': 'multiclass','boosting': 'gbdt','n_jobs': -1,'verbosity': -1, 'num_classes':7} \n\nmodel_list=[]\n\ngkf = StratifiedKFold(11)\nfor fold, (train_idx, valid_idx) in enumerate(gkf.split(X_new,y)):   \n\n    tr_dataset = lgb.Dataset(X_new.iloc[train_idx],y.iloc[train_idx],feature_name = feats)\n    vl_dataset = lgb.Dataset(X_new.iloc[valid_idx],y.iloc[valid_idx],feature_name = feats)\n    \n    model = lgb.train(params = params_lgb, \n                train_set = tr_dataset, \n                valid_sets =  vl_dataset, \n                num_boost_round = 5000, \n                callbacks=[ lgb.early_stopping(stopping_rounds=300, verbose=False), lgb.log_evaluation(period=200)])  \n    \n    model_list.append(model) ","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_preds=0\nfor model in model_list:\n    lgb_preds+=model.predict(df_new[feats])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T17:26:48.091788Z","iopub.execute_input":"2022-07-15T17:26:48.092193Z","iopub.status.idle":"2022-07-15T17:28:43.448273Z","shell.execute_reply.started":"2022-07-15T17:26:48.092158Z","shell.execute_reply":"2022-07-15T17:28:43.447316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels=np.argmax(lgb_preds,axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T16:53:46.666011Z","iopub.execute_input":"2022-07-15T16:53:46.666408Z","iopub.status.idle":"2022-07-15T16:53:46.673921Z","shell.execute_reply.started":"2022-07-15T16:53:46.666377Z","shell.execute_reply":"2022-07-15T16:53:46.672973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(15,8))\nax = plt.subplot(1,2,1, projection='3d', label=\"bla\")\nax.scatter(PCA_ds['col1'],PCA_ds['col2'],PCA_ds['col3'], s=40, marker='o', cmap = 'rainbow' )\nax.set_title(\"Before clustering\")\nax = plt.subplot(1,2,2, projection='3d', label=\"bla\")\nax.scatter(PCA_ds['col1'],PCA_ds['col2'],PCA_ds['col3'], s=40, c=PCA_ds[\"Clusters\"], marker='o',cmap=\"rainbow\")\nax.set_title(\"After Clustering\")\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T17:37:36.473741Z","iopub.execute_input":"2022-07-15T17:37:36.474455Z","iopub.status.idle":"2022-07-15T17:37:41.305602Z","shell.execute_reply.started":"2022-07-15T17:37:36.474410Z","shell.execute_reply":"2022-07-15T17:37:41.304610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss.Predicted=np.argmax(lgb_preds,axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T13:13:00.007119Z","iopub.execute_input":"2022-07-15T13:13:00.007575Z","iopub.status.idle":"2022-07-15T13:13:00.017121Z","shell.execute_reply.started":"2022-07-15T13:13:00.007536Z","shell.execute_reply":"2022-07-15T13:13:00.015610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ploting the count plot for cluster distribution","metadata":{}},{"cell_type":"code","source":"pl = sns.countplot(x=np.argmax(lgb_preds,axis=1))\npl.set_title(\"Distribution Of The Clusters\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T13:13:13.811265Z","iopub.execute_input":"2022-07-15T13:13:13.811698Z","iopub.status.idle":"2022-07-15T13:13:14.054626Z","shell.execute_reply.started":"2022-07-15T13:13:13.811664Z","shell.execute_reply":"2022-07-15T13:13:14.053313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T13:13:23.960570Z","iopub.execute_input":"2022-07-15T13:13:23.961062Z","iopub.status.idle":"2022-07-15T13:13:24.126835Z","shell.execute_reply.started":"2022-07-15T13:13:23.961017Z","shell.execute_reply":"2022-07-15T13:13:24.125612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}