{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<a id=\"load-libraries\"></a>\n# Load necessary libraries","metadata":{}},{"cell_type":"code","source":"# pip install scikit-lego","metadata":{"execution":{"iopub.status.busy":"2022-07-29T10:10:31.082188Z","iopub.execute_input":"2022-07-29T10:10:31.082659Z","iopub.status.idle":"2022-07-29T10:10:43.239908Z","shell.execute_reply.started":"2022-07-29T10:10:31.082557Z","shell.execute_reply":"2022-07-29T10:10:43.238634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import PowerTransformer, StandardScaler\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.cluster import KMeans, AgglomerativeClustering\nfrom sklearn.mixture import GaussianMixture, BayesianGaussianMixture\n# from sklego.mixture import BayesianGMMClassifier\n# from sklego.naive_bayes import BayesianGaussianMixtureNB\n\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\n\nimport plotly.express as px\nimport umap\nfrom tqdm import tqdm\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\npd.set_option('display.max_columns', 100)\npd.set_option('display.float_format', '{:.2f}'.format)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-29T10:27:38.541263Z","iopub.execute_input":"2022-07-29T10:27:38.542043Z","iopub.status.idle":"2022-07-29T10:27:38.550165Z","shell.execute_reply.started":"2022-07-29T10:27:38.542007Z","shell.execute_reply":"2022-07-29T10:27:38.549041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"data-reading\"></a>\n# Read the data","metadata":{}},{"cell_type":"code","source":"data_df = pd.read_csv('../input/tabular-playground-series-jul-2022/data.csv', index_col='id')\nprint('Data Shape: ', data_df.shape)\ndata_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T10:11:06.893171Z","iopub.execute_input":"2022-07-29T10:11:06.893939Z","iopub.status.idle":"2022-07-29T10:11:08.101318Z","shell.execute_reply.started":"2022-07-29T10:11:06.893906Z","shell.execute_reply":"2022-07-29T10:11:08.100085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.read_csv('../input/tabular-playground-series-jul-2022/sample_submission.csv')\nprint('Submission Shape: ', submission_df.shape)\nsubmission_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T10:11:08.104446Z","iopub.execute_input":"2022-07-29T10:11:08.104981Z","iopub.status.idle":"2022-07-29T10:11:08.136262Z","shell.execute_reply.started":"2022-07-29T10:11:08.104936Z","shell.execute_reply":"2022-07-29T10:11:08.135308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of missing values: ', data_df.isnull().sum().sum())","metadata":{"execution":{"iopub.status.busy":"2022-07-29T10:11:08.137556Z","iopub.execute_input":"2022-07-29T10:11:08.137941Z","iopub.status.idle":"2022-07-29T10:11:08.150532Z","shell.execute_reply.started":"2022-07-29T10:11:08.137903Z","shell.execute_reply":"2022-07-29T10:11:08.149544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T10:11:08.151907Z","iopub.execute_input":"2022-07-29T10:11:08.152729Z","iopub.status.idle":"2022-07-29T10:11:08.178686Z","shell.execute_reply.started":"2022-07-29T10:11:08.152696Z","shell.execute_reply":"2022-07-29T10:11:08.177827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_df.describe().T\\\n        .style.bar(subset=['mean'], color=px.colors.qualitative.G10[0])\\\n        .background_gradient(subset=['std'], cmap='Greens')\\\n        .background_gradient(subset=['50%'], cmap='BuGn')","metadata":{"execution":{"iopub.status.busy":"2022-07-29T10:11:08.182138Z","iopub.execute_input":"2022-07-29T10:11:08.182427Z","iopub.status.idle":"2022-07-29T10:11:08.426012Z","shell.execute_reply.started":"2022-07-29T10:11:08.182402Z","shell.execute_reply":"2022-07-29T10:11:08.425171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"float_cols = [col for col in data_df.columns if data_df[col].dtype == 'float64']\nint_cols = [col for col in data_df.columns if data_df[col].dtype == 'int64']\nprint(f'Number of all columns: {len(data_df.columns)}\\nNumber of float columns: {len(float_cols)}\\nNumber of int columns: {len(int_cols)}')","metadata":{"execution":{"iopub.status.busy":"2022-07-29T10:11:08.427121Z","iopub.execute_input":"2022-07-29T10:11:08.427383Z","iopub.status.idle":"2022-07-29T10:11:08.433889Z","shell.execute_reply.started":"2022-07-29T10:11:08.427360Z","shell.execute_reply":"2022-07-29T10:11:08.432842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"Ploting\"></a>\n# Plot some diagrams","metadata":{}},{"cell_type":"code","source":"_, ax = plt.subplots(2, 11, figsize=(14, 3))\nplt.tight_layout()\n\nfor i , col in tqdm(zip(range(22), float_cols)):\n    num_row = int((i - (i % 11)) / 11)\n    num_col = i % 11\n    sns.kdeplot(data=data_df, x=col, shade=True, ax=ax[num_row, num_col])","metadata":{"execution":{"iopub.status.busy":"2022-07-29T10:11:08.434979Z","iopub.execute_input":"2022-07-29T10:11:08.435351Z","iopub.status.idle":"2022-07-29T10:11:20.600156Z","shell.execute_reply.started":"2022-07-29T10:11:08.435312Z","shell.execute_reply":"2022-07-29T10:11:20.599109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, ax = plt.subplots(1, 7, figsize=(14, 2))\nplt.tight_layout()\n\nfor i , col in tqdm(zip(range(7), int_cols)):\n    sns.countplot(data=data_df, x=col, ax=ax[i])\n    ax[i].set_xticks([])\n    ax[i].set_yticks([])","metadata":{"execution":{"iopub.status.busy":"2022-07-29T10:11:20.602736Z","iopub.execute_input":"2022-07-29T10:11:20.603047Z","iopub.status.idle":"2022-07-29T10:11:21.718410Z","shell.execute_reply.started":"2022-07-29T10:11:20.603020Z","shell.execute_reply":"2022-07-29T10:11:21.717280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, ax =  plt.subplots(figsize=(10, 10))\n\ncolormap = plt.cm.PuRd\nsns.heatmap(data_df.corr(),\n            annot=True,\n            fmt=\".2f\",\n#             square=False,\n            cmap=colormap,\n            annot_kws={\"size\": 7, 'color': 'black'},\n            cbar_kws={\"shrink\": .4},\n            vmin=-1 ,\n            vmax=1,\n            ax=ax\n           )","metadata":{"execution":{"iopub.status.busy":"2022-07-29T10:11:21.719993Z","iopub.execute_input":"2022-07-29T10:11:21.720309Z","iopub.status.idle":"2022-07-29T10:11:24.979404Z","shell.execute_reply.started":"2022-07-29T10:11:21.720281Z","shell.execute_reply":"2022-07-29T10:11:24.978347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"Elbow\"></a>\n# Elbow method","metadata":{}},{"cell_type":"code","source":"%%time\n\ncols_ex = ['f_07', 'f_08', 'f_09', 'f_10', 'f_11', 'f_12', 'f_13', 'f_22', 'f_23', 'f_24', 'f_25', 'f_26', 'f_27', 'f_28', ]\ndata = data_df[cols_ex]\ninertia_list = np.empty(10)\nfor i in tqdm(range(1,10)):\n    kmeans = KMeans(n_clusters=i)\n    kmeans.fit(data)\n    inertia_list[i] = kmeans.inertia_\n    \nplt.figure(figsize=(10, 10))\nplt.plot(range(0,10),inertia_list,'-o')\nplt.xlabel('Number of cluster')\nplt.ylabel('Inertia')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T10:11:24.981230Z","iopub.execute_input":"2022-07-29T10:11:24.981969Z","iopub.status.idle":"2022-07-29T10:12:06.056321Z","shell.execute_reply.started":"2022-07-29T10:11:24.981926Z","shell.execute_reply":"2022-07-29T10:12:06.055069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"fitting\"></a>\n# Fit the model","metadata":{}},{"cell_type":"code","source":"transformer = PowerTransformer()\n\ndata_scaled = pd.DataFrame(transformer.fit_transform(data), columns=data.columns, index=data.index)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T10:12:06.057850Z","iopub.execute_input":"2022-07-29T10:12:06.058737Z","iopub.status.idle":"2022-07-29T10:12:07.666307Z","shell.execute_reply.started":"2022-07-29T10:12:06.058703Z","shell.execute_reply":"2022-07-29T10:12:07.665392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nbgm_model = BayesianGaussianMixture(\n                    n_components=7,\n#                         random_state = i+1,\n#                         covariance_type='full', #{‘full’, ‘tied’, ‘diag’, ‘spherical’}, default=’full’\n                        max_iter=300, # int, default=100\n                        n_init=10,\n#                         init_params='k-means++', # {‘kmeans’, ‘k-means++’, ‘random’, ‘random_from_data’}, default=’kmeans’\n#                         warm_start=True, # bool, default=False\n)\nclusters = bgm_model.fit_predict(data_scaled)\nclusters_prob = bgm_model.predict_proba(data_scaled)\n\ndata['classes'] = clusters\ndata['predict_prob'] = np.max(clusters_prob, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T10:12:07.667696Z","iopub.execute_input":"2022-07-29T10:12:07.668448Z","iopub.status.idle":"2022-07-29T10:26:57.021478Z","shell.execute_reply.started":"2022-07-29T10:12:07.668414Z","shell.execute_reply":"2022-07-29T10:26:57.020294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = StandardScaler()\n\nprob_value = 0.7\ntrain = data.loc[data['predict_prob'] > prob_value, :]\nX = train.drop('predict_prob', axis=1)\ntarget = X.pop('classes')\ntest_data = data.loc[data['predict_prob'] <= prob_value, :].drop(['predict_prob', 'classes'], axis=1)\n\nX = scaler.fit_transform(X)\ntest_data = scaler.transform(test_data)\n\nX_train, X_valid, y_train, y_valid = train_test_split(X, target)\ny_train = np.array(y_train).reshape(-1, 1)\ny_valid = np.array(y_valid).reshape(-1, 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T10:41:42.780699Z","iopub.execute_input":"2022-07-29T10:41:42.781670Z","iopub.status.idle":"2022-07-29T10:41:42.838852Z","shell.execute_reply.started":"2022-07-29T10:41:42.781610Z","shell.execute_reply":"2022-07-29T10:41:42.837922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_model(X_train, X_valid, y_train, y_valid, test_data):\n            \n    lr_start = 0.01\n    \n    lr = ReduceLROnPlateau(monitor = 'val_loss', factor = 0.7, patience = 4, verbose = 1)\n    es = EarlyStopping(monitor = 'val_loss',patience = 12, verbose = 1, mode = 'min', restore_best_weights = True)\n    callbacks = [lr, es]\n    \n    model = keras.Sequential([\n        layers.Dense(64, activation='relu', input_shape=(len(cols_ex),)),\n#         layers.BatchNormalization(axis=1),\n        layers.Dense(128, activation='relu'),\n#         layers.BatchNormalization(axis=1),\n        layers.Dense(64, activation='relu'),\n        layers.Dense(32, activation='relu'),\n#         layers.BatchNormalization(axis=1),\n        layers.Dense(7, activation='softmax'),\n    ])\n    \n    optimizer_func = keras.optimizers.Adam(learning_rate = lr_start)\n    loss_func = keras.losses.MeanSquaredError()\n    \n    model.compile(optimizer = optimizer_func, loss = loss_func, metrics=[keras.metrics.RootMeanSquaredError()])\n    \n    validation_data = (X_valid, y_valid)\n    \n    model.fit(X_train, \n              y_train, \n              validation_data = validation_data, \n              epochs          = 100,\n              verbose         = 2,\n              batch_size      = 1024,\n              shuffle         = True,\n              callbacks       = callbacks\n            )\n    \n    callbacks, es, lr = None, None, None\n    \n    test_preds = model.predict(data[cols_ex])\n    \n    return test_preds","metadata":{"execution":{"iopub.status.busy":"2022-07-29T10:42:11.151806Z","iopub.execute_input":"2022-07-29T10:42:11.152368Z","iopub.status.idle":"2022-07-29T10:42:11.161397Z","shell.execute_reply.started":"2022-07-29T10:42:11.152337Z","shell.execute_reply":"2022-07-29T10:42:11.160686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = fit_model(X_train, X_valid, y_train, y_valid, test_data)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T10:42:14.110681Z","iopub.execute_input":"2022-07-29T10:42:14.111240Z","iopub.status.idle":"2022-07-29T10:42:17.789413Z","shell.execute_reply.started":"2022-07-29T10:42:14.111208Z","shell.execute_reply":"2022-07-29T10:42:17.788636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data.loc[data['predict_prob'] <= prob_value, 'classes'] = np.argmax(test_preds, axis=1)\ndata['classes'] = np.argmax(test_preds, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T10:42:53.792543Z","iopub.execute_input":"2022-07-29T10:42:53.793286Z","iopub.status.idle":"2022-07-29T10:42:53.800924Z","shell.execute_reply.started":"2022-07-29T10:42:53.793250Z","shell.execute_reply":"2022-07-29T10:42:53.800152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# bgmc_model = BayesianGMMClassifier(\n#                                    n_components=7,\n#                                    covariance_type='full',\n#                                    max_iter=500,\n#                                    n_init=7,\n#                                    init_params='kmeans',\n#                                     )\n# bgmc_model.fit(X, target)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"umap\"></a>\n# Plot the results using UMAP","metadata":{}},{"cell_type":"code","source":"%%time\n\nmanifold = umap.UMAP().fit(data[cols_ex], data['classes'])\ndata_reduced = manifold.transform(data[cols_ex])\nprint('Shape of reduced data: ', data_reduced.shape)\nplt.figure(figsize=(10, 10))\nplt.scatter(data_reduced[:, 0], data_reduced[:, 1], s=0.5, c=data['classes'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T10:46:12.810977Z","iopub.execute_input":"2022-07-29T10:46:12.811394Z","iopub.status.idle":"2022-07-29T10:47:43.485019Z","shell.execute_reply.started":"2022-07-29T10:46:12.811362Z","shell.execute_reply":"2022-07-29T10:47:43.484050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"submission\"></a>\n# Make submission","metadata":{}},{"cell_type":"code","source":"submission_df['Predicted'] = data['classes']\ndisplay(submission_df.head())\nsubmission_df.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}