{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib as mat\nimport matplotlib.pyplot as plt    \nimport seaborn as sns\nimport random\nimport os\nimport gc\nfrom sklearn import cluster\nfrom sklearn.cluster import KMeans\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.preprocessing import  LabelEncoder, RobustScaler , MinMaxScaler ,StandardScaler\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom tensorflow import keras\nimport os\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold,GroupKFold\n\nimport warnings\nwarnings.filterwarnings('ignore')\nsns.set_style(\"whitegrid\")\nfrom tqdm.notebook import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-04T14:03:57.561643Z","iopub.execute_input":"2022-07-04T14:03:57.562742Z","iopub.status.idle":"2022-07-04T14:04:00.715027Z","shell.execute_reply.started":"2022-07-04T14:03:57.562614Z","shell.execute_reply":"2022-07-04T14:04:00.713648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler,RobustScaler\nsc = StandardScaler()\nrb = RobustScaler()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:04:00.717261Z","iopub.execute_input":"2022-07-04T14:04:00.718520Z","iopub.status.idle":"2022-07-04T14:04:00.725583Z","shell.execute_reply.started":"2022-07-04T14:04:00.718466Z","shell.execute_reply":"2022-07-04T14:04:00.723501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Loading the Data**","metadata":{}},{"cell_type":"code","source":"data = pd.read_csv(\"../input/tabular-playground-series-jul-2022/data.csv\")\nsubmission = pd.read_csv(\"../input/tabular-playground-series-jul-2022/sample_submission.csv\")\n\ndata.shape, submission.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:04:00.728309Z","iopub.execute_input":"2022-07-04T14:04:00.729347Z","iopub.status.idle":"2022-07-04T14:04:01.953926Z","shell.execute_reply.started":"2022-07-04T14:04:00.729271Z","shell.execute_reply":"2022-07-04T14:04:01.952175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:04:01.957139Z","iopub.execute_input":"2022-07-04T14:04:01.957596Z","iopub.status.idle":"2022-07-04T14:04:01.994476Z","shell.execute_reply.started":"2022-07-04T14:04:01.957555Z","shell.execute_reply":"2022-07-04T14:04:01.993131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.drop(\"id\", axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:04:01.996651Z","iopub.execute_input":"2022-07-04T14:04:01.997076Z","iopub.status.idle":"2022-07-04T14:04:02.015917Z","shell.execute_reply.started":"2022-07-04T14:04:01.997036Z","shell.execute_reply":"2022-07-04T14:04:02.014771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Data Analysis**","metadata":{}},{"cell_type":"code","source":"cat_columns = [x for (x, y) in data.dtypes.items() if y == \"int64\"]\nnum_columns = [x for (x, y) in data.dtypes.items() if y == \"float64\"]\n\nnum_data = data[num_columns]\ncat_data = data[cat_columns]\nprint('Categorical columns -> ', cat_columns)\nprint('\\nNumerical columns -> ', num_columns)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:04:02.017425Z","iopub.execute_input":"2022-07-04T14:04:02.018493Z","iopub.status.idle":"2022-07-04T14:04:02.048317Z","shell.execute_reply.started":"2022-07-04T14:04:02.018419Z","shell.execute_reply":"2022-07-04T14:04:02.046853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Heat Map\nplt.subplots(figsize=(25,20))\nsns.heatmap(data.corr(), annot= True, cmap=\"RdYlGn\", fmt = '0.1f', vmin=-0.6, vmax=0.6, cbar=False);","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:04:02.050074Z","iopub.execute_input":"2022-07-04T14:04:02.050609Z","iopub.status.idle":"2022-07-04T14:04:05.691671Z","shell.execute_reply.started":"2022-07-04T14:04:02.050565Z","shell.execute_reply":"2022-07-04T14:04:05.690183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Correlation matrix\ncorrMatrix =data.corr(method='pearson', min_periods=1)\ncorrMatrix.style.background_gradient(axis=None)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:04:05.694075Z","iopub.execute_input":"2022-07-04T14:04:05.694846Z","iopub.status.idle":"2022-07-04T14:04:06.099220Z","shell.execute_reply.started":"2022-07-04T14:04:05.694789Z","shell.execute_reply":"2022-07-04T14:04:06.097795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\nfeatures = data.columns.values\nsns.distplot(data[features].mean(axis=1), color='red', kde=True, label='train')\nplt.title('Distribution of mean per row in the dataset')","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:04:06.100967Z","iopub.execute_input":"2022-07-04T14:04:06.101375Z","iopub.status.idle":"2022-07-04T14:04:06.977326Z","shell.execute_reply.started":"2022-07-04T14:04:06.101342Z","shell.execute_reply":"2022-07-04T14:04:06.975891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\nfeatures = data.columns.values\nsns.distplot(data[features].std(axis=1), color='red', kde=True, label='train')\nplt.title('Distribution of std per row in the dataset')","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:04:06.979181Z","iopub.execute_input":"2022-07-04T14:04:06.979567Z","iopub.status.idle":"2022-07-04T14:04:07.910686Z","shell.execute_reply.started":"2022-07-04T14:04:06.979534Z","shell.execute_reply":"2022-07-04T14:04:07.909043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\nfeatures = data.columns.values\nsns.distplot(data[features].mean(axis=0), color='cyan', kde=True, label='train')\nplt.title('Distribution of mean per column in the dataset')","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:04:07.912430Z","iopub.execute_input":"2022-07-04T14:04:07.912863Z","iopub.status.idle":"2022-07-04T14:04:08.329342Z","shell.execute_reply.started":"2022-07-04T14:04:07.912826Z","shell.execute_reply":"2022-07-04T14:04:08.327927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\nfeatures = data.columns.values\nsns.distplot(data[features].std(axis=0), color='cyan', kde=True, label='train')\nplt.title('Distribution of std per column in the dataset')","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:04:08.330903Z","iopub.execute_input":"2022-07-04T14:04:08.331375Z","iopub.status.idle":"2022-07-04T14:04:09.018572Z","shell.execute_reply.started":"2022-07-04T14:04:08.331334Z","shell.execute_reply":"2022-07-04T14:04:09.017320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Scaling the Data**","metadata":{}},{"cell_type":"code","source":"sc.fit(data)\ndata = sc.transform(data)\n\ndata.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:04:09.023315Z","iopub.execute_input":"2022-07-04T14:04:09.024656Z","iopub.status.idle":"2022-07-04T14:04:09.120170Z","shell.execute_reply.started":"2022-07-04T14:04:09.024592Z","shell.execute_reply":"2022-07-04T14:04:09.118567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Main Model DNN","metadata":{}},{"cell_type":"code","source":"def build_model():\n    \n    x_input = layers.Input(shape=(29,))\n    \n    x1 = layers.Dense(512, activation='relu')(x_input)\n    x2 = layers.Dense(1024, activation='relu')(x1)\n    x3 = layers.Dense(512, activation='relu')(x2)\n    x4 = layers.Dense(256, activation='relu')(x3)\n    x5 = layers.Dense(128, activation='relu')(x4)\n    \n    z2 = layers.Dense(512, activation='relu')(x2)\n    \n    z31 = layers.Multiply()([x3, z2])\n    z31 = layers.BatchNormalization()(z31)\n    z3 = layers.Dense(256, activation='relu')(z31)\n    \n    z41 = layers.Multiply()([x4, z3])\n    z41 = layers.BatchNormalization()(z41)\n    z4 = layers.Dense(128, activation='relu')(z41)\n    \n    z51 = layers.Multiply()([x5, z4])\n    z51 = layers.BatchNormalization()(z51)\n    z5 = layers.Dense(64, activation='relu')(z51)\n    \n    x = layers.Concatenate()([x5, z2, z3, z4, z5])\n    \n    x = layers.Dense(units=128, activation='relu')(x)\n    \n    x_output = layers.Dense(units=7, activation='softmax')(x)\n\n    model = tf.keras.Model(inputs=x_input, outputs=x_output, name='model')\n    \n    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001), loss=tf.keras.losses.SparseCategoricalCrossentropy(), metrics=['accuracy'])\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:04:09.122118Z","iopub.execute_input":"2022-07-04T14:04:09.122661Z","iopub.status.idle":"2022-07-04T14:04:09.140565Z","shell.execute_reply.started":"2022-07-04T14:04:09.122608Z","shell.execute_reply":"2022-07-04T14:04:09.139043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Model Aircturature**","metadata":{}},{"cell_type":"code","source":"tf.keras.utils.plot_model(build_model())","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:04:09.141981Z","iopub.execute_input":"2022-07-04T14:04:09.142599Z","iopub.status.idle":"2022-07-04T14:04:09.553913Z","shell.execute_reply.started":"2022-07-04T14:04:09.142550Z","shell.execute_reply":"2022-07-04T14:04:09.552424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **K-Mean** | **K-Fold Model**\n\n#### The K-means clustering algorithm is used to find groups which have not been explicitly labeled in the data.","metadata":{}},{"cell_type":"code","source":"kf = KFold(n_splits=5, shuffle=True)\n\nfor fold, (train_idx, val_idx) in enumerate(kf.split(data)):\n    print('-'*30, '>', f'Fold {fold+1}', '<', '-'*30)\n    train_data, val_data = np.asarray(data)[train_idx], np.asarray(data)[val_idx]\n    \n    n_clusters = 7\n    \n    train_kmeans = KMeans(n_clusters = n_clusters, n_init = 20).fit(train_data)\n    val_kmeans = KMeans(n_clusters = n_clusters, n_init = 20).fit(val_data)\n    \n    train_target = train_kmeans.labels_\n    val_target = val_kmeans.labels_\n    \n    model = build_model()\n    \n    save = tf.keras.callbacks.ModelCheckpoint(f\"Model{fold+1}.h5\", verbose=1, monitor='accuracy',save_best_only=True)\n    \n    model.fit(train_data, train_target, epochs=5, batch_size=256, shuffle=True, callbacks=[save])","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:05:15.138324Z","iopub.execute_input":"2022-07-04T14:05:15.138811Z","iopub.status.idle":"2022-07-04T14:13:21.871623Z","shell.execute_reply.started":"2022-07-04T14:05:15.138777Z","shell.execute_reply":"2022-07-04T14:13:21.869462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Load Models**","metadata":{}},{"cell_type":"code","source":"models_paths = [\"./Model1.h5\",\"./Model2.h5\",\"./Model3.h5\",\"./Model4.h5\",\"./Model5.h5\"]","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:13:21.874194Z","iopub.execute_input":"2022-07-04T14:13:21.875352Z","iopub.status.idle":"2022-07-04T14:13:21.880942Z","shell.execute_reply.started":"2022-07-04T14:13:21.875302Z","shell.execute_reply":"2022-07-04T14:13:21.879830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = [tf.keras.models.load_model(model_path) for model_path in models_paths]","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:13:21.882571Z","iopub.execute_input":"2022-07-04T14:13:21.883745Z","iopub.status.idle":"2022-07-04T14:13:23.471217Z","shell.execute_reply.started":"2022-07-04T14:13:21.883681Z","shell.execute_reply":"2022-07-04T14:13:23.469745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Model Prediction**","metadata":{}},{"cell_type":"code","source":"tmp_pred = []","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:13:43.393899Z","iopub.execute_input":"2022-07-04T14:13:43.394436Z","iopub.status.idle":"2022-07-04T14:13:43.401232Z","shell.execute_reply.started":"2022-07-04T14:13:43.394394Z","shell.execute_reply":"2022-07-04T14:13:43.400022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for model in models:\n    pred = model.predict(data, verbose=1)\n    pred = np.argmax(pred, axis=1)\n    tmp_pred.append(np.asarray(pred))","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:13:43.774446Z","iopub.execute_input":"2022-07-04T14:13:43.775640Z","iopub.status.idle":"2022-07-04T14:15:17.776011Z","shell.execute_reply.started":"2022-07-04T14:13:43.775555Z","shell.execute_reply":"2022-07-04T14:15:17.774963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp_pred","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:16:47.708111Z","iopub.execute_input":"2022-07-04T14:16:47.708550Z","iopub.status.idle":"2022-07-04T14:16:47.718815Z","shell.execute_reply.started":"2022-07-04T14:16:47.708516Z","shell.execute_reply":"2022-07-04T14:16:47.717537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Prediction of Cluster**","metadata":{}},{"cell_type":"code","source":"pd.DataFrame(pred)[0].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:28:52.620243Z","iopub.execute_input":"2022-07-04T14:28:52.620725Z","iopub.status.idle":"2022-07-04T14:28:52.633037Z","shell.execute_reply.started":"2022-07-04T14:28:52.620691Z","shell.execute_reply":"2022-07-04T14:28:52.631946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"submission['Predicted'] = kmeans.labels_\nsubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}