{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# TPS JUL 22 - Simple Autoencoder\n\n__Idea:__\n\n- Generate a low-dimensional encoding for the training data using an autoencoder. \n- Is there discernable structure in the low-dim encoding?\n\n__Notes__:\n- Fit the encoder to 3 latent dimensions\n- Use the f_00:f_07 and f_22:f_28 features with a MinMaxScaler\n- Little effort given to optimize/tune the encoder\n\n__TLDR:__\n- looks like the encoder fitted a reasonable representation in 3 latent dimensions\n- unfortunately, visualizing the 3-D encoding doesn't seem to produce discernible clusters\n","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Model, Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, GaussianNoise\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau,EarlyStopping\n\nimport numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport random\n\nINPUT = Path('../input/tabular-playground-series-jul-2022')\nSEED = 420\n\npd.set_option('display.max_columns', None)\npd.set_option('display.float_format', '{:.3f}'.format)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-07-15T15:47:54.865862Z","iopub.execute_input":"2022-07-15T15:47:54.866234Z","iopub.status.idle":"2022-07-15T15:47:54.873341Z","shell.execute_reply.started":"2022-07-15T15:47:54.866204Z","shell.execute_reply":"2022-07-15T15:47:54.872285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load training data","metadata":{}},{"cell_type":"code","source":"data = pd.read_csv(INPUT / 'data.csv', index_col='id')\ndisplay(data.head(2))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:47:54.879709Z","iopub.execute_input":"2022-07-15T15:47:54.879990Z","iopub.status.idle":"2022-07-15T15:47:55.434393Z","shell.execute_reply.started":"2022-07-15T15:47:54.879967Z","shell.execute_reply":"2022-07-15T15:47:55.433345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Select features and scale","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler, MinMaxScaler\nfrom sklearn.model_selection import train_test_split\n\ndf = data[['f_07','f_08','f_09','f_10','f_11','f_12','f_13',\n           'f_22','f_23','f_24','f_25','f_26','f_27','f_28']]\nscaler = MinMaxScaler()\nX = scaler.fit_transform(df.to_numpy())\nX_train, X_test = train_test_split(X, test_size=0.1)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:47:55.436560Z","iopub.execute_input":"2022-07-15T15:47:55.436896Z","iopub.status.idle":"2022-07-15T15:47:55.907293Z","shell.execute_reply.started":"2022-07-15T15:47:55.436861Z","shell.execute_reply":"2022-07-15T15:47:55.906286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define and compile autoencoder","metadata":{}},{"cell_type":"code","source":"input_dim = X_train.shape[1]\nlatent_dim = 3\n\nactivation = 'swish'\n\nencoder = Sequential([\n    GaussianNoise(0.25, input_shape=(input_dim,)),\n    Dense(128, activation=activation, input_shape=(input_dim,)),\n    Dense(128, activation=activation,),\n    Dense(128, activation=activation, ),\n    Dense(latent_dim, activation=activation, name='encoder_output')\n])\n\ndecoder = Sequential([\n    Dense(128, activation=activation, input_shape=(latent_dim,)),\n    Dense(128, activation=activation, ),\n    Dense(128, activation=activation, ),\n    Dense(input_dim, activation=None)\n])\n\nautoencoder = Model(inputs=encoder.input, outputs=decoder(encoder.output))\nautoencoder.compile(loss='mse', optimizer='adam')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:47:55.908730Z","iopub.execute_input":"2022-07-15T15:47:55.909119Z","iopub.status.idle":"2022-07-15T15:47:59.026995Z","shell.execute_reply.started":"2022-07-15T15:47:55.909083Z","shell.execute_reply":"2022-07-15T15:47:59.025193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Utility function to compare actual vs prediction","metadata":{}},{"cell_type":"code","source":"def plot_orig_vs_recon(title='', n_samples=3):\n    fig = plt.figure(figsize=(10,6))\n    plt.suptitle(title)\n    for i in range(3):\n        plt.subplot(3, 1, i+1)\n        idx = random.sample(range(X_test.shape[0]), 1)\n        plt.plot(autoencoder.predict(X_test[idx]).squeeze(), label='reconstructed' if i == 0 else '')\n        plt.plot(X_test[idx].squeeze(), label='original' if i == 0 else '')\n        plt.grid(True)\n        if i == 0: plt.legend();\n\nplot_orig_vs_recon('Before Training')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:47:59.028480Z","iopub.execute_input":"2022-07-15T15:47:59.028833Z","iopub.status.idle":"2022-07-15T15:48:00.700273Z","shell.execute_reply.started":"2022-07-15T15:47:59.028797Z","shell.execute_reply":"2022-07-15T15:48:00.699350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \nmodel_history = autoencoder.fit(X_train, X_train, \n                                epochs=50, \n                                batch_size=512,\n                                shuffle=True,\n                                validation_split=.10,\n                                callbacks=[\n                                    ReduceLROnPlateau(monitor='val_loss',\n                                                      mode='min',\n                                                      factor=0.5,patience=3),\n                                    EarlyStopping(monitor='val_loss', \n                                                  mode=\"min\", \n                                                  patience=6,\n                                                  restore_best_weights=True)\n                                ])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:48:00.702830Z","iopub.execute_input":"2022-07-15T15:48:00.703732Z","iopub.status.idle":"2022-07-15T15:48:13.269153Z","shell.execute_reply.started":"2022-07-15T15:48:00.703686Z","shell.execute_reply":"2022-07-15T15:48:13.268057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(model_history.history[\"loss\"])\nplt.title(\"Loss vs. Epoch\")\nplt.ylabel(\"Loss\")\nplt.xlabel(\"Epoch\")\nplt.grid(True)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:48:13.271539Z","iopub.execute_input":"2022-07-15T15:48:13.271916Z","iopub.status.idle":"2022-07-15T15:48:21.951198Z","shell.execute_reply.started":"2022-07-15T15:48:13.271879Z","shell.execute_reply":"2022-07-15T15:48:21.950283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Compare autoencoder reconstruction to original values in test data","metadata":{}},{"cell_type":"code","source":"plot_orig_vs_recon('After training')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:48:21.952590Z","iopub.execute_input":"2022-07-15T15:48:21.953123Z","iopub.status.idle":"2022-07-15T15:48:22.410595Z","shell.execute_reply.started":"2022-07-15T15:48:21.953086Z","shell.execute_reply":"2022-07-15T15:48:22.409559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Examine encoded representation\n\n__Conclusion__: no obvious structure is visible :-(","metadata":{}},{"cell_type":"code","source":"X_enc = encoder(X_test)\nfig = plt.figure(figsize=(12,12))\nax = fig.add_subplot(111, projection = '3d')\n\nax.scatter(X_enc[:,0],X_enc[:,1],X_enc[:,2], s=2,alpha=0.2)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:48:22.411895Z","iopub.execute_input":"2022-07-15T15:48:22.412273Z","iopub.status.idle":"2022-07-15T15:48:22.803718Z","shell.execute_reply.started":"2022-07-15T15:48:22.412236Z","shell.execute_reply":"2022-07-15T15:48:22.802769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}