{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"input_path = '../input/tabular-playground-series-apr-2022/'\noutput_path = './'","metadata":{"execution":{"iopub.status.busy":"2022-07-22T03:04:56.850059Z","iopub.execute_input":"2022-07-22T03:04:56.850505Z","iopub.status.idle":"2022-07-22T03:04:56.875668Z","shell.execute_reply.started":"2022-07-22T03:04:56.850432Z","shell.execute_reply":"2022-07-22T03:04:56.875193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ndef load_data(train_or_test='train'):\n    file_name = f'{input_path}/{train_or_test}.csv'\n    df = pd.read_csv(file_name)\n    sensors = df.loc[:, 'sensor_00':'sensor_12'].values.reshape((-1, 60, 13))\n    subj_nums = df[df['step'] == 0]['subject'].values\n    return sensors, subj_nums\n\ndef load_label(train_or_test='train'):\n    file_name = input_path + ('train_labels.csv' if train_or_test=='train' else 'sample_submission.csv')\n    df = pd.read_csv(file_name)\n    return df['state']\n\ndef submit(arr):\n    df = load_label('test')\n    df['state'] = arr\n    df.to_csv(f'{output_path}/submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T03:04:56.909116Z","iopub.execute_input":"2022-07-22T03:04:56.909433Z","iopub.status.idle":"2022-07-22T03:04:56.915179Z","shell.execute_reply.started":"2022-07-22T03:04:56.909411Z","shell.execute_reply":"2022-07-22T03:04:56.914452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GroupShuffleSplit\n\nsensors, subj_nums = load_data('train')\ny = load_label('train')\nX = sensors / (np.linalg.norm(sensors, axis=1, keepdims=True) + 1e-10)\n\nsplitter = GroupShuffleSplit(random_state=42)\nfor train_idxes, test_idxes in splitter.split(X, y, subj_nums):\n    X_train, y_train = X[train_idxes], y[train_idxes]\n    X_test, y_test = X[test_idxes], y[test_idxes]\n    break\n\nX_norm = X / (np.linalg.norm(X, axis=1, keepdims=True) + 1e-10)\nnoise = 1e-2 * np.random.random(X.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T03:04:56.959526Z","iopub.execute_input":"2022-07-22T03:04:56.961111Z","iopub.status.idle":"2022-07-22T03:05:07.702880Z","shell.execute_reply.started":"2022-07-22T03:04:56.961070Z","shell.execute_reply":"2022-07-22T03:05:07.702147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\n\nclass TSAE(keras.Model):\n    def __init__(self, dim=13, innermost=8):\n        super(TSAE, self).__init__()\n        self.encoder = keras.Sequential(\n        [\n            keras.layers.Conv1D(filters=10, kernel_size=13, activation='elu'), \n#             keras.layers.MaxPool1D(2),\n            keras.layers.Conv1D(filters=8, kernel_size=13, activation='elu'), \n#             keras.layers.MaxPool1D(2),\n            keras.layers.Conv1D(filters=8, kernel_size=13, activation='elu'), \n#             keras.layers.MaxPool1D(2),\n            keras.layers.Conv1D(filters=innermost, kernel_size=7, activation='elu'), \n            keras.layers.Flatten(),\n            keras.layers.Dense(units=144, activation='elu'),\n        ])\n        self.decoder = keras.Sequential(\n        [\n            keras.layers.Reshape((18, innermost)),\n#             keras.layers.UpSampling1D(2),\n            keras.layers.Conv1DTranspose(filters=8, kernel_size=7, activation='elu'),\n#             keras.layers.UpSampling1D(2), \n            keras.layers.Conv1DTranspose(filters=8, kernel_size=13, activation='elu'),\n#             keras.layers.UpSampling1D(2),\n            keras.layers.Conv1DTranspose(filters=10, kernel_size=13, activation='elu'),\n#             keras.layers.UpSampling1D(2), \n            keras.layers.Conv1DTranspose(filters=dim, kernel_size=13)\n        ])\n    \n    def encode(self, inputs):\n        return self.encoder(inputs)\n    \n    def decode(self, inputs):\n        return self.decoder(inputs)\n    \n    def call(self, inputs):\n        return self.decode(self.encode(inputs))\n\n    \nclass ConvAE(keras.Model):\n    def __init__(self):\n        super(ConvAE, self).__init__()\n        self.encoder = keras.Sequential(\n        [\n            keras.layers.Conv1D(filters=5, kernel_size=1, padding='same', activation='elu'), \n            keras.layers.SeparableConv1D(filters=10, kernel_size=8, padding='same', activation='elu'),\n            keras.layers.MaxPool1D(2), \n            keras.layers.SeparableConv1D(filters=20, kernel_size=8, padding='same', activation='elu'),\n            keras.layers.MaxPool1D(2), \n            keras.layers.SeparableConv1D(filters=10, kernel_size=8, padding='same', activation='elu'),\n#             keras.layers.MaxPool1D(3),\n#             keras.layers.MaxPool1D(2),\n            keras.layers.Flatten(),\n#             keras.layers.Dense(120, activation='elu'), \n        ])\n        self.decoder = keras.Sequential(\n        [\n#             keras.layers.Dense(480, activation='elu'),\n            keras.layers.Reshape((15, 10)),\n#             keras.layers.UpSampling1D(3), \n            keras.layers.SeparableConv1D(filters=20, kernel_size=8, padding='same', activation='elu'),\n            keras.layers.UpSampling1D(2), \n            keras.layers.SeparableConv1D(filters=10, kernel_size=8, padding='same', activation='elu'),\n            keras.layers.UpSampling1D(2), \n            keras.layers.SeparableConv1D(filters=5, kernel_size=8, padding='same', activation='elu'), \n            keras.layers.Conv1D(filters=13, kernel_size=1, padding='same')\n        ])\n    \n    def encode(self, inputs):\n        return self.encoder(inputs)\n    \n    def decode(self, inputs):\n        return self.decoder(inputs)\n    \n    def call(self, inputs):\n        return self.decode(self.encode(inputs))\n\n\nclass SeparatedConvAE(keras.Model):\n    def __init__(self):\n        super(SeparatedConvAE, self).__init__()\n        self.encoding_branches = [\n            keras.Sequential([\n                keras.layers.Conv1D(filters=1, kernel_size=1, padding='same', activation='elu'), \n                keras.layers.SeparableConv1D(filters=5, kernel_size=8, padding='same', activation='elu'), \n                keras.layers.MaxPool1D(2), \n                keras.layers.SeparableConv1D(filters=5, kernel_size=8, padding='same', activation='elu'), \n                keras.layers.MaxPool1D(2), \n                keras.layers.SeparableConv1D(filters=1, kernel_size=4, padding='same', activation='elu')\n            ])\n            for _ in range(4)\n        ]\n        self.decoding_branches = [\n            keras.Sequential([\n                keras.layers.SeparableConv1D(filters=5, kernel_size=4, padding='same', activation='elu'),\n                keras.layers.UpSampling1D(2),\n                keras.layers.SeparableConv1D(filters=5, kernel_size=8, padding='same', activation='elu'),\n                keras.layers.UpSampling1D(2),\n                keras.layers.SeparableConv1D(filters=1, kernel_size=8, padding='same', activation='elu'), \n            ])\n        ]\n        self.decoding_final = keras.layers.Conv1D(filters=13, kernel_size=1, padding='same')\n        \n    def encode(self, inputs):\n        hidden = tf.concat([layer(inputs) for layer in self.encoding_branches], axis=-1)\n        return tf.reshape(hidden, (-1, 60))\n    \n    def decode(self, hidden):\n        outputs = tf.reshape(hidden, (-1, 15, 4))\n        outputs = tf.concat([layer(outputs[..., i:i+1]) for i, layer in enumerate(self.decoding_branches)], axis=-1)\n        return self.decoding_final(outputs)\n    \n    def call(self, inputs):\n        return self.decode(self.encode(inputs))\n    \nclass LSTMAE(keras.Model):\n    def __init__(self):\n        super(LSTMAE, self).__init__()\n        self.encoder = keras.Sequential(\n        [\n            keras.layers.RNN(\n                keras.layers.StackedRNNCells([\n                    keras.layers.LSTMCell(units=128),\n                ]), \n            ),\n#             keras.layers.Flatten(), \n#             keras.layers.Dense(100, activation='relu'),\n        ])\n        self.decoder = keras.Sequential(\n        [\n            keras.layers.RepeatVector(60), \n            keras.layers.RNN(\n                keras.layers.StackedRNNCells([\n                    keras.layers.LSTMCell(units=128),\n                ]), \n                return_sequences=True\n            ), \n            keras.layers.TimeDistributed(keras.layers.Dense(13)),  \n        ])\n    \n    def encode(self, inputs):\n        return self.encoder(inputs)\n    \n    def decode(self, inputs):\n        return self.decoder(inputs)\n    \n    def call(self, inputs):\n        return self.decode(self.encode(inputs))\n\n\nclass SobolevLossDeg1(keras.losses.Loss):\n    def call(self, y_true, y_pred):\n        y_diff = y_true - y_pred\n        rms = tf.math.reduce_sum(tf.math.square(y_diff), axis=1)\n        y_diff_d1 = y_diff[:, 1:] - y_diff[:, :-1]\n        rms_d1 = 50.* tf.math.reduce_sum(tf.math.square(y_diff_d1), axis=1)\n        return rms + rms_d1","metadata":{"execution":{"iopub.status.busy":"2022-07-22T03:05:07.704198Z","iopub.execute_input":"2022-07-22T03:05:07.704420Z","iopub.status.idle":"2022-07-22T03:05:14.525048Z","shell.execute_reply.started":"2022-07-22T03:05:07.704394Z","shell.execute_reply":"2022-07-22T03:05:14.524416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keras.backend.clear_session()\n\ncallbacks = [\n    keras.callbacks.EarlyStopping(patience=1000, restore_best_weights=True)\n]\n\n# tsae = SeparatedConvAE()\n# tsae = ConvAE()\n# tsae = LSTMAE()\ntsae = TSAE(dim=13, innermost=8)\ntsae.compile(\n    loss=SobolevLossDeg1(),\n    optimizer=keras.optimizers.Adam(5e-3))\ntsae.fit((X_norm + noise), X_norm, \n         epochs=100, \n         batch_size=1024,\n         callbacks=callbacks,\n        )\n\n# idxs_set_1 = [0, 1, 3, 4, 6, 7, 9, 10, 11]\n# idxs_set_2 = [2, 5, 8, 12]\n# tsae_2 = TSAE(dim=len(idxs_set_2), innermost=8)\n# tsae_2.compile(\n#     loss=SobolevLossDeg1(),\n#     optimizer=keras.optimizers.Adam(5e-3))\n# tsae_2.fit(\n#     (X_norm+noise)[:, :, idxs_set_2],\n#     X_norm[:, :, idxs_set_2],\n#     epochs=100, \n#     batch_size=1024,\n#     callbacks=callbacks,)\n\n# tsae1.compile(\n#     loss=SobolevLossDeg1(),\n#     optimizer=keras.optimizers.Adam(5e-3))\n# tsae1.fit(Xn_1, X_1, \n#           epochs=100, \n#           batch_size=1024,\n#           callbacks=callbacks,\n#          )\n# tsae2.compile(\n#     loss=SobolevLossDeg1(),\n#     optimizer=keras.optimizers.Adam(5e-3))\n# tsae2.fit(Xn_2, X_2, \n#           epochs=100, \n#           batch_size=1024,\n#           callbacks=callbacks,\n#          )\n\nX_norm_recon = tsae.predict(X_norm)\n# X_norm_recon_2 = tsae_2.predict(X_norm[:, :, idxs_set_2])","metadata":{"execution":{"iopub.status.busy":"2022-07-22T03:05:14.525981Z","iopub.execute_input":"2022-07-22T03:05:14.526423Z","iopub.status.idle":"2022-07-22T03:08:36.361706Z","shell.execute_reply.started":"2022-07-22T03:05:14.526395Z","shell.execute_reply":"2022-07-22T03:08:36.361243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nidx = 1823 # 1445\narr = X_norm[idx]\nbrr = X_norm_recon_2[idx]\n\nfig, axs = plt.subplots(nrows=5, ncols=3, figsize=(24, 6))\naxs = axs.ravel()\nfor i in range(13):\n    ax = axs[i]\n    ax.plot(arr[:, i])\n    ax.set_title(f'sensor_{i:0>2}')\nfor j in range(len(idxs_set_2)):\n    i = idxs_set_2[j]\n    ax = axs[i]\n    ax.plot(brr[:, j])\nfig.suptitle(f'index={idx}')\nplt.show(); plt.close();","metadata":{"execution":{"iopub.status.busy":"2022-07-22T03:08:36.365001Z","iopub.execute_input":"2022-07-22T03:08:36.365167Z","iopub.status.idle":"2022-07-22T03:08:36.607811Z","shell.execute_reply.started":"2022-07-22T03:08:36.365147Z","shell.execute_reply":"2022-07-22T03:08:36.606825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nidx = 1823 # 1445\narr = X_norm[idx]\nbrr = X_norm_recon[idx]\n\nfig, axs = plt.subplots(nrows=5, ncols=3, figsize=(24, 6))\naxs = axs.ravel()\nfor i in range(13):\n    ax = axs[i]\n    ax.plot(arr[:, i])\n    ax.plot(brr[:, i])\n#     if i in [0, 1, 3, 4, 6, 7, 9, 10, 11]:\n#         ax.plot(brr[:, [0, 1, 3, 4, 6, 7, 9, 10, 11].index(i)])\n    ax.set_title(f'sensor_{i:0>2}')\nfig.suptitle(f'index={idx}')\nplt.show(); plt.close();","metadata":{"execution":{"iopub.status.busy":"2022-07-22T03:09:24.929573Z","iopub.execute_input":"2022-07-22T03:09:24.929962Z","iopub.status.idle":"2022-07-22T03:09:26.577480Z","shell.execute_reply.started":"2022-07-22T03:09:24.929926Z","shell.execute_reply":"2022-07-22T03:09:26.576905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GroupShuffleSplit\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.model_selection import train_test_split\n\nencoded = tsae.encode(X_norm).numpy()[:, ::10]\n# encoded = tsae_2.encode(X_norm[:, :, idxs_set_2]).numpy()\n# encoded = np.concatenate([\n#     tsae.encode(X_norm).numpy(),\n#     tsae_2.encode(X_norm[:, :, idxs_set_2]).numpy()\n# ])\n\nsplitter = GroupShuffleSplit(random_state=42)\nfor train_idxes, test_idxes in splitter.split(X, y, subj_nums):\n    X_train, y_train = encoded[train_idxes], y[train_idxes]\n    X_test, y_test = encoded[test_idxes], y[test_idxes]\n    break\nclfs = [\n    LogisticRegression(max_iter=1000), \n#     DecisionTreeClassifier(max_depth=10), \n#     RandomForestClassifier(max_depth=10, min_samples_split=20, min_samples_leaf=10), \n    MLPClassifier(\n        hidden_layer_sizes=(100, ), \n        alpha=5e-2,\n        max_iter=1000,\n    ),\n#     GradientBoostingClassifier(max_depth=10), \n]\nfor clf in clfs:\n    print(type(clf))\n    clf.fit(X_train, y_train)\n    print(np.mean(y_train == clf.predict(X_train)))\n    print(np.mean(y_test == clf.predict(X_test)))","metadata":{"execution":{"iopub.status.busy":"2022-07-22T03:14:45.413328Z","iopub.execute_input":"2022-07-22T03:14:45.414069Z","iopub.status.idle":"2022-07-22T03:14:49.375829Z","shell.execute_reply.started":"2022-07-22T03:14:45.414029Z","shell.execute_reply":"2022-07-22T03:14:49.375124Z"},"trusted":true},"execution_count":null,"outputs":[]}]}