{"cells":[{"metadata":{},"cell_type":"markdown","source":"# AE","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## import libralies","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#from google.colab import drive\n#drive.mount('/content/gdrive')\nimport numpy as np\nimport pandas as pd\nimport random\nfrom numba.decorators import jit\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pylab import rcParams\n\nfrom keras.models import Model, load_model\nfrom keras.layers import Input, Dense\nfrom keras import regularizers\nfrom sklearn.preprocessing import StandardScaler, StandardScaler\nfrom keras.layers import Dense, BatchNormalization, Activation\nimport tensorflow as tf\nfrom keras.models import Model, load_model\nfrom keras.layers import Input, Dense, Layer, InputSpec\nfrom keras.callbacks import ModelCheckpoint, TensorBoard\nfrom keras import regularizers, activations, initializers, constraints, Sequential\nfrom keras import backend as K\nfrom keras.constraints import UnitNorm, Constraint\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping\n\nfrom sklearn.metrics import (confusion_matrix, precision_recall_curve, auc,\n                             roc_curve, recall_score, classification_report, f1_score,\n                             precision_recall_fscore_support)\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\n\nfrom sklearn.metrics import roc_auc_score\nimport lightgbm as lgb\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Autoencoder Optimization\nclass DenseTied(Layer):\n    def __init__(self, units,\n                 activation=None,\n                 use_bias=True,\n                 kernel_initializer='glorot_uniform',\n                 bias_initializer='zeros',\n                 kernel_regularizer=None,\n                 bias_regularizer=None,\n                 activity_regularizer=None,\n                 kernel_constraint=None,\n                 bias_constraint=None,\n                 tied_to=None,\n                 **kwargs):\n        self.tied_to = tied_to\n        if 'input_shape' not in kwargs and 'input_dim' in kwargs:\n            kwargs['input_shape'] = (kwargs.pop('input_dim'),)\n        super().__init__(**kwargs)\n        self.units = units\n        self.activation = activations.get(activation)\n        self.use_bias = use_bias\n        self.kernel_initializer = initializers.get(kernel_initializer)\n        self.bias_initializer = initializers.get(bias_initializer)\n        self.kernel_regularizer = regularizers.get(kernel_regularizer)\n        self.bias_regularizer = regularizers.get(bias_regularizer)\n        self.activity_regularizer = regularizers.get(activity_regularizer)\n        self.kernel_constraint = constraints.get(kernel_constraint)\n        self.bias_constraint = constraints.get(bias_constraint)\n        self.input_spec = InputSpec(min_ndim=2)\n        self.supports_masking = True\n                \n    def build(self, input_shape):\n        assert len(input_shape) >= 2\n        input_dim = input_shape[-1]\n\n        if self.tied_to is not None:\n            self.kernel = K.transpose(self.tied_to.kernel)\n            self._non_trainable_weights.append(self.kernel)\n        else:\n            self.kernel = self.add_weight(shape=(input_dim, self.units),\n                                          initializer=self.kernel_initializer,\n                                          name='kernel',\n                                          regularizer=self.kernel_regularizer,\n                                          constraint=self.kernel_constraint)\n        if self.use_bias:\n            self.bias = self.add_weight(shape=(self.units,),\n                                        initializer=self.bias_initializer,\n                                        name='bias',\n                                        regularizer=self.bias_regularizer,\n                                        constraint=self.bias_constraint)\n        else:\n            self.bias = None\n        self.input_spec = InputSpec(min_ndim=2, axes={-1: input_dim})\n        self.built = True\n\n    def compute_output_shape(self, input_shape):\n        assert input_shape and len(input_shape) >= 2\n        output_shape = list(input_shape)\n        output_shape[-1] = self.units\n        return tuple(output_shape)\n\n    def call(self, inputs):\n        output = K.dot(inputs, self.kernel)\n        if self.use_bias:\n            output = K.bias_add(output, self.bias, data_format='channels_last')\n        if self.activation is not None:\n            output = self.activation(output)\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class WeightsOrthogonalityConstraint (Constraint):\n    def __init__(self, encoding_dim, weightage = 1.0, axis = 0):\n        self.encoding_dim = encoding_dim\n        self.weightage = weightage\n        self.axis = axis\n        \n    def weights_orthogonality(self, w):\n        if(self.axis==1):\n            w = K.transpose(w)\n        if(self.encoding_dim > 1):\n            m = K.dot(K.transpose(w), w) - K.eye(self.encoding_dim)\n            return self.weightage * K.sqrt(K.sum(K.square(m)))\n        else:\n            m = K.sum(w ** 2) - 1.\n            return m\n\n    def __call__(self, w):\n        return self.weights_orthogonality(w)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class UncorrelatedFeaturesConstraint (Constraint):\n    \n    def __init__(self, encoding_dim, weightage = 1.0):\n        self.encoding_dim = encoding_dim\n        self.weightage = weightage\n    \n    def get_covariance(self, x):\n        x_centered_list = []\n\n        for i in range(self.encoding_dim):\n            x_centered_list.append(x[:, i] - K.mean(x[:, i]))\n        \n        x_centered = tf.stack(x_centered_list)\n        covariance = K.dot(x_centered, K.transpose(x_centered)) / tf.cast(x_centered.get_shape()[0], tf.float32)\n        \n        return covariance\n            \n    # Constraint penalty\n    def uncorrelated_feature(self, x):\n        if(self.encoding_dim <= 1):\n            return 0.0\n        else:\n            output = K.sum(K.square(self.covariance - K.dot(self.covariance, K.eye(self.encoding_dim))))\n            return output\n\n    def __call__(self, x):\n        self.covariance = self.get_covariance(x)\n        return self.weightage * self.uncorrelated_feature(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#TensorFlowがGPUを認識しているか確認\nfrom tensorflow.python.client import device_lib\ndevice_lib.list_local_devices()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/eff09445/train_output_0.9445983706153531.csv\")\ntest  = pd.read_csv(\"../input/eff09445/test_output_0.9445983706153531.csv\")\n\nuseful = [t for t in train.columns.to_list() if (\"count\" not in t) and \n          (\"w\" not in t) and (\"rows\" not in t) \n          and (\"umap_y\" not in t)]\nfeature_names = list(set(useful) - set(['image_name', \"patient_id\", 'target']))\nycol = [\"target\"]\nprint(len(feature_names))\n\n#mm = MinMaxScaler()\nmm = StandardScaler()\nmm.fit(train[feature_names])\ntrain_mm = mm.transform(train[feature_names])\ntest_mm  = mm.transform(test[feature_names])\n\ntrain[feature_names] = train_mm\ntest[feature_names] = test_mm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nb_epoch = 600\nbatch_size = 2048\n\ninput_dim = len(feature_names)\nencoding_dim = 10\nlearning_rate = 1e-3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@jit\ndef noise(array):\n  print('now noising') \n  height = len(array)\n  width = len(array[0])\n  print('start rand')  \n  rands = np.random.uniform(0, 1, (height, width) )\n  print('finish rand')  \n  copy  = np.copy(array)\n  for h in range(height):\n    for w in range(width):\n      if rands[h, w] <= 0.10:\n        swap_target_h = random.randint(0,h)\n        copy[h, w] = array[swap_target_h, w]\n  print('finish noising') \n  return copy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_vanila_AE():\n\n    # baseline_model\n    encoder = Dense(encoding_dim, activation=\"relu\", input_shape=(input_dim,), use_bias = True) \n    decoder = Dense(input_dim, activation=\"relu\", use_bias = True)\n\n    autoencoder_vanila = Sequential()\n    autoencoder_vanila.add(encoder)\n    autoencoder_vanila.add(decoder)\n    \n    autoencoder_vanila.compile(metrics=['mae'],\n                    loss='mean_squared_error',\n                    optimizer='adam')\n    #autoencoder_vanila.summary()\n    return autoencoder_vanila","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_full_AE():\n    encoder = Dense(encoding_dim, activation=\"relu\", input_shape=(input_dim,), use_bias = True, kernel_constraint=UnitNorm(axis=0)) \n    decoder = Dense(input_dim, activation=\"relu\", use_bias = True, kernel_constraint=UnitNorm(axis=1))\n    autoencoder_full = Sequential()\n    autoencoder_full.add(encoder)\n    autoencoder_full.add(decoder)\n    autoencoder_full.compile(metrics=['accuracy'],\n                        loss='mean_squared_error',\n                          optimizer='adam')\n    return autoencoder_full","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_AE(autoencoder, train_data, valid_data, \n             batch_size, epoch, callbacks, DO_TRAINING):\n  if DO_TRANING:\n     history = autoencoder.fit(train_data, train_data,\n                               epochs=nb_epoch,\n                               batch_size=batch_size,\n                               shuffle=True,\n                               validation_data=(valid_data, valid_data),\n                               verbose=1,\n                               callbacks=callbacks).history\n\n     # Model loss\n     plt.plot(history['loss'])\n     plt.plot(history['val_loss'])\n     plt.title('model loss')\n     plt.ylabel('acc')\n     plt.xlabel('epoch')\n     plt.legend(['train', 'val'], loc='upper right');\n     plt.show()\n     \n  return(autoencoder)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)\noof_dfs = []\npredicted_tests = []\nfor fold_id, (trn_idx, val_idx) in enumerate(skf.split(train, train[ycol])):\n        # set callbacks\n        DO_TRANING = True\n        checkpointer = ModelCheckpoint(filepath=\"model_{}.h5\".format(fold_id),\n                                       verbose=0,\n                                       save_best_only=True)\n        earlystoping = EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=0, mode='auto')\n        callbacks=[checkpointer, earlystoping]\n        print(\"Fold \", fold_id)\n        train_data = train.iloc[trn_idx]\n        train_data = train_data[train_data[\"target\"]==0.0][feature_names]\n        train_data = pd.DataFrame(train_data.values, columns=feature_names)\n        # add noise\n        #train_data = pd.DataFrame(noise(train_data[feature_names].values), columns=feature_names)\n        \n        valid_data = train[feature_names].iloc[val_idx]\n\n        autoencoder = get_vanila_AE()\n        autoencoder_trained = train_AE(autoencoder, \n                                      train_data, \n                                      valid_data, \n                                      batch_size, \n                                      nb_epoch, \n                                      callbacks,\n                                      DO_TRANING)\n        \n        #predicted_valid = autoencoder_trained.predict(valid_data)\n        #predicted_test  = autoencoder_trained.predict(test[feature_names].values)\n        \n        #oof_dfs.append(pd.DataFrame(np.concatenate([predicted_valid, val_idx.reshape(len(val_idx), 1)], 1), \n        #                            columns=feature_names+[\"original_idx\"]) \n        #                )\n        #predicted_tests.append(predicted_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#oof_df = pd.concat(oof_dfs)\n#oof_df[\"original_idx\"] = oof_df[\"original_idx\"].astype(int)\n#oof_df = oof_df.sort_values(\"original_idx\").reset_index()\n#oof_df = pd.concat([train[[\"image_name\", \"patient_id\", \"target\"]], oof_df[feature_names]], axis=1)\n#oof_df.to_csv(\"oof_AE_with_noise.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sum(oof_df.groupby(\"target\").mean().diff().iloc[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#test_output = pd.DataFrame(sum(predicted_tests) / len(predicted_tests), columns=feature_names)#\n#test_output.to_csv(\"test_AE_with_noise.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}