{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"scrolled":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input/vsb-power-line-fault-detection\"))\nprint(os.listdir(\"../input/dataprocessing\"))\n# Any results you write to the current directory are nsaved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1cbf574d81a83448cfd11c13ffa374c2d1d04c9a"},"cell_type":"markdown","source":"**Outlier Detection**"},{"metadata":{"trusted":true,"_uuid":"523c6140dcf7cd8aef564f200f64334dbc1d463e"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom scipy import stats\n\n\ndef get_outliers(df, outlier_z_score_abs_threshold=5, outlier_feature_fraction=0.3):\n    # features is level 1. ts is level 0.\n    f1 = df.groupby(df.columns.get_level_values(1), axis=1).mean().T\n    f2 = df.abs().groupby(df.columns.get_level_values(1), axis=1).mean().T\n    f3 = df.groupby(df.columns.get_level_values(1), axis=1).std().T\n    f4 = df.groupby(df.columns.get_level_values(1), axis=1).quantile(0.5).T\n    f5 = df.groupby(df.columns.get_level_values(1), axis=1).quantile(0.1).T\n    f6 = df.groupby(df.columns.get_level_values(1), axis=1).quantile(0.9).T\n\n    print('Features', f1.shape, f1.columns[:3])\n    if 'diff_smoothend_by_1 Quant-0.0' in f1.columns:\n        f1 = f1.drop(['diff_smoothend_by_1 Quant-0.0'], axis=0)\n        f2 = f2.drop(['diff_smoothend_by_1 Quant-0.0'], axis=0)\n        f3 = f3.drop(['diff_smoothend_by_1 Quant-0.0'], axis=0)\n        f4 = f4.drop(['diff_smoothend_by_1 Quant-0.0'], axis=0)\n        f5 = f5.drop(['diff_smoothend_by_1 Quant-0.0'], axis=0)\n        f6 = f6.drop(['diff_smoothend_by_1 Quant-0.0'], axis=0)\n\n    outlier_feature_count = int(f1.shape[0] * outlier_feature_fraction)\n    fs = [f1, f2, f3, f4, f5, f6]\n    zscores = list(map(lambda f: stats.zscore(f, axis=1), fs))\n    outliers = list(map(lambda zscore: np.abs(zscore) > outlier_z_score_abs_threshold, zscores))\n    examplewise_outliers = list(map(lambda outlier: np.sum(outlier, axis=0), outliers))\n    # print('Shape of example_outliers', examplewise_outliers[0].shape)\n    outlier_filters = []\n    for i, ex_out in enumerate(examplewise_outliers):\n        outlier_filter = ex_out > outlier_feature_count\n        outlier_filters.append(outlier_filter)\n        # zero_percent = round(ex_out[ex_out == 0].shape[0] / ex_out.shape[0] * 100, 2)\n        # outlier_count = ex_out[outlier_filter].shape[0]\n        # print('FeatureIndex', i, 'zero percent', zero_percent)\n        # print('FeatureIndex', i, 'outlier count', outlier_count)\n\n    outlier_filter = np.sum(outlier_filters, axis=0)\n    outlier_filter = outlier_filter >= len(outlier_filters) // 2\n    print('Outlier percent:', round(outlier_filter.sum() / outlier_filter.shape[0] * 100, 2))\n    print('Outlier count:', outlier_filter.sum())\n\n    output_df = pd.Series(outlier_filter, index=f1.columns).to_frame('outliers')\n    # output_df.index = output_df.index.astype(int)\n    return output_df\n\n\ndef target_class_outlier_distribution_grid_search(df, meta_fname):\n    meta_df = pd.read_csv(meta_fname).set_index('signal_id')[['target']]\n    thresholds = [4, 5, 6]\n    fractions = [0.3, 0.5, 0.7]\n    vc = round(meta_df['target'].value_counts().loc[1] / meta_df.shape[0] * 100, 2)\n    print('In original data, target class 1 is {}%'.format(vc))\n\n    target_one_percent = pd.DataFrame([], columns=thresholds, index=fractions)\n    target_one_percent.index.name = 'outlier_feature_fraction'\n    target_one_percent.columns.name = 'outlier_z_score_abs_threshold'\n    for thresh in thresholds:\n        for frac in fractions:\n            outliers_df = get_outliers(df, thresh, frac)\n            outliers_df = outliers_df.join(meta_df, how='left')\n            vc = outliers_df[outliers_df['outliers'] == True]['target'].value_counts()\n            if 1 not in vc.index:\n                target_one_percent.loc[frac, thresh] = 0\n            else:\n                percnt = round(vc.loc[1] / vc.sum() * 100, 2)\n                target_one_percent.loc[frac, thresh] = percnt\n                print('In outliers with thresh:{}, frac:{}, target class 1 is {}%'.format(thresh, frac, percnt))\n\n    return target_one_percent\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"03506ea578d03ee491387b7dddacb37c6381a0b7"},"cell_type":"code","source":"# Taken from https://keunwoochoi.wordpress.com/2017/08/24/tip-fit_generator-in-keras-how-to-parallelise-correctly/\nimport threading\n\n\nclass ThreadSafeIter:\n    def __init__(self, iter):\n        self._itr = iter\n        self.lock = threading.Lock()\n\n    def __iter__(self):\n        return self\n\n    def __next__(self):\n        with self.lock:\n            return self._itr.__next__()\n\n\ndef threadsafe(f):\n    \"\"\"A decorator that takes a generator function and makes it thread-safe.\n    \"\"\"\n\n    def g(*a, **kw):\n        return ThreadSafeIter(f(*a, **kw))\n\n    return g\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e2c13c157cc3b637150010134deaf8a4884d39f"},"cell_type":"code","source":"from typing import Tuple\nimport numpy as np\nimport pandas as pd\nfrom keras import backend as K\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.layers import (Bidirectional, Dense, Input, CuDNNLSTM, Activation, BatchNormalization, LeakyReLU, Dropout)\nfrom keras.models import Model\nfrom sklearn.metrics import matthews_corrcoef\nfrom sklearn.model_selection import StratifiedKFold\nfrom keras import regularizers\n\n# from threadsafe_iterator import threadsafe\n\n\n# It is the official metric used in this competition\n# below is the declaration of a function used inside the keras model, calculation with K (keras backend / thensorflow)\ndef matthews_correlation(y_true, y_pred):\n    '''Calculates the Matthews correlation coefficient measure for quality\n    of binary classification problems.\n    '''\n    y_pred_pos = K.round(K.clip(y_pred, 0, 1))\n    y_pred_neg = 1 - y_pred_pos\n\n    y_pos = K.round(K.clip(y_true, 0, 1))\n    y_neg = 1 - y_pos\n\n    tp = K.sum(y_pos * y_pred_pos)\n    tn = K.sum(y_neg * y_pred_neg)\n\n    fp = K.sum(y_neg * y_pred_pos)\n    fn = K.sum(y_pos * y_pred_neg)\n\n    numerator = (tp * tn - fp * fn)\n    denominator = K.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))\n\n    return numerator / (denominator + K.epsilon())\n\n\nclass LSTModel:\n    def __init__(\n            self,\n            units: int,\n            dense_count: int,\n            train_fname='/home/ashesh/Documents/initiatives/kaggle_competitions/vsb_powerline/data/transformed_train.csv',\n            meta_train_fname='/home/ashesh/Documents/initiatives/kaggle_competitions/vsb_powerline/data/metadata_train.csv',\n            skip_fraction: float = 0,\n            data_aug_num_shifts=1,\n            data_aug_flip=False,\n            dropout_fraction=0.3,\n            remove_outliers_from_training=False,\n            outlier_removal_kwargs={},\n            plot_stats=True):\n        \"\"\"\n        Args:\n            skip_fraction: initial fraction of timestamps can be ignored.\n        \"\"\"\n        self._units = units\n        self._dense_c = dense_count\n        self._data_fname = train_fname\n        self._meta_fname = meta_train_fname\n        self._skip_fraction = skip_fraction\n        self._data_aug_num_shifts = data_aug_num_shifts\n        self._data_aug_flip = data_aug_flip\n        self._plot_stats = plot_stats\n        self._dropout_fraction = dropout_fraction\n        self._remove_outliers_from_training = remove_outliers_from_training\n        self._outlier_removal_kwargs = outlier_removal_kwargs\n\n        self._skip_features = [\n            'diff_smoothend_by_1 Quant-0.25', 'diff_smoothend_by_1 Quant-0.75', 'diff_smoothend_by_1 abs_mean',\n            'diff_smoothend_by_1 mean', 'diff_smoothend_by_16 Quant-0.25', 'diff_smoothend_by_16 Quant-0.75',\n            'diff_smoothend_by_16 abs_mean', 'diff_smoothend_by_16 mean', 'diff_smoothend_by_2 Quant-0.25',\n            'diff_smoothend_by_2 Quant-0.75', 'diff_smoothend_by_4 Quant-0.25', 'diff_smoothend_by_4 Quant-0.75',\n            'diff_smoothend_by_8 Quant-0.25', 'diff_smoothend_by_8 Quant-0.5', 'signal_Quant-0.25', 'signal_Quant-0.75'\n        ]\n\n        self._n_splits = 3\n        self._feature_c = None\n        self._ts_c = None\n        self._train_batch_size = None\n        # validation score is saved here.\n        self._val_score = -1\n        # normalization is done using this.\n        self._n_split_scales = []\n        # a value between 0 and 1. a prediction greater than this value is considered as 1.\n        self.threshold = None\n\n    def get_model(self):\n        inp = Input(shape=(\n            self._ts_c,\n            self._feature_c,\n        ))\n#         x = Dropout(self._dropout_fraction, noise_shape=(self._train_batch_size, 1, self._feature_c))(inp)\n        x = Bidirectional(\n            CuDNNLSTM(\n                self._units,\n                return_sequences=False,\n                #                 kernel_regularizer=regularizers.l1(0.001),\n                # activity_regularizer=regularizers.l1(0.01),\n                # bias_regularizer=regularizers.l1(0.01)\n            ))(inp)\n\n        #         x = Bidirectional(CuDNNLSTM(self._units, return_sequences=False))(inp)\n        #         x = Bidirectional(CuDNNLSTM(self._units // 2, return_sequences=False,\n        #                                    kernel_regularizer=regularizers.l1(0.001),))(x)\n        x = Dropout(self._dropout_fraction)(x)\n        x = Dense(self._dense_c)(x)\n        x = BatchNormalization()(x)\n        x = LeakyReLU()(x)\n        x = Dense(1, activation='sigmoid')(x)\n        model = Model(inputs=inp, outputs=x)\n        model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])\n        return model\n\n    @staticmethod\n    def skip_quantile_features(cols, quantiles):\n        filt_cols1 = LSTModel._skip_quantiles(cols, quantiles, '_')\n        filt_cols2 = LSTModel._skip_quantiles(cols, quantiles, '-')\n        cols3 = list(set(filt_cols1) & set(filt_cols2))\n        cols3.sort()\n        return cols3\n\n    @staticmethod\n    def _skip_quantiles(cols, quantiles, delimiter):\n        filtered_cols = []\n        for col in cols:\n            try:\n                val = float(col.split(delimiter)[-1])\n                if val in quantiles:\n                    continue\n            except:\n                pass\n            filtered_cols.append(col)\n        return filtered_cols\n\n    def get_processed_data_df(self, fname: str):\n        processed_data_df = pd.read_csv(fname, compression='gzip', index_col=[0, 1])\n        processed_data_df = processed_data_df.T\n        processed_data_df = processed_data_df.swaplevel(axis=1).sort_index(axis=1)\n        if 'Unnamed: 0' in processed_data_df.index:\n            processed_data_df = processed_data_df.drop('Unnamed: 0', axis=0)\n\n        processed_data_df.index = list(map(int, processed_data_df.index))\n\n        # skip unnecessary columns\n        # feature_cols = LSTModel.skip_quantile_features(processed_data_df.columns.levels[1], [0.25, 0.75])\n        feature_cols = list(set(processed_data_df.columns.levels[1]) - set(self._skip_features))\n        processed_data_df = processed_data_df.iloc[:, processed_data_df.columns.get_level_values(1).isin(feature_cols)]\n\n        # skip first few timestamps. (from paper.)\n        ts_units = len(processed_data_df.columns.levels[0])\n        skip_end_ts_index = int(ts_units * self._skip_fraction) - 1\n        if skip_end_ts_index > 0:\n            print('Skipping first ', skip_end_ts_index + 1, 'timestamp units out of total ', ts_units, ' units')\n            col_filter = processed_data_df.columns.get_level_values(0) > skip_end_ts_index\n            processed_data_df = processed_data_df.iloc[:, col_filter]\n\n        return processed_data_df.sort_index(axis=0)\n\n    def get_y_df(self):\n        fname = self._meta_fname\n        df = pd.read_csv(fname)\n        return df.set_index('signal_id')\n\n    def add_phase_data(self, processed_data_df, meta_fname):\n        return processed_data_df\n\n        print('Phase data is about to be added')\n        metadata_df = pd.read_csv(meta_fname).set_index('signal_id')\n        processed_data_df = processed_data_df.join(metadata_df[['id_measurement']], how='left')\n        assert not processed_data_df.isna().any().any()\n\n        # pandas does not have a cyclic shift facility. therefore copying it.\n        temp_df = pd.concat([processed_data_df, processed_data_df])\n        grp = temp_df.groupby('id_measurement')\n\n        data_1 = grp.shift(0)\n        data_1 = data_1[~data_1.index.duplicated(keep='first')]\n\n        data_2 = grp.shift(-1)\n        data_2 = data_2[~data_2.index.duplicated(keep='first')]\n\n        data_3 = grp.shift(-2)\n        data_3 = data_3[~data_3.index.duplicated(keep='first')]\n        del grp\n        del temp_df\n\n        assert set(data_1.index.tolist()) == (set(data_2.index.tolist()))\n        assert set(data_1.index.tolist()) == (set(data_3.index.tolist()))\n\n        # change indicators name to ensure uniqueness of columns\n        feat_names = ['Phase1-' + e for e in data_1.columns.levels[1].tolist()]\n        data_1.columns.set_levels(feat_names, level=1, inplace=True)\n\n        feat_names = ['Phase2-' + e for e in data_2.columns.levels[1].tolist()]\n        data_2.columns.set_levels(feat_names, level=1, inplace=True)\n\n        feat_names = ['Phase3-' + e for e in data_3.columns.levels[1].tolist()]\n        data_3.columns.set_levels(feat_names, level=1, inplace=True)\n\n        processed_data_df = pd.concat([data_1, data_2, data_3], axis=1)\n        print(processed_data_df.shape)\n        print('Phase data added')\n        return processed_data_df\n\n    @staticmethod\n    def add_ts_segment_feature(processed_data_df):\n        # add segment feature. from paper. (it is said that the 2nd and 4th component of sine wave has information and\n        # first and 3rd are non-informative )\n        segment_size = len(processed_data_df.columns.levels[0]) // 4\n        print('Adding segment data in one-hot encoding form')\n        for ts_index in processed_data_df.columns.levels[0]:\n            segment = ts_index // segment_size\n            segment_0 = 'segment_0'\n            segment_1 = 'segment_1'\n            segment_2 = 'segment_2'\n            segment_3 = 'segment_3'\n\n            processed_data_df[ts_index, segment_0] = int(segment == 0)\n            processed_data_df[ts_index, segment_1] = int(segment == 1)\n            processed_data_df[ts_index, segment_2] = int(segment == 2)\n            processed_data_df[ts_index, segment_3] = int(segment >= 3)\n\n            assert processed_data_df.iloc[0][ts_index][[segment_0, segment_1, segment_2, segment_3]].sum() == 1\n\n        return processed_data_df.sort_index(axis=1)\n\n    def get_X_df(self, fname, meta_fname):\n        processed_data_df = self.get_processed_data_df(fname)\n\n        # NOTE: there are 8 columns which are being zero. one needs to fix it.\n        assert processed_data_df.isna().any(axis=0).sum() <= 9, processed_data_df.isna().any(axis=0).sum()\n        assert processed_data_df.isna().all(axis=0).sum() <= 9, processed_data_df.isna().all(axis=0).sum()\n\n        processed_data_df = processed_data_df.fillna(0)\n        processed_data_df = self.add_phase_data(processed_data_df, meta_fname)\n        assert not processed_data_df.isna().any().any(), 'Training data has nan'\n\n        processed_data_df.columns = processed_data_df.columns.remove_unused_levels()\n        return LSTModel.add_ts_segment_feature(processed_data_df)\n\n    def get_X_in_parts_df(self, fname, meta_fname):\n        processed_data_df = self.get_processed_data_df(fname)\n\n        # NOTE: there are 8 columns which are being zero. one needs to fix it.\n        assert processed_data_df.isna().any(axis=0).sum() <= 9\n        assert processed_data_df.isna().all(axis=0).sum() <= 9\n\n        processed_data_df = processed_data_df.fillna(0)\n        meta_df = pd.read_csv(meta_fname)\n        chunksize = 2 * 999\n        s_index = 0\n        e_index = chunksize\n        sz = processed_data_df.shape[0]\n        while e_index < sz:\n            last_accesible_id = meta_df.iloc[e_index - 1]['id_measurement']\n            first_inaccesible_id = meta_df.iloc[e_index]['id_measurement']\n            while e_index < sz and last_accesible_id == first_inaccesible_id:\n                e_index += 1\n                last_accesible_id = meta_df.iloc[e_index - 1]['id_measurement']\n                first_inaccesible_id = meta_df.iloc[e_index]['id_measurement']\n\n            # making all three phases data available.\n            data_df = self.add_phase_data(processed_data_df.iloc[s_index:e_index], meta_fname)\n            assert not data_df.isna().any().any(), 'Training data has nan'\n            s_index = e_index\n            e_index = s_index + chunksize\n            print('Completed Test data preprocessing', round(e_index / sz * 100), '%')\n            yield LSTModel.add_ts_segment_feature(data_df)\n\n        data_df = self.add_phase_data(processed_data_df.iloc[s_index:], meta_fname)\n        assert not data_df.isna().any().any(), 'Training data has nan'\n        yield LSTModel.add_ts_segment_feature(data_df)\n\n    def get_X_y(self):\n        \"\"\"\n        Returns:\n            Tuple(X,y):\n                X.shape should be: (#examples,#ts,#features)\n                y.shape should be: (#examples,)\n        \"\"\"\n        processed_train_df = self.get_X_df(self._data_fname, self._meta_fname)\n\n        y_df = self.get_y_df()\n        y_df = y_df.loc[processed_train_df.index]\n\n        if self._remove_outliers_from_training:\n            outlier_filter = get_outliers(processed_train_df, **self._outlier_removal_kwargs)['outliers']\n            processed_train_df = processed_train_df.loc[~outlier_filter]\n            y_df = y_df.loc[processed_train_df.index]\n            print('Removed', outlier_filter.sum(), 'many outlier entries from training data ')\n\n        examples_c = processed_train_df.shape[0]\n        self._ts_c = len(processed_train_df.columns.levels[0])\n        self._feature_c = len(processed_train_df.columns.levels[1])\n\n        print('#examples', examples_c)\n        print('#ts', self._ts_c)\n        print('#features', self._feature_c)\n        print('data shape', processed_train_df.shape)\n\n        X = processed_train_df.values.reshape(examples_c, self._ts_c, self._feature_c)\n        y = y_df.target.values\n\n        assert X.shape == (examples_c, self._ts_c, self._feature_c)\n        assert y.shape == (examples_c, )\n        return X, y\n\n    def predict(self, fname: str, meta_fname: str):\n        ser = self._predict(fname, meta_fname)\n        ser.index.name = 'signal_id'\n\n        ser = ser.to_frame('prediction')\n        meta_df = pd.read_csv(meta_fname).set_index('signal_id')\n        df = ser.join(meta_df[['id_measurement']], how='left')\n        ser = df.groupby('id_measurement').transform(np.mean)['prediction']\n        ser[ser >= 0.5] = 1\n        ser[ser < 0.5] = 0\n        ser = ser.astype(int)\n\n        return ser\n\n    def _predict(self, fname: str, meta_fname):\n        \"\"\"\n        Using the self._n_splits(5) models, it returns a pandas.Series with values belonging to {0,1}\n        \"\"\"\n        output = []\n        output_index = []\n        for df in self.get_X_in_parts_df(fname, meta_fname):\n            examples_c = df.shape[0]\n            X = df.values.reshape(examples_c, self._ts_c, self._feature_c)\n\n            pred_array = []\n            for split_index in range(self._n_splits):\n                weight_fname = 'weights_{}.h5'.format(split_index)\n                model = self.get_model()\n                model.load_weights(weight_fname)\n\n                scale = self._n_split_scales[split_index]\n                pred_array.append(model.predict(X / scale, batch_size=128))\n\n            # Take average value over different models.\n            pred_array = np.array(pred_array).reshape(len(pred_array), -1)\n            pred_array = (pred_array > self.threshold).astype(int)\n            pred = np.mean(np.array(pred_array), axis=0)\n            # majority prediction\n            pred = (pred > 0.5).astype(int)\n            assert pred.shape[0] == X.shape[0]\n\n            output.append(pred)\n            output_index.append(df.index.tolist())\n        return pd.Series(np.squeeze(np.concatenate(output)), index=np.concatenate(output_index))\n\n    def fit_threshold(self, prediction, actual, start=0.08, end=0.98, n_count=20, center_alignment_offset=0.01):\n        best_score = -1\n        self.threshold = 0\n        scores = []\n        thresholds = np.linspace(start, end, n_count)\n        for threshold in thresholds:\n            score = matthews_corrcoef(actual, (prediction > threshold).astype(np.float64))\n            scores.append(score)\n            center_alignment = 1 if threshold > (1 - self.threshold) else -1\n            if score > best_score + center_alignment * center_alignment_offset:\n                best_score = score\n                self.threshold = threshold\n\n        if self._plot_stats:\n            import matplotlib.pyplot as plt\n\n            plt.plot(thresholds, scores)\n            plt.title('Fitting threshold')\n            plt.ylabel('mathews correlation coef')\n            plt.xlabel('threshold')\n            plt.show()\n\n        print('Matthews correlation on train set is ', best_score, ' with threshold:', self.threshold)\n\n    @staticmethod\n    def get_generator(train_X: np.array, train_y: np.array, batch_size: int, flip: bool, num_shifts: int = 2):\n\n        shifts = list(map(int, np.linspace(0, train_X.shape[1] * 0.2, num_shifts + 1)[1:-1]))\n        shifts = [0] + shifts\n        flip_ts = [1, -1] if flip else [1]\n\n        @threadsafe\n        def augument_by_timestamp_shifts() -> Tuple[np.array, np.array]:\n            \"\"\"\n            num_shifts: factor by which the training data is to be increased.\n            We shift the timestamps to get more data to train. It assumes timestamp is in 2nd dimension of\n            train_X\n            \"\"\"\n            num_times = len(flip_ts) * num_shifts\n            print('After data augumentation, training data has become ', num_times, ' times its original size.')\n            # generator = DataGenerator('training_data_augumented.csv', batch_size, train_X.shape[1], train_X.shape[2],)\n            # generator.add(train_X, train_y)\n            # 1 time is the original data itself.\n            while True:\n                for shift_amount in shifts:\n                    for flip_direction in flip_ts:\n                        flipped_X = train_X[:, ::flip_direction, :]\n                        train_X_shifted = np.roll(flipped_X, shift_amount, axis=1)\n                        for index in range(0, train_X_shifted.shape[0], batch_size):\n                            X = train_X_shifted[index:(index + batch_size), :, :]\n                            y = train_y[index:(index + batch_size)]\n                            if X.shape[0] < batch_size:\n                                continue\n                            assert X.shape[0] == batch_size\n                            yield (X, y)\n\n        steps_per_epoch = len(flip_ts) * len(shifts) * train_X.shape[0] // batch_size\n        return augument_by_timestamp_shifts, steps_per_epoch\n\n    def _plot_acc_loss(self, history):\n        import matplotlib.pyplot as plt\n\n        plt.plot(history.history['matthews_correlation'])\n        plt.plot(history.history['val_matthews_correlation'])\n        plt.title('model accuracy')\n        plt.ylabel('accuracy')\n        plt.xlabel('epoch')\n        plt.legend(['train', 'test'], loc='upper left')\n        plt.show()\n        # summarize history for loss\n        plt.plot(history.history['loss'])\n        plt.plot(history.history['val_loss'])\n        plt.title('model loss')\n        plt.ylabel('loss')\n        plt.xlabel('epoch')\n        plt.legend(['train', 'test'], loc='upper left')\n        plt.show()\n\n    def train(self, batch_size=128, epoch=50):\n\n        # to be used in get_model()\n        self._train_batch_size = batch_size\n\n        X, y = self.get_X_y()\n        print('X shape', X.shape)\n        print('Y shape', y.shape)\n\n        splits = list(StratifiedKFold(n_splits=self._n_splits, shuffle=True).split(X, y))\n\n        preds_array = []\n        y_array = []\n        # Then, iteract with each fold\n        # If you dont know, enumerate(['a', 'b', 'c']) returns [(0, 'a'), (1, 'b'), (2, 'c')]\n        for idx, (train_idx, val_idx) in enumerate(splits):\n            K.clear_session()  # I dont know what it do, but I imagine that it \"clear session\" :)\n            print(\"Beginning fold {}\".format(idx + 1))\n            # use the indexes to extract the folds in the train and validation data\n            train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]\n\n            # We should get scale using normalization on train data only.\n            # axis 0 is #examples, 1 is #timestamps, 2 is features.\n            scale = np.abs(np.max(train_X, axis=(0, 1)))\n            scale[scale == 0] = 1\n\n            train_X = train_X / scale\n            val_X = val_X / scale\n            self._n_split_scales.append(scale)\n\n            # data augumentation\n            generator, steps_per_epoch = LSTModel.get_generator(\n                train_X,\n                train_y,\n                batch_size,\n                self._data_aug_flip,\n                num_shifts=self._data_aug_num_shifts,\n            )\n            # print('Train X shape', train_X.shape)\n            # print('Val X shape', val_X.shape)\n            # print('Train Y shape', train_y.shape)\n            # print('Val y shape', val_y.shape)\n\n            model = self.get_model()\n            print(model.summary())\n            # This checkpoint helps to avoid overfitting. It just save the weights of the model if it delivered an\n            # validation matthews_correlation greater than the last one.\n            ckpt = ModelCheckpoint(\n                'weights_{}.h5'.format(idx),\n                save_best_only=True,\n                save_weights_only=True,\n                verbose=0,\n                monitor='val_matthews_correlation',\n                mode='max',\n            )\n\n            # Train\n            history = model.fit_generator(\n                generator(),\n                epochs=epoch,\n                validation_data=[val_X, val_y],\n                callbacks=[ckpt],\n                steps_per_epoch=steps_per_epoch,\n                # workers=2,\n                # use_multiprocessing=True,\n                verbose=0,\n            )\n\n            if self._plot_stats:\n                self._plot_acc_loss(history)\n\n            # loads the best weights saved by the checkpoint\n            model.load_weights('weights_{}.h5'.format(idx))\n            # Add the predictions of the validation to the list preds_val\n            preds_array.append(model.predict(val_X, batch_size=20))\n            y_array.append(val_y)\n\n        prediction = np.concatenate(preds_array)\n        actual = np.concatenate(y_array)\n\n        self.fit_threshold(model.predict(train_X), train_y)\n        self._val_score = matthews_corrcoef(actual, (prediction > self.threshold).astype(np.float64))\n        print('On validation data, score is:', self._val_score)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de54051df1937e8232d79f13cf621033be20e154"},"cell_type":"code","source":"import pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a949cd8f9205991e6691114a94c4c332847e4a78","scrolled":true},"cell_type":"code","source":"pd.read_csv('../input/vsb-power-line-fault-detection/sample_submission.csv').head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"646b2bf31b0cc9c49b7d542804a254d8488fe9df"},"cell_type":"code","source":"# fname = '../input/dataprocessing/test_data.csv'\n# processed_data_df2 = pd.read_csv(fname, compression='gzip', index_col=[0, 1])\n# fname = '../input/dataprocessing/train_data_0.95_10_200.csv'\n# processed_data_df1 = pd.read_csv(fname, compression='gzip', index_col=[0, 1])\n# a = processed_data_df1.abs().groupby('features').quantile(0.9).mean(axis=1)\n# b = processed_data_df2.abs().groupby('features').quantile(0.9).mean(axis=1)\n# percentage_deviation = ((b-a)/a * 100 ).round(2)\n# percentage_deviation[percentage_deviation.abs() > 10].abs().sort_values()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91047c7099160642242f576b9f28730dd135fee4","scrolled":false},"cell_type":"code","source":"# model = LSTModel(128, 64, train_fname='../input/dataprocessing/train_data_3_0.95_15.csv', \n#                  meta_train_fname='../input/vsb-power-line-fault-detection/metadata_train.csv',\n#                 skip_fraction=0)\n# # df = model.get_X_df('../input/dataprocessing/train_data.csv',\n# #                    '../input/vsb-power-line-fault-detection/metadata_train.csv')\n# model.train(epoch=40)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3e11c73718aff29cc5be678cb2b536f71c64d33","scrolled":true},"cell_type":"code","source":"fnames = ['train_data_0.25_10_150_100.csv',\n'train_data_0.75_10_150_50.csv',\n'train_data_0.75_10_150_100.csv',\n'train_data_0.25_10_150_50.csv']\nfor data_aug_num_times in [3]:\n    for dropout_fraction in [0.1]:\n        for fname in fnames:\n            fname = '../input/dataprocessing/' + fname\n            model = LSTModel(128, 64, train_fname=fname, \n                         meta_train_fname='../input/vsb-power-line-fault-detection/metadata_train.csv',\n                        skip_fraction=0,\n                        data_aug_num_shifts=data_aug_num_times,\n                         dropout_fraction=dropout_fraction,\n                            data_aug_flip=False,\n                            remove_outliers_from_training=True)\n            model.train(epoch=25)\n            print(fname, ' ', data_aug_num_times, ' ', model._val_score)\n            del model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce0cc45b80d0ee63fd9d5c22e37f306cdf5924bc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b3d3852ff5b9758496bd35c2beef888d8c9c02e"},"cell_type":"code","source":"# norm_train_df = model.get_X_df(model._data_fname, model._meta_fname)\n\n# y_df = model.get_y_df()\n# y_df = y_df.loc[norm_train_df.index]\n\n# feat1 = norm_train_df.groupby(axis=1, level=1).mean()\n# feat2 = norm_train_df.abs().groupby(axis=1, level=1).mean()\n# feat3 = norm_train_df.groupby(axis=1, level=1).std()\n# feat4 = norm_train_df.groupby(axis=1, level=1).max()\n\n# from sklearn.feature_selection import mutual_info_classif\n# dfs= []\n# for index, feat in enumerate([feat1, feat2, feat3, feat4]):\n#     mut_df = pd.Series(mutual_info_classif(feat, y_df['target']), index=feat.columns)\n#     dfs.append(mut_df.to_frame(index))\n# mutual_information_df = pd.concat(dfs, axis=1).max(axis=1).sort_values()\n# columns_to_skip = mutual_information_df.head(40).index.tolist()\n# columns_to_skip.sort()\n\n# features_to_skip = list(set(map(lambda x: '-'.join(x.split('-')[1:]), columns_to_skip)))\n# features_to_skip.sort()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f139b8b56c1d5203ea664360d2ce6e639186249e"},"cell_type":"code","source":"# mutual_information_df = pd.concat(dfs, axis=1).max(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a6c28ec1074a8ac0d1d585c395ca8df7e4b09a59"},"cell_type":"code","source":"# cols = mutual_information_df.index.tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a74a087781c0a6409bdf5f9f7d2b698ff75b043","scrolled":true},"cell_type":"code","source":"# df = model.predict('../input/dataprocessing/test_data.csv', '../input/vsb-power-line-fault-detection/metadata_test.csv')\n# df.index.name = 'signal_id'\n# df = df.to_frame('target').reset_index()\n# df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ba84d8084421d66b7510e17e501cc8bdb52cc02"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}