{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport pyarrow.parquet as pq\n%matplotlib inline\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6aeeeb33fd95805d9bf387b2a6124a077438228"},"cell_type":"code","source":"meta_train_df = pd.read_csv('../input/metadata_train.csv')\nmeta_test_df = pd.read_csv('../input/metadata_test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"faee4faa285152f4b0073f97f9d771659a23f74f"},"cell_type":"code","source":"signal_ids_for_one = meta_train_df[meta_train_df.target ==1].sample(300).signal_id.tolist()\nif '1577' not in signal_ids_for_one:\n    signal_ids_for_one.append('1577')\nsignal_ids_for_zero = meta_train_df[meta_train_df.target ==0].sample(300).signal_id.tolist()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"cols_one = list(map(str, signal_ids_for_one))\ndf_one = pq.read_pandas('../input/train.parquet', columns=cols_one).to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc04d1c264bc93791f7c6f65af696e648156b5dd"},"cell_type":"code","source":"cols_zero = list(map(str, signal_ids_for_zero))\ndf_zero = pq.read_pandas('../input/train.parquet', columns=cols_zero).to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e1dc2eff75345ab1d09eeaf8e318da470a384ae"},"cell_type":"code","source":"df_one['1577'].plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2fc8c69bb7e39350bf8304d8e0da8406f2211776"},"cell_type":"code","source":"peak_threshold = 40\ncorona_max_distance=10\ncorona_max_height_ratio=0.1\ncorona_cleanup_distance=20\nclean_df = DataProcessor.remove_corona_discharge(df_one['1577'], \n                                      peak_prominence=peak_threshold,\n                                     corona_max_distance=corona_max_distance,\n                                     corona_max_height_ratio=corona_max_height_ratio,\n                                     corona_cleanup_distance=corona_cleanup_distance)\nclean_df.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9a43e4beed2135ca71fabf389169bf2cefcd4b2"},"cell_type":"code","source":"(df_one['1577'] - clean_df).plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ef5aa366ad9b8ea6f168d3d53b15ccedafd4387"},"cell_type":"code","source":"df_one['1577'].iloc[:216400].plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8faf5aa40865a243af9c7d42226dee8aa7f48dcd"},"cell_type":"code","source":"corona_max_distance","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6bbcf74807397040f2239b1da3abdc28a7ffc3ac"},"cell_type":"code","source":"clean_df = DataProcessor.remove_corona_discharge(df_one['1577'].iloc[:216400], \n                                      peak_prominence=peak_threshold,\n                                     corona_max_distance=corona_max_distance,\n                                     corona_max_height_ratio=corona_max_height_ratio,\n                                     corona_cleanup_distance=corona_cleanup_distance)\nclean_df.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"108e253ebdb0e47b27b7cd782b97b7f7bf672a87"},"cell_type":"code","source":"#(118142, 118143), (124161, 124163), (127994, 127996)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1cfdedd32437198e67fa7abd75f9b6742c319b6"},"cell_type":"code","source":"df_one['1577'].iloc[124150:124250].plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3903863e2cee006ec1cb809e13a3f653f579cfc7"},"cell_type":"code","source":"peaks, peak_data =find_peaks(df_one['1577'].iloc[124150:124250],prominence=20)\npeak_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ce3c311efeeb5e5f5b6320b50fbe144c4f4fb19"},"cell_type":"code","source":"peaks, peak_data =find_peaks(-1*df_one['1577'].iloc[124150:124250],prominence=20)\npeak_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac1179d42c23f39da57eae789ee6781b09c47630"},"cell_type":"code","source":"peaks","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60b4532a976ec2cbae871a007af469b8accb8966"},"cell_type":"code","source":"clean_df.iloc[124150:124250].plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0873621a0a811f4c4ae68fe37644bc009f711d7c"},"cell_type":"code","source":"removed_data = (df_one['1577'].iloc[:216400] - clean_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b07ba58388f472300f6130ebde444f24f85701e"},"cell_type":"code","source":"(df_one['1577'].iloc[:216300] - clean_df).iloc[124150:124250].plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a91bcc88559a04dcbe7a4059448a12c96109d0d4"},"cell_type":"code","source":"clean_df.iloc[216200:216250].plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e305f938b766dc77d99ece255c60ede9a2734c9"},"cell_type":"code","source":"from scipy.signal import find_peaks, peak_prominences\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"402e7e073452bfb3caaa382500f273873f99133a"},"cell_type":"code","source":"peaks, dict_ = find_peaks(clean_df.iloc[216200:216250], prominence=peak_threshold)\nprint(peak_prominences(clean_df.iloc[216200:216250],peaks)[0])\n\npeaks, _ = find_peaks(-1*clean_df.iloc[216200:216250], prominence=peak_threshold)\nprint(peak_prominences(-1* clean_df.iloc[216200:216250],peaks)[0])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"181b09e9d6ce492f1facab689380ea77cf56a64f"},"cell_type":"code","source":"df_one.astype(float).diff().abs().quantile([0.1, 0.5, 0.9, 0.95,0.98, 0.99, 1]).max(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2282b63850ba567bf06a90b39856b2fd87876eef"},"cell_type":"code","source":"df_zero.astype(float).diff().abs().quantile([0.1, 0.5, 0.9, 0.95, 0.98, 0.99, 1]).max(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e2f3ae3536786fa8742947d050bebd0a9ea249c"},"cell_type":"code","source":"ax = df['6'].plot()\ndf['6'].rolling(4).mean().plot(ax=ax)\n\ndf['7'].rolling(4).mean().plot(ax=ax)\ndf['7'].plot(ax=ax)\n\ndf['8'].rolling(4).mean().plot(ax=ax)\ndf['8'].plot(ax=ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"41c1e6680a479da583f76dfbee688b5bafafdad7"},"cell_type":"code","source":"ax = df['0'].plot()\ndf['0'].rolling(3).mean().plot(ax=ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9bb645014aee1d8bb5eea9ee1228e778e686842e"},"cell_type":"code","source":"from typing import Dict, List, Tuple\nfrom multiprocessing import Pool\nfrom scipy.signal import find_peaks\nimport numpy as np\nimport pandas as pd\nimport pyarrow.parquet as pq\n\nimport numpy as np\n\n\nclass DataPipeline:\n    \"\"\"\n    Top level class which takes in the parquet file and converts it to a low dimensional dataframe.\n    \"\"\"\n\n    def __init__(\n            self,\n            parquet_fname: str,\n            data_processor_args,\n            start_row_num: int,\n            num_rows: int,\n            concurrency: int = 100,\n    ):\n        self._fname = parquet_fname\n        self._concurrency = concurrency\n        self._nrows = num_rows\n        self._start_row_num = start_row_num\n        self._processor_args = data_processor_args\n        self._process_count = 4\n\n    @staticmethod\n    def run_one_chunk(arg_tuple):\n        fname, processor_args, start_row, end_row, num_rows = arg_tuple\n        cols = [str(i) for i in range(start_row, end_row)]\n        data = pq.read_pandas(fname, columns=cols)\n        data_df = data.to_pandas()\n        processor = DataProcessor(**processor_args)\n        output_df = processor.transform(data_df)\n        print('Another ', round((end_row - start_row) / num_rows * 100, 2), '% Complete')\n        del processor\n        del data_df\n        del data\n        return output_df\n\n    def run(self):\n        outputs = []\n\n        args_list = []\n        for s_index in range(self._start_row_num, self._start_row_num + self._nrows, self._concurrency):\n            e_index = s_index + self._concurrency\n            args_this_chunk = [self._fname, self._processor_args, s_index, e_index, self._nrows]\n            args_list.append(args_this_chunk)\n\n        pool = Pool(self._process_count)\n        outputs = pool.map(DataPipeline.run_one_chunk, args_list)\n        pool.close()\n        pool.join()\n\n        final_output_df = pd.concat(outputs, axis=1)\n        return final_output_df\n\n\nclass DataProcessor:\n    def __init__(\n            self,\n            intended_time_steps: int,\n            original_time_steps: int,\n            peak_prominence: int,\n            smoothing_window: int = 3,\n            remove_corona=False,\n            corona_max_distance=5,\n            corona_max_height_ratio=0.3,\n            corona_cleanup_distance=100,\n            num_processes: int = 7,\n    ):\n        self._o_steps = original_time_steps\n        self._steps = intended_time_steps\n        # 50 is a confident smoothed version\n        # resulting signal after subtracting the smoothened version has some noise along with signal.\n        # with 10, smoothened version seems to have some signals as well.\n        self._smoothing_window = smoothing_window\n        self._num_processes = num_processes\n        self._peak_prominence = peak_prominence\n        self._remove_corona = remove_corona\n        self._corona_max_distance = corona_max_distance\n        self._corona_max_height_ratio = corona_max_height_ratio\n        self._corona_cleanup_distance = corona_cleanup_distance\n\n    def get_noise(self, X_df: pd.DataFrame):\n        \"\"\"\n        TODO: we need to keep the noise. However, we don't want very high freq jitter.\n        band pass filter is what is needed.\n        \"\"\"\n        msg = 'Expected len:{}, found:{}'.format(self._o_steps, len(X_df))\n        assert self._o_steps == X_df.shape[0], msg\n        smoothe_df = X_df.rolling(self._smoothing_window, min_periods=1).mean()\n        noise_df = X_df - smoothe_df\n        del smoothe_df\n        return noise_df\n\n    @staticmethod\n    def peak_data(\n            ser: pd.Series,\n            prominence: float,\n            quantiles=[0, 0.1, 0.5, 0.95, 0.99, 1],\n    ) -> Dict[str, np.array]:\n        maxima_peak_indices, maxima_data_dict = find_peaks(ser, prominence=prominence, width=0)\n        maxima_width = maxima_data_dict['widths']\n        maxima_height = maxima_data_dict['prominences']\n\n        minima_peak_indices, minima_data_dict = find_peaks(-1 * ser, prominence=prominence, width=0)\n        minima_width = minima_data_dict['widths']\n        minima_height = minima_data_dict['prominences']\n\n        peak_indices = np.concatenate([maxima_peak_indices, minima_peak_indices])\n        peak_width = np.concatenate([maxima_width, minima_width])\n        peak_height = np.concatenate([maxima_height, minima_height])\n        maxima_minima = np.concatenate([np.array([1] * len(maxima_height)), np.array([-1] * len(minima_height))])\n\n        index_ordering = np.argsort(peak_indices)\n        peak_width = peak_width[index_ordering]\n        peak_height = peak_height[index_ordering]\n        peak_indices = peak_indices[index_ordering]\n        maxima_minima = maxima_minima[index_ordering]\n        return {\n            'width': peak_width,\n            'height': peak_height,\n            'maxima_minima': maxima_minima,\n            'indices': peak_indices,\n        }\n\n    @staticmethod\n    def corona_discharge_index_pairs(\n            ser: pd.Series,\n            peak_prominence: float,\n            corona_max_distance: int,\n            corona_max_height_ratio: float,\n    ) -> List[Tuple[int, int]]:\n        \"\"\"\n        Args:\n            ser: time series data.\n            peak_prominence: for detecting peaks, if elevation is more than this value, then consider it a peak.\n            corona_max_distance: maximum distance between consequitive alternative peaks for it to be a corona discharge.\n            corona_max_height_ratio: the alternate peaks should have similar peak heights.\n        Returns:\n            List of (peak1, peak2) indices. Note that these peaks are consequitive and have opposite sign\n        \"\"\"\n        data = DataProcessor.peak_data(ser, peak_prominence)\n        corona_indices = []\n        for index, data_index in enumerate(data['indices']):\n            if index < len(data['indices']) - 1:\n                opposite_peaks = data['maxima_minima'][index] * data['maxima_minima'][index + 1] == -1\n                nearby_peaks = data['indices'][index + 1] - data['indices'][index] < corona_max_distance\n\n                # for height ratio, divide smaller by larger height\n                h1 = data['height'][index]\n                h2 = data['height'][index + 1]\n                height_ratio = (h1 / h2 if h1 < h2 else h2 / h1)\n                similar_height = height_ratio > corona_max_height_ratio\n                if opposite_peaks and nearby_peaks and similar_height:\n                    corona_indices.append((data_index, data['indices'][index + 1]))\n        return corona_indices\n\n    @staticmethod\n    def remove_corona_discharge(\n            ser: pd.Series,\n            peak_prominence: float,\n            corona_max_distance: int,\n            corona_max_height_ratio: float,\n            corona_cleanup_distance: int,\n    ) -> pd.Series:\n        \"\"\"\n        Args:\n            ser: time series data.\n            peak_prominence: for detecting peaks, if elevation is more than this value, then consider it a peak.\n            corona_max_distance: maximum distance between consequitive alternative peaks for it to be a corona discharge.\n            corona_max_height_ratio: the alternate peaks should have similar peak heights.\n            corona_cleanup_distance: how many indices after the corona discharge should the data be removed.\n        Returns:\n            ser: cleaned up time series data.\n        \"\"\"\n        pairs = DataProcessor.corona_discharge_index_pairs(\n            ser,\n            peak_prominence,\n            corona_max_distance,\n            corona_max_height_ratio,\n        )\n        print('[Corona discharge peaks removal]', len(pairs), 'many peaks removed')\n        ser = ser.copy()\n        for start_index, end_index in pairs:\n            smoothing_start_index = max(0, start_index - 1)\n            smoothing_end_index = min(end_index + corona_cleanup_distance, ser.index[-1])\n            start_val = ser.iloc[smoothing_start_index]\n            end_val = ser.iloc[smoothing_end_index]\n            count = smoothing_end_index - smoothing_start_index\n\n            ser.iloc[smoothing_start_index:smoothing_end_index] = np.linspace(start_val, end_val, count)\n\n        return ser\n\n    @staticmethod\n    def peak_stats(ser: pd.Series, prominence, quantiles=[0, 0.1, 0.5, 0.95, 0.99, 1]):\n        \"\"\"\n        Returns quantiles of peak width, height, distances from next peak.\n        \"\"\"\n        data = DataProcessor.peak_data(ser, prominence, quantiles=quantiles)\n        peak_indices = data['indices']\n        peak_width = data['width']\n        peak_height = data['height']\n\n        if len(peak_indices) == 0:\n            # no peaks\n            width_stats = [0] * len(quantiles)\n            height_stats = [0] * len(quantiles)\n            distance_stats = [0] * len(quantiles)\n        else:\n            peak_distances = np.diff(peak_indices)\n\n            peak_width[peak_width > 100] = 100\n\n            width_stats = np.quantile(peak_width, quantiles)\n            height_stats = np.quantile(peak_height, quantiles)\n\n            # for just one peak, distance will be empty array.\n            if len(peak_distances) == 0:\n                assert len(peak_indices) == 1\n                distance_stats = [ser.shape[0]] * len(quantiles)\n            else:\n                distance_stats = np.quantile(peak_distances, quantiles)\n\n        width_names = ['peak_width_' + str(i) for i in quantiles]\n        height_names = ['peak_height_' + str(i) for i in quantiles]\n        distance_names = ['peak_distances_' + str(i) for i in quantiles]\n\n        index = width_names + height_names + distance_names + ['peak_count']\n        data = np.concatenate([width_stats, height_stats, distance_stats, [len(peak_indices)]])\n\n        return pd.Series(data, index=index)\n\n    @staticmethod\n    def get_peak_stats_df(df, peak_prominence):\n        \"\"\"\n        Args:\n            df:\n                columns are different examples.\n                axis is time series.\n        \"\"\"\n        return df.apply(lambda x: DataProcessor.peak_stats(x, peak_prominence), axis=0)\n\n    @staticmethod\n    def pandas_describe(df, quantiles=[0, 0.1, 0.5, 0.95, 0.99, 1]):\n        output_df = df.quantile(quantiles, axis=0)\n        output_df.index = list(map(lambda x: 'Quant-' + str(x), output_df.index.tolist()))\n        abs_mean_df = df.abs().mean().to_frame('abs_mean')\n        mean_df = df.mean().to_frame('mean')\n        std_df = df.std().to_frame('std')\n        return pd.concat([output_df, abs_mean_df.T, mean_df.T, std_df.T])\n\n    @staticmethod\n    def transform_chunk(signal_time_series_df: pd.DataFrame, peak_prominence: float) -> pd.DataFrame:\n        \"\"\"\n        It sqashes the time series to a single point multi featured vector.\n        \"\"\"\n        df = signal_time_series_df\n        # mean, var, percentile.\n        # NOTE pandas.describe() is the costliest computation with 95% time of the function.\n        metrics_df = DataProcessor.pandas_describe(df)\n        peak_metrics_df = DataProcessor.get_peak_stats_df(df, peak_prominence)\n\n        metrics_df.index = list(map(lambda x: 'signal_' + x, metrics_df.index))\n        temp_metrics = [metrics_df, peak_metrics_df]\n\n        for smoothener in [1, 2, 4, 8, 16, 32]:\n            diff_df = df.rolling(smoothener).mean()[::smoothener].diff().abs()\n            temp_df = DataProcessor.pandas_describe(diff_df)\n\n            temp_df.index = list(map(lambda x: 'diff_smoothend_by_' + str(smoothener) + ' ' + x, temp_df.index))\n            temp_metrics.append(temp_df)\n\n        df = pd.concat(temp_metrics)\n        df.index.name = 'features'\n        return df\n\n    def transform(self, X_df: pd.DataFrame):\n        \"\"\"\n        Args:\n            X_df: dataframe with each column being one data point. Rows are timestamps.\n        \"\"\"\n\n        def cleanup_corona(x: pd.Series):\n            return DataProcessor.remove_corona_discharge(\n                x,\n                self._peak_prominence,\n                self._corona_max_distance,\n                self._corona_max_height_ratio,\n                self._corona_cleanup_distance,\n            )\n\n        # Corona removed.\n        if self._remove_corona:\n            X_df = X_df.apply(cleanup_corona)\n\n        # Remove the smoothened version of the data so as to work with noise.\n        if self._smoothing_window > 0:\n            X_df = self.get_noise(X_df)\n\n        # stepsize many consequitive timestamps are compressed to form one timestamp.\n        # this will ensure we are left with self._steps many timestamps.\n        stepsize = self._o_steps // self._steps\n        transformed_data = []\n        for s_tm_index in range(0, self._o_steps, stepsize):\n            e_tm_index = s_tm_index + stepsize\n            # NOTE: dask was leading to memory leak.\n            #one_data_point = delayed(DataProcessor.transform_chunk)(X_df.iloc[s_tm_index:e_tm_index, :])\n            one_data_point = DataProcessor.transform_chunk(X_df.iloc[s_tm_index:e_tm_index, :], self._peak_prominence)\n            transformed_data.append(one_data_point)\n\n        # transformed_data = dd.compute(*transformed_data, scheduler='processes', num_workers=self._num_processes)\n        # Add timestamp\n        for ts in range(0, len(transformed_data)):\n            transformed_data[ts]['ts'] = ts\n\n        df = pd.concat(transformed_data, axis=0).set_index(['ts'], append=True)\n        df.columns.name = 'Examples'\n        return df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"17fef3d7ddf60b6ff432783c8f39d5b98ff8a745","scrolled":false},"cell_type":"code","source":"\ncorona_cleanup_distance = 10\nintended_time_steps = 150\npeak_prominence_arr = [50, 100]\nfor peak_prominence in peak_prominence_arr:\n    for max_height_ratio in [0.25, 0.75]:\n    \n        dp_dict = {'intended_time_steps':intended_time_steps,'original_time_steps':800000,\n                   'smoothing_window':3,\n                  'peak_prominence':peak_prominence,'remove_corona':True,\n                   'corona_max_height_ratio':max_height_ratio,\n                  'corona_cleanup_distance':corona_cleanup_distance}\n        start_row_num = meta_train_df.signal_id[0]\n        num_rows = meta_train_df.shape[0]\n        pipeline = DataPipeline('../input/train.parquet', dp_dict,start_row_num, num_rows)\n        df = pipeline.run()\n        fname = 'train_data_{}_{}_{}_{}.csv'.format(max_height_ratio, corona_cleanup_distance, intended_time_steps,\n                                                peak_prominence)\n        df.to_csv(fname, compression='gzip')\n        del df\n        del pipeline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"630a8f0676309b9b502d480ec3ea2de786ae0266"},"cell_type":"code","source":"# dp_dict = {'intended_time_steps':150,'original_time_steps':800000,\n#            'smoothing_window':3,\n#           'peak_threshold':15,'remove_corona':True,\n#            'corona_max_height_ratio':0.95,\n#           'corona_cleanup_distance':50}\n# start_row_num = meta_train_df.signal_id[0]\n# num_rows = meta_train_df.shape[0]\n# pipeline = DataPipeline('../input/train.parquet', dp_dict,start_row_num, num_rows)\n# df = pipeline.run()\n# fname = 'train_data.csv'\n# df.to_csv(fname, compression='gzip')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea1c9431ac0b72214c1e3257e15f04f016c6ac6d"},"cell_type":"code","source":"ax = df['3'].plot(figsize=(20,10))\ndf['4'].plot(ax=ax)\ndf['5'].plot(ax=ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ec245d9aac1e7631c408b0c468cb225853438b1"},"cell_type":"code","source":"ax = df['0'].plot(figsize=(20,10))\ndf['1'].plot(ax=ax)\ndf['2'].plot(ax=ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a5a4041bc5b641407e390c5184bdb0ca94e87c5b"},"cell_type":"code","source":"\n# start_row_num = meta_test_df.signal_id[0]\n# num_rows = meta_test_df.shape[0]\n# pipeline = DataPipeline('../input/test.parquet', dp_dict,start_row_num, num_rows)\n# df = pipeline.run()\n# df.to_csv('test_data.csv', compression='gzip')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d989a56eefd28b7fb520368bd1612837d42df2b2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}