{"cells":[{"metadata":{"_uuid":"6258f7cbae746eb8c2a24813980df28cd86b676a"},"cell_type":"markdown","source":"# Outputs features of signals divided in parts (e.g. 200 parts of 4000 steps) for both train and test set in +- five minutes."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom scipy import sparse\nfrom tqdm import tqdm_notebook as tqdm\nfrom numba import jit\nfrom matplotlib import pyplot as plt\nfrom scipy import signal as scipy_signal\nimport gc\n%matplotlib inline\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"edae63bac4a7191068049bd1fbf0e924b83a9812"},"cell_type":"code","source":"N_TRAIN = 8712  # nr of samples in training set\nN_TEST = 20337\n# N_TRAIN = 210  # to try out, also comment out save_data(1) below\nP = 3  # nr of phases in signal\nADDRESS_TRAIN = '../input/train.parquet'\nADDRESS_TEST = '../input/test.parquet'\nSPLITS_TRAIN = 3  # signal splits have to be divisable by 3 for align()\nSPLITS_TEST = 3\nLENGTH = 800000\nN_PARTS = 200","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"407df76b3aef423c20a99d992009df719caeac5a"},"cell_type":"code","source":"def align(signals):\n    window = np.ones(32) / 32\n    for i in range(0, signals.shape[0], 3):\n        signal = np.convolve(signals[i], window)[16:-15]  # this smoothing operation is slow with numba\n        argmax = np.argmax(signal)\n        signals[i:i+3] = np.concatenate((signals[i:i+3, argmax:], signals[i:i+3, :argmax]), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e874546512497c641f886e29ed4294b47a31da1d"},"cell_type":"code","source":"@jit(nopython=True)\ndef get_feats(signal, n, splits):\n    part_n = N_PARTS\n    part_length = LENGTH // part_n\n    new_sig = np.empty((n//splits, part_n, 3))  \n    for i in range(part_n):\n        segment = np.ascontiguousarray(  # numba asks for this\n            signal[:, i*part_length:(i+1)*part_length]).astype(np.int16)\n        mean_max_ent = np.empty((n//splits, 3))  # three features\n        for j in range(n//splits):\n            mean_max_ent[j, 0] = segment[j].mean()\n            segment_diff = np.absolute(np.diff(segment[j]))\n            mean_max_ent[j, 1] = np.log1p(segment_diff.max())  # log for outliers\n            histogram = np.histogram(\n                segment[j], bins=15, range=(-128., 127.))[0].astype(np.float64)\n            histogram = histogram / histogram.sum()\n            entropy = 0\n            for k in range(15):\n                if histogram[k] > 0:\n                    entropy = entropy + histogram[k] * np.log(histogram[k])\n            mean_max_ent[j, 2] = np.log1p(-entropy)  # log1p for outliers\n        max_mean = np.max(mean_max_ent[:, 0])\n        min_mean = np.absolute(np.min(mean_max_ent[:, 0]))\n        mean_max_ent[:, 0] = mean_max_ent[:, 0] / (2*np.max(np.array([max_mean, min_mean])))\n        new_sig[:, i] = mean_max_ent\n    return new_sig","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8413c988efc10b3eb09badd616cd1bbf0ba44ccb"},"cell_type":"code","source":"def save_data(train_test):\n    if not train_test:\n        address = ADDRESS_TRAIN\n        n = N_TRAIN\n        splits = SPLITS_TRAIN\n        test_adjust = 0\n    else:\n        address = ADDRESS_TEST\n        n = N_TEST\n        splits = SPLITS_TEST\n        test_adjust = N_TRAIN\n        \n    def get_signals():\n        new_sigs = np.empty((n, N_PARTS, 3))\n        for i in tqdm(range(splits)):\n            start_index = i*(n//splits)\n            end_index = (i+1)*(n//splits)\n            cols = [str(k+test_adjust) for k in range(start_index, end_index)]\n            signals_split = pd.read_parquet(address, columns=cols).values.T    \n#             align(signals_split)  # this starts signals in groups of three phases at the same point\n            new_sigs[start_index:end_index] = get_feats(signals_split, n, splits)\n            del signals_split\n            gc.collect()\n        for i in range(1, 3):\n            if np.max(new_sigs[:, :, i]) != 0:\n                new_sigs[:, :, i] = new_sigs[:, :, i] / np.max(new_sigs[:, :, i])\n                plt.hist(new_sigs[:, :, i].flatten())\n                plt.show()\n        plt.hist(new_sigs[:, :, 0].flatten())\n        plt.show()\n        return new_sigs\n\n    comb_feats = get_signals()\n    np.save('aligned_comb_feats_{}.npy'.format(str(train_test)), comb_feats)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a42508975e45269a66e30c4933bafb7b13e642fd"},"cell_type":"code","source":"%%time\nsave_data(1)  # test set\nsave_data(0)  # train set","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"405d8967195e128dffe1a73496c7c26d89d31743"},"cell_type":"code","source":"signals = np.load('aligned_comb_feats_0.npy')\nprint(signals.shape)\nsignals = signals.reshape((N_PARTS*N_TRAIN//P, 9))\nprint(signals.shape)\nsignals = pd.DataFrame(signals)\nsignals.describe()  # three features for three phases is nine features per signal","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f3b6b209bada2db9f6e89abc41081d1bbba285a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}