{"cells":[{"metadata":{"_uuid":"f23906e5de95b946320de79d8520b1769e3a3cff"},"cell_type":"markdown","source":"This kernel just creates and saves features based on this kernel: https://www.kaggle.com/junkoda/handmade-features\nIt's sole purpose is to be used with other kernels for further modelin."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Load training data\nimport numpy as np\nimport pandas as pd\nimport pyarrow.parquet as pq\nfrom tqdm import tqdm_notebook as tqdm\n\nv_raw_train = pq.read_pandas('../input/train.parquet').to_pandas().values\nmeta_train = np.loadtxt('../input/metadata_train.csv', skiprows=1, delimiter=',')\ny_train = meta_train[:, 3].astype(bool)\n\nprint(v_raw_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3921e4dae3d484d59f2a53d330fea35691df84e7"},"cell_type":"code","source":"def compute_spectra(v_raw, *, m = 1000):\n    \"\"\"\n    compute mean and percentile - mean for every chunk of m data\n    \n    Args:\n      v_raw (array): 800,000 x n_sample; the input\n      m (int): the chunk size\n    \n    Returns: d (dict)\n      d['mean']: mean in each chunks\n      d['percentile']: percentile - mean\n    \"\"\"\n    percentile = (100, 99, 95, 0, 1, 5)\n    \n    n = v_raw.shape[1] # number of samples\n    length = v_raw.shape[0] // m # 800,000 -> 800\n    n_spectra = len(percentile)\n    \n    mean_signal = np.zeros((n, length), dtype='float32') # mean in each chunk\n    percentile_spectra = np.zeros((n, length, n_spectra), dtype='float32')\n    \n    # compute spectra\n    print('computing spectra...', flush=True)\n    for i in tqdm(range(n)):\n        v = v_raw[:, i].astype('float32').reshape(-1, m) / 128.0\n        \n        mean = np.mean(v, axis=1)        \n        s = np.abs(np.percentile(v, percentile, axis=1) - mean)\n        \n        # subtract baseline\n        h = np.percentile(s, 5.0)\n        s = np.maximum(0.0, s - h)\n\n        mean_signal[i, :] = mean\n        percentile_spectra[i, :, :] = s.T\n            \n    d = {}\n    d['mean'] = mean_signal\n    d['percentile'] = percentile_spectra\n    \n    return d\n\nspec_train = compute_spectra(v_raw_train)\nprint('done.')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import tensorflow as tf\n\ndef max_windowed(spec, *, width=150, stride=10):\n    \"\"\"\n    Smooth the spectrum with a tophat window function and find the\n    peak inteval that maximises the smoothed spectrum.\n    \n    Returns: d(dict)\n      d['w'] (array): smoothed max - mean spectrum\n      d['ibegin'] (array): the left edge index of the peak interval\n    \"\"\"\n    n = spec.shape[0]\n    length = spec.shape[1] # 800\n    nspec = spec.shape[2] # 6 spectra\n\n    n_triplet = n // 3\n\n    # Reorganize the max spectrum from 8712 data to 2904 triplets with 3 phases\n    max_spec3 = np.empty((n_triplet, length, 3))\n    for i_triplet in range(n_triplet):\n        max_spec3[i_triplet, :, 0] = spec[3*i_triplet, :, 0] # phase 0\n        max_spec3[i_triplet, :, 1] = spec[3*i_triplet + 1, :, 0] # phase 1\n        max_spec3[i_triplet, :, 2] = spec[3*i_triplet + 2, :, 0] # phase 2\n\n    x = tf.placeholder(tf.float32, [None, length, 3]) # input spectra before smoothing\n    # 800 -> 80: static convolaution\n    # convolution but not CNN, the kernel is static\n    # smoothing/convolution kernel\n    # tophat window function\n    # shape (3, 1) adds up 3 phases to one output\n    K = np.ones((width, 3, 1), dtype='float32') / width\n\n    W_conv1 = tf.constant(K)\n    h_conv1 = tf.nn.conv1d(x, W_conv1, stride=stride, padding='VALID')\n    \n    with tf.Session() as sess:\n        w = sess.run(h_conv1, feed_dict={x:max_spec3})\n\n    imax = np.argmax(w[:, :, 0], axis=1) # index of maximum smoothed spectrum\n    \n    d = {}\n    d['w'] = w # smoothed max spectrum\n    d['ibegin'] = imax*stride\n    \n    return d\n\npeaks = max_windowed(spec_train['percentile'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"677c71340b5225dafe2c4f6fba58195c55c8d31d"},"cell_type":"code","source":"def compute_features(v_raw, spec=None):\n    \"\"\"\n    Args:\n      v_raw (array): The original 800,000 x 8712 training data\n      spec (dict): The result of compute_spectra() if already computed.\n                   If it is None, it will be computed automatically.\n    \n    Returns:\n       X (array): Feature vector of shape (2904, 57)\n                  2904 triplets, 57 features\n    \"\"\"\n    if spec is None:\n        spec = compute_spectra(v_raw)\n    \n    v_spec = spec['percentile']\n    shape = v_spec.shape\n    n = shape[0] # number of data\n    length = shape[1]\n    nspec = shape[2]\n    \n    n_triplet = n // 3\n    \n    # Reorder to i_triplet, phase\n    spec3 = np.empty((n_triplet, length, nspec, 3))\n    \n    for i_triplet in range(n_triplet):\n        spec3[i_triplet, :, :, 0] = v_spec[3*i_triplet, :, :] # phase 0\n        spec3[i_triplet, :, :, 1] = v_spec[3*i_triplet + 1, :, :] # phase 1\n        spec3[i_triplet, :, :, 2] = v_spec[3*i_triplet + 2, :, :] # phase 2\n\n    # extract \"max-windowed\" from the spectra\n    width = 150\n    peaks = max_windowed(v_spec, width=width)\n    \n    # Feature vector\n    n_feature4 = 3\n    X = np.empty((n_triplet, n_feature4*nspec*3 + 3))\n    \n    # features for each percentile and phase\n    X4 = np.empty((n_triplet, n_feature4, nspec, 3)) # triplet, figure, spec type, phase\n        \n    for i_triplet in range(n_triplet):       \n        # Maximum of the spectra in the full range\n        # 18 features (6 percentiles x 3 phases)\n        X4[i_triplet, 0, :, :] = np.max(spec3[i_triplet, :, :, :], axis=0)\n        \n        # Peak interval\n        ibegin = peaks['ibegin'][i_triplet]\n        iend = ibegin + width\n        imid = ibegin + width // 2\n    \n        # Mean of the spectra in the peak inteval 18 features\n        X4[i_triplet, 1, :, :] = np.mean(spec3[i_triplet, ibegin:iend, :, :], axis=0)\n        \n        # Max of the spectra in the peak inteval (18 features)\n        X4[i_triplet, 2, :, :] = np.max(spec3[i_triplet, ibegin:iend, :, :], axis=0)\n        \n        # Mean signal at the midpoint of the interval (3 features)\n        X[i_triplet, 0] = spec['mean'][3*i_triplet,     imid]\n        X[i_triplet, 1] = spec['mean'][3*i_triplet + 1, imid]\n        X[i_triplet, 2] = spec['mean'][3*i_triplet + 2, imid]\n    \n    shape = X4.shape\n    \n    # Flatten the X4 tensor\n    # 3 + 18x3 = 57 features\n    X[:, 3:] = X4.reshape(shape[0], shape[1]*shape[2]*shape[3])\n    \n    return X\n\nX_all3 = compute_features(v_raw_train, spec_train)\n\n# The label for the triple\n# True iff two or more labels in 3 phases are True\ny_all3 = np.sum(y_train.reshape(-1, 3), axis=1) >= 2\n\nprint('Three phases are combined into one training data; the shapes are, therefore,')\nprint(X_all3.shape, y_all3.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc1347a4e94ae49bc03cab0619d413be9c633061"},"cell_type":"code","source":"train = pd.DataFrame(data=X_all3, columns=['col_'+str(i) for i in range(57)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e306c16eddba1a5a8fa36d6c0b9b8e2707e3021"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f43f002c3913e0ebd509cc2716fbb0eb801a860d"},"cell_type":"code","source":"train['target'] = y_all3*1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2456edede924206138d3c2d5e110efb9a76d9da"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"531be28fe3d2c33f7c8d0f9b37875949c8e55d52"},"cell_type":"code","source":"# Release RAM of the training data \nif 'v_raw_train' in globals():\n    del v_raw_train\n\n# Load test data\nid_test = np.loadtxt('../input/metadata_test.csv', skiprows=1, delimiter=',')[:, 0].astype(int)\nn_test = len(id_test)\n\nX_tests = []\n\n# Load test data and compute the feature vector\n# The test data is split into 4 to fit it into RAM\nn_subset = 4\nnread = 0\n\nfor i_subset in range(n_subset):\n    # signal_id range in the test data; 8712 is the first data in the test.parquet\n    ibegin = 8712 + 3*int(n_test // 3 * (i_subset/n_subset))\n    iend = 8712 + 3*int(n_test // 3 * ((i_subset + 1)/n_subset))\n    \n    print('Loading %d/%d; signal_id %d - %d...' % (i_subset, n_subset, ibegin, iend))\n    v_raw_test = pq.read_pandas('../input/test.parquet',\n                                columns=[str(i) for i in range(ibegin, iend)]).to_pandas().values\n    \n    nread += v_raw_test.shape[1]\n    X = compute_features(v_raw_test)\n    X_tests.append(X)\n    print('%d/%d test data processed.' % (nread, n_test))\n\n    del v_raw_test\n\nX_test = np.concatenate(X_tests, axis=0)\nassert(X_test.shape[0] == id_test.shape[0] // 3)\n\ndel X_tests\n\nprint('X_test computation done. shape', X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79ad00dbea1d89c2ddeeb84697aee7816a241299"},"cell_type":"code","source":"test = pd.DataFrame(data=X_test, columns=['col_'+str(i) for i in range(57)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4fb3892191a8d354e307e3abbb06e89fce6437e9"},"cell_type":"code","source":"train.to_csv('train.csv', index=False)\ntest.to_csv('test.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c15b78dd9235fb63a04b3fa05d137c4d2f3f9543"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}