{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-07T11:35:39.609638Z","iopub.execute_input":"2022-01-07T11:35:39.610152Z","iopub.status.idle":"2022-01-07T11:35:39.639641Z","shell.execute_reply.started":"2022-01-07T11:35:39.610056Z","shell.execute_reply":"2022-01-07T11:35:39.638765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mục đích notebook\n\nThực hiện biến đổi các đặc trưng và lưu lại để tiện sử dụng sau này.","metadata":{}},{"cell_type":"code","source":"%%time\n# Tổng số dòng dữ liệu\ntotal = 629145481 \n# Số dữ liệu được tính mỗi bước.\nchunksize = 150_000\n\n# Tải dữ liệu sử dụng chunk, do nếu tải hết dữ liệu vào trong lúc Editing Noteboong thì sẽ báo lỗi.\nchunks = pd.read_csv('../input/LANL-Earthquake-Prediction/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64}, chunksize = chunksize)\n\nc = 1\n\nsegments = total // chunksize\n# for debug\n# segments = 10\n    \n# Segment là số lượng đoạn được lấy từ input.\n# Các chunk được load sau đó tiền xử lý luôn luôn.","metadata":{"execution":{"iopub.status.busy":"2022-01-07T11:35:39.641755Z","iopub.execute_input":"2022-01-07T11:35:39.642879Z","iopub.status.idle":"2022-01-07T11:35:39.66596Z","shell.execute_reply.started":"2022-01-07T11:35:39.642822Z","shell.execute_reply":"2022-01-07T11:35:39.664841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Hiển thị đến độ chính xác 15 chữ số sau dấu phẩy.\npd.options.display.precision = 15","metadata":{"execution":{"iopub.status.busy":"2022-01-07T11:35:39.667198Z","iopub.execute_input":"2022-01-07T11:35:39.667791Z","iopub.status.idle":"2022-01-07T11:35:39.673711Z","shell.execute_reply.started":"2022-01-07T11:35:39.667744Z","shell.execute_reply":"2022-01-07T11:35:39.671858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Chuyển đổi, làm sạch dữ liệu","metadata":{}},{"cell_type":"markdown","source":"Nạp các thư viện.","metadata":{}},{"cell_type":"code","source":"%%time\n# Load các thư viện liên quan đến biến đổi tín hiệu và thống kê sẽ được sủ dụng.\nfrom scipy import stats\nfrom scipy.signal import hilbert\nfrom scipy.signal import hann\nfrom scipy.signal import convolve\n\n# Load thư viện về hồi quy tuyến tính đơn giản để tìm xu hướng của dữ liệu.\nfrom sklearn.linear_model import LinearRegression\nfrom tqdm import tqdm_notebook as tqdm\n\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2022-01-07T08:40:07.213897Z","iopub.execute_input":"2022-01-07T08:40:07.214204Z","iopub.status.idle":"2022-01-07T08:40:07.304428Z","shell.execute_reply.started":"2022-01-07T08:40:07.214173Z","shell.execute_reply":"2022-01-07T08:40:07.303485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Định nghĩa làm STA và LTA","metadata":{}},{"cell_type":"code","source":"%%time\nimport warnings\nwarnings.filterwarnings('ignore')\n\ndef classic_sta_lta(x, length_sta, length_lta):\n    \"\"\"\n    Input: x là một mảng số.\n    STA: Short Term Avg.\n    LTA: Long Term Avg.\n    \n    Term Average: được tính bằng trung bình của các phần tử liên tiếp nhau.\n    \"\"\"\n    sta = np.cumsum(x ** 2)\n\n    # Convert to float\n    sta = np.require(sta, dtype=np.float)\n\n    # Copy for LTA\n    lta = sta.copy()\n\n    # Compute the STA and the LTA\n    sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]\n    sta /= length_sta\n    lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]\n    lta /= length_lta\n\n    # Pad zeros\n    sta[:length_lta - 1] = 0\n\n    # Avoid division by zero by setting zero values to tiny float\n    dtiny = np.finfo(0.0).tiny\n    idx = lta < dtiny\n    lta[idx] = dtiny\n    \n    return sta / lta","metadata":{"execution":{"iopub.status.busy":"2022-01-07T08:40:07.305849Z","iopub.execute_input":"2022-01-07T08:40:07.306738Z","iopub.status.idle":"2022-01-07T08:40:07.316455Z","shell.execute_reply.started":"2022-01-07T08:40:07.30669Z","shell.execute_reply":"2022-01-07T08:40:07.315723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Tạo một mảng các đặc trưng. Nội dung của mảng được lấy từ [notebook Độ quan trọng đặc trưng](https://www.kaggle.com/ngodang/int3405e-20-lanl-feature-importance).","metadata":{}},{"cell_type":"code","source":"keep_cols = ['change_mean', 'abs_rolling_std_10_percentage_0.8',\n       'abs_rolling_std_100_percentage_0.99',\n       'abs_rolling_std_50_percentage_0.5',\n       'abs_rolling_std_100_percentage_0.7',\n       'abs_rolling_std_10_percentage_0.99',\n       'abs_rolling_std_100_percentage_0.95',\n       'abs_rolling_std_100_percentage_0.8',\n       'abs_rolling_std_50_percentage_0.3',\n       'abs_rolling_std_50_percentage_0.8',\n       'abs_rolling_std_50_percentage_0.4',\n       'abs_rolling_std_50_percentage_0.6',\n       'abs_rolling_std_50_percentage_0.9',\n       'abs_rolling_std_10_percentage_0.85',\n       'abs_rolling_std_50_percentage_0.99', 'abs_quantile_0.85',\n       'abs_rolling_std_200_percentage_0.95',\n       'abs_rolling_std_5000_percentage_0.01',\n       'abs_rolling_std_5000_percentage_0.1',\n       'abs_rolling_std_5000_percentage_0.85',\n       'abs_rolling_std_200_percentage_0.99',\n       'abs_rolling_std_10000_percentage_0.4',\n       'abs_rolling_std_500_percentage_0.7',\n       'abs_rolling_std_200_percentage_0.5',\n       'abs_rolling_std_200_percentage_0.8',\n       'abs_rolling_std_10000_percentage_0.95',\n       'abs_rolling_std_5000_percentage_0.99', 'hmean',\n       'abs_rolling_std_50_percentage_0.95',\n       'abs_rolling_std_5000_percentage_0.15',\n       'abs_rolling_std_5000_percentage_0.2',\n       'abs_rolling_std_100_percentage_0.9',\n       'abs_rolling_std_10000_percentage_0.15',\n       'abs_rolling_std_10000_percentage_0.7',\n       'abs_rolling_std_500_percentage_0.9',\n       'abs_rolling_std_50_percentage_0.85',\n       'abs_rolling_std_10_percentage_0.6', 'rolling_min_std_10000',\n       'abs_rolling_std_100_percentage_0.85',\n       'abs_rolling_std_5000_percentage_0.7',\n       'abs_rolling_std_100_percentage_0.4',\n       'abs_rolling_std_50_percentage_0.2',\n       'imag_abs_rolling_min_mean_200',\n       'abs_rolling_std_500_percentage_0.99',\n       'abs_rolling_std_5000_percentage_0.9',\n       'abs_rolling_std_10000_percentage_0.1',\n       'abs_rolling_std_100_percentage_0.6',\n       'abs_rolling_std_200_percentage_0.7', 'abs_quantile_0.5',\n       'abs_rolling_std_10_percentage_0.95',\n       'abs_rolling_std_200_percentage_0.4',\n       'abs_rolling_std_100_percentage_0.5',\n       'abs_rolling_std_10_percentage_0.9',\n       'abs_rolling_std_500_percentage_0.05',\n       'abs_rolling_std_5000_percentage_0.05',\n       'abs_rolling_std_5000_percentage_0.3',\n       'abs_rolling_std_10000_percentage_0.99', 'real_abs_min',\n       'abs_rolling_std_10000_percentage_0.8',\n       'imag_abs_rolling_std_50_percentage_0.2',\n       'abs_rolling_std_500_percentage_0.8', 'real_dif_mean',\n       'abs_rolling_std_5000_percentage_0.5',\n       'abs_rolling_std_1000_percentage_0.99',\n       'abs_rolling_std_10000_percentage_0.5',\n       'abs_rolling_std_5000_percentage_0.4',\n       'abs_rolling_std_50_percentage_0.7',\n       'abs_rolling_std_200_percentage_0.2',\n       'abs_rolling_std_200_percentage_0.15', 'iqr',\n       'real_abs_rolling_max_mean_10',\n       'abs_rolling_std_10_percentage_0.4',\n       'abs_rolling_std_500_percentage_0.6', 'imag_abs_percentile_0.15',\n       'abs_rolling_std_500_percentage_0.85',\n       'abs_rolling_std_10_percentage_0.5',\n       'abs_rolling_std_50_percentage_0.05',\n       'abs_rolling_std_200_percentage_0.05',\n       'abs_rolling_std_5000_percentage_0.95',\n       'abs_rolling_std_100_percentage_0.15',\n       'abs_rolling_std_10000_percentage_0.05',\n       'abs_rolling_std_1000_percentage_0.9',\n       'abs_rolling_std_200_percentage_0.3',\n       'abs_rolling_std_500_percentage_0.4', 'imag_hann_window_5000',\n       'real_abs_rolling_std_10_percentage_0.01',\n       'real_abs_rolling_mean_10000_percentage_0.15',\n       'imag_abs_rolling_std_10_percentage_0.01', 'ave_div_0', 'mad',\n       'real_abs_percentile_0.85', 'imag_change_mean',\n       'abs_rolling_std_100_percentage_0.2', 'imag_skew', 'imag_coef_',\n       'ave_div_2', 'gmean', 'real_abs_rolling_max_mean_100',\n       'abs_rolling_mean_5000_percentage_0.05', 'imag_ave_div_2',\n       'real_rolling_max_ave_10', 'std_div_1',\n       'abs_rolling_std_50_percentage_0.15', 'real_abs_percentile_0.1',\n       'real_abs_rolling_std_100_percentage_0.2',\n       'abs_rolling_std_5000_percentage_0.6', 'imag_rolling_min_ave_5000',\n       'min_div_4', 'abs_rolling_std_10000_percentage_0.9',\n       'real_abs_rolling_mean_10_percentage_0.01',\n       'abs_rolling_max_mean_10000', 'min_max_diff_div_3',\n       'real_abs_percentile_0.05', 'max_div_2',\n       'real_rolling_max_std_5000', 'abs_rolling_std_50_percentage_0.01',\n       'imag_abs_percentile_0.1',\n       'real_abs_rolling_std_10_percentage_0.2',\n       'abs_rolling_mean_50_percentage_0.4', 'imag_std_div_3']","metadata":{"execution":{"iopub.status.busy":"2022-01-07T08:40:07.317733Z","iopub.execute_input":"2022-01-07T08:40:07.317984Z","iopub.status.idle":"2022-01-07T08:40:07.333066Z","shell.execute_reply.started":"2022-01-07T08:40:07.317953Z","shell.execute_reply":"2022-01-07T08:40:07.332033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Thực hiện xử lý dữ liệu và xây dựng đặc trưng. Tuy nhiên, đã bỏ đi các đặc trưng và các nhóm chính không có trong tập 120 đặc trưng được chọn.","metadata":{}},{"cell_type":"code","source":"%%time\ntrain = []\ny = pd.DataFrame(index=range(segments), dtype=np.float64, columns=['time_to_failure'])\n\nrows = 150_000\n\nfor segment in tqdm(range(segments)):\n    chunk = next(chunks)\n    ad = chunk['acoustic_data']\n    y.loc[segment, 'time_to_failure'] = chunk['time_to_failure'].values[-1]\n    \n    ft = np.fft.fft(ad)\n    real_ad = pd.Series(np.real(ft))\n    imag_ad = pd.Series(np.imag(ft))\n    \n    features = dict()\n    \n    # Tính các feature với ad\n    #########################################################################################################\n    quantiles = [0.5, 0.85]\n    for quantile in quantiles:\n        features['abs_quantile_' + str(quantile)] = np.quantile(np.abs(ad), quantile)\n        \n    div = 5\n    for i in range(div):\n        features['ave_div_' + str(i)] = ad[rows // div * i: rows // div * (i + 1)].mean()\n        features['std_div_' + str(i)] = ad[rows // div * i: rows // div * (i + 1)].std()\n        features['max_div_' + str(i)] = ad[rows // div * i: rows // div * (i + 1)].max()\n        features['min_div_' + str(i)] = ad[rows // div * i: rows // div * (i + 1)].min()\n        features['min_max_diff_div_' + str(i)] = features['max_div_' + str(i)] - features['min_div_' + str(i)]\n\n\n    windows = [10, 50, 100, 200, 500, 1000, 5000, 10000]\n    percentages = [0.01, 0.05, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 0.99]\n    for window in windows:\n        roll_mean = ad.rolling(window).mean().dropna()\n        roll_std = ad.rolling(window).std().dropna()\n\n        features['rolling_min_std_' + str(window)] = roll_std.min()\n        \n        features['abs_rolling_max_mean_' + str(window)] = np.abs(roll_mean).max()\n        \n        for percentage in percentages:\n            features['abs_rolling_mean_{}_percentage_{}'.format(window, percentage)] = np.percentile(roll_mean, percentage)\n            features['abs_rolling_std_{}_percentage_{}'.format(window, percentage)] = np.percentile(roll_std, percentage)\n            \n    dif = ad.diff().dropna().values\n    change = (dif / ad.values[:-1])\n    change = change[np.nonzero(change)[0]]\n    change = change[~np.isnan(change)]\n    change = change[change != -np.inf]\n    change = change[change != np.inf]\n    \n    features['change_mean'] = change.mean()\n    \n    features['hmean'] = stats.hmean(np.abs(ad[ad != 0]))\n    features['gmean'] = stats.gmean(np.abs(ad[ad != 0]))\n    \n    features['iqr'] = np.subtract(*np.percentile(ad, [75, 25]))\n    \n    features['mad'] = np.mean(np.abs(ad - np.mean(ad)))\n    \n    del change\n    del dif\n    \n    # Tính các feature với real_ad\n    #########################################################################################################\n    del ad\n    ad = real_ad\n    \n    prefix = \"real_\"\n\n    features[prefix + 'abs_min'] = np.abs(ad).min()\n        \n    features[prefix + 'abs_percentile_' + str(0.05)] = np.percentile(np.abs(ad), 0.05)\n    features[prefix + 'abs_percentile_' + str(0.1)] = np.percentile(np.abs(ad), 0.1)\n    features[prefix + 'abs_percentile_' + str(0.85)] = np.percentile(np.abs(ad), 0.85)\n    \n    # Tính các đặc tính theo tổng liên tiếp\n    \n    roll_mean = ad.rolling(10).mean().dropna()\n    features[prefix + 'abs_rolling_max_mean_' + str(10)] = np.abs(roll_mean).max()\n    roll_mean = ad.rolling(100).mean().dropna()\n    features[prefix + 'abs_rolling_max_mean_' + str(100)] = np.abs(roll_mean).max()\n    \n    roll_mean = ad.rolling(10).mean().dropna()\n    features[prefix + 'abs_rolling_mean_{}_percentage_{}'.format(10, 0.01)] = np.percentile(roll_mean, 0.01)\n    roll_mean = ad.rolling(10000).mean().dropna()\n    features[prefix + 'abs_rolling_mean_{}_percentage_{}'.format(10000, 0.15)] = np.percentile(roll_mean, 0.15)\n    \n    roll_std = ad.rolling(10).std().dropna()\n    features[prefix + 'abs_rolling_std_{}_percentage_{}'.format(10, 0.01)] = np.percentile(roll_std, 0.01)\n    features[prefix + 'abs_rolling_std_{}_percentage_{}'.format(10, 0.2)] = np.percentile(roll_std, 0.2)\n    roll_std = ad.rolling(100).std().dropna()\n    features[prefix + 'abs_rolling_std_{}_percentage_{}'.format(100, 0.2)] = np.percentile(roll_std, 0.2)\n    \n    roll_mean = ad.rolling(10).mean().dropna()\n    features[prefix + 'rolling_max_ave_' + str(10)] = roll_mean.max()\n    roll_std = ad.rolling(5000).std().dropna()\n    features[prefix + 'rolling_max_std_' + str(5000)] = roll_std.max()\n            \n    dif = ad.diff().dropna().values\n    features[prefix + 'dif_mean'] = dif.mean()\n    \n    del dif\n    \n    # Tính các feature với imag_ad\n    #########################################################################################################\n    del ad\n    ad = imag_ad\n    \n    prefix = \"imag_\"\n    \n    # find trend using linear regression.\n    model = LinearRegression().fit(np.arange(len(ad)).reshape(-1, 1), ad)\n\n    features[prefix + 'coef_'] = model.coef_[0]\n    \n    # divide into smaller segment.\n    \n    features[prefix + 'ave_div_' + str(2)] = ad[rows // div * 2: rows // div * (2 + 1)].mean()\n    features[prefix + 'std_div_' + str(3)] = ad[rows // div * 3: rows // div * (3 + 1)].std()\n    \n    # some statistic distribution characteristic\n    \n    features[prefix + 'abs_percentile_' + str(0.1)] = np.percentile(np.abs(ad), 0.1)\n    features[prefix + 'abs_percentile_' + str(0.15)] = np.percentile(np.abs(ad), 0.15)\n        \n    features[prefix + 'skew'] = stats.skew(ad)\n    \n    features[prefix + 'hann_window_' + str(5000)] = (convolve(ad, hann(5000), mode='same') / sum(hann(5000))).mean()\n        \n    roll_mean = ad.rolling(5000).mean().dropna()\n    features[prefix + 'rolling_min_ave_' + str(5000)] = roll_mean.min()\n    \n    roll_mean = ad.rolling(200).mean().dropna()\n    features[prefix + 'abs_rolling_min_mean_' + str(200)] = np.abs(roll_mean).min()\n    \n    roll_std = ad.rolling(10).std().dropna()\n    features[prefix + 'abs_rolling_std_{}_percentage_{}'.format(10, 0.01)] = np.percentile(roll_std, 0.01)\n    \n    roll_std = ad.rolling(50).std().dropna()\n    features[prefix + 'abs_rolling_std_{}_percentage_{}'.format(50, 0.2)] = np.percentile(roll_std, 0.2)\n        \n    # change rate \n    dif = ad.diff().dropna().values\n    \n    change = (dif / ad.values[:-1])\n    change = change[np.nonzero(change)[0]]\n    change = change[~np.isnan(change)]\n    change = change[change != -np.inf]\n    change = change[change != np.inf]\n    \n    features[prefix + 'change_mean'] = change.mean()\n\n    del change\n    del dif\n    \n    train.append(features)\n    \n    del ad\n    del chunk\n    del real_ad\n    del imag_ad\n    del ft\n    \nX = pd.DataFrame(train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X[keep_cols]\nX.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-03T18:06:46.367498Z","iopub.execute_input":"2022-01-03T18:06:46.367861Z","iopub.status.idle":"2022-01-03T18:06:46.401303Z","shell.execute_reply.started":"2022-01-03T18:06:46.36783Z","shell.execute_reply":"2022-01-03T18:06:46.400579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Do thời gian chạy code chuyển đổi thông tin rất lâu, lưu lại dữ liệu đã xử lý","metadata":{}},{"cell_type":"code","source":"X.to_csv(\"X_train.csv\", index=False)\ny.to_csv(\"y_train.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-03T18:06:46.402414Z","iopub.execute_input":"2022-01-03T18:06:46.402771Z","iopub.status.idle":"2022-01-03T18:06:46.414977Z","shell.execute_reply.started":"2022-01-03T18:06:46.402741Z","shell.execute_reply":"2022-01-03T18:06:46.414015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt \n\nplt.plot(range(len(y)), y)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-03T18:06:46.416332Z","iopub.execute_input":"2022-01-03T18:06:46.416782Z","iopub.status.idle":"2022-01-03T18:06:46.618787Z","shell.execute_reply.started":"2022-01-03T18:06:46.416745Z","shell.execute_reply":"2022-01-03T18:06:46.617911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Biểu đồ các giá trị của y. Ta thấy rằng do số mẫu rất nhỏ so với tổng số mẫu, nên tính liên tục của y không bị ảnh hưởng quá nhiều.","metadata":{}},{"cell_type":"markdown","source":"## Xây dựng dữ liệu từ bộ test","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv', index_col='seg_id')\n\ntest = []\n\nfor seg_id in tqdm(range(len(submission.index))):\n    seg = pd.read_csv('../input/LANL-Earthquake-Prediction/test/' + submission.index[seg_id] + '.csv')\n\n    ad = seg['acoustic_data']\n\n    ft = np.fft.fft(ad)\n    real_ad = pd.Series(np.real(ft))\n    imag_ad = pd.Series(np.imag(ft))\n    \n    features = dict()\n    \n    # Tính các feature với ad\n    #########################################################################################################\n    quantiles = [0.5, 0.85]\n    for quantile in quantiles:\n        features['abs_quantile_' + str(quantile)] = np.quantile(np.abs(ad), quantile)\n        \n    div = 5\n    for i in range(div):\n        features['ave_div_' + str(i)] = ad[rows // div * i: rows // div * (i + 1)].mean()\n        features['std_div_' + str(i)] = ad[rows // div * i: rows // div * (i + 1)].std()\n        features['max_div_' + str(i)] = ad[rows // div * i: rows // div * (i + 1)].max()\n        features['min_div_' + str(i)] = ad[rows // div * i: rows // div * (i + 1)].min()\n        features['min_max_diff_div_' + str(i)] = features['max_div_' + str(i)] - features['min_div_' + str(i)]\n\n\n    windows = [10, 50, 100, 200, 500, 1000, 5000, 10000]\n    percentages = [0.01, 0.05, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 0.99]\n    for window in windows:\n        roll_mean = ad.rolling(window).mean().dropna()\n        roll_std = ad.rolling(window).std().dropna()\n\n        features['rolling_min_std_' + str(window)] = roll_std.min()\n        \n        features['abs_rolling_max_mean_' + str(window)] = np.abs(roll_mean).max()\n        \n        for percentage in percentages:\n            features['abs_rolling_mean_{}_percentage_{}'.format(window, percentage)] = np.percentile(roll_mean, percentage)\n            features['abs_rolling_std_{}_percentage_{}'.format(window, percentage)] = np.percentile(roll_std, percentage)\n            \n    dif = ad.diff().dropna().values\n    change = (dif / ad.values[:-1])\n    change = change[np.nonzero(change)[0]]\n    change = change[~np.isnan(change)]\n    change = change[change != -np.inf]\n    change = change[change != np.inf]\n    \n    features['change_mean'] = change.mean()\n    \n    features['hmean'] = stats.hmean(np.abs(ad[ad != 0]))\n    features['gmean'] = stats.gmean(np.abs(ad[ad != 0]))\n    \n    features['iqr'] = np.subtract(*np.percentile(ad, [75, 25]))\n    \n    features['mad'] = np.mean(np.abs(ad - np.mean(ad)))\n    \n    del change\n    del dif\n    \n    # Tính các feature với real_ad\n    #########################################################################################################\n    del ad\n    ad = real_ad\n    \n    prefix = \"real_\"\n\n    features[prefix + 'abs_min'] = np.abs(ad).min()\n        \n    features[prefix + 'abs_percentile_' + str(0.05)] = np.percentile(np.abs(ad), 0.05)\n    features[prefix + 'abs_percentile_' + str(0.1)] = np.percentile(np.abs(ad), 0.1)\n    features[prefix + 'abs_percentile_' + str(0.85)] = np.percentile(np.abs(ad), 0.85)\n    \n    # Tính các đặc tính theo tổng liên tiếp\n    \n    roll_mean = ad.rolling(10).mean().dropna()\n    features[prefix + 'abs_rolling_max_mean_' + str(10)] = np.abs(roll_mean).max()\n    roll_mean = ad.rolling(100).mean().dropna()\n    features[prefix + 'abs_rolling_max_mean_' + str(100)] = np.abs(roll_mean).max()\n    \n    roll_mean = ad.rolling(10).mean().dropna()\n    features[prefix + 'abs_rolling_mean_{}_percentage_{}'.format(10, 0.01)] = np.percentile(roll_mean, 0.01)\n    roll_mean = ad.rolling(10000).mean().dropna()\n    features[prefix + 'abs_rolling_mean_{}_percentage_{}'.format(10000, 0.15)] = np.percentile(roll_mean, 0.15)\n    \n    roll_std = ad.rolling(10).std().dropna()\n    features[prefix + 'abs_rolling_std_{}_percentage_{}'.format(10, 0.01)] = np.percentile(roll_std, 0.01)\n    features[prefix + 'abs_rolling_std_{}_percentage_{}'.format(10, 0.2)] = np.percentile(roll_std, 0.2)\n    roll_std = ad.rolling(100).std().dropna()\n    features[prefix + 'abs_rolling_std_{}_percentage_{}'.format(100, 0.2)] = np.percentile(roll_std, 0.2)\n    \n    roll_mean = ad.rolling(10).mean().dropna()\n    features[prefix + 'rolling_max_ave_' + str(10)] = roll_mean.max()\n    roll_std = ad.rolling(5000).std().dropna()\n    features[prefix + 'rolling_max_std_' + str(5000)] = roll_std.max()\n            \n    dif = ad.diff().dropna().values\n    features[prefix + 'dif_mean'] = dif.mean()\n    \n    del dif\n    \n    # Tính các feature với imag_ad\n    #########################################################################################################\n    del ad\n    ad = imag_ad\n    \n    prefix = \"imag_\"\n    \n    # find trend using linear regression.\n    model = LinearRegression().fit(np.arange(len(ad)).reshape(-1, 1), ad)\n\n    features[prefix + 'coef_'] = model.coef_[0]\n    \n    # divide into smaller segment.\n    \n    features[prefix + 'ave_div_' + str(2)] = ad[rows // div * 2: rows // div * (2 + 1)].mean()\n    features[prefix + 'std_div_' + str(3)] = ad[rows // div * 3: rows // div * (3 + 1)].std()\n    \n    # some statistic distribution characteristic\n    \n    features[prefix + 'abs_percentile_' + str(0.1)] = np.percentile(np.abs(ad), 0.1)\n    features[prefix + 'abs_percentile_' + str(0.15)] = np.percentile(np.abs(ad), 0.15)\n        \n    features[prefix + 'skew'] = stats.skew(ad)\n    \n    features[prefix + 'hann_window_' + str(5000)] = (convolve(ad, hann(5000), mode='same') / sum(hann(5000))).mean()\n        \n    roll_mean = ad.rolling(5000).mean().dropna()\n    features[prefix + 'rolling_min_ave_' + str(5000)] = roll_mean.min()\n    \n    roll_mean = ad.rolling(200).mean().dropna()\n    features[prefix + 'abs_rolling_min_mean_' + str(200)] = np.abs(roll_mean).min()\n    \n    roll_std = ad.rolling(10).std().dropna()\n    features[prefix + 'abs_rolling_std_{}_percentage_{}'.format(10, 0.01)] = np.percentile(roll_std, 0.01)\n    \n    roll_std = ad.rolling(50).std().dropna()\n    features[prefix + 'abs_rolling_std_{}_percentage_{}'.format(50, 0.2)] = np.percentile(roll_std, 0.2)\n        \n    # change rate \n    dif = ad.diff().dropna().values\n    \n    change = (dif / ad.values[:-1])\n    change = change[np.nonzero(change)[0]]\n    change = change[~np.isnan(change)]\n    change = change[change != -np.inf]\n    change = change[change != np.inf]\n    \n    features[prefix + 'change_mean'] = change.mean()\n    \n    test.append(features)\n    \nX_test = pd.DataFrame(test)","metadata":{"execution":{"iopub.status.busy":"2022-01-03T18:06:47.498901Z","iopub.execute_input":"2022-01-03T18:06:47.499135Z","iopub.status.idle":"2022-01-03T18:06:48.17618Z","shell.execute_reply.started":"2022-01-03T18:06:47.499108Z","shell.execute_reply":"2022-01-03T18:06:48.175241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = X_test[keep_cols]\nX_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-03T18:06:48.177912Z","iopub.execute_input":"2022-01-03T18:06:48.178539Z","iopub.status.idle":"2022-01-03T18:06:48.204727Z","shell.execute_reply.started":"2022-01-03T18:06:48.178491Z","shell.execute_reply":"2022-01-03T18:06:48.203817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Do thời gian chạy code chuyển đổi thông tin rất lâu, lưu lại dữ liệu đã xử lý (2)","metadata":{}},{"cell_type":"code","source":"X_test.to_csv(\"X_test.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-03T18:06:48.205874Z","iopub.execute_input":"2022-01-03T18:06:48.206112Z","iopub.status.idle":"2022-01-03T18:06:48.213965Z","shell.execute_reply.started":"2022-01-03T18:06:48.206083Z","shell.execute_reply":"2022-01-03T18:06:48.213268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(X_test))\nprint(len(X_test.columns))","metadata":{"execution":{"iopub.status.busy":"2022-01-03T17:38:31.253885Z","iopub.execute_input":"2022-01-03T17:38:31.254471Z","iopub.status.idle":"2022-01-03T17:38:31.263286Z","shell.execute_reply.started":"2022-01-03T17:38:31.254423Z","shell.execute_reply":"2022-01-03T17:38:31.262271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Kết quả\n\nDữ liệu đã xử lý được lưu trong [bộ dữ liệu](https://www.kaggle.com/ngodang/lanl-u-vo-chun-ha).","metadata":{}}]}