{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q python-speech-features\n\nimport os\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nfrom python_speech_features import mfcc, logfbank\nfrom matplotlib import pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-01T06:52:50.961248Z","iopub.execute_input":"2021-07-01T06:52:50.962313Z","iopub.status.idle":"2021-07-01T06:53:02.041066Z","shell.execute_reply.started":"2021-07-01T06:52:50.962018Z","shell.execute_reply":"2021-07-01T06:53:02.039547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLDS = 16\nSEED = 2809","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-01T06:53:02.042864Z","iopub.execute_input":"2021-07-01T06:53:02.043163Z","iopub.status.idle":"2021-07-01T06:53:02.047926Z","shell.execute_reply.started":"2021-07-01T06:53:02.043132Z","shell.execute_reply":"2021-07-01T06:53:02.046775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"0\"></a>\n## 0. EDA","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/g2net-gravitational-wave-detection/training_labels.csv')\ntrain_df['path'] = train_df['id'].apply(lambda x: f'../input/g2net-gravitational-wave-detection/train/{x[0]}/{x[1]}/{x[2]}/{x}.npy')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-01T06:53:02.050123Z","iopub.execute_input":"2021-07-01T06:53:02.050412Z","iopub.status.idle":"2021-07-01T06:53:03.001456Z","shell.execute_reply.started":"2021-07-01T06:53:02.050385Z","shell.execute_reply":"2021-07-01T06:53:03.000459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-01T06:53:03.002981Z","iopub.execute_input":"2021-07-01T06:53:03.003436Z","iopub.status.idle":"2021-07-01T06:53:03.026051Z","shell.execute_reply.started":"2021-07-01T06:53:03.003401Z","shell.execute_reply":"2021-07-01T06:53:03.025259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GlobalMinMaxScaler:\n    def __init__(self, size=256):\n        self.__min = []\n        self.__max = []\n        self.__is_initialized = False\n        \n    def fit(self, x):\n        for i in range(3):\n            if not self.__is_initialized:\n                self.__min.append(np.min(x[..., i]))\n                self.__max.append(np.max(x[..., i]))\n            else:\n                self.__min[i] = np.min([self.__min[i], np.min(x[..., i])])\n                self.__max[i] = np.max([self.__max[i], np.max(x[..., i])])\n        self.__is_initialized = True\n        \n    def transform(self, x):\n        if self.__is_initialized:\n            for i in range(3):\n                x[..., i] = (x[..., i] - self.__min[i]) / (self.__max[i] - self.__min[i])\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-07-01T06:53:03.027187Z","iopub.execute_input":"2021-07-01T06:53:03.027595Z","iopub.status.idle":"2021-07-01T06:53:03.038543Z","shell.execute_reply.started":"2021-07-01T06:53:03.027564Z","shell.execute_reply":"2021-07-01T06:53:03.037125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_data(path):\n    return np.load(path)\n\ndef get_power(path):\n    power_vec = []\n    data = get_data(path)\n    for i in range(3):\n        vec = data[i]\n        power = logfbank(vec, samplerate=2., winlen=256, winstep=8, nfft=512, nfilt=256, preemph=0.)\n        power_vec.append(power)\n    return np.dstack(power_vec)\n\ndef get_img_power(path, scaler):\n    power_vec = []\n    data = get_data(path)\n    for i in range(3):\n        vec = data[i]\n        power = logfbank(vec, samplerate=2., winlen=256, winstep=8, nfft=512, nfilt=256, preemph=0.)\n        power_vec.append(power)\n    power_vec = np.dstack(power_vec)\n    power_vec = scaler.transform(power_vec)\n    return (power_vec * 255).astype(np.uint8)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-01T06:53:03.040040Z","iopub.execute_input":"2021-07-01T06:53:03.040345Z","iopub.status.idle":"2021-07-01T06:53:03.055510Z","shell.execute_reply.started":"2021-07-01T06:53:03.040315Z","shell.execute_reply":"2021-07-01T06:53:03.054083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = GlobalMinMaxScaler()\nweight = 0\nfor p, t in tqdm(zip(train_df.path.values[:10_000], train_df.target.values[:10_000])):\n    scaler.fit(get_power(p))\n    weight += t\nprint('Scaler Balance Weight', weight / 10_000)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T06:53:27.240508Z","iopub.execute_input":"2021-07-01T06:53:27.240875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = get_data(train_df.path.values[0])\n\nfig, ax = plt.subplots(3, 1, figsize=(21, 21))\n\nfor i in range(3):\n    ax[i].plot(data[i])\nplt.show();","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"power = get_img_power(train_df.path.values[1], scaler)\n\nfig, ax = plt.subplots(1, 3, figsize=(24, 8))\n\n\nfor i in range(3):\n    ax[i].imshow(power[..., i].T)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"1\"></a>\n## 1. Grouped by Target","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nskf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=SEED)\ntrain_df['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(skf.split(train_df, train_df['target'])):\n    train_df.loc[val_idx,'fold'] = fold\ntrain_df.groupby(['fold', 'target'])['id'].count()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"1\"></a>\n## 2. Save in TFRecords","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\n\ndef _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n    \"\"\"Returns a float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_serialize_example(feature0, feature1, feature2):\n    feature = {\n      'image'         : _bytes_feature(feature0),\n      'image_id'      : _bytes_feature(feature1),   \n      'target'        : _int64_feature(feature2),\n  }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show=True\nfolds = sorted(train_df.fold.unique().tolist())\nfor fold in tqdm(folds):\n    if fold not in range(0, 4):\n        continue\n    fold_df = train_df[train_df.fold==fold]\n    if show:\n        print(); print('Writing TFRecord of fold %i :'%(fold))  \n    with tf.io.TFRecordWriter('train%.2i-%i.tfrec'%(fold,fold_df.shape[0])) as writer:\n        samples = fold_df.shape[0]\n        it = tqdm(range(samples)) if show else range(samples)\n        for k in it:\n            row = fold_df.iloc[k,:]\n            image      = get_img_power(row['path'], scaler)[...,::-1]\n            image_id   = row['id']\n            target     = np.array(row['target'], dtype=np.uint8)\n            example  = train_serialize_example(\n                cv2.imencode('.png', image)[1].tobytes(),\n                str.encode(image_id),\n                target,\n                )\n            writer.write(example)\n        if show:\n            filepath = 'train%.2i-%i.tfrec'%(fold,fold_df.shape[0])\n            filename = filepath.split('/')[-1]\n            filesize = os.path.getsize(filepath)/10**6\n            print(filename,':',np.around(filesize, 2),'MB')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}