{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<h2> Hi guys.. I just to come back to share about \"sampling\" </h2>\n\nnormally we use 50k train data, it is seperate into 4 groups.\n  > 1. True Positive (Target 1, Prediction 1) <br>\n  > 2. True Negative (Target 0, Prediction 0) <br>\n3. False Positive (Target 0, Prediction 1) <br>\n4. False Negative (Target 1, Prediction 0) <br>\n\n\n<h2> Important thing is we should avoid False Postive (FP), False Negative (FN). </h2>\n<h2> if not, that will ruined our beautiful Classifier :(</h2>\n","metadata":{}},{"cell_type":"markdown","source":"<h3> So i made the pipeline for extract those sample datas. </h3>\n<h3> and if you want you can also get data directly </h3>\n\nplease refered below, if it helpled please upvote :)\n > https://www.kaggle.com/leemop/amb04 <br> \n > https://www.kaggle.com/leemop/amb59 <br>\n > https://www.kaggle.com/leemop/amb1014 <br>\n > https://www.kaggle.com/leemop/amb1519 <br>","metadata":{}},{"cell_type":"markdown","source":"1. Prepare pretrained model\n > We can train model in our own ways. <br>\n > or, you can add pretrained model in dataset: https://www.kaggle.com/leemop/b3-10epoch-fold0-alldata (if it helped please upvote :) )\n\n2. at this kernel, using Pretrained Model, we filtered ultimate both FP, FN.\n3. and saved to sampling_train*.tfrecords.\n4. By doing that, You guys can add sampled data in your own Kernel.\n5. And Please share great techniques for others :)","metadata":{}},{"cell_type":"markdown","source":"<h1> If it helped or informatives, please upvote for me. Thanks </h1>","metadata":{}},{"cell_type":"code","source":"!pip install efficientnet tensorflow_addons > /dev/null\nimport os\nimport math\nimport random\nimport re\nimport warnings\nfrom pathlib import Path\nfrom typing import Optional, Tuple\n\nimport efficientnet.tfkeras as efn\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom scipy.signal import get_window\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\n\nNUM_FOLDS = 5\nIMAGE_SIZE = 256\nBATCH_SIZE = 32\nEFFICIENTNET_SIZE = 3\nWEIGHTS = \"imagenet\"\n\nfrom kaggle_datasets import KaggleDatasets\nfrom typing import Optional, Tuple\n\nMIXUP_PROB = 0.0\nEPOCHS = 5\nR_ANGLE = 0 / 180 * np.pi\nS_SHIFT = 0.0\nT_SHIFT = 0.0\nLABEL_POSITIVE_SHIFT = 0.99\n\ndef get_datapath():\n    gcs_paths = []    \n    for i, j in [(0, 4), (5, 9), (10, 14), (15, 19)]:        \n        GCS_path = KaggleDatasets().get_gcs_path(f\"g2net-waveform-tfrecords-train-{i}-{j}\")        \n        gcs_paths.append(GCS_path)\n        print(GCS_path)\n\n    all_files = []\n    for path in gcs_paths:\n        all_files.extend(np.sort(np.array(tf.io.gfile.glob(path + \"/train*.tfrecords\"))))        \n           \n    print(\"train_files: \", len(all_files))    \n    return all_files\n\n### Dataset Preperation\n\ndef create_cqt_kernels(\n    q: float,\n    fs: float,\n    fmin: float,\n    n_bins: int = 84,\n    bins_per_octave: int = 12,\n    norm: float = 1,\n    window: str = \"hann\",\n    fmax: Optional[float] = None,\n    topbin_check: bool = True\n) -> Tuple[np.ndarray, int, np.ndarray, float]:\n    fft_len = 2 ** _nextpow2(np.ceil(q * fs / fmin))\n    \n    if (fmax is not None) and (n_bins is None):\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    elif (fmax is None) and (n_bins is not None):\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    else:\n        warnings.warn(\"If nmax is given, n_bins will be ignored\", SyntaxWarning)\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n        \n    if np.max(freqs) > fs / 2 and topbin_check:\n        raise ValueError(f\"The top bin {np.max(freqs)} Hz has exceeded the Nyquist frequency, \\\n                           please reduce the `n_bins`\")\n    \n    kernel = np.zeros((int(n_bins), int(fft_len)), dtype=np.complex64)\n    \n    length = np.ceil(q * fs / freqs)\n    for k in range(0, int(n_bins)):\n        freq = freqs[k]\n        l = np.ceil(q * fs / freq)\n        \n        if l % 2 == 1:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0)) - 1\n        else:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0))\n\n        sig = get_window(window, int(l), fftbins=True) * np.exp(\n            np.r_[-l // 2:l // 2] * 1j * 2 * np.pi * freq / fs) / l\n        \n        if norm:\n            kernel[k, start:start + int(l)] = sig / np.linalg.norm(sig, norm)\n        else:\n            kernel[k, start:start + int(l)] = sig\n    return kernel, fft_len, length, freqs\n\n\ndef _nextpow2(a: float) -> int:\n    return int(np.ceil(np.log2(a)))\n\n\ndef prepare_cqt_kernel(\n    sr=22050,\n    hop_length=512,\n    fmin=32.70,\n    fmax=None,\n    n_bins=84,\n    bins_per_octave=12,\n    norm=1,\n    filter_scale=1,\n    window=\"hann\"\n):\n    q = float(filter_scale) / (2 ** (1 / bins_per_octave) - 1)\n    print(q)\n    return create_cqt_kernels(q, sr, fmin, n_bins, bins_per_octave, norm, window, fmax) \n\n\nHOP_LENGTH = 16\ncqt_kernels, KERNEL_WIDTH, lengths, _ = prepare_cqt_kernel(\n                                        sr=2048,\n                                        hop_length=HOP_LENGTH,\n                                        fmin=20,\n                                        fmax=1024,\n                                        bins_per_octave=24)\nLENGTHS = tf.constant(lengths, dtype=tf.float32)\nCQT_KERNELS_REAL = tf.constant(np.swapaxes(cqt_kernels.real[:, np.newaxis, :], 0, 2))\nCQT_KERNELS_IMAG = tf.constant(np.swapaxes(cqt_kernels.imag[:, np.newaxis, :], 0, 2))\nPADDING = tf.constant([[0, 0],\n            [KERNEL_WIDTH // 2, KERNEL_WIDTH // 2],\n            [0, 0]])\n            \ndef create_cqt_image(wave, hop_length=16, cqtCFG = None):\n    CQTs = []\n    for i in range(3):\n        x = wave[i]\n        x = tf.expand_dims(tf.expand_dims(x, 0), 2)\n        x = tf.pad(x, PADDING, \"REFLECT\")\n\n        CQT_real = tf.nn.conv1d(x, CQT_KERNELS_REAL, stride=hop_length, padding=\"VALID\")\n        CQT_imag = -tf.nn.conv1d(x, CQT_KERNELS_IMAG, stride=hop_length, padding=\"VALID\")\n        CQT_real *= tf.math.sqrt(LENGTHS)\n        CQT_imag *= tf.math.sqrt(LENGTHS)\n\n        CQT = tf.math.sqrt(tf.pow(CQT_real, 2) + tf.pow(CQT_imag, 2))\n        CQTs.append(CQT[0])\n    return tf.stack(CQTs, axis=2)      \n\ndef read_id_label_tfrecord(example):\n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_image(example[\"wave\"], IMAGE_SIZE), example[\"wave_id\"], tf.reshape(tf.cast(example[\"target\"], tf.float32), [1])\n\n\ndef read_labeled_tfrecord(example):    \n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)    \n    return prepare_image(example[\"wave\"], IMAGE_SIZE), tf.reshape(tf.cast(example[\"target\"], tf.float32), [1])\n\n\ndef read_unlabeled_tfrecord(example, return_image_id):\n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_image(example[\"wave\"], IMAGE_SIZE), example[\"wave_id\"] if return_image_id else 0\n\n\ndef count_data_items(fileids):\n    return len(fileids) * 28000\n\n\ndef count_data_items_test(fileids):\n    return len(fileids) * 22600\n\n\ndef prepare_image(wave, dim=256):    \n    wave = tf.reshape(tf.io.decode_raw(wave, tf.float64), (3, 4096))    \n    normalized_waves = []\n    for i in range(3):\n        normalized_wave = wave[i] / tf.math.reduce_max(wave[i])\n        normalized_waves.append(normalized_wave)\n    wave = tf.stack(normalized_waves)\n    wave = tf.cast(wave, tf.float32)\n    image = create_cqt_image(wave, HOP_LENGTH)\n    image = tf.image.resize(image, size=(dim, dim))    \n    return tf.reshape(image, (dim, dim, 3))\n\n\ndef get_dataset(files, batch_size=16, repeat=False, shuffle=False, aug=True, labeled=True, return_image_ids=True):    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO, compression_type=\"GZIP\")\n    ds = ds.cache() # dataset을 캐시 함으로서, 로컬에 저장하여 효율성 높임. 각 에폭에서만 적용?\n\n    if repeat:\n        ds = ds.repeat()\n\n    if shuffle:\n        ds = ds.shuffle(1024 * 2)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n\n    \n    # https://www.tensorflow.org/api_docs/python/tf/data/TFRecordDataset#map\n    if labeled == \"sampling\":\n        ds = ds.map(read_id_label_tfrecord, num_parallel_calls=AUTO)\n    elif labeled:\n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)        \n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_ids), num_parallel_calls=AUTO)\n\n    ds = ds.batch(batch_size * REPLICAS)\n    \n    if aug:\n        ds = ds.map(lambda x, y: aug_f(x, y, batch_size * REPLICAS), num_parallel_calls=AUTO)    \n    \n    ds = ds.prefetch(AUTO)\n    \n    print(ds)\n    return ds  \n\nimport os\nimport math\nimport random\nimport re\nimport warnings\nfrom pathlib import Path\nfrom typing import Optional, Tuple\n\nimport efficientnet.tfkeras as efn\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom scipy.signal import get_window\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\n\ndef build_model(size=256, efficientnet_size=0, weights=\"imagenet\", count=0):\n    inputs = tf.keras.layers.Input(shape=(size, size, 3))\n    \n    efn_string= f\"EfficientNetB{efficientnet_size}\"\n    efn_layer = getattr(efn, efn_string)(input_shape=(size, size, 3), weights=weights, include_top=False) # getattr(efn, efn_string) == efn.efn_string 같지만 활용도가 좋음\n\n    x = efn_layer(inputs)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dropout(0.2)(x)\n    x = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n    model = tf.keras.Model(inputs=inputs, outputs=x)\n\n    lr_decayed_fn = tf.keras.experimental.CosineDecay(1e-3, count) # learning rate schedule\n    opt = tfa.optimizers.AdamW(lr_decayed_fn, learning_rate=1e-4)\n    loss = tf.keras.losses.BinaryCrossentropy()\n    model.compile(optimizer=opt, loss=loss, metrics=[\"AUC\"])\n    model.summary()\n    return model\n    \n        \ndef get_lr_callback(batch_size=8, replicas=8):\n    lr_start   = 1e-4\n    lr_max     = 0.000015 * replicas * batch_size\n    lr_min     = 1e-7\n    lr_ramp_ep = 3\n    lr_sus_ep  = 0\n    lr_decay   = 0.7\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n    return lr_callback   \n\ndef set_seed(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\ndef auto_select_accelerator(): # TPU Setting\n    TPU_DETECTED = False\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n        TPU_DETECTED = True\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n\n    return strategy, TPU_DETECTED\n\n\nset_seed(1213)\n\nstrategy, tpu_detected = auto_select_accelerator()\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\n\nall_files = get_datapath()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfiles_train_all = np.array(all_files[:5])\nprint(files_train_all)\nkf = KFold(n_splits=5, shuffle=True, random_state=1213)\n\nfp = []\nfn = []\ntp = []\ntn = []\n    \ntptest = []\ntntest = []\n    \ncnt = 0\n    \ntpn = 0\ntnn = 0\nfpn = 0\nfnn = 0\n\nfor fold, (trn_idx, val_idx) in enumerate(kf.split(files_train_all)):\n    files_train = files_train_all[trn_idx]     \n    files_valid = files_train_all[val_idx]\n\n    train_image_count = count_data_items(files_train) \n    valid_image_count = count_data_items(files_valid) \n    \n    print(fold, trn_idx, val_idx, train_image_count, valid_image_count)    \n    tf.keras.backend.clear_session() \n    strategy, tpu_detected = auto_select_accelerator()\n    with strategy.scope():\n        model = build_model(\n                size=IMAGE_SIZE, \n                efficientnet_size=EFFICIENTNET_SIZE,\n                weights=WEIGHTS, \n                count=train_image_count // BATCH_SIZE // REPLICAS // 4)\n        \n\n    model.load_weights(\"../input/b3-10epoch-fold0-alldata/alltrain_b3_10epoch_fold0.h5\")\n    ds_trEval = get_dataset(files_train, labeled=\"sampling\", return_image_ids=False, repeat=False, shuffle=False, batch_size=BATCH_SIZE, aug=False)        \n    \n    endure_threshold = 0.2\n    fnignore = 0\n    fpignore = 0\n    for wave, wave_id, target in ds_trEval:\n        pred = model.predict(wave, verbose=1)\n        #print(len(pred), len(wave_id), len(target))        \n        cal = (target-pred).numpy()        \n        for idx, value in enumerate(cal):        \n            if 0.5 < value <= 1: # (1-endure_threshold) fn, if target = 1, predition < 0.1 then ignore\n                if value < 1 - endure_threshold:\n                    fnn += 1\n                    fn.append(wave_id[idx].numpy())\n                else:\n                    fnignore += 1\n            elif 0 < value <= 0.5: # tp\n                tpn += 1\n                tp.append(wave_id[idx].numpy())\n            elif 0 > value > -0.5: # tn\n                tnn += 1\n                tn.append(wave_id[idx].numpy())    \n            elif -1 <= value <= -0.5: # fp, if target = 0, prediction > 0.9 then ignore\n                if value > -1 + endure_threshold:\n                    fpn +=1\n                    fp.append(wave_id[idx].numpy())                \n                else:\n                    fpignore += 1\n            elif value == 0:\n                if target == 0: #tn\n                    tnn += 1\n                    tn.append(wave_id[idx].numpy())\n                elif target == 1: #tp\n                    tpn += 1\n                    tp.append(wave_id[idx].numpy())\n                tpn += 1                    \n            else:\n                print(\"Error\", value, target, idx)\n                import sys\n                sys.exit(0)\n                \n    print(\"tp, tn, fp, fn\")\n    print(len(tp),len(tn),len(fp),len(fn))\n    print(tpn, tnn, fpn, fnn)\n    print(tpn + tnn + fpn + fnn)\n    print(fpignore, fnignore)\n    break     ","metadata":{"execution":{"iopub.status.busy":"2021-09-07T06:41:31.395026Z","iopub.execute_input":"2021-09-07T06:41:31.395421Z","iopub.status.idle":"2021-09-07T06:46:06.91319Z","shell.execute_reply.started":"2021-09-07T06:41:31.395387Z","shell.execute_reply":"2021-09-07T06:46:06.912138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow.python as tfp\nfrom tqdm import tqdm\n\ndef get_train_file_path(image_id):\n    return \"../input/g2net-gravitational-wave-detection/train/{}/{}/{}/{}.npy\".format(\n        image_id[0], image_id[1], image_id[2], image_id)\n\ndef _bytes_feature(value):\n    if isinstance(value, tfp.framework.ops.EagerTensor):\n        value = value.numpy()\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n\ndef _float_feature(value):\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\n\ndef _int64_feature(value):\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\n\ndef create_tf_example(wave_id: str, wave: bytes, target: int) -> tf.train.Example:\n    feature = {\n        \"wave_id\": _bytes_feature(wave_id),\n        \"wave\": _bytes_feature(wave),\n        \"target\": _int64_feature(target)\n    }\n    return tf.train.Example(features=tf.train.Features(feature=feature))\n\n\ndef write_tfrecord(samlist_pos, samlist_neg, filename: str):    \n    options = tf.io.TFRecordOptions(\"GZIP\")\n    with tf.io.TFRecordWriter(filename, options=options) as writer:\n        print(len(samlist_pos), len(samlist_neg))\n        sample_list = dict()\n        for i in samlist_pos:\n            sample_list[i] = 1\n        for i in samlist_neg:\n            sample_list[i] = 0        \n        sl = list(sample_list.keys())\n        random.shuffle(sl)\n        \n        for i in tqdm(sl):\n            wave_id = i # for byte\n            wave_dir = get_train_file_path(str(wave_id).split('\\'')[1])            \n            wave = np.load(wave_dir).tobytes()\n            target = sample_list[i]\n            tf_example = create_tf_example(wave_id, wave, target)            \n            writer.write(tf_example.SerializeToString())\n                \ntrain_samples_per_file = 28000\n#print(len(fp+fn+tp+tn))\ntrain_number_of_files = len(fp+fn+tp+tn) // train_samples_per_file\n#print(train_number_of_files)\n\ntprate = round(len(tp) / len(tp+tn+fp+fn) * 100) \ntnrate = round(len(tn) / len(tp+tn+fp+fn) * 100) \nfprate = round(len(fp) / len(tp+tn+fp+fn) * 100) \nfnrate = round(len(fn) / len(tp+tn+fp+fn) * 100) \nprint(tprate, tnrate, fprate, fnrate)\ntp_prev = 0\ntn_prev = 0\nfp_prev = 0\nfn_prev = 0\n\nprint(train_number_of_files)\n#print(len(tp+tn+fp+fn))\nfor i in range(train_number_of_files):\n    tp_cur = tp_prev + int(tprate * train_samples_per_file * 0.01)\n    tn_cur = tn_prev + int(tnrate * train_samples_per_file * 0.01)\n    fp_cur = fp_prev + int(fprate * train_samples_per_file * 0.01)\n    fn_cur = fn_prev + int(fnrate * train_samples_per_file * 0.01)\n    \n    print(tp_cur, tn_cur, fp_cur, fn_cur)\n    filename = f\"sampling_train{i}.tfrecords\"                                                                           \n    a = write_tfrecord(fn[fn_prev:fn_cur] + tp[tp_prev:tp_cur], fp[fp_prev:fp_cur] + tn[tn_prev:tn_cur], filename)    \n    tp_prev = tp_cur\n    tn_prev = tn_cur\n    fp_prev = fp_cur\n    fn_prev = fn_cur","metadata":{"execution":{"iopub.status.busy":"2021-09-07T06:46:06.914526Z","iopub.execute_input":"2021-09-07T06:46:06.914798Z","iopub.status.idle":"2021-09-07T06:54:11.991383Z","shell.execute_reply.started":"2021-09-07T06:46:06.914773Z","shell.execute_reply":"2021-09-07T06:54:11.989763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h1> Enjoy Kaggle ! Thanks </h1>","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}