{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install efficientnet tensorflow_addons > /dev/null\nimport os\nimport math\nimport random\nimport re\nimport warnings\nfrom pathlib import Path\nfrom typing import Optional, Tuple\n\nimport efficientnet.tfkeras as efn\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom scipy.signal import get_window\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\n\nNUM_FOLDS = 5\nIMAGE_SIZE = 256\nBATCH_SIZE = 32\nEFFICIENTNET_SIZE = 7\nWEIGHTS = \"imagenet\"\n","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:46:41.506224Z","iopub.execute_input":"2021-09-07T13:46:41.506823Z","iopub.status.idle":"2021-09-07T13:46:59.647554Z","shell.execute_reply.started":"2021-09-07T13:46:41.506786Z","shell.execute_reply":"2021-09-07T13:46:59.646171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SAVEDIR = Path(\"models\")\nSAVEDIR.mkdir(exist_ok=True)\n\nOOFDIR = Path(\"oof\")\nOOFDIR.mkdir(exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:46:59.649331Z","iopub.execute_input":"2021-09-07T13:46:59.649727Z","iopub.status.idle":"2021-09-07T13:46:59.655692Z","shell.execute_reply.started":"2021-09-07T13:46:59.649688Z","shell.execute_reply":"2021-09-07T13:46:59.654142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\nfrom typing import Optional, Tuple\n\nMIXUP_PROB = 0.0\nEPOCHS = 10\nR_ANGLE = 0 / 180 * np.pi\nS_SHIFT = 0.0\nT_SHIFT = 0.0\nLABEL_POSITIVE_SHIFT = 0.99\n\ndef get_datapath():\n gcs_paths = [] \n for i, j in [(0, 4), (5, 9), (10, 14), (15, 19)]: \n #for i, j in [(0, 4)]: # only use 25% for training \n #GCS_path = KaggleDatasets().get_gcs_path(f\"g2net-waveform-tfrecords-train-{i}-{j}\")\n #GCS_path = KaggleDatasets().get_gcs_path(f\"bftfrec{i}{j}\")\n GCS_path = KaggleDatasets().get_gcs_path(f\"amb{i}{j}\")\n gcs_paths.append(GCS_path)\n print(GCS_path)\n \n #GCS_path = KaggleDatasets().get_gcs_path(\"sampling-train\")\n #GCS_path = KaggleDatasets().get_gcs_path(\"del-amb-5-8\")\n #gcs_paths.append(GCS_path)|\n print(GCS_path)\n\n all_files = []\n for path in gcs_paths:\n #all_files.extend(np.sort(np.array(tf.io.gfile.glob(path + \"/train*.tfrecords\"))))\n all_files.extend(np.sort(np.array(tf.io.gfile.glob(path + \"/sampling_train*.tfrecords\"))))\n #all_files.extend(np.sort(np.array(tf.io.gfile.glob(path + \"/bf_train*.tfrecords\"))))\n \n print(\"train_files: \", len(all_files))\n #print(all_files[:2], len(all_files[:2]))\n return all_files # 10%\n\n### Dataset Preperation\n\ndef create_cqt_kernels(\n q: float,\n fs: float,\n fmin: float,\n n_bins: int = 84,\n bins_per_octave: int = 12,\n norm: float = 1,\n window: str = \"hann\",\n fmax: Optional[float] = None,\n topbin_check: bool = True\n) -> Tuple[np.ndarray, int, np.ndarray, float]:\n fft_len = 2 ** _nextpow2(np.ceil(q * fs / fmin))\n \n if (fmax is not None) and (n_bins is None):\n n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n elif (fmax is None) and (n_bins is not None):\n freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n else:\n warnings.warn(\"If nmax is given, n_bins will be ignored\", SyntaxWarning)\n n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n \n if np.max(freqs) > fs / 2 and topbin_check:\n raise ValueError(f\"The top bin {np.max(freqs)} Hz has exceeded the Nyquist frequency, \\\n please reduce the `n_bins`\")\n \n kernel = np.zeros((int(n_bins), int(fft_len)), dtype=np.complex64)\n \n length = np.ceil(q * fs / freqs)\n for k in range(0, int(n_bins)):\n freq = freqs[k]\n l = np.ceil(q * fs / freq)\n \n if l % 2 == 1:\n start = int(np.ceil(fft_len / 2.0 - l / 2.0)) - 1\n else:\n start = int(np.ceil(fft_len / 2.0 - l / 2.0))\n\n sig = get_window(window, int(l), fftbins=True) * np.exp(\n np.r_[-l // 2:l // 2] * 1j * 2 * np.pi * freq / fs) / l\n \n if norm:\n kernel[k, start:start + int(l)] = sig / np.linalg.norm(sig, norm)\n else:\n kernel[k, start:start + int(l)] = sig\n return kernel, fft_len, length, freqs\n\n\ndef _nextpow2(a: float) -> int:\n return int(np.ceil(np.log2(a)))\n\n\n\ndef butter_bandpass(low_cut, high_cut, fs, order=5):\n nyq = 0.5 * fs\n\n # design filter\n low = low_cut / nyq\n high = high_cut / nyq\n b, a = butter(order, [low, high], btype='band')\n\n # returns the filter coefficients: numerator and denominator\n return b, a \n\ndef butter_bandpass_filter(data, lowcut=20, highcut=350, fs=44100, order=5): \n b,a = butter_bandpass(lowcut, highcut, fs, order=order) \n y = lfilter(b,a, data) \n \n return y\n\n\ndef tf_bp_filter(input): \n y = tf.py_function(butter_bandpass_filter, [input], tf.float64)\n print(y) \n return y\n\n\ndef prepare_cqt_kernel(\n sr=22050,\n hop_length=512,\n fmin=32.70,\n fmax=None,\n n_bins=84,\n bins_per_octave=12,\n norm=1,\n filter_scale=1,\n window=\"hann\"\n):\n q = float(filter_scale) / (2 ** (1 / bins_per_octave) - 1)\n print(q)\n return create_cqt_kernels(q, sr, fmin, n_bins, bins_per_octave, norm, window, fmax) \n\n\nHOP_LENGTH = 16\ncqt_kernels, KERNEL_WIDTH, lengths, _ = prepare_cqt_kernel(\n sr=2048,\n hop_length=HOP_LENGTH,\n fmin=20,\n fmax=1024,\n bins_per_octave=24)\nLENGTHS = tf.constant(lengths, dtype=tf.float32)\nCQT_KERNELS_REAL = tf.constant(np.swapaxes(cqt_kernels.real[:, np.newaxis, :], 0, 2))\nCQT_KERNELS_IMAG = tf.constant(np.swapaxes(cqt_kernels.imag[:, np.newaxis, :], 0, 2))\nPADDING = tf.constant([[0, 0],\n [KERNEL_WIDTH // 2, KERNEL_WIDTH // 2],\n [0, 0]])\n \ndef create_cqt_image(wave, hop_length=16, cqtCFG = None):\n CQTs = []\n for i in range(3):\n x = wave[i]\n x = tf.expand_dims(tf.expand_dims(x, 0), 2)\n x = tf.pad(x, PADDING, \"REFLECT\")\n\n CQT_real = tf.nn.conv1d(x, CQT_KERNELS_REAL, stride=hop_length, padding=\"VALID\")\n CQT_imag = -tf.nn.conv1d(x, CQT_KERNELS_IMAG, stride=hop_length, padding=\"VALID\")\n CQT_real *= tf.math.sqrt(LENGTHS)\n CQT_imag *= tf.math.sqrt(LENGTHS)\n\n CQT = tf.math.sqrt(tf.pow(CQT_real, 2) + tf.pow(CQT_imag, 2))\n CQTs.append(CQT[0])\n return tf.stack(CQTs, axis=2) \n\ndef read_id_label_tfrecord(example):\n tfrec_format = {\n \"wave\": tf.io.FixedLenFeature([], tf.string),\n \"wave_id\": tf.io.FixedLenFeature([], tf.string),\n \"target\": tf.io.FixedLenFeature([], tf.int64)\n }\n example = tf.io.parse_single_example(example, tfrec_format)\n return prepare_image(example[\"wave\"], IMAGE_SIZE), example[\"wave_id\"], tf.reshape(tf.cast(example[\"target\"], tf.float32), [1])\n\n\ndef read_labeled_tfrecord(example): \n tfrec_format = {\n \"wave\": tf.io.FixedLenFeature([], tf.string),\n \"wave_id\": tf.io.FixedLenFeature([], tf.string),\n \"target\": tf.io.FixedLenFeature([], tf.int64)\n }\n example = tf.io.parse_single_example(example, tfrec_format) \n return prepare_image(example[\"wave\"], IMAGE_SIZE), tf.reshape(tf.cast(example[\"target\"], tf.float32), [1])\n\n\ndef read_unlabeled_tfrecord(example, return_image_id):\n tfrec_format = {\n \"wave\": tf.io.FixedLenFeature([], tf.string),\n \"wave_id\": tf.io.FixedLenFeature([], tf.string)\n }\n example = tf.io.parse_single_example(example, tfrec_format)\n return prepare_image(example[\"wave\"], IMAGE_SIZE), example[\"wave_id\"] if return_image_id else 0\n\n\ndef count_data_items(fileids):\n return len(fileids) * 28000\n #return len(fileids) * 7000\n\n\ndef count_data_items_test(fileids):\n return len(fileids) * 22600\n\n\ndef mixup(image, label, probability=0.5, aug_batch=64 * 8):\n imgs = []\n labs = []\n for j in range(aug_batch):\n p = tf.cast(tf.random.uniform([], 0, 1) <= probability, tf.float32)\n k = tf.cast(tf.random.uniform([], 0, aug_batch), tf.int32)\n a = tf.random.uniform([], 0, 1) * p\n\n img1 = image[j]\n img2 = image[k]\n imgs.append((1 - a) * img1 + a * img2)\n lab1 = label[j]\n lab2 = label[k]\n labs.append((1 - a) * lab1 + a * lab2)\n image2 = tf.reshape(tf.stack(imgs), (aug_batch, IMAGE_SIZE, IMAGE_SIZE, 3))\n label2 = tf.reshape(tf.stack(labs), (aug_batch,))\n return image2, label2\n\n\ndef time_shift(img, shift=T_SHIFT):\n if shift > 0:\n T = IMAGE_SIZE\n P = tf.random.uniform([],0,1)\n SHIFT = tf.cast(T * P, tf.int32)\n return tf.concat([img[-SHIFT:], img[:-SHIFT]], axis=0)\n return img\n\n\ndef rotate(img, angle=R_ANGLE):\n if angle > 0:\n P = tf.random.uniform([],0,1)\n A = tf.cast(angle * P, tf.float32)\n return tfa.image.rotate(img, A)\n return img\n\n\ndef spector_shift(img, shift=S_SHIFT):\n if shift > 0:\n T = IMAGE_SIZE\n P = tf.random.uniform([],0,1)\n SHIFT = tf.cast(T * P, tf.int32)\n return tf.concat([img[:, -SHIFT:], img[:, :-SHIFT]], axis=1)\n return img\n\ndef img_aug_f(img):\n img = time_shift(img)\n img = spector_shift(img)\n # img = rotate(img)\n return img\n\n\ndef imgs_aug_f(imgs, batch_size):\n _imgs = []\n DIM = IMAGE_SIZE\n for j in range(batch_size):\n _imgs.append(img_aug_f(imgs[j]))\n return tf.reshape(tf.stack(_imgs),(batch_size,DIM,DIM,3))\n\n\ndef label_positive_shift(labels):\n return labels * LABEL_POSITIVE_SHIFT\n\n\ndef aug_f(imgs, labels, batch_size):\n imgs, label = mixup(imgs, labels, MIXUP_PROB, batch_size)\n imgs = imgs_aug_f(imgs, batch_size)\n return imgs, label_positive_shift(label)\n\n\ndef prepare_image(wave, dim=256): \n wave = tf.reshape(tf.io.decode_raw(wave, tf.float64), (3, 4096)) \n normalized_waves = []\n for i in range(3):\n normalized_wave = wave[i] / tf.math.reduce_max(wave[i])\n normalized_waves.append(normalized_wave)\n wave = tf.stack(normalized_waves)\n wave = tf.cast(wave, tf.float32)\n image = create_cqt_image(wave, HOP_LENGTH)\n image = tf.image.resize(image, size=(dim, dim)) \n return tf.reshape(image, (dim, dim, 3))\n\n\ndef get_dataset(files, batch_size=16, repeat=False, shuffle=False, aug=True, labeled=True, return_image_ids=True): \n ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO, compression_type=\"GZIP\")\n ds = ds.cache() # dataset을 캐시 함으로서, 로컬에 저장하여 효율성 높임. 각 에폭에서만 적용?\n\n if repeat:\n ds = ds.repeat()\n\n if shuffle:\n ds = ds.shuffle(1024 * 2)\n opt = tf.data.Options()\n opt.experimental_deterministic = False\n ds = ds.with_options(opt)\n\n \n # https://www.tensorflow.org/api_docs/python/tf/data/TFRecordDataset#map\n if labeled == \"sampling\":\n print(\"sampling\")\n ds = ds.map(read_id_label_tfrecord, num_parallel_calls=AUTO)\n elif labeled:\n print(\"labeled\")\n ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO) \n else:\n print(\"else\")\n ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_ids), num_parallel_calls=AUTO)\n\n ds = ds.batch(batch_size * REPLICAS)\n \n if aug:\n ds = ds.map(lambda x, y: aug_f(x, y, batch_size * REPLICAS), num_parallel_calls=AUTO) \n \n ds = ds.prefetch(AUTO)\n \n print(ds)\n return ds ","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:46:59.658857Z","iopub.execute_input":"2021-09-07T13:46:59.659863Z","iopub.status.idle":"2021-09-07T13:46:59.803348Z","shell.execute_reply.started":"2021-09-07T13:46:59.659806Z","shell.execute_reply":"2021-09-07T13:46:59.802217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport math\nimport random\nimport re\nimport warnings\nfrom pathlib import Path\nfrom typing import Optional, Tuple\n\nimport efficientnet.tfkeras as efn\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom scipy.signal import get_window\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\n\ndef build_model(size=256, efficientnet_size=0, weights=\"imagenet\", count=0):\n inputs = tf.keras.layers.Input(shape=(size, size, 3))\n \n efn_string= f\"EfficientNetB{efficientnet_size}\"\n efn_layer = getattr(efn, efn_string)(input_shape=(size, size, 3), weights=weights, include_top=False) # getattr(efn, efn_string) == efn.efn_string 같지만 활용도가 좋음\n\n x = efn_layer(inputs)\n x = tf.keras.layers.GlobalAveragePooling2D()(x)\n x = tf.keras.layers.Dropout(0.2)(x)\n x = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n model = tf.keras.Model(inputs=inputs, outputs=x)\n\n lr_decayed_fn = tf.keras.experimental.CosineDecay(1e-3, count) # learning rate schedule\n opt = tfa.optimizers.AdamW(lr_decayed_fn, learning_rate=1e-4)\n loss = tf.keras.losses.BinaryCrossentropy()\n model.compile(optimizer=opt, loss=loss, metrics=[\"AUC\"])\n model.summary()\n return model\n \n \ndef get_lr_callback(batch_size=8, replicas=8):\n lr_start = 1e-4\n #lr_start = 1e-1\n lr_max = 0.000015 * replicas * batch_size\n lr_min = 1e-7\n #lr_min = 1e-5\n lr_ramp_ep = 3\n lr_sus_ep = 0\n lr_decay = 0.7\n \n def lrfn(epoch):\n if epoch < lr_ramp_ep:\n lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n \n elif epoch < lr_ramp_ep + lr_sus_ep:\n lr = lr_max\n \n else:\n lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n \n return lr\n\n lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n return lr_callback ","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:46:59.805024Z","iopub.execute_input":"2021-09-07T13:46:59.805350Z","iopub.status.idle":"2021-09-07T13:46:59.821953Z","shell.execute_reply.started":"2021-09-07T13:46:59.805315Z","shell.execute_reply":"2021-09-07T13:46:59.820400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=42):\n random.seed(seed)\n os.environ[\"PYTHONHASHSEED\"] = str(seed)\n np.random.seed(seed)\n tf.random.set_seed(seed)\n\ndef auto_select_accelerator(): # TPU Setting\n TPU_DETECTED = False\n try:\n tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n tf.config.experimental_connect_to_cluster(tpu)\n tf.tpu.experimental.initialize_tpu_system(tpu)\n strategy = tf.distribute.experimental.TPUStrategy(tpu)\n print(\"Running on TPU:\", tpu.master())\n TPU_DETECTED = True\n except ValueError:\n strategy = tf.distribute.get_strategy()\n print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n\n return strategy, TPU_DETECTED\n\n\nset_seed(1213)\n\nstrategy, tpu_detected = auto_select_accelerator()\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\n\nall_files = get_datapath()","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:46:59.824294Z","iopub.execute_input":"2021-09-07T13:46:59.824791Z","iopub.status.idle":"2021-09-07T13:47:06.751111Z","shell.execute_reply.started":"2021-09-07T13:46:59.824745Z","shell.execute_reply":"2021-09-07T13:47:06.749912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nimport librosa\nimport librosa.display\n\n!pip install -q nnAudio -qq\nimport torch\nfrom nnAudio.Spectrogram import CQT1992v2\n\ndef convert_image_id_2_path(image_id: str, is_train: bool = True) -> str:\n folder = \"train\" if is_train else \"test\"\n return \"../input/g2net-gravitational-wave-detection/{}/{}/{}/{}/{}.npy\".format(\n folder, image_id[0], image_id[1], image_id[2], image_id \n )\n\nQ_TRANSFORM = CQT1992v2(sr=2048, fmin=20, fmax=1024, hop_length=32)\n\ndef visualize_sample_qtransform(\n _id, \n signal_names=(\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"),\n sr=2048,\n):\n x = np.load(convert_image_id_2_path(_id))\n plt.figure(figsize=(16, 5))\n for i in range(3):\n waves = x[i] / np.max(x[i])\n print(type(waves), len(waves))\n waves = torch.from_numpy(waves).float()\n image = Q_TRANSFORM(waves)\n print(np.shape(image))\n plt.subplot(1, 3, i + 1)\n plt.imshow(image.squeeze())\n plt.title(signal_names[i], fontsize=14)\n\n plt.suptitle(f\"id: {_id}\", fontsize=16)\n plt.show() \n\"\"\" ","metadata":{"execution":{"iopub.status.busy":"2021-09-04T06:48:24.490879Z","iopub.execute_input":"2021-09-04T06:48:24.491188Z","iopub.status.idle":"2021-09-04T06:48:35.053856Z","shell.execute_reply.started":"2021-09-04T06:48:24.491159Z","shell.execute_reply":"2021-09-04T06:48:35.052474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls models","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:54:09.258013Z","iopub.execute_input":"2021-09-04T14:54:09.258425Z","iopub.status.idle":"2021-09-04T14:54:10.105915Z","shell.execute_reply.started":"2021-09-04T14:54:09.258388Z","shell.execute_reply":"2021-09-04T14:54:10.104658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"%%time\n# Test kernel\nfiles_train_all = np.array(all_files)\nprint(files_train_all)\nkf = KFold(n_splits=2, shuffle=True, random_state=1213)\n\nfp = []\nfn = []\ntp = []\ntn = []\n \ntptest = []\ntntest = []\n \ncnt = 0\n \ntpn = 0\ntnn = 0\nfpn = 0\nfnn = 0\nfor fold, (trn_idx, val_idx) in enumerate(kf.split(files_train_all)):\n files_train = files_train_all[trn_idx] \n files_valid = files_train_all[val_idx]\n\n train_image_count = count_data_items(files_train) \n valid_image_count = count_data_items(files_valid) \n \n print(fold, trn_idx, val_idx, train_image_count, valid_image_count)\n \n tf.keras.backend.clear_session() # model의 복잡도로 올라간 memory 등을 초기화 함\n strategy, tpu_detected = auto_select_accelerator()\n with strategy.scope():\n model = build_model(\n size=IMAGE_SIZE, \n efficientnet_size=EFFICIENTNET_SIZE,\n weights=WEIGHTS, \n count=train_image_count // BATCH_SIZE // REPLICAS // 4)\n \n\n #model.load_weights(str(SAVEDIR / f\"fold{fold}.h5\"))\n #model.load_weights(\"../input/b0-20traindata-10epoch-cv4-fold0model/models/fold0.h5\")\n #model.load_weights(\"../input/b3-10epoch-fold0-alldata/alltrain_b3_10epoch_fold0.h5\")\n model.load_weights(\"models/sample_fold0.h5\")\n ds_trEval = get_dataset(files_train[0], labeled=\"sampling\", return_image_ids=False, repeat=False, shuffle=False, batch_size=BATCH_SIZE, aug=False) \n \n for wave, wave_id, target in ds_trEval:\n pred = model.predict(wave, verbose=2)#[:count_data_items(files_train_all)]\n #print(len(pred), len(wave_id), len(target)) \n cal = (target-pred).numpy() \n for idx, value in enumerate(cal): \n if value >= 0.5: # fn\n fnn += 1\n if value > 0.75:\n fn.append(wave_id[idx].numpy()) \n elif 0 < value < 0.5: # tp\n tpn += 1\n if value > 0.25:\n tp.append(wave_id[idx].numpy()) \n elif value < 0.1: # will be deleted\n tptest.append(wave_id[idx].numpy())\n elif 0 > value > -0.5: # tn\n tnn += 1\n if value < -0.25:\n tn.append(wave_id[idx].numpy())\n elif value > -0.1: # will be deleted\n tntest.append(wave_id[idx].numpy())\n elif value <= -0.5: #fp\n fpn +=1\n if value < -0.75:\n fp.append(wave_id[idx].numpy())\n \n \"\"\"\n # 드물게 value = 0일때 tn, fp가 뭐인지 현재 상황에서는 중요하지 않음\n elif value == 0:\n if target == 0: #tn\n tnn += 1\n else target == 1: #tp\n tpn += 1 \n \"\"\" \n print(\"tp, tn, fp, fn\")\n print(len(tp),len(tn),len(fp),len(fn))\n print(tpn, tnn, fpn, fnn)\n print(tpn + tnn + fpn + fnn)\n break \n\n\"\"\"\ntfrecord0\n1670 5545 350 1337\n9832 12592 1565 4011\n28000\n\ntfrecords1\n1724 5562 417 1385\n9920 12359 1643 4078\n28000\n\"\"\"","metadata":{}},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-09-04T15:46:18.361878Z","iopub.execute_input":"2021-09-04T15:46:18.36222Z","iopub.status.idle":"2021-09-04T15:48:46.44631Z","shell.execute_reply.started":"2021-09-04T15:46:18.362189Z","shell.execute_reply":"2021-09-04T15:48:46.445088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow.python as tfp\nfrom tqdm import tqdm\n\ndef get_train_file_path(image_id):\n return \"../input/g2net-gravitational-wave-detection/train/{}/{}/{}/{}.npy\".format(\n image_id[0], image_id[1], image_id[2], image_id)\n\ndef _bytes_feature(value):\n if isinstance(value, tfp.framework.ops.EagerTensor):\n value = value.numpy()\n return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n\ndef _float_feature(value):\n return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\n\ndef _int64_feature(value):\n return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\n\ndef create_tf_example(wave_id: str, wave: bytes, target: int) -> tf.train.Example:\n feature = {\n \"wave_id\": _bytes_feature(wave_id),\n \"wave\": _bytes_feature(wave),\n \"target\": _int64_feature(target)\n }\n return tf.train.Example(features=tf.train.Features(feature=feature))\n\n\ndef write_tfrecord(samlist_pos, samlist_neg, filename: str): \n options = tf.io.TFRecordOptions(\"GZIP\")\n with tf.io.TFRecordWriter(filename, options=options) as writer:\n print(len(samlist_pos), len(samlist_neg))\n for i in tqdm(range(len(samlist_pos))):\n wave_id = samlist_pos[i] # for byte\n wave_dir = get_train_file_path(str(wave_id).split('\\'')[1]) \n wave = np.load(wave_dir).tobytes()\n target = 1\n tf_example = create_tf_example(wave_id, wave, target)\n writer.write(tf_example.SerializeToString())\n \n for i in tqdm(range(len(samlist_neg))):\n wave_id = samlist_neg[i] \n wave_dir = get_train_file_path(str(wave_id).split('\\'')[1]) \n wave = np.load(wave_dir).tobytes()\n target = 0\n tf_example = create_tf_example(wave_id, wave, target)\n writer.write(tf_example.SerializeToString())\n \ntrain_samples_per_file = 7000\n#print(len(fp+fn+tp+tn))\ntrain_number_of_files = len(fp+fn+tp+tn) // train_samples_per_file\n#print(train_number_of_files)\n\ntprate = round(len(tp) / len(tp+tn+fp+fn) * 100) \ntnrate = round(len(tn) / len(tp+tn+fp+fn) * 100) \nfprate = round(len(fp) / len(tp+tn+fp+fn) * 100) \nfnrate = round(len(fn) / len(tp+tn+fp+fn) * 100) \n#print(tprate, tnrate, fprate, fnrate)\ntp_prev = 0\ntn_prev = 0\nfp_prev = 0\nfn_prev = 0\n\n#print(len(tp+tn+fp+fn))\nfor i in range(train_number_of_files):\n tp_cur = tp_prev + int(tprate * train_samples_per_file * 0.01)\n tn_cur = tn_prev + int(tnrate * train_samples_per_file * 0.01)\n fp_cur = fp_prev + int(fprate * train_samples_per_file * 0.01)\n fn_cur = fn_prev + int(fnrate * train_samples_per_file * 0.01)\n \n #print(tp_cur, tn_cur, fp_cur, fn_cur)\n filename = f\"sampling_train{i}.tfrecords\" \n write_tfrecord(fn[fn_prev:fn_cur] + tp[tp_prev:tp_cur], fp[fp_prev:fp_cur] + tn[tn_prev:tn_cur], filename) \n tp_prev = tp_cur\n tn_prev = tn_cur\n fp_prev = fp_cur\n fn_prev = fn_cur\nprint(\"Done\")","metadata":{}},{"cell_type":"markdown","source":"print(\"test true positive\")\nfor idx, val in enumerate(tptest):\n if idx > 0:\n break\n visualize_sample_qtransform(str(val).split('\\'')[1])\n \nprint(\"test true negative\")\nfor idx, val in enumerate(tntest):\n if idx > 0:\n break\n visualize_sample_qtransform(str(val).split('\\'')[1]) \n \nprint(\"false positive\")\nfor val in fp:\n visualize_sample_qtransform(str(val).split('\\'')[1]) ","metadata":{}},{"cell_type":"code","source":"kf = KFold(n_splits=6, shuffle=True, random_state=1213)\noof_pred = []\noof_target = []\n\nfiles_train_all = np.array(all_files)\nprint(all_files, NUM_FOLDS)\nfor fold, (trn_idx, val_idx) in enumerate(kf.split(files_train_all)):\n files_train = files_train_all[trn_idx] \n files_valid = files_train_all[val_idx]\n\n print(\"=\" * 120)\n print(f\"Fold {fold}\")\n print(\"=\" * 120)\n\n train_image_count = count_data_items(files_train) \n valid_image_count = count_data_items(files_valid) \n print(train_image_count)\n tf.keras.backend.clear_session() # model의 복잡도로 올라간 memory 등을 초기화 함\n\n strategy, tpu_detected = auto_select_accelerator()\n with strategy.scope():\n model = build_model(\n size=IMAGE_SIZE, \n efficientnet_size=EFFICIENTNET_SIZE,\n weights=WEIGHTS, \n count=train_image_count // BATCH_SIZE // REPLICAS // 4)\n \n model_ckpt = tf.keras.callbacks.ModelCheckpoint(\n str(SAVEDIR / f\"sample_fold{fold}.h5\"), monitor=\"val_auc\", verbose=1, save_best_only=True,\n save_weights_only=True, mode=\"max\", save_freq=\"epoch\"\n )\n #model.load_weights(\"../input/b3-10epoch-fold0-alldata/alltrain_b3_10epoch_fold0.h5\")\n history = model.fit( \n get_dataset(files_train, batch_size=BATCH_SIZE, shuffle=True, repeat=True, aug=True), \n epochs=30, \n #epochs=EPOCHS, \n callbacks=[model_ckpt, get_lr_callback(BATCH_SIZE, REPLICAS)],\n steps_per_epoch=train_image_count // BATCH_SIZE // REPLICAS // 4,\n validation_data=get_dataset(files_valid, batch_size=BATCH_SIZE * 4, repeat=False, shuffle=False, aug=False),\n verbose=1\n )\n ### hj start\n \"\"\"\n ds_trEval = get_dataset(files_train, labeled=\"sampling\", return_image_ids=False, repeat=True, shuffle=False, batch_size=BATCH_SIZE, aug=False)\n STEPS = count_data_items(files_train_all) / BATCH_SIZE / 2 / REPLICAS \n for wave, wave_id, target in ds_trEval:\n pred = model.predict(wave, verbose=1, batch_size=256)\n \"\"\" \n \n \n \n ### hj end\n \n print(\"Loading best model...\")\n model.load_weights(str(SAVEDIR / f\"sample_fold{fold}.h5\"))\n\n ds_valid = get_dataset(files_valid, labeled=False, return_image_ids=False, repeat=True, shuffle=False, batch_size=BATCH_SIZE * 2, aug=False)\n STEPS = valid_image_count / BATCH_SIZE / 2 / REPLICAS\n pred = model.predict(ds_valid, steps=STEPS, verbose=0)[:valid_image_count]\n \n oof_pred.append(np.mean(pred.reshape((valid_image_count, 1), order=\"F\"), axis=1))\n\n ds_valid = get_dataset(files_valid, repeat=False, labeled=True, return_image_ids=True, aug=False)\n oof_target.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))\n\n plt.figure(figsize=(8, 6))\n sns.distplot(oof_pred[-1])\n plt.show()\n \n plt.figure(figsize=(15, 5))\n plt.plot(\n np.arange(len(history.history[\"auc\"])),\n history.history[\"auc\"],\n \"-o\",\n label=\"Train auc\",\n color=\"#ff7f0e\")\n plt.plot(\n np.arange(len(history.history[\"auc\"])),\n history.history[\"val_auc\"],\n \"-o\",\n label=\"Val auc\",\n color=\"#1f77b4\")\n \n x = np.argmax(history.history[\"val_auc\"])\n y = np.max(history.history[\"val_auc\"])\n\n xdist = plt.xlim()[1] - plt.xlim()[0]\n ydist = plt.ylim()[1] - plt.ylim()[0]\n\n plt.scatter(x, y, s=200, color=\"#1f77b4\")\n plt.text(x - 0.03 * xdist, y - 0.13 * ydist, f\"max auc\\n{y}\", size=14)\n\n plt.ylabel(\"auc\", size=14)\n plt.xlabel(\"Epoch\", size=14)\n plt.legend(loc=2)\n\n plt2 = plt.gca().twinx()\n plt2.plot(\n np.arange(len(history.history[\"auc\"])),\n history.history[\"loss\"],\n \"-o\",\n label=\"Train Loss\",\n color=\"#2ca02c\")\n plt2.plot(\n np.arange(len(history.history[\"auc\"])),\n history.history[\"val_loss\"],\n \"-o\",\n label=\"Val Loss\",\n color=\"#d62728\")\n \n x = np.argmin(history.history[\"val_loss\"])\n y = np.min(history.history[\"val_loss\"])\n \n ydist = plt.ylim()[1] - plt.ylim()[0]\n\n plt.scatter(x, y, s=200, color=\"#d62728\")\n plt.text(x - 0.03 * xdist, y + 0.05 * ydist, \"min loss\", size=14)\n\n plt.ylabel(\"Loss\", size=14)\n plt.title(f\"Fold {fold + 1} - Image Size {IMAGE_SIZE}, EfficientNetB{EFFICIENTNET_SIZE}\", size=18)\n\n plt.legend(loc=3)\n plt.savefig(OOFDIR / f\"fig{fold}.png\")\n plt.show() \n break","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:47:06.752875Z","iopub.execute_input":"2021-09-07T13:47:06.753288Z","iopub.status.idle":"2021-09-07T14:22:43.647859Z","shell.execute_reply.started":"2021-09-07T13:47:06.753243Z","shell.execute_reply":"2021-09-07T14:22:43.647129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"files_train_all = np.array(all_files)\nfiles_train = files_train_all[0] \nprint(files_train)\n#data = get_dataset(files_train[0], batch_size=BATCH_SIZE, labeled=True, shuffle=True, repeat=True, aug=False)\n#ds_trEval = get_dataset(files_train[0], labeled=\"sampling\", return_image_ids=False, repeat=False, shuffle=False, batch_size=BATCH_SIZE, aug=False) \nds_trEval = get_dataset(files_train, labeled=True, return_image_ids=False, repeat=False, shuffle=False, batch_size=BATCH_SIZE, aug=False)\nfor a,b in ds_trEval:\n #print(b)\n print(b.numpy().sum()) \n ","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:26:18.371055Z","iopub.execute_input":"2021-09-04T14:26:18.371675Z","iopub.status.idle":"2021-09-04T14:26:22.630065Z","shell.execute_reply.started":"2021-09-04T14:26:18.371639Z","shell.execute_reply":"2021-09-04T14:26:22.628471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gcs_paths = []\nfor i, j in [(0, 4), (5, 9)]:\n#for i, j in [(0, 4)]:\n GCS_path = KaggleDatasets().get_gcs_path(f\"g2net-waveform-tfrecords-test-{i}-{j}\")\n #GCS_path = KaggleDatasets().get_gcs_path(f\"bftfrectest{i}{j}\")\n gcs_paths.append(GCS_path)\n print(GCS_path)\n\nall_files = []\nfor path in gcs_paths:\n all_files.extend(np.sort(np.array(tf.io.gfile.glob(path + \"/test*.tfrecords\"))))\n #all_files.extend(np.sort(np.array(tf.io.gfile.glob(path + \"/bf_test*.tfrecords\")))) \n\nprint(\"test_files: \", len(all_files)) \n\nfiles_test_all = np.array(all_files)\nall_test_preds = []\n\nwith strategy.scope():\n model = build_model(\n size=IMAGE_SIZE,\n efficientnet_size=EFFICIENTNET_SIZE,\n weights=WEIGHTS,\n count=0)","metadata":{"execution":{"iopub.status.busy":"2021-09-07T14:34:14.253773Z","iopub.execute_input":"2021-09-07T14:34:14.254176Z","iopub.status.idle":"2021-09-07T14:34:39.534516Z","shell.execute_reply.started":"2021-09-07T14:34:14.254140Z","shell.execute_reply":"2021-09-07T14:34:39.533099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#weights_dir = Path(\"../input/g2net-tf-on-the-fly-cqt-tpu-training/models/\")\nweights_dir = Path(\"models/\")\n#for i in range(4):\nfor i in range(1):\n print(f\"Load weight for Fold {i + 1} model\")\n model.load_weights(weights_dir / f\"sample_fold{i}.h5\")\n \n ds_test = get_dataset(files_test_all, batch_size=BATCH_SIZE * 2, repeat=True, shuffle=False, aug=False, labeled=False, return_image_ids=False)\n STEPS = count_data_items_test(files_test_all) / BATCH_SIZE / 2 / REPLICAS\n pred = model.predict(ds_test, verbose=1, steps=STEPS)[:count_data_items_test(files_test_all)]\n all_test_preds.append(pred.reshape(-1)) \n print(type(all_test_preds[0]), len(all_test_preds[0]), all_test_preds)","metadata":{"execution":{"iopub.status.busy":"2021-09-07T14:34:39.536727Z","iopub.execute_input":"2021-09-07T14:34:39.537204Z","iopub.status.idle":"2021-09-07T14:37:35.232677Z","shell.execute_reply.started":"2021-09-07T14:34:39.537152Z","shell.execute_reply":"2021-09-07T14:37:35.231492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_test = get_dataset(files_test_all, batch_size=BATCH_SIZE * 2, repeat=False, shuffle=False, aug=False, labeled=False, return_image_ids=True)\nfile_ids = np.array([target.numpy() for img, target in iter(ds_test.unbatch())])\ntest_pred = np.zeros_like(all_test_preds[0])\nfor i in range(len(all_test_preds)):\n test_pred += all_test_preds[i] / len(all_test_preds)\n \ntest_df = pd.DataFrame({\n \"id\": [i.decode(\"UTF-8\") for i in file_ids],\n \"target\": test_pred\n})\n\ntest_df.head()\ntest_df.to_csv(\"submission_test_aug.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-07T14:37:35.234310Z","iopub.execute_input":"2021-09-07T14:37:35.234668Z","iopub.status.idle":"2021-09-07T14:45:44.036025Z","shell.execute_reply.started":"2021-09-07T14:37:35.234633Z","shell.execute_reply":"2021-09-07T14:45:44.034765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Inference

","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}}]}