{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":44224,"databundleVersionId":5188730,"sourceType":"competition"},{"sourceId":5163568,"sourceType":"datasetVersion","datasetId":2987335}],"dockerImageVersionId":30408,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-14T09:27:43.474276Z","iopub.execute_input":"2023-03-14T09:27:43.474772Z","iopub.status.idle":"2023-03-14T09:27:43.497933Z","shell.execute_reply.started":"2023-03-14T09:27:43.474735Z","shell.execute_reply":"2023-03-14T09:27:43.496997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport scipy as sp\nimport librosa\nimport json\nimport gc\nimport tensorflow as tf\nfrom keras import utils\nfrom keras import layers\nfrom keras import models\nfrom keras import Input\nfrom keras import activations\nfrom keras import optimizers\nfrom keras import losses\nfrom keras import metrics\nfrom keras import callbacks\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import average_precision_score\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-03-14T09:27:43.500050Z","iopub.execute_input":"2023-03-14T09:27:43.500442Z","iopub.status.idle":"2023-03-14T09:27:51.295853Z","shell.execute_reply.started":"2023-03-14T09:27:43.500404Z","shell.execute_reply":"2023-03-14T09:27:51.294787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    kernel_size = 1024\n    window_size = 10\n\n    SR = 32000\n    num_cls = 264\n\n    hidden_size = 384\n    num_heads = 4\n    num_blocks = 2\n    drop_rate = 0.1\n    norm_eps = 1e-6\n    ffn_size = hidden_size * 2\n\n    cls_hidden = 512\n    cls_drop = 0.2\n\n    base_dir = '/kaggle/input/birdclef-2023/train_audio/'\n    audio_type = 'ogg'\n\n    k_fold = 6\n    seed = 430\n    tra_bz = 340\n    val_bz = 512\n    lr = 1e-3\n    epochs = 200\n\n    normal_lr = 0.0005\n    warmup_steps = 600\n    init_lr = 0.00005\n    total_steps = 5000\n\n\nCFG = Config()\n# gpu = tf.config.list_physical_devices('GPU')\n# tf.config.experimental.set_memory_growth(gpu[0], True)\nfile_label = pd.read_csv('/kaggle/input/bfk-config-file/file_label.csv', index_col=False, header=0)\nfile_label['label'] = file_label['label'].map(str)\nfile_label = file_label.values","metadata":{"execution":{"iopub.status.busy":"2023-03-14T09:27:51.298117Z","iopub.execute_input":"2023-03-14T09:27:51.299330Z","iopub.status.idle":"2023-03-14T09:27:51.343849Z","shell.execute_reply.started":"2023-03-14T09:27:51.299290Z","shell.execute_reply":"2023-03-14T09:27:51.342681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gpu = tf.config.list_physical_devices('GPU')\nprint(gpu)","metadata":{"execution":{"iopub.status.busy":"2023-03-14T09:27:51.346787Z","iopub.execute_input":"2023-03-14T09:27:51.347886Z","iopub.status.idle":"2023-03-14T09:27:51.556849Z","shell.execute_reply.started":"2023-03-14T09:27:51.347847Z","shell.execute_reply":"2023-03-14T09:27:51.555681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 从较小的初始学习率，按照steps线性增长，直到正常学习率\nclass LinearWarmupCosineLR(tf.keras.optimizers.schedules.LearningRateSchedule):\n    def __init__(self, normal_learning_rate, warmup_steps, init_learning_rate, total_steps):\n        super(LinearWarmupCosineLR, self).__init__()\n        self.normal_learning_rate = normal_learning_rate\n        self.warmup_steps = warmup_steps\n        self.init_learning_rate = init_learning_rate\n        self.total_steps = total_steps\n        self.Pi = 3.14159265359\n\n    def __call__(self, global_step):\n        global_step = tf.cast(global_step, dtype=tf.float32)\n        normal_lr = self.normal_learning_rate\n        warmup_steps = self.warmup_steps\n        init_lr = self.init_learning_rate\n        tt_steps = self.total_steps\n        warmup_rate = init_lr + (normal_lr - init_lr) * global_step / warmup_steps\n        cosine_rate = init_lr + (normal_lr - init_lr) * tf.cos(\n            self.Pi * (global_step - warmup_steps) / (tt_steps - warmup_steps) / 2.0)\n        learning_rate = tf.where(global_step < warmup_steps, warmup_rate, cosine_rate)\n        learning_rate = tf.where(learning_rate < self.init_learning_rate,\n                                 self.init_learning_rate,\n                                 learning_rate)\n        return learning_rate\n\n    def get_config(self):\n        return dict(init_learning_rate=self.init_learning_rate,\n                    normal_learning_rate=self.normal_learning_rate,\n                    warmup_steps=self.warmup_steps)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-14T09:27:51.558615Z","iopub.execute_input":"2023-03-14T09:27:51.559128Z","iopub.status.idle":"2023-03-14T09:27:51.570802Z","shell.execute_reply.started":"2023-03-14T09:27:51.559079Z","shell.execute_reply":"2023-03-14T09:27:51.569572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_audio(file_name, aud_type, SR):\n    tmp_path = CFG.base_dir + file_name[:-3] + aud_type\n    if aud_type != 'wav':\n        audio = librosa.load(tmp_path, sr=SR)[0]\n        return audio\n    audio = sp.io.wavfile.read(tmp_path)[1]\n    return audio\n\n\ndef generator(fi_label, aud_type, infer):\n    if not infer:\n        np.random.shuffle(fi_label)\n    num = fi_label.shape[0]\n    ind = 0\n    data_len = CFG.window_size * CFG.SR\n    aud_type = aud_type.decode('utf-8')\n    while ind < num:\n        data = get_audio(fi_label[ind][0].decode('utf-8'), aud_type=aud_type, SR=CFG.SR)\n        if data.shape[0] > data_len:\n            clip = np.random.randint(0, data.shape[0] - data_len + 1)\n            data = data[clip:clip + data_len]\n        label = np.zeros(shape=(CFG.num_cls, ))\n        label[int(fi_label[ind][1])] = 1\n        ind += 1\n        if ind == num:\n            if not infer:\n                ind = 0\n                np.random.shuffle(fi_label)\n            else:\n                ind = num + 1\n        yield data, label\n\n\ndef preprocess(data, label):\n    dmin = tf.reduce_min(data)\n    dmax = tf.reduce_max(data)\n    data = 2 * (data - dmin) / (dmax - dmin) - 1\n    data = tf.expand_dims(data, axis=1)\n    return data, label\n\n\ndef DataLoader(fi_label, aud_type, infer=False, batch_size=128):\n    tune = tf.data.experimental.AUTOTUNE\n    data_signature = (tf.TensorSpec(shape=(None, ), dtype=tf.float32),\n                      tf.TensorSpec(shape=(CFG.num_cls,), dtype=tf.float32))\n    ds = tf.data.Dataset.from_generator(generator, args=(fi_label, aud_type, infer),\n                                        output_signature=data_signature)\n    ds = ds.map(preprocess, num_parallel_calls=tune, deterministic=False)\n    ds = ds.padded_batch(batch_size=batch_size, padded_shapes=([CFG.window_size * CFG.SR, 1], [None]))\n    ds = ds.prefetch(tune)\n    return ds\n\n\ndef padded_cmap(y_true, y_pred, padding_factor=5):\n    pad_rows = np.ones(shape=(padding_factor, CFG.num_cls))\n    y_true = np.concatenate([y_true, pad_rows])\n    y_pred = np.concatenate([y_pred, pad_rows])\n    score = average_precision_score(y_true, y_pred, average='macro',)\n    return score\n","metadata":{"execution":{"iopub.status.busy":"2023-03-14T09:27:51.572414Z","iopub.execute_input":"2023-03-14T09:27:51.573093Z","iopub.status.idle":"2023-03-14T09:27:51.589300Z","shell.execute_reply.started":"2023-03-14T09:27:51.573053Z","shell.execute_reply":"2023-03-14T09:27:51.588306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def TransformerBlock(inputs, ind, recurrent=0, n=1):\n    mha = layers.MultiHeadAttention(num_heads=CFG.num_heads, key_dim=CFG.hidden_size,\n                                    name=f'TE_{ind}_att')\n    drop_1 = layers.Dropout(CFG.drop_rate, name=f'TE_{ind}_drop_1')\n    drop_2 = layers.Dropout(CFG.drop_rate, name=f'TE_{ind}_drop_2')\n    norm_1 = layers.LayerNormalization(epsilon=CFG.norm_eps, name=f'TE_{ind}_norm_1')\n    norm_2 = layers.LayerNormalization(epsilon=CFG.norm_eps, name=f'TE_{ind}_norm_2')\n    ffn_1 = layers.Dense(CFG.ffn_size, activation=activations.gelu, name=f'TE_{ind}_ffn_1')\n    ffn_2 = layers.Dense(CFG.hidden_size, name=f'TE_{ind}_ffn_2')\n    add_1 = layers.Add(name=f'TE_{ind}_add_1')\n    add_2 = layers.Add(name=f'TE_{ind}_add_2')\n    p = tf.constant(0.25, dtype=tf.float32)\n    alpha = tf.pow(2 * (n + 1), p)\n\n    for _ in range(recurrent):\n        x = mha(inputs, inputs)\n        x = drop_1(x)\n        x = add_1([inputs * alpha, x])\n        inputs = norm_1(x)\n        x = ffn_1(inputs)\n        x = ffn_2(x)\n        x = drop_2(x)\n        x = add_2([inputs * alpha, x])\n        x = norm_2(x)\n        inputs = x\n    return x\n\n\ndef build_model():\n    aud_input = Input(shape=(CFG.window_size * CFG.SR, 1),\n                      dtype='float32', name='aud_signal_input')\n    x = layers.Conv1D(filters=CFG.hidden_size,\n                      kernel_size=CFG.kernel_size,\n                      strides=CFG.kernel_size,\n                      use_bias=False,\n                      padding='valid',\n                      name='aud_tokenizer')(aud_input)\n    x = layers.Masking(mask_value=0, name='pad_mask')(x)\n    x = layers.Dense(CFG.hidden_size,\n                     activation=activations.gelu,\n                     name='projection_head')(x)\n    # for i in range(CFG.num_blocks):\n    #     x = TransformerBlock(x, i, n=i)\n    x = TransformerBlock(x, 1, recurrent=2, n=1)\n    x = layers.GlobalAveragePooling1D()(x)\n    x = layers.Dense(CFG.cls_hidden)(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Activation(activations.gelu)(x)\n    x = layers.Dropout(CFG.cls_drop)(x)\n    output = layers.Dense(CFG.num_cls, activation=activations.softmax)(x)\n\n    model = models.Model(inputs=[aud_input], outputs=[output])\n    print(model.summary())\n    model.compile(loss=losses.CategoricalCrossentropy(name='cce_loss'),\n                  metrics=[metrics.AUC(curve='PR', name='auc'),\n                           metrics.CategoricalAccuracy(name='c_acc')],\n                  optimizer=optimizers.Adam(learning_rate=LinearWarmupCosineLR(CFG.normal_lr,\n                                                                               CFG.warmup_steps,\n                                                                               CFG.init_lr,\n                                                                               CFG.total_steps)))\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-03-14T09:27:51.590947Z","iopub.execute_input":"2023-03-14T09:27:51.591501Z","iopub.status.idle":"2023-03-14T09:27:51.608501Z","shell.execute_reply.started":"2023-03-14T09:27:51.591465Z","shell.execute_reply":"2023-03-14T09:27:51.607403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"k_fold = StratifiedKFold(n_splits=CFG.k_fold, shuffle=True, random_state=CFG.seed)\ncmap = []\nfor k, (train_ind, valid_ind) in enumerate(k_fold.split(file_label[:, 0], file_label[:, 1])):\n    train_ds = DataLoader(file_label[train_ind], CFG.audio_type, batch_size=CFG.tra_bz)\n    valid_ds = DataLoader(file_label[valid_ind], CFG.audio_type, infer=True, batch_size=CFG.val_bz)\n\n    callback = [callbacks.EarlyStopping(monitor='val_auc', patience=20, verbose=1,\n                                        restore_best_weights=True)]\n    \n#     tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()\n#     tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)\n#     with tpu_strategy.scope():\n    model = build_model()\n        \n    utils.plot_model(model)\n    history =    .fit(train_ds,\n                        validation_data=valid_ds,\n                        steps_per_epoch=train_ind.shape[0] // CFG.tra_bz + 1,\n                        epochs=CFG.epochs,\n                        callbacks=callback,\n                        verbose=1)\n    valid_ds = DataLoader(file_label[valid_ind], CFG.audio_type, infer=True, batch_size=CFG.val_bz)\n    y_pred = model.predict(valid_ds)\n    y_true = utils.to_categorical(file_label[valid_ind][:, 1].astype('int'), num_classes=CFG.num_cls)\n    cur_cmap = padded_cmap(y_true, y_pred)\n    cmap.append(cur_cmap)\n    print('\\nCmAP: {}\\n'.format(cur_cmap))\n    model.save_weights(f'../weights/model_albert_384_fold_{k}_{str(cur_cmap)[:6]}.h5')\n    with open(f'../log/model_albert_384_log_{k}_.json', 'w') as tmp_file:\n        json.dump(history.history, tmp_file)\n    if k == 2:\n        break\nprint('\\n ave CmAP: {}\\n'.format(np.mean(cmap)))\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-03-14T09:27:51.610121Z","iopub.execute_input":"2023-03-14T09:27:51.610716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}