{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\n\nimport numpy as np\nimport pandas as pd\nimport librosa\nimport tensorflow as tf\n\n\nimport librosa.display\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport IPython.display as ipd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BASE_INPUT_DIR = '/kaggle/input/rfcx-species-audio-detection/'\nTRAIN_INPUT_DIR = os.path.join(BASE_INPUT_DIR, 'train')\nTEST_INPUT_DIR = os.path.join(BASE_INPUT_DIR, 'test')\nTRAIN_TF_RECORDS = os.path.join(BASE_INPUT_DIR, 'tfrecords/train')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_tp = pd.read_csv(os.path.join(BASE_INPUT_DIR, 'train_tp.csv'))\ntrain_fp = pd.read_csv(os.path.join(BASE_INPUT_DIR, 'train_tp.csv'))\nsubmission = pd.read_csv(os.path.join(BASE_INPUT_DIR, 'sample_submission.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_files = os.listdir(TRAIN_INPUT_DIR)\ntrain_files = [os.path.join(TRAIN_INPUT_DIR, f) for f in train_files]\n\ntrain_tf_records = os.listdir(TRAIN_TF_RECORDS)\ntrain_tf_records = [os.path.join(TRAIN_TF_RECORDS, f) for f in train_tf_records]\n\ntest_files = os.listdir(TEST_INPUT_DIR)\ntest_files = [os.path.join(TEST_INPUT_DIR, f) for f in test_files]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model 1"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Config\n\nFMIN = 40.0\nFMAX = 24000.0\n\nSR = 48000\nN_MELS = 224\n\nIMG_SIZE = (224, 512)\nIMG_HEIGHT = IMG_SIZE[0]\nIMG_WIDTH = IMG_SIZE[1]\n\nCLIP_DURATION = 60\nSEGMENT_DURATION = 10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Constants\n\nNUM_SPECIES = 24\nSPECIES_ID = list(range(NUM_SPECIES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model 1 Prep"},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_audio(recording_id, train=True):\n    filepath = os.path.join(TRAIN_INPUT_DIR if train else TEST_INPUT_DIR, recording_id + '.flac')\n    data, _ = librosa.load(filepath, sr=SR)\n    return data\n\n\ndef cut_audio(audio_data, tmin, tmax, sr=SR, segment_duration=SEGMENT_DURATION):\n    clip_duration = len(audio_data)/sr\n    extra_time = max(0, segment_duration - (tmax - tmin)) / 2\n    left_extension = right_extension = extra_time\n    if tmin - left_extension < 0:\n        right_extension += left_extension - tmin\n    if tmax + right_extension > clip_duration:\n        left_extension += tmax + right_extension - clip_duration\n    tmin = max(0, tmin - left_extension)\n    tmax = min(clip_duration, tmax + right_extension)\n    \n    min_sample, max_sample = librosa.time_to_samples([tmin, tmax], sr=sr)\n    return audio_data[min_sample:(max_sample + 1)]\n    \n\ndef get_mel_spec_img(audio_data):\n    mel_spec = librosa.feature.melspectrogram(audio_data, sr=SR, n_mels=N_MELS)\n    log_mel_spec = librosa.power_to_db(mel_spec)\n    img = tf.expand_dims(log_mel_spec, -1)\n    img = tf.image.resize(img, IMG_SIZE)\n    img = tf.image.per_image_standardization(img)\n    return img, log_mel_spec\n\n\ndef cut_and_get_mel_spec_img(sample):\n    audio_data = load_audio(sample['recording_id'])\n    audio_data = cut_audio(audio_data, sample['t_min'], sample['t_max'])\n    return get_mel_spec_img(audio_data)\n    \n\ndef cut_and_save_image(sample):\n    img, _ = cut_and_get_mel_spec_img(sample)\n    idx = sample.name\n    output_file = os.path.join(f'./train/', sample['recording_id'] + f'_{idx}.npy')\n    np.save(output_file, img.numpy())\n   \n    \ndef get_rgb_spec_img(img):\n    img_min = tf.reduce_min(img)\n    img_max = tf.reduce_max(img)\n    img = (img - img_min)/(img_max - img_min)*255\n    return tf.image.grayscale_to_rgb(img)\n\n\ndef time_to_img_pos(sample):\n    tmin = sample['t_min']\n    tmax = sample['t_max']\n    extra_time = max(0, SEGMENT_DURATION - (tmax - tmin)) / 2\n    seg_min = max(0, tmin - extra_time)\n    seg_max = min(CLIP_DURATION, tmax + extra_time)\n    \n    tmin_frame, tmax_frame = librosa.time_to_frames([tmin, tmax], sr=SR)\n    segmin_frame, segmax_frame = librosa.time_to_frames([seg_min, seg_max], sr=SR)\n    img_scale_factor = IMG_WIDTH/(segmax_frame - segmin_frame)\n    \n    tmin_img_pos = (tmin_frame - segmin_frame)*img_scale_factor\n    tmax_img_pos = (tmax_frame - segmin_frame)*img_scale_factor\n    return tmin_img_pos, tmax_img_pos\n\n\ndef random_time_crop(img, sample):\n    img_size = tf.shape(img)[:2]\n    tmin_img_pos, tmax_img_pos = time_to_img_pos(sample)\n    \n    min_left = 0\n    max_left = tmin_img_pos\n    min_right = tmax_img_pos\n    max_right = img_size[1]\n    \n    left = tf.random.uniform([], minval=min_left, maxval=max_left, dtype=tf.int32)\n    right = tf.random.uniform([], minval=min_right, maxval=max_right, dtype=tf.int32)\n    return tf.image.resize(img[:, left:right, :], img_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sample = train_tp.loc[9, :]\n# cut_and_save_image(sample)\n# spec_img = np.load(f'./train/{sample.recording_id}_{sample.name}.npy')\n\n# fig, ax = plt.subplots(figsize=(25, 5))\n# plt.imshow(get_rgb_spec_img(spec_img).numpy()/255)\n\n# print(sample['t_min'], sample['t_max'])\n# tmin_img_pos, tmax_img_pos = time_to_img_pos(sample)\n# ax.axvline(tmin_img_pos, color='g')\n# ax.axvline(tmax_img_pos, color='r')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# img = random_time_crop(spec_img, sample)\n\n# fig, ax = plt.subplots(figsize=(25, 5))\n# plt.imshow(get_rgb_spec_img(img).numpy()/255)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generate Numpy files"},{"metadata":{"trusted":true},"cell_type":"code","source":"# import dask.dataframe as dd\n\n# train_tp_dd = dd.from_pandas(train_tp, npartitions=8)\n# train_tp_dd.map_partitions(lambda df: df.apply(cut_and_save_image, axis=1), meta=(None, object)).compute()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !zip -r melspec_img_raw.zip ./train","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generate TFRecords"},{"metadata":{"trusted":true},"cell_type":"code","source":"# def generator():\n#     for i in range(train_tp.shape[0]):\n#         sample = train_tp.loc[i, :]\n#         spec_img = np.load(f'./train/{sample.recording_id}_{sample.name}.npy')\n#         spec_img = tf.constant(spec_img, dtype=tf.float32)\n#         yield spec_img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# melspec_ds = tf.data.Dataset.from_generator(generator, tf.float32).prefetch(1).map(tf.io.serialize_tensor)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# filename = 'train.melspec-img.tfrecord'\n# writer = tf.data.experimental.TFRecordWriter(filename)\n# writer.write(melspec_ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# raw_dataset = tf.data.TFRecordDataset(filename)\n\n# for raw_record in raw_dataset.take(1):\n#     spec_img = tf.io.parse_tensor(raw_record, tf.float32)\n#     plt.imshow(get_rgb_spec_img(spec_img).numpy()/255)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model 2"},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_2_generator():\n    for rec_id, samples in train_tp.groupby('recording_id'):\n        audio_wav = load_audio(rec_id)\n        sr = SR\n        clip_duration = len(audio_wav)/sr\n        assert clip_duration == CLIP_DURATION\n        segment_duration = SEGMENT_DURATION\n        indices = []\n        cut_tmin_lst = []\n        overlap_windows = []\n        overlap_species = []\n        \n        for _, sample in samples.iterrows():\n            tmin = sample['t_min']\n            tmax = sample['t_max']\n\n            extra_time = max(0.0, segment_duration - (tmax - tmin)) / 2\n            cut_tmin = max(0.0, tmin - extra_time)\n            if tmax + extra_time > clip_duration:\n                cut_tmin = max(0.0, cut_tmin - (tmax + extra_time - clip_duration))\n            cut_tmin_lst.append(cut_tmin)\n            indices.append(sample.name)\n            \n        for cut_tmin in cut_tmin_lst:\n            cut_tmax = cut_tmin + segment_duration\n            cut_overlap_windows = []\n            cut_overlap_species = []\n            for _, sample in samples.iterrows():\n                tmin = sample['t_min']\n                tmax = sample['t_max']\n                if cut_tmin <= tmin <= cut_tmax:\n                    cut_overlap_windows.append([tmin - cut_tmin, min(tmax, cut_tmax) - cut_tmin])\n                    cut_overlap_species.append(sample['species_id'])\n            overlap_windows.append(cut_overlap_windows)\n            overlap_species.append(cut_overlap_species)\n            \n        for idx, cut_tmin, windows, species in zip(indices, cut_tmin_lst, overlap_windows, overlap_species):\n            cut_tmin_sample = int(np.floor(cut_tmin * sr))\n            cut_tmax_sample = cut_tmin_sample + segment_duration * sr\n            _sample = {\n                'idx': tf.constant(idx, dtype=tf.int32),\n                'audio_wav': tf.reshape(audio_wav[cut_tmin_sample:cut_tmax_sample], [SR*SEGMENT_DURATION]),\n                'recording_id': tf.constant(rec_id, dtype=tf.string),\n                'cut_tmin': tf.constant(cut_tmin, dtype=tf.float32),\n                'windows': tf.constant(windows, dtype=tf.float32),\n                'species': tf.constant(species, dtype=tf.int32)\n            }\n            yield _sample","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features_dtypes = {\n    'idx': tf.int32,\n    'audio_wav': tf.float32,\n    'recording_id': tf.string,\n    'cut_tmin': tf.float32,\n    'windows': tf.float32,\n    'species': tf.int32\n}\n\nparsed_dataset = tf.data.Dataset.from_generator(model_2_generator, features_dtypes).prefetch(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _tensor_feature(value):\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[tf.io.serialize_tensor(value).numpy()]))\n\n\ndef serialize_example(idx, audio_wav, recording_id, cut_tmin, windows, species):\n    feature = {\n        'idx': _tensor_feature(idx),\n        'audio_wav': _tensor_feature(audio_wav),\n        'recording_id': _tensor_feature(recording_id),\n        'cut_tmin': _tensor_feature(cut_tmin),\n        'windows': _tensor_feature(windows),\n        'species': _tensor_feature(species),\n    }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()\n\n\ndef tf_serialize_example(data):\n    pos_data = (\n        data['idx'],\n        data['audio_wav'],\n        data['recording_id'],\n        data['cut_tmin'],\n        data['windows'],\n        data['species']\n    )\n    tf_string = tf.py_function(serialize_example, pos_data, tf.string)\n    return tf.reshape(tf_string, ())\n\n\nserialized_dataset = parsed_dataset.map(tf_serialize_example).prefetch(50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filename = 'train.cut_audio.tfrecord'\nwriter = tf.data.experimental.TFRecordWriter(filename)\nwriter.write(serialized_dataset)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}