{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"![](\"https://raw.githubusercontent.com/dimitreOliveira/MachineLearning/master/Kaggle/Rainforest%20Connection%20Species%20Audio%20Detection/banner.png\")
\n
\nRainforest - Audio classification Tensorflow starter
\n
\n\n#### References:\n- [Simple audio recognition: Recognizing keywords](https://www.tensorflow.org/tutorials/audio/simple_audio)\n- [RFCX: train resnet50 with TPU](https://www.kaggle.com/yosshi999/rfcx-train-resnet50-with-tpu)\n- [Getting Started: Rainforest EDA with TFRecords](https://www.kaggle.com/jessemostipak/getting-started-rainforest-eda-with-tfrecords)","metadata":{"papermill":{"duration":0.033843,"end_time":"2020-12-15T16:30:31.108718","exception":false,"start_time":"2020-12-15T16:30:31.074875","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Dependencies","metadata":{"papermill":{"duration":0.039314,"end_time":"2020-12-15T16:30:31.307081","exception":false,"start_time":"2020-12-15T16:30:31.267767","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install --quiet efficientnet","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":11.342642,"end_time":"2020-12-15T16:30:42.681143","exception":false,"start_time":"2020-12-15T16:30:31.338501","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, re, warnings, random, glob\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom IPython import display as Idisplay\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.metrics import classification_report, confusion_matrix\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras import optimizers, losses, metrics, Model\nfrom tensorflow.keras.callbacks import EarlyStopping\nimport efficientnet.tfkeras as efn\nimport tensorflow_addons as tfa\n\ndef seed_everything(seed=0):\n random.seed(seed)\n np.random.seed(seed)\n tf.random.set_seed(seed)\n os.environ['PYTHONHASHSEED'] = str(seed)\n os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 0\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":8.53675,"end_time":"2020-12-15T16:30:51.250248","exception":false,"start_time":"2020-12-15T16:30:42.713498","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Hardware configuration","metadata":{"papermill":{"duration":0.031658,"end_time":"2020-12-15T16:30:51.313854","exception":false,"start_time":"2020-12-15T16:30:51.282196","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# TPU or GPU detection\n# Detect hardware, return appropriate distribution strategy\ntry:\n tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n tpu = None\n\nif tpu:\n tf.config.experimental_connect_to_cluster(tpu)\n tf.tpu.experimental.initialize_tpu_system(tpu)\n strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","metadata":{"_kg_hide-input":true,"papermill":{"duration":29.579927,"end_time":"2020-12-15T16:31:20.925583","exception":false,"start_time":"2020-12-15T16:30:51.345656","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model parameters","metadata":{"papermill":{"duration":0.031914,"end_time":"2020-12-15T16:31:20.990217","exception":false,"start_time":"2020-12-15T16:31:20.958303","status":"completed"},"tags":[]}},{"cell_type":"code","source":"BATCH_SIZE = 16 * REPLICAS\nLEARNING_RATE = 1e-3 * REPLICAS\nEPOCHS = 25\nHEIGHT = 467\nWIDTH = 512\nCHANNELS = 3\nN_CLASSES = 24\nES_PATIENCE = 5\nN_FOLDS = 5\nFOLDS_USED = 5\nTTA_STEPS = 15 # Do TTA if > 0\nMIN_CROP_SIZE = 10\nCROP_SIZE = 6","metadata":{"papermill":{"duration":0.044397,"end_time":"2020-12-15T16:31:21.067877","exception":false,"start_time":"2020-12-15T16:31:21.02348","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data\n\nWe have two different `.csv` files, `train_tp.csv` has the information of all true positive species labels and `train_fp.csv` has the information of all false positive species labels.","metadata":{"papermill":{"duration":0.034407,"end_time":"2020-12-15T16:31:21.135327","exception":false,"start_time":"2020-12-15T16:31:21.10092","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def count_data_items(filenames):\n n = [int(re.compile(r'-([0-9]*)\\.').search(filename).group(1)) for filename in filenames]\n return np.sum(n)\n\n\ndatabase_base_path = '/kaggle/input/rfcx-species-audio-detection/'\ntrain_fp = pd.read_csv(f'{database_base_path}train_fp.csv')\ntrain_tp = pd.read_csv(f'{database_base_path}train_tp.csv')\n\nprint(f'Train false positive samples: {len(train_fp)}')\ndisplay(train_fp.head())\nprint(f'Train true positive samples: {len(train_tp)}')\ndisplay(train_tp.head())\n\nGCS_PATH = KaggleDatasets().get_gcs_path('rfcx-species-audio-detection')\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/tfrecords/train/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/tfrecords/test/*.tfrec')\n\nNUM_TRAINING_SAMPLES = count_data_items(TRAINING_FILENAMES)\nNUM_TEST_SAMPLES = count_data_items(TEST_FILENAMES)\n\nprint(f'GCS: train samples: {NUM_TRAINING_SAMPLES}')\nprint(f'GCS: test samples: {NUM_TEST_SAMPLES}')","metadata":{"_kg_hide-input":true,"papermill":{"duration":2.368874,"end_time":"2020-12-15T16:31:23.536952","exception":false,"start_time":"2020-12-15T16:31:21.168078","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Augmentation","metadata":{}},{"cell_type":"code","source":"def data_augment(sample):\n image = sample['audio_wav']\n \n gau = L.GaussianNoise(0.3)\n image = tf.cond(tf.random.uniform([]) < 0.5, lambda: gau(image, training=True), lambda: image)\n image = tf.image.random_brightness(image, 0.2)\n image = tf.image.random_flip_left_right(image)\n \n sample['audio_wav'] = image\n return sample","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Auxiliary functions","metadata":{"papermill":{"duration":0.034562,"end_time":"2020-12-15T16:31:23.782284","exception":false,"start_time":"2020-12-15T16:31:23.747722","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Audio cropping diagram\n\n#### Workflow\n1. Apply `crop_audio()` to the audio.\n - If the audio has less than the `min_crop_size` pad will be added.\n2. Apply `random_crop_audio()` to the audio.\n - The resulting audio will have length equal to `crop_size`.","metadata":{}},{"cell_type":"markdown","source":"### 1st example\n\nIf possible, equal padding is applied to both side of the crop.\n\n\n\n### 2nd example\n\nIf equal padding is not possible, more padding will be added to one side.\n\n","metadata":{}},{"cell_type":"code","source":"def crop_audio(audio, tmin, tmax, min_crop_size=MIN_CROP_SIZE, sample_rate=48000, max_size=60.):\n \"\"\"\n Crops a 'waveform' file to have {min_crop_size} size given, {tmin}, {tmax}, {sample_rate} and {max_size}.\n \"\"\"\n label_size = tmax - tmin\n \n if label_size >= min_crop_size: # No padding needed\n cut_min = tmin\n cut_max = tmax \n else: # Needs padding\n pad_start = (min_crop_size - label_size) / 2\n cut_min = tf.maximum(0., (tmin - pad_start))\n \n pad_end = (min_crop_size - (label_size - cut_min))\n cut_max = tf.minimum(max_size, (tmax + pad_end))\n \n cut_size = cut_max - cut_min\n \n if cut_size < min_crop_size:\n cut_min = tf.maximum(0., (cut_min - (min_crop_size - cut_size)))\n \n cut_size = cut_max - cut_min\n \n # Casting tensors\n cut_min = tf.cast((cut_min * sample_rate), tf.int32)\n cut_max = tf.cast((cut_max * sample_rate), tf.int32)\n cut_size = tf.cast((cut_size * sample_rate), tf.int32)\n \n audio = audio[cut_min:cut_max] # croping the audio\n audio = audio[:cut_size] # making sure it has the max size\n \n audio = tf.reshape(audio, [cut_size]) # making sure it has the expected shape\n return audio\n\ndef random_crop_audio(audio, crop_size=CROP_SIZE, sample_rate=48000, max_size=60):\n \"\"\"\n Randomly crops a 'waveform' file to have {crop_size} size given, {sample_rate} and {max_size}.\n \"\"\"\n start = tf.random.uniform([], minval=0, \n maxval=(max_size - crop_size), \n dtype=tf.int32)\n cut_min = start * sample_rate\n cut_max = (start + crop_size) * sample_rate\n \n audio_size = len(audio)\n if cut_max > audio_size:\n cut_min -= cut_max - audio_size\n cut_max = cut_min + (crop_size * sample_rate)\n \n # Casting tensors\n cut_min = tf.cast(cut_min, tf.int32)\n cut_max = tf.cast(cut_max, tf.int32)\n cut_size = tf.cast((crop_size*sample_rate), tf.int32)\n \n audio = audio[cut_min:cut_max] # croping the audio\n audio = audio[:cut_size] # making sure it has the max size\n \n audio = tf.reshape(audio, [cut_size]) # making sure it has the expected shape\n return audio","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Datasets utility functions\ndef decode_audio(audio_binary):\n \"\"\"\n Decode a 16-bit PCM WAV file to a float tensor.\n \"\"\"\n audio, _ = tf.audio.decode_wav(audio_binary)\n return tf.squeeze(audio, axis=-1)\n return audio\n\ndef string_split_semicolon(column):\n split_labels_sc = tf.strings.split(column, sep=';')\n return split_labels_sc\n\ndef string_split_comma(column):\n split_labels_c = tf.strings.split(column, sep=',')\n return split_labels_c\n\ndef get_label_info(label_info):\n first_split = string_split_semicolon(label_info)\n remove_quotes = tf.strings.regex_replace(first_split, '\"', \"\")\n label_info = string_split_comma(remove_quotes)\n return label_info\n\ndef get_spectrogram(waveform, padding=False, min_padding=48000):\n \"\"\"\n Transforms a 'waveform' into a 'spectrogram', adding padding if needed.\n \"\"\"\n waveform = tf.cast(waveform, tf.float32)\n if padding:\n # Padding for files with less than {min_padding} samples\n zero_padding = tf.zeros([min_padding] - tf.shape(waveform), dtype=tf.float32)\n # Concatenate audio with padding so that all audio clips will be of the same length\n waveform = tf.concat([waveform, zero_padding], 0)\n spectrogram = tf.signal.stft(waveform, frame_length=2048, frame_step=512, fft_length=2048)\n spectrogram = tf.abs(spectrogram)\n return spectrogram\n \ndef get_spectrogram_tf(example):\n \"\"\"\n Transforms a 'waveform' tensor into a 'spectrogram'.\n Applied to a Tensorflow dataset.\n \"\"\"\n audio = example['audio_wav']\n spectrogram = get_spectrogram(audio)\n example['audio_wav'] = spectrogram\n return example\n\ndef prepare_sample(example):\n \"\"\"\n 1. Resize samples to the expected size.\n 2. Convert gray scales (1 channel) images to RGB (3 channels) format.\n \"\"\"\n sample = example['audio_wav']\n \n sample = tf.expand_dims(sample, axis=-1)\n sample = tf.image.resize(sample, [HEIGHT, WIDTH])\n sample = tf.image.grayscale_to_rgb(sample)\n sample = sample / 255. # Scalse\n \n example['audio_wav'] = sample\n return example\n\ndef read_tfrecord(example, labeled=True):\n \"\"\"\n 1. Parse data based on the 'TFREC_FORMAT' map.\n 2. Decode PCM WAV file.\n 3. Break down the information from 'label_info' into other features.\n 4. Crop the 'audio' waveform if needed.\n 5. Returns the features as a dictionary.\n \"\"\"\n TFREC_FORMAT = {\n 'audio_wav': tf.io.FixedLenFeature([], tf.string), \n 'recording_id': tf.io.FixedLenFeature([], tf.string), \n 'label_info': tf.io.FixedLenFeature([], tf.string, default_value='-1,-1,0,0,0,0,1'), \n }\n \n example = tf.io.parse_single_example(example, TFREC_FORMAT)\n audio = decode_audio(example['audio_wav'])\n \n # Break down 'label_info' into the data columns\n label_info = get_label_info(example['label_info'])\n species_id = tf.strings.to_number(tf.gather_nd(label_info, [0, 0]), tf.int32)\n# songtype_id = tf.strings.to_number(tf.gather_nd(label_info, [0, 1]), tf.int32)\n tmin = tf.strings.to_number(tf.gather_nd(label_info, [0, 2]))\n# fmin = tf.strings.to_number(tf.gather_nd(label_info, [0, 3]))\n tmax = tf.strings.to_number(tf.gather_nd(label_info, [0, 4]))\n# fmax = tf.strings.to_number(tf.gather_nd(label_info, [0, 5]))\n is_tp = tf.strings.to_number(tf.gather_nd(label_info, [0, 6]), tf.int32)\n \n if labeled:\n audio = crop_audio(audio, tmin, tmax)\n \n audio = random_crop_audio(audio) \n \n features = {'audio_wav': audio, \n 'recording_id': example['recording_id'], \n# 'species_id': species_id, \n 'species_id': tf.one_hot(tf.cast(species_id, tf.int32), N_CLASSES), \n 'is_tp': is_tp\n }\n return features\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n \"\"\"\n Load and parse the TFRecords.\n \"\"\"\n ignore_order = tf.data.Options()\n if not ordered:\n ignore_order.experimental_deterministic = False\n dataset = tf.data.Dataset.list_files(filenames)\n dataset = dataset.interleave(tf.data.TFRecordDataset, num_parallel_calls=AUTO)\n else:\n dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n dataset = dataset.with_options(ignore_order) \n dataset = dataset.map(lambda x: read_tfrecord(x, labeled=labeled), num_parallel_calls=AUTO)\n return dataset\n\ndef conf_output(sample, labeled=True):\n \"\"\"\n Configure the output of the dataset.\n \"\"\"\n output = ({'input_audio': sample['audio_wav']}, sample['species_id'])\n return output\n\ndef _filtterTP(example):\n return example['is_tp'] == 1\n\ndef get_dataset(filenames, labeled=True, ordered=False, repeated=False, augment=False):\n \"\"\"\n 1. Load TFRecord files, parse and generate features (waveform and meta-data).\n 2. Filter the dataset to contain only true positive samples.\n 3. Create 'spectrogram' from the 'waveform'.\n 4. Prepare image for the model.\n 5. Configure data to have the expected output format.\n 6. Apply Tensorflow data functions to optimize training.\n \n Returns a Tensorflow dataset ready for training or inference.\n \"\"\"\n dataset = load_dataset(filenames, labeled=labeled)\n \n if labeled:\n dataset = dataset.filter(_filtterTP)\n \n dataset = dataset.map(get_spectrogram_tf, num_parallel_calls=AUTO)\n dataset = dataset.map(prepare_sample, num_parallel_calls=AUTO)\n \n if augment:\n dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n \n dataset = dataset.map(lambda x: conf_output(x, labeled=labeled), num_parallel_calls=AUTO)\n \n if not ordered:\n dataset = dataset.shuffle(256)\n if repeated:\n dataset = dataset.repeat()\n \n dataset = dataset.batch(BATCH_SIZE)\n dataset = dataset.prefetch(AUTO)\n return dataset","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":0.071773,"end_time":"2020-12-15T16:31:24.017413","exception":false,"start_time":"2020-12-15T16:31:23.94564","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualization utility functions\ndef plot_spectrogram(spectrogram, ax):\n # Convert to frequencies to log scale and transpose so that the time is represented in the x-axis (columns).\n log_spec = np.log(spectrogram.T)\n height = log_spec.shape[0]\n X = np.arange(spectrogram.shape[0])\n Y = range(height)\n ax.pcolormesh(X, Y, log_spec)\n \ndef display_waveforms(ds, n_rows=3, n_cols=3, figsize=(20, 16)):\n n = n_rows*n_cols\n fig, axes = plt.subplots(n_rows, n_cols, figsize=figsize)\n for i, sample in enumerate(ds.take(n)):\n r = i // n_cols\n c = i % n_cols\n ax = axes[r][c]\n ax.plot(sample['audio_wav'].numpy())\n ax.set_yticks(np.arange(-1.2, 1.2, 0.2))\n label = sample['species_id'].numpy()\n recording_id = sample['recording_id'].numpy().decode()\n ax.set_title(f'{recording_id} - {label}')\n plt.show()\n \ndef display_spectrograms(ds, n_rows=3, n_cols=3, figsize=(20, 16)):\n n = n_rows*n_cols\n fig, axes = plt.subplots(n_rows, n_cols, figsize=figsize)\n for i, sample in enumerate(ds.take(n)):\n r = i // n_cols\n c = i % n_cols\n ax = axes[r][c]\n plot_spectrogram(np.squeeze(sample['audio_wav'].numpy()), ax)\n label = sample['species_id'].numpy()\n recording_id = sample['recording_id'].numpy().decode()\n ax.set_title(f'{recording_id} - {label}')\n plt.show()\n \ndef inspect_preds(features, labels, preds, n_rows=3, n_cols=2, figsize=(20, 16)):\n n = n_rows*n_cols\n fig, axes = plt.subplots(n_rows, n_cols, figsize=figsize)\n for i, (feature, label, pred) in enumerate(zip(features, labels, preds)):\n r = i // n_cols\n c = i % n_cols\n ax = axes[r][c]\n plot_spectrogram(np.squeeze(feature), ax)\n if pred == label:\n color = 'black'\n title = f'{pred} [True]'\n else:\n color = 'red'\n title = f'{pred} [False, should be {label}]'\n ax.set_title(title, fontsize=14, color=color)\n plt.show()\n \ndef display_waveforms_audio_spectrogram(ds, n_samples=1, sample_rate=48000):\n for sample in ds.take(n_samples):\n waveform = sample['audio_wav']\n label = sample['species_id'].numpy()\n recording_id = sample['recording_id'].numpy().decode()\n spectrogram = get_spectrogram(waveform)\n\n print(f'Name: {recording_id}')\n print(f'Label: {label}')\n print(f'Waveform shape: {waveform.shape}')\n print(f'Spectrogram shape: {spectrogram.shape}')\n print(f'Audio playback')\n Idisplay.display(Idisplay.Audio(waveform, rate=sample_rate))\n \n fig, axes = plt.subplots(2, figsize=(12, 8))\n timescale = np.arange(waveform.shape[0])\n axes[0].plot(timescale, waveform.numpy())\n axes[0].set_title('Waveform')\n axes[0].set_xlim([0, waveform.shape[0]])\n plot_spectrogram(spectrogram.numpy(), axes[1])\n axes[1].set_title('Spectrogram')\n plt.show()\n \ndef inspect_preds(features, labels, preds, n_rows=3, n_cols=2, figsize=(20, 16)):\n n = n_rows*n_cols\n fig, axes = plt.subplots(n_rows, n_cols, figsize=figsize)\n for i, (feature, label, pred) in enumerate(zip(features, labels, preds)):\n r = i // n_cols\n c = i % n_cols\n ax = axes[r][c]\n \n feature = tf.image.rgb_to_grayscale(feature).numpy()\n plot_spectrogram(np.squeeze(feature), ax)\n if pred == label:\n color = 'black'\n title = f'{pred} [True]'\n else:\n color = 'red'\n title = f'{pred} [False, should be {label}]'\n ax.set_title(title, fontsize=14, color=color)\n plt.show()\n \n# Model evaluation\ndef plot_metrics(history):\n fig, axes = plt.subplots(2, 1, sharex='col', figsize=(20, 8))\n axes = axes.flatten()\n \n axes[0].plot(history['loss'], label='Train loss')\n axes[0].plot(history['val_loss'], label='Validation loss')\n axes[0].legend(loc='best', fontsize=16)\n axes[0].set_title('Loss')\n axes[0].axvline(np.argmin(history['loss']), linestyle='dashed')\n axes[0].axvline(np.argmin(history['val_loss']), linestyle='dashed', color='orange')\n \n axes[1].plot(history['categorical_accuracy'], label='Train accuracy')\n axes[1].plot(history['val_categorical_accuracy'], label='Validation accuracy')\n axes[1].legend(loc='best', fontsize=16)\n axes[1].set_title('Accuracy')\n axes[1].axvline(np.argmax(history['categorical_accuracy']), linestyle='dashed')\n axes[1].axvline(np.argmax(history['val_categorical_accuracy']), linestyle='dashed', color='orange')\n\n plt.xlabel('Epochs', fontsize=16)\n sns.despine()\n plt.show()","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.092055,"end_time":"2020-12-15T16:31:24.144293","exception":false,"start_time":"2020-12-15T16:31:24.052238","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.116466,"end_time":"2020-12-15T16:36:40.857129","exception":false,"start_time":"2020-12-15T16:36:40.740663","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def model_fn(input_shape, N_CLASSES):\n inputs = L.Input(shape=input_shape, name='input_audio')\n base_model = efn.EfficientNetB2(input_tensor=inputs, \n include_top=False, \n weights='noisy-student')\n\n x = L.GlobalAveragePooling2D()(base_model.output)\n x = L.Dropout(.5)(x)\n output = L.Dense(N_CLASSES, activation='softmax', name='output')(x)\n \n model = Model(inputs=inputs, outputs=output)\n\n return model","metadata":{"papermill":{"duration":0.132119,"end_time":"2020-12-15T16:36:41.105573","exception":false,"start_time":"2020-12-15T16:36:40.973454","status":"completed"},"tags":[],"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"papermill":{"duration":0.118064,"end_time":"2020-12-15T16:36:41.348655","exception":false,"start_time":"2020-12-15T16:36:41.230591","status":"completed"},"tags":[]}},{"cell_type":"code","source":"skf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=seed)\noof_pred = []; oof_labels = []; history_list = [];\n\nfor fold,(idxT, idxV) in enumerate(skf.split(np.arange(32))):\n if fold >= FOLDS_USED:\n break\n print(f'\\nFOLD: {fold+1}')\n print(f'TRAIN: {idxT} VALID: {idxV}')\n\n # Create train and validation sets\n TRAIN_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/tfrecords/train/%.2i*.tfrec' % x for x in idxT])\n VALID_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/tfrecords/train/%.2i*.tfrec' % x for x in idxV])\n np.random.shuffle(TRAINING_FILENAMES)\n ct_train = count_data_items(TRAIN_FILENAMES)\n steps_per_epoch = 64 #(ct_train // BATCH_SIZE)\n\n ## MODEL\n with strategy.scope():\n model = model_fn((None, None, CHANNELS), N_CLASSES)\n\n model.compile(optimizer=tfa.optimizers.RectifiedAdam(lr=LEARNING_RATE, \n min_lr=1e-8, \n total_steps=int(steps_per_epoch*EPOCHS), \n warmup_proportion=0.2), \n loss=losses.CategoricalCrossentropy(label_smoothing=.3), \n metrics=[metrics.CategoricalAccuracy()])\n\n es = EarlyStopping(monitor='val_categorical_accuracy', mode='max', \n patience=ES_PATIENCE, restore_best_weights=True, verbose=1)\n\n ## TRAIN\n history = model.fit(x=get_dataset(TRAIN_FILENAMES, labeled=True, ordered=False, repeated=True, augment=True), \n validation_data=get_dataset(VALID_FILENAMES, labeled=True, ordered=True), \n steps_per_epoch=steps_per_epoch, \n epochs=EPOCHS, \n callbacks=[es], \n verbose=2).history\n \n history_list.append(history)\n\n # Save last model weights\n model.save_weights(f'model_{fold}.h5')\n\n # OOF predictions\n ds_valid = get_dataset(VALID_FILENAMES, labeled=True, ordered=True, repeated=False)\n oof_labels.append(np.argmax([target.numpy() for x, target in iter(ds_valid.unbatch())], axis=-1))\n x_oof = ds_valid.map(lambda x, target: x)\n oof_pred.append(np.argmax(model.predict(x_oof), axis=-1))\n\n ## RESULTS\n print(f\"#### Accuracy = {np.max(history['val_categorical_accuracy']):.3f}\")","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":4883.276789,"end_time":"2020-12-15T17:58:04.742648","exception":false,"start_time":"2020-12-15T16:36:41.465859","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model loss graph","metadata":{"papermill":{"duration":0.157888,"end_time":"2020-12-15T17:58:05.058708","exception":false,"start_time":"2020-12-15T17:58:04.90082","status":"completed"},"tags":[]}},{"cell_type":"code","source":"for fold, history in enumerate(history_list):\n print(f'\\nFOLD: {fold+1}')\n plot_metrics(history)","metadata":{"_kg_hide-input":true,"papermill":{"duration":3.358115,"end_time":"2020-12-15T17:58:08.575321","exception":false,"start_time":"2020-12-15T17:58:05.217206","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model evaluation\n\nNow we can evaluate the performance of the model, first, we can evaluate the usual metrics like, `accuracy`, `precision`, `recall`, and `f1-score`, `scikit-learn` provides the perfect function for this `classification_report`.\n\n## OOF metrics","metadata":{"papermill":{"duration":0.166729,"end_time":"2020-12-15T17:58:08.923208","exception":false,"start_time":"2020-12-15T17:58:08.756479","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(f\"Average OOF accuracy: {np.mean([np.max(history['val_categorical_accuracy']) for history in history_list]):.3f}\")\ny_true = np.concatenate(oof_labels)\ny_pred = np.concatenate(oof_pred)\n\nprint(classification_report(y_true, y_pred))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Confusion matrix\n\nLet's also take a look at the confusion matrix, this will give us an idea about what classes the model is mixing or having a hard time.","metadata":{"papermill":{"duration":0.170907,"end_time":"2020-12-15T17:58:10.058673","exception":false,"start_time":"2020-12-15T17:58:09.887766","status":"completed"},"tags":[]}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(20, 12))\ncfn_matrix = confusion_matrix(y_true, y_pred)\ncfn_matrix = (cfn_matrix.T / cfn_matrix.sum(axis=1)).T\ndf_cm = pd.DataFrame(cfn_matrix)\nax = sns.heatmap(df_cm, cmap='Blues', annot=True, fmt='.2f', linewidths=.5)\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-12-15T17:58:10.461706Z","iopub.status.busy":"2020-12-15T17:58:10.460514Z","iopub.status.idle":"2020-12-15T17:58:11.007202Z","shell.execute_reply":"2020-12-15T17:58:11.007903Z"},"papermill":{"duration":0.759856,"end_time":"2020-12-15T17:58:11.0081","exception":false,"start_time":"2020-12-15T17:58:10.248244","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize predictions\n\nFinally, it is a good practice to always inspect some of the model's prediction by looking at the data, this can give an idea if the model is getting some predictions wrong because the data is really hard, or if it is because the model is actually bad.","metadata":{"papermill":{"duration":0.173347,"end_time":"2020-12-15T17:58:11.3631","exception":false,"start_time":"2020-12-15T17:58:11.189753","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_dataset = get_dataset(TRAINING_FILENAMES, ordered=True)\n\nfor features, label in train_dataset.take(1):\n preds = np.argmax(model.predict(features['input_audio']), axis=-1)[:6]\n labels = np.argmax(label.numpy()[:6], axis=-2)\n batch_features = features['input_audio'].numpy()[:6]\n \ninspect_preds(batch_features, labels, preds)","metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-12-15T17:58:11.72601Z","iopub.status.busy":"2020-12-15T17:58:11.725119Z","iopub.status.idle":"2020-12-15T17:59:22.423348Z","shell.execute_reply":"2020-12-15T17:59:22.424004Z"},"papermill":{"duration":70.88803,"end_time":"2020-12-15T17:59:22.424177","exception":false,"start_time":"2020-12-15T17:58:11.536147","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test set predictions","metadata":{}},{"cell_type":"code","source":"model_path_list = glob.glob('/kaggle/working/*.h5')\nmodel_path_list.sort()\n\nprint('Models to predict:')\nprint(*model_path_list, sep='\\n')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_size = count_data_items(TEST_FILENAMES)\ntest_preds = np.zeros((test_size, N_CLASSES))\n\nfor model_path in model_path_list:\n print(model_path)\n K.clear_session()\n model.load_weights(model_path)\n \n test_ds = get_dataset(TEST_FILENAMES, labeled=False, ordered=True, repeated=False).repeat()\n ct_steps = TTA_STEPS * ((test_size/BATCH_SIZE) + 1)\n x_test = test_ds.map(lambda features, recording_id: features['input_audio'])\n preds = model.predict(x_test, steps=ct_steps, verbose=1)[:(test_size * TTA_STEPS)]\n preds = np.mean(preds.reshape(test_size, TTA_STEPS, N_CLASSES, order='F'), axis=1)\n test_preds += preds / len(model_path_list)\n\nnames_test_ds = load_dataset(TEST_FILENAMES, labeled=False, ordered=True)\nnames = [features['recording_id'].numpy().decode('utf-8') for features in iter(names_test_ds)]","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({'recording_id': names})\nfor column in range(N_CLASSES):\n submission[f's{column}'] = test_preds[:, column]\n\nsubmission.to_csv('submission.csv', index=False)\ndisplay(submission.head(10))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predicted classes distribution","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(18, 8))\nax = sns.countplot(y=test_preds.argmax(axis=-1), palette='viridis')\nax.tick_params(labelsize=16)\n\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]}]}