{"cells":[{"metadata":{},"cell_type":"markdown","source":"# My Training Script\nThis is the training script I used to approach the competition. If you have any questions, feel free to reach out and ask."},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n\nimport pylab as pl\nimport numpy as np\nimport pandas as pd\nimport librosa as lb\nimport tensorflow as tf\nimport tensorflow.keras as keras\nimport tensorflow.keras.layers as layers\n\nfrom tqdm import tqdm\nfrom random import shuffle, randrange, sample\nfrom sklearn.metrics import label_ranking_average_precision_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"annotations = pd.read_csv(\"../input/rfcx-species-audio-detection/train_tp.csv\")\nannotations_nl = pd.read_csv(\"../input/rfcx-species-audio-detection/train_fp.csv\")\ncrop_length = 10\n\nsample_rate = 48000\nn_mels = 384\n\nf_min = 50\nf_max = sample_rate // 2\n\nmels_per_bin = 2595 * tf.math.log(1. + f_max/700.) / n_mels\n\nmodels = { 'inception': keras.applications.InceptionV3,\n           'efficientnet': keras.applications.EfficientNetB3,\n           'resnet': keras.applications.ResNet50 }","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Preprocessing\nChanging the input data from 1D sound signals to 2D Mel Spectrograms, cropped to where the labels are located along the time (x) axis, and label gathering.\n\nThe 1D sound signals are being converted to Mel Spectrograms because neural network architectures in image classification are well established while the data conversion doesn't lead to any loss of information, so it makes sense to use better tools for what is essentially the same data.\n\nWe're cropping the input images to the locations of the labels because feeding instead a 60-second clip would provide too much information for the model to learn properly what is what. e.g. if the label is tf.one_hot([0, 3, 15, 23], 24) given the entire clip, then distinguishing which part is which class is much more difficult than feeding the model each instance of the label directly."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_labels_numpy(recording_id, start_time, end_time, annotations):\n    info = np.array(annotations.loc[annotations['recording_id'] == recording_id])\n    \n    total_labels = np.zeros((24,))\n    if info.shape[0] == 0:\n        return total_labels\n    \n    label_species_ids = info[:,1]\n    label_start_times = info[:,3]\n    label_end_times = info[:,5]\n    for i, s, e in zip(label_species_ids, label_start_times, label_end_times):\n        if e > start_time and s < end_time:\n            total_labels += tf.one_hot(i, 24)\n            \n    total_labels = tf.minimum(total_labels, 1)\n    \n    return tf.cast(total_labels, tf.float32)\n\ndef get_labels_validation_numpy(recording_id, times, frequencies, annotations):\n    info = np.array(annotations.loc[annotations['recording_id'] == recording_id])\n    \n    start_time, end_time = times\n    lower_freq, upper_freq = frequencies\n\n    total_labels = np.zeros((24,))\n    if info.shape[0] == 0:\n        return total_labels\n    \n    label_species_ids = info[:,1]\n    label_lower_freqs = info[:,2]\n    label_start_times = info[:,3]\n    label_upper_freqs = info[:,4]\n    label_end_times = info[:,5]\n    for i, lf, st, uf, et in zip(label_species_ids,\n                                 label_lower_freqs,\n                                 label_start_times,\n                                 label_upper_freqs,\n                                 label_end_times):\n        if et > start_time and st < end_time and uf > lower_freq and lf < upper_freq:\n            total_labels += tf.one_hot(i, 24)\n            \n    total_labels = tf.minimum(total_labels, 1)\n    \n    return tf.cast(total_labels, tf.float32)\n\ndef frequency_to_mel_bin(frequency):\n    mel = 2595. * tf.math.log(1. + frequency/700.)\n\n    return int(mel / mels_per_bin)\n\ndef mel_bin_to_frequency(mel_bin):\n    freq = mel_bin * mels_per_bin\n    freq = 700 * (tf.math.exp(freq/700.) - 1)\n\n    return freq\n\n\n# load a signal and convert it to a mel-spectrogram\n# NOTE: crop_length is given in seconds, all signals are 60 seconds long\ndef load_spec(filename, index=-1, crop_length=crop_length):\n    # signal.shape == [<signal_length>,]\n    signal, _ = lb.load(filename, sr=sample_rate)\n    \n    filename = filename[-14:-5]\n    label = annotations.loc[annotations['recording_id'] == filename]\n\n    if index == -1:\n        label_start = label['t_min']\n        idx = randrange(label_start.shape[0])\n        freq_min = float(label['f_min'].iloc[idx])\n        label_start = int(label_start.iloc[idx])\n        label_end = label['t_max']\n        freq_max = float(label['f_max'].iloc[idx])\n        freq_max = freq_max + randrange(f_max-int(freq_max))\n        label_end = int(label_end.iloc[idx])\n\n        label_length = label_end - label_start\n    else:\n        freq_max = f_max\n\n    m = signal.shape[0]\n    \n    # helper functions for converting to and from signal length to seconds\n    ratio = lambda x: int(x/60 * m)\n    anti_ratio = lambda x: x / m * 60\n    \n    # randomly crop each signal at a point where the label is to be found,\n    # with a slight offset\n    # TODO: is this offset helping?\n    if index == -1:\n        start_offset = randrange(ratio(crop_length - (crop_length/4)))\n        start = max(ratio(label_start) - start_offset, 1)\n    else:\n        start = ratio(index)\n    \n    end = start + ratio(crop_length)\n    \n    signal = signal[start:end]\n    \n    melspec = lb.feature.melspectrogram(signal, sr=sample_rate, n_mels=n_mels, fmin=f_min, fmax=f_max)\n    melspec = lb.power_to_db(melspec).astype(np.float32)\n\n    # this function should honestly just return the labels themselves\n    # instead of the times at which the labels are found\n    start, end = anti_ratio(start), anti_ratio(end)\n    \n    if index == -1:\n        return melspec, start, end\n    else:\n        return melspec\n\n# see https://arxiv.org/pdf/1710.09412.pdf\ndef mixup(inp, targ):\n    indice = tf.range(len(inp))\n    indice = tf.random.shuffle(indice)\n    sinp = tf.gather(inp, indice, axis=0)\n    starg = tf.gather(targ, indice, axis=0)\n    \n    alpha = 0.1\n    t = tf.compat.v1.distributions.Beta(alpha, alpha).sample([len(inp)])\n    tx = tf.reshape(t, [-1, 1, 1, 1])\n    ty = tf.reshape(t, [-1, 1])\n    x = inp * tx + sinp * (1-tx)\n    y = targ * ty + starg * (1-ty)\n\n    return x, y\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset Generation\nActual making of the TensorFlow Datasets using generators to fetch the data. The created datasets are then divided into five different folds. If you're unfamiliar with k-fold validation, see https://en.wikipedia.org/wiki/Cross-validation_(statistics).\n\nSomething I tried to do for some cross-validation and leaderboard correlation was to similarize the validation and inference methods i.e. validation follows inference in slicing and stacking each image then taking the max prediction for each slice to get the prediction for the entire image."},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 8\n\ntrain_path = \"../input/rfcx-species-audio-detection/train/\"\n#train_path = \"../input/rcfx-spectrograms-32-khz/train/\"\n# only training on the signals which contain true positive labels\ntrain_files = [s for s in annotations['recording_id']]\nclass_count = 24\n\ndef generate_dataset(batch_size, fold, train_files, validation_files):\n    def train_generator():\n        for i in range(len(train_files)):\n            image, start, end, recording_id = get_data_index(i)\n            images = tf.expand_dims(image, -1)\n            images = augment(images)\n\n            labels_pl = get_labels_numpy(recording_id, start, end, annotations)\n            labels_nl = get_labels_numpy(recording_id, start, end, annotations_nl)\n\n            labels = labels_pl\n            \n            yield images, labels\n            \n    def validation_generator():\n        for i in range(len(validation_files)):\n            recording_id = validation_files[i].split('/')[-1][:-5]\n\n            images = []\n            labels = []\n            image = tf.expand_dims(load_spec(validation_files[i], 0, 60), axis=-1)\n\n            x_step = (image.shape[1]//60*(crop_length))\n            for j in range(0, image.shape[1]-(image.shape[1]//60*crop_length), x_step):\n                im = image[:,j:j+x_step]\n                im = augment(im, training=False)\n                images.append(im)\n\n                labels.append(tf.cast(get_labels_numpy(recording_id,\n                                                       j/image.shape[1]*60, (j+x_step)/image.shape[1]*60,\n                                                       annotations), tf.float32))\n\n            images = tf.stack(images)\n            labels = tf.stack(labels)\n\n            yield images, labels\n            \n            \n    train_dataset = tf.data.Dataset.from_generator(train_generator, output_types=(tf.float32,\n                                                                                  tf.float32)).repeat().batch(batch_size).map(mixup)\n\n    validation_dataset = tf.data.Dataset.from_generator(validation_generator, output_types=(tf.float32,\n                                                                                            tf.float32))#.batch(batch_size)\n    \n    return train_dataset, validation_dataset\n\n# Creating each fold\n# Note that this is deterministic and could likely be improved by shuffling\n# the lists containing each class before selecting recording_ids for the folds\nfolds = 5\ndatasets = []\nfor fold in range(folds):\n    train_files = []\n    validation_files = []\n    for c in range(class_count):\n        class_records = annotations.loc[annotations['species_id'] == c]['recording_id']\n        class_ratio = class_records.shape[0] // folds \n        class_records = list(class_records)\n        train_files += class_records[:class_ratio*fold] + class_records[class_ratio*(fold+1):]\n        validation_files += class_records[class_ratio*fold:class_ratio*(fold+1)]\n        \n    shuffle(train_files)\n    shuffle(validation_files)\n    \n    train_files = [train_path+s+\".flac\" for s in train_files]\n    validation_files = [train_path+s+\".flac\" for s in validation_files]\n        \n    datasets.append(generate_dataset(BATCH_SIZE, fold, train_files, validation_files))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Augmentations\nVarious augmentations to improve the generalization of the model. I wasn't able to find any success in using them in this notebook."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# size in seconds\n# images includes batch dimension\ndef time_dropout(images, cuts=3, size=0.1):\n    cut_size = int(images.shape[2] * size)\n    images = images.numpy()\n    for c in range(cuts):\n        begin = randrange(images.shape[2]-cut_size)\n        end = begin + cut_size\n        \n        images[:,:,begin:end] = np.zeros(list(images.shape[:2])+[end-begin])\n        \n    return images\n\ndef frequency_masking(mel_spectrogram):\n    \n    frequency_masking_para = 80, \n    frequency_mask_num = 2\n    \n    fbank_size = tf.shape(mel_spectrogram)\n#     print(fbank_size)\n    n, v = fbank_size[0], fbank_size[1]\n\n    for i in range(frequency_mask_num):\n        f = tf.random.uniform([], minval=0, maxval=tf.squeeze(frequency_masking_para), dtype=tf.int32)\n        v = tf.cast(v, dtype=tf.int32)\n        f0 = tf.random.uniform([], minval=0, maxval=tf.squeeze(v-f), dtype=tf.int32)\n\n        # warped_mel_spectrogram[f0:f0 + f, :] = 0\n        mask = tf.concat((tf.ones(shape=(n, v - f0 - f,1)),\n                          tf.zeros(shape=(n, f,1)),\n                          tf.ones(shape=(n, f0,1)),\n                          ),1)\n        mel_spectrogram = mel_spectrogram * mask\n    return tf.cast(mel_spectrogram, dtype=tf.float32)\n\n\ndef time_masking(mel_spectrogram):\n    time_masking_para = 40, \n    time_mask_num = 1\n    \n    fbank_size = tf.shape(mel_spectrogram)\n    n, v = fbank_size[0], fbank_size[1]\n\n   \n    for i in range(time_mask_num):\n        t = tf.random.uniform([], minval=0, maxval=tf.squeeze(time_masking_para), dtype=tf.int32)\n        t0 = tf.random.uniform([], minval=0, maxval=n-t, dtype=tf.int32)\n\n        # mel_spectrogram[:, t0:t0 + t] = 0\n        mask = tf.concat((tf.ones(shape=(n-t0-t, v,1)),\n                          tf.zeros(shape=(t, v,1)),\n                          tf.ones(shape=(t0, v,1)),\n                          ), 0)\n        \n        mel_spectrogram = mel_spectrogram * mask\n    return tf.cast(mel_spectrogram, dtype=tf.float32)\n\n\ndef random_brightness(image):\n    return tf.image.random_brightness(image, 0.2)\n\ndef random_gamma(image):\n    return tf.image.random_contrast(image, lower=0.1, upper=0.3)\n\ndef random_flip_right(image):\n    return tf.image.random_flip_left_right(image)\n\ndef random_flip_up_down(image):\n    return tf.image.random_flip_left_right(image)\n\navailable_ops = [\n          #frequency_masking,\n          #time_masking, \n          #random_brightness, \n          #random_flip_up_down,\n          #random_flip_right \n         ]\n\ndef apply_augmentation(image):\n    num_layers = int(np.random.uniform(low=0, high=3))\n    \n    for layer_num in range(num_layers):\n        op_to_select = tf.random.uniform([], maxval=len(available_ops), dtype=tf.int32, seed=1)\n        for (i, op_name) in enumerate(available_ops):\n            image = tf.cond(\n            tf.equal(i, op_to_select),\n            lambda selected_func=op_name,: selected_func(\n                image),\n            lambda: image)\n    return image\n\ndef augment(images, training=True):\n    #if training:\n    #    images = apply_augmentation(images)\n        \n    images = tf.image.grayscale_to_rgb(images)\n    images = tf.image.per_image_standardization(images)\n    images = tf.image.resize(images, (n_mels//2, int(n_mels)))\n    \n    return images\n\ndef get_data_index(index):\n    s = train_files[index].split('/')[-1][:-5]\n    f, start, end = load_spec(train_files[index])\n    \n    return f, start, end, s\n\ndef get_validation_data(index):\n    f, start, end = load_spec(validation_files[index])\n    s = validation_files[index].split('/')[-1][:-5]\n\n    return f, start, end, s","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Definition\nAfter being sent through the model, the output is averaged across the frequency-axis (y-axis) then sent through an attention mechanism before the final predictions are made."},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, _ in datasets[0][0].take(1):\n    spec_shape = i[0].shape\n    print(spec_shape)\n\nfor _ in datasets[0][1].take(1):\n    print(_[0].shape)\n\ninput_shape = (spec_shape[0], spec_shape[1], 3)\ndef get_model(model='inception'):\n    backbone = models[model](include_top=False, weights='imagenet', pooling=None, input_shape=input_shape)\n            \n    inp = keras.Input(input_shape)\n    \n    x = backbone(inp)\n    x = tf.math.reduce_mean(x, axis=1)\n    x = layers.Dropout(rate=0.25)(x)\n    norm_att = tf.keras.layers.Conv1D(class_count, 1)(x)\n    norm_att = tf.nn.softmax(tf.clip_by_value(norm_att, -10, 10), axis=1)\n    conv = tf.keras.layers.Conv1D(class_count, 1)(x)\n    x = tf.math.reduce_sum(norm_att * conv, axis=1)\n    x = tf.nn.sigmoid(x)\n    \n    return tf.keras.Model(inp, x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Competition metric\nSee https://stackoverflow.com/questions/55881642/how-to-interpret-label-ranking-average-precision-score.\n \nCode here taken from https://www.kaggle.com/ashusma/training-rfcx-tensorflow-tpu-effnet-b2#Competition-Metric"},{"metadata":{"trusted":true},"cell_type":"code","source":"def _one_sample_positive_class_precisions(example):\n    y_true, y_pred = example\n    y_true = tf.reshape(y_true, tf.shape(y_pred))\n    retrieved_classes = tf.argsort(y_pred, direction='DESCENDING')\n    class_rankings = tf.argsort(retrieved_classes)\n    retrieved_class_true = tf.gather(y_true, retrieved_classes)\n    retrieved_cumulative_hits = tf.math.cumsum(tf.cast(retrieved_class_true, tf.float32))\n\n    idx = tf.where(y_true)[:, 0]\n    i = tf.boolean_mask(class_rankings, y_true)\n    r = tf.gather(retrieved_cumulative_hits, i)\n    c = 1 + tf.cast(i, tf.float32)\n    precisions = r / c\n\n    dense = tf.scatter_nd(idx[:, None], precisions, [y_pred.shape[0]])\n    return dense\n\nclass LWLRAP(tf.keras.metrics.Metric):\n    def __init__(self, num_classes, name='lwlrap'):\n        super().__init__(name=name)\n\n        self._precisions = self.add_weight(\n            name='per_class_cumulative_precision',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n        self._counts = self.add_weight(\n            name='per_class_cumulative_count',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        y_true = y_true[:,:class_count]\n        y_pred = y_pred[:,:class_count]\n\n        y_true = tf.math.reduce_max(y_true, axis=0, keepdims=True)\n        y_pred = tf.math.reduce_max(y_pred, axis=0, keepdims=True)\n\n        precisions = tf.map_fn(\n            fn=_one_sample_positive_class_precisions,\n            elems=(y_true, y_pred),\n            dtype=(tf.float32),\n        )\n\n        increments = tf.cast(precisions > 0, tf.float32)\n        total_increments = tf.reduce_sum(increments, axis=0)\n        total_precisions = tf.reduce_sum(precisions, axis=0)\n\n        self._precisions.assign_add(total_precisions)\n        self._counts.assign_add(total_increments)        \n\n    def result(self):\n        per_class_lwlrap = self._precisions / tf.maximum(self._counts, 1.0)\n        per_class_weight = self._counts / tf.reduce_sum(self._counts)\n        overall_lwlrap = tf.reduce_sum(per_class_lwlrap * per_class_weight)\n        return overall_lwlrap\n\n    def reset_states(self):\n        self._precisions.assign(self._precisions * 0)\n        self._counts.assign(self._counts * 0)\n\n\nclass Precision(keras.metrics.Metric):\n    def __init__(self):\n        super().__init__()\n\n        self.met = keras.metrics.Precision()\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        self.met.update_state(y_true[:,:class_count], y_pred[:,:class_count], sample_weight)\n\n    def result(self):\n        return self.met.result()\n\n    def reset_states(self):\n        self.met.reset_states()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Learning Rate Schedule\nA changing learning rate over time typically performs better than a stagnant one. Here we're using the schedule used to train the [Transformer model](https://arxiv.org/pdf/1706.03762.pdf%EF%BC%89%E6%8F%8F%E8%BF%B0%E4%BA%86%E8%BF%99%E6%A0%B7%E5%81%9A%E7%9A%84%E5%8E%9F%E5%9B%A0%E3%80%82)."},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 30\nsteps_per_epoch = len(train_files)//BATCH_SIZE\nlearning_rate_base = 1.5e-3\ntotal_steps = steps_per_epoch * epochs\nwarmup_learning_rate = 1e-5\nwarmup_steps = (epochs // 10) * steps_per_epoch\n\n\n@tf.function\ndef cosine_decay_with_warmup(global_step,\n                             hold_base_rate_steps=0):\n\n    if total_steps < warmup_steps:\n        raise ValueError('total_steps must be larger or equal to '\n                     'warmup_steps.')\n    learning_rate = 0.5 * learning_rate_base * (1 + tf.cos(\n        np.pi *\n        (tf.cast(global_step, tf.float32) - warmup_steps - hold_base_rate_steps\n        ) / float(total_steps - warmup_steps - hold_base_rate_steps)))\n    if hold_base_rate_steps > 0:\n        learning_rate = tf.where(\n          global_step > warmup_steps + hold_base_rate_steps,\n          learning_rate, learning_rate_base)\n    if warmup_steps > 0:\n        if learning_rate_base < warmup_learning_rate:\n            raise ValueError('learning_rate_base must be larger or equal to '\n                         'warmup_learning_rate.')\n        slope = (learning_rate_base - warmup_learning_rate) / warmup_steps\n        warmup_rate = slope * tf.cast(global_step,\n                                    tf.float32) + warmup_learning_rate\n        learning_rate = tf.where(global_step < warmup_steps, warmup_rate,\n                               learning_rate)\n    return tf.where(global_step > total_steps, 0.0, learning_rate,\n                    name='learning_rate')\n\nclass LRSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n    def __call__(self, step):\n        return cosine_decay_with_warmup(step)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training\nNested loop for training all five folds for each model. The loop is wrapped in a function to make sure this notebook can submit; otherwise it would take too long and time out."},{"metadata":{"trusted":true},"cell_type":"code","source":"lf = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.2, reduction=tf.keras.losses.Reduction.NONE)\nloss = lambda target, logits: tf.nn.compute_average_loss(lf(target, logits))\n\nsubmissions = []\n\ndef train():\n    for m in models.keys():\n        for i, fold in enumerate(datasets):\n            print(f\"FOLD {i}\")\n\n            train_dataset, validation_dataset = fold\n\n            model = get_model(m)\n            model.compile(optimizer=keras.optimizers.Adam(learning_rate=LRSchedule()),\n                          loss=loss,\n                          metrics=[LWLRAP(24)])\n\n            checkpointing = tf.keras.callbacks.ModelCheckpoint(\n                                f\"fold_{m}_{i}.h5\", monitor='val_loss', verbose=0, save_best_only=False,\n                                save_weights_only=True, mode='min', save_freq='epoch')\n\n            # Train the model\n            model.fit(train_dataset,\n                      validation_data=validation_dataset,\n                      epochs=epochs,\n                      steps_per_epoch=steps_per_epoch,\n                      callbacks=[checkpointing])\n\n            # Use the trained model to create the submission for this fold\n            test_filepath = \"data/raw/test/\"\n            test_files = [test_filepath+s for s in os.listdir(test_filepath)]\n\n            submission = { 'recording_id': [] }\n            for x in range(24):\n                submission[f's{x}'] = []\n\n            for filepath in tqdm(test_files):\n                recording_id = filepath.split('/')[-1][:-5]\n                logitses = []\n                inp = []\n\n                # For inference, we evaluate the model on each {crop_length}-second segment of the signal\n                # and take the maximum of each of its predictions \n                # for a given class to make the image-wide prediction.\n                #\n                # Each batch consists of a single image, sliced and stacked.\n                image = tf.expand_dims(load_spec(filepath, 0, 60), axis=-1)\n                x_step = (image.shape[1]//60*(crop_length))\n                for j in range(0, image.shape[1]-(image.shape[1]//60*crop_length), x_step):\n                    im = image[:,j:j+x_step]\n                    im = tf.image.resize(im, (n_mels//2, int(n_mels)))\n                    inp.append(im)\n\n                inp = tf.stack(inp)\n                inp = augment(inp, training=False)\n                logitses = tf.squeeze(model(inp, training=False))\n                logits = tf.math.reduce_max(logitses, axis=0).numpy()[:class_count]\n\n                submission['recording_id'].append(recording_id)\n                for x, species in enumerate(logits):\n                    submission[f's{x}'].append(species)\n\n            submissions.append(submission)\n\n            pd.DataFrame.from_dict(submission).to_csv(f\"submission_{m}_{i}.csv\", index=False)\n\n        # Merge the submissions from each fold by using the average prediction for each recording_id\n        submissions = [pd.DataFrame.from_dict(sub) for sub in submissions]\n        submission_values = np.mean([np.array(sub.drop('recording_id', axis=1)) for sub in submissions], axis=0)\n        submission = submissions[0]\n        submission[[f's{i}' for i in range(class_count)]] = submission_values\n\n        submission.to_csv(f\"submission_{m}.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}