{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pickle\nfrom sklearn.preprocessing import MultiLabelBinarizer\nimport matplotlib.pyplot as plt\nimport librosa\nimport librosa.display\nfrom scipy import signal\nimport random\nfrom sklearn.model_selection import StratifiedKFold\nimport sklearn\nimport math\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input/freesound-audio-tagging-2019\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Author: Trent J. Bradberry <trentjason@hotmail.com>\n# License: BSD 3 clause\n\nimport numpy as np\n\nfrom sklearn.utils import check_random_state\nfrom sklearn.utils.validation import _num_samples, check_array\nfrom sklearn.utils.multiclass import type_of_target\n\nfrom sklearn.model_selection._split import _BaseKFold, _RepeatedSplits, \\\n    BaseShuffleSplit, _validate_shuffle_split\n\n\ndef IterativeStratification(labels, r, random_state):\n    \"\"\"This function implements the Iterative Stratification algorithm described\n    in the following paper:\n    Sechidis K., Tsoumakas G., Vlahavas I. (2011) On the Stratification of\n    Multi-Label Data. In: Gunopulos D., Hofmann T., Malerba D., Vazirgiannis M.\n    (eds) Machine Learning and Knowledge Discovery in Databases. ECML PKDD\n    2011. Lecture Notes in Computer Science, vol 6913. Springer, Berlin,\n    Heidelberg.\n    \"\"\"\n\n    n_samples = labels.shape[0]\n    test_folds = np.zeros(n_samples, dtype=int)\n\n    # Calculate the desired number of examples at each subset\n    c_folds = r * n_samples\n\n    # Calculate the desired number of examples of each label at each subset\n    c_folds_labels = np.outer(r, labels.sum(axis=0))\n\n    labels_not_processed_mask = np.ones(n_samples, dtype=bool)\n\n    while np.any(labels_not_processed_mask):\n        # Find the label with the fewest (but at least one) remaining examples,\n        # breaking ties randomly\n        num_labels = labels[labels_not_processed_mask].sum(axis=0)\n\n        # Handle case where only all-zero labels are left by distributing\n        # across all folds as evenly as possible (not in original algorithm but\n        # mentioned in the text). (By handling this case separately, some\n        # code redundancy is introduced; however, this approach allows for\n        # decreased execution time when there are a relatively large number\n        # of all-zero labels.)\n        if num_labels.sum() == 0:\n            sample_idxs = np.where(labels_not_processed_mask)[0]\n\n            for sample_idx in sample_idxs:\n                fold_idx = np.where(c_folds == c_folds.max())[0]\n\n                if fold_idx.shape[0] > 1:\n                    fold_idx = fold_idx[random_state.choice(fold_idx.shape[0])]\n\n                test_folds[sample_idx] = fold_idx\n                c_folds[fold_idx] -= 1\n\n            break\n\n        label_idx = np.where(num_labels == num_labels[np.nonzero(num_labels)].min())[0]\n        if label_idx.shape[0] > 1:\n            label_idx = label_idx[random_state.choice(label_idx.shape[0])]\n\n        sample_idxs = np.where(np.logical_and(labels[:, label_idx].flatten(), labels_not_processed_mask))[0]\n\n        for sample_idx in sample_idxs:\n            # Find the subset(s) with the largest number of desired examples\n            # for this label, breaking ties by considering the largest number\n            # of desired examples, breaking further ties randomly\n            label_folds = c_folds_labels[:, label_idx]\n            fold_idx = np.where(label_folds == label_folds.max())[0]\n\n            if fold_idx.shape[0] > 1:\n                temp_fold_idx = np.where(c_folds[fold_idx] ==\n                                         c_folds[fold_idx].max())[0]\n                fold_idx = fold_idx[temp_fold_idx]\n\n                if temp_fold_idx.shape[0] > 1:\n                    fold_idx = fold_idx[random_state.choice(temp_fold_idx.shape[0])]\n\n            test_folds[sample_idx] = fold_idx\n            labels_not_processed_mask[sample_idx] = False\n\n            # Update desired number of examples\n            c_folds_labels[fold_idx, labels[sample_idx]] -= 1\n            c_folds[fold_idx] -= 1\n\n    return test_folds\n\n\nclass MultilabelStratifiedKFold(_BaseKFold):\n    \"\"\"Multilabel stratified K-Folds cross-validator\n    Provides train/test indices to split multilabel data into train/test sets.\n    This cross-validation object is a variation of KFold that returns\n    stratified folds for multilabel data. The folds are made by preserving\n    the percentage of samples for each label.\n    Parameters\n    ----------\n    n_splits : int, default=3\n        Number of folds. Must be at least 2.\n    shuffle : boolean, optional\n        Whether to shuffle each stratification of the data before splitting\n        into batches.\n    random_state : int, RandomState instance or None, optional, default=None\n        If int, random_state is the seed used by the random number generator;\n        If RandomState instance, random_state is the random number generator;\n        If None, the random number generator is the RandomState instance used\n        by `np.random`. Unlike StratifiedKFold that only uses random_state\n        when ``shuffle`` == True, this multilabel implementation\n        always uses the random_state since the iterative stratification\n        algorithm breaks ties randomly.\n    Examples\n    --------\n    >>> from iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n    >>> import numpy as np\n    >>> X = np.array([[1,2], [3,4], [1,2], [3,4], [1,2], [3,4], [1,2], [3,4]])\n    >>> y = np.array([[0,0], [0,0], [0,1], [0,1], [1,1], [1,1], [1,0], [1,0]])\n    >>> mskf = MultilabelStratifiedKFold(n_splits=2, random_state=0)\n    >>> mskf.get_n_splits(X, y)\n    2\n    >>> print(mskf)  # doctest: +NORMALIZE_WHITESPACE\n    MultilabelStratifiedKFold(n_splits=2, random_state=0, shuffle=False)\n    >>> for train_index, test_index in mskf.split(X, y):\n    ...    print(\"TRAIN:\", train_index, \"TEST:\", test_index)\n    ...    X_train, X_test = X[train_index], X[test_index]\n    ...    y_train, y_test = y[train_index], y[test_index]\n    TRAIN: [0 3 4 6] TEST: [1 2 5 7]\n    TRAIN: [1 2 5 7] TEST: [0 3 4 6]\n    Notes\n    -----\n    Train and test sizes may be slightly different in each fold.\n    See also\n    --------\n    RepeatedMultilabelStratifiedKFold: Repeats Multilabel Stratified K-Fold\n    n times.\n    \"\"\"\n\n    def __init__(self, n_splits=3, shuffle=False, random_state=None):\n        super(MultilabelStratifiedKFold, self).__init__(n_splits, shuffle, random_state)\n\n    def _make_test_folds(self, X, y):\n        y = np.asarray(y, dtype=bool)\n        type_of_target_y = type_of_target(y)\n\n        if type_of_target_y != 'multilabel-indicator':\n            raise ValueError(\n                'Supported target type is: multilabel-indicator. Got {!r} instead.'.format(type_of_target_y))\n\n        num_samples = y.shape[0]\n\n        rng = check_random_state(self.random_state)\n        indices = np.arange(num_samples)\n\n        if self.shuffle:\n            rng.shuffle(indices)\n            y = y[indices]\n\n        r = np.asarray([1 / self.n_splits] * self.n_splits)\n\n        test_folds = IterativeStratification(labels=y, r=r, random_state=rng)\n\n        return test_folds[np.argsort(indices)]\n\n    def _iter_test_masks(self, X=None, y=None, groups=None):\n        test_folds = self._make_test_folds(X, y)\n        for i in range(self.n_splits):\n            yield test_folds == i\n\n    def split(self, X, y, groups=None):\n        \"\"\"Generate indices to split data into training and test set.\n        Parameters\n        ----------\n        X : array-like, shape (n_samples, n_features)\n            Training data, where n_samples is the number of samples\n            and n_features is the number of features.\n            Note that providing ``y`` is sufficient to generate the splits and\n            hence ``np.zeros(n_samples)`` may be used as a placeholder for\n            ``X`` instead of actual training data.\n        y : array-like, shape (n_samples, n_labels)\n            The target variable for supervised learning problems.\n            Multilabel stratification is done based on the y labels.\n        groups : object\n            Always ignored, exists for compatibility.\n        Returns\n        -------\n        train : ndarray\n            The training set indices for that split.\n        test : ndarray\n            The testing set indices for that split.\n        Notes\n        -----\n        Randomized CV splitters may return different results for each call of\n        split. You can make the results identical by setting ``random_state``\n        to an integer.\n        \"\"\"\n        y = check_array(y, ensure_2d=False, dtype=None)\n        return super(MultilabelStratifiedKFold, self).split(X, y, groups)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_overall_lwlrap_sklearn(truth, scores):\n    \"\"\"Calculate the overall lwlrap using sklearn.metrics.lrap.\"\"\"\n    # sklearn doesn't correctly apply weighting to samples with no labels, so just skip them.\n    sample_weight = np.sum(truth > 0, axis=1)\n    nonzero_weight_sample_indices = np.flatnonzero(sample_weight > 0)\n    overall_lwlrap = sklearn.metrics.label_ranking_average_precision_score(\n                                                                truth[nonzero_weight_sample_indices, :] > 0, \n                                                                scores[nonzero_weight_sample_indices, :], \n                                                                sample_weight=sample_weight[nonzero_weight_sample_indices])\n    return overall_lwlrap","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Models"},{"metadata":{"trusted":true},"cell_type":"code","source":"# models\nfrom keras import losses, models\nfrom keras.models import Sequential, Model\nfrom keras.layers import (Input, Conv2D, GlobalAveragePooling2D, BatchNormalization, Flatten, Dense,\n                          GlobalMaxPooling2D, MaxPooling2D, concatenate, Activation, ZeroPadding2D, Dropout, add, ReLU)\nfrom keras.callbacks import (LearningRateScheduler, ReduceLROnPlateau)\nfrom keras.optimizers import Adam, SGD\nfrom keras.regularizers import l2\n\ndef cnn_6(lr, do=0, l2reg=0):\n    \n    inp = Input(shape=(64,128,1))\n    \n    x = BatchNormalization()(inp)\n    x = ReLU()(x)\n    x = Conv2D(filters=32, kernel_size=(5,5), strides=(1,1), kernel_regularizer=l2(l2reg), padding='same')(x)\n    x = MaxPooling2D(pool_size=(3,3), strides=(2,2), padding='same')(x)\n    \n    x = BatchNormalization()(x)\n    x = ReLU()(x)\n    x = Conv2D(filters=32, kernel_size=(3,3), strides=(1,1), kernel_regularizer=l2(l2reg), padding='same')(x)\n    \n    x = BatchNormalization()(x)\n    x = ReLU()(x)\n    x = Conv2D(filters=64, kernel_size=(3,3), strides=(2,2), kernel_regularizer=l2(l2reg), padding='same')(x)\n    \n    x = BatchNormalization()(x)\n    x = ReLU()(x)\n    x = Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), kernel_regularizer=l2(l2reg), padding='same')(x)\n    \n    x = BatchNormalization()(x)\n    x = ReLU()(x)\n    x = Conv2D(filters=128, kernel_size=(3,3), strides=(2,2), kernel_regularizer=l2(l2reg), padding='same')(x)\n    \n    x = BatchNormalization()(x)\n    x = ReLU()(x)\n    x = Conv2D(filters=128, kernel_size=(3,3), strides=(1,1), kernel_regularizer=l2(l2reg), padding='same')(x)\n    \n    x = GlobalMaxPooling2D()(x)\n    #x = Dense(2048, activation='relu')(x)\n    #x = Dropout(do)(x)\n    x = Dense(1024, activation='relu')(x)\n    x = Dropout(do)(x)\n    out = Dense(80, activation='sigmoid')(x)\n\n    model = Model(inputs=inp, outputs=out)\n    opt = Adam(lr=lr)\n\n    model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=[])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creates parent-label mappings.\n#===========================================================================================================================================\n# For splitting with MultilabelStratifiedKFold (iterative stratification):\n\n# 1. \n# 'train_labels_dict': dict, stores actual labels of each parent file, where each label may contain more than one positive class.\n# key             : value\n# parent_filename : complete_label\n# '0c28d31c.wav'  : [0,0,1,0, ..., 1] (1D array of length 80, with num positive classes >= 1)\n\n# 2.\n# 'all_files': array, 1D (N), stores names of parent files. For splitting using indices by StratifiedKFold and MultilabelStratifiedKFold.\n# [parent_filename1, parent_filename2, ...]\n# [ '0006ae4e.wav',   '0019ef41.wav',  ...]\n# also used for approximate stratification.\n\n# 3.\n# 'y_split_iterative': array, 2D (N, C), stores the complete label of each file in 'all_files' (order must equal to that of all_files)\n# [ [0,0,1,...,0] \n#   [0,1,0,...,1]\n#       ...\n#   [1,0,1,...,0] ]\n#===========================================================================================================================================\n# For splitting with StratifiedKFold (approximate stratification)\n\n# 4. \n# 'labels_for_splitting': dict, stores a random positive element from each parent file's label. For stratification approximation purposes.\n# key              : value\n# parent_filename  : single_label\n# '0c28d31c.wav_0' : [0,0,1,0, ..., 0] (1D array of length 80, with num positive classes == 1)\n\n# 5. y_split: array, 2D (N, C), stores the label from labels_for_splitting (single-class labels only) of each file in 'all_files' \n# (order must equal to that of all_files)\n# [ [0,0,1,...,0] \n#   [0,1,0,...,0]\n#       ...\n#   [1,0,0,...,0] ]\n#===========================================================================================================================================\n# ACTUAL LABELS:\n# stored in train-Labels_dict\n\n# load train\ntrain = pd.read_csv(\"../input/freesound-audio-tagging-2019/train_curated.csv\") # just change this target to train_noisy if needed\nprint(train.head(10))\nprint()\n\n# binarize labels\nlabels = train['labels'].tolist()\nsplit_labels = [] # create a list of sets\nfor label in labels:\n    split_labels.append(set(label.split(',')))\nmlb = MultiLabelBinarizer()\nonehot_labels = mlb.fit_transform(split_labels)\nprint('train set labels shape:', onehot_labels.shape)\nprint('found', len(mlb.classes_), 'unique classes')\nprint()\n\n# create dict mapping between filenames and labels\ntrain_labels_dict = {}\nfor i, filename in enumerate(train['fname'].tolist()):\n    train_labels_dict[filename] = onehot_labels[i]\n\n# explore distribution of number of labels\nprint('distribution:')\nn_labels = np.sum(onehot_labels, axis=1)\nn_labels = pd.Series(n_labels)\nfor series_name, series in n_labels.groupby(n_labels):\n    print(series_name, 'labels:', len(series))\n#===========================================================================================================================================\n# LABELS FOR SPLITTING\n# stored in labels_for_splitting\n\n# split labels into sets\nlabels = train['labels'].tolist()\nsplit_labels = [] # create a list of sets\nfor label in labels:\n    split_labels.append(set(label.split(',')))\n\n# create dict mapping between filenames and labels\nlabels_complete = {}\nfor i, filename in enumerate(train['fname'].tolist()):\n    labels_complete[filename] = split_labels[i]\n\n# pick a random label for every training sample\nrandom.seed(1001)\nlabels_one = {}\nfor file, label_set in labels_complete.items():\n    labels_one[file] = random.sample(label_set, 1)\n\n# binarized version\nsingle_labels = []\nfor file, label in labels_one.items():\n    single_labels.append(set(label))\nsingle_labels = mlb.transform(single_labels)\nlabels_for_splitting = {}\nfor i, file in enumerate(train['fname'].tolist()):\n    labels_for_splitting[file] = single_labels[i]\n#===========================================================================================================================================\n# FILE NAMES LIST\n# saved in all_files\n# y_split for stratification approximation\nall_files = np.empty(len(train), dtype='U12')\ny_split = np.zeros((len(train), len(mlb.classes_)))\ni = 0\nfor file, label in labels_for_splitting.items():\n    all_files[i] = file\n    y_split[i] = label\n    i += 1\ny_split = np.argmax(y_split, axis=-1)\n#===========================================================================================================================================\n# y_split_iterative for stratification using iterative approach\ny_split_iterative = np.zeros((len(train), len(mlb.classes_)))\ni = 0\nfor file, label in train_labels_dict.items():\n    y_split_iterative[i] = label\n    i += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Stores MFCC arrays of children files.\n#============================================================================================================================================\n# 1. \n# 'training_data': list, len N, stores each child's (filename, MFCC array, label) of training data.\n# [(filename1, MFCC_array1, label1), (filename2, MFCC_array2, label2), ...]\n# shuffles the list after constructing it.\n\n# 2.\n# 'test_data': list, len N_test, stores each child's (filename, MFCC array) of test data.\n# [(filename1, MFCC_array1), (filename2, MFCC_array2), ...]\n#============================================================================================================================================\n# Config variables and utilities\n\nSAMPLE_DURATION = 3.0\nSAMPLE_RATE = 44100\nL = int(SAMPLE_RATE * SAMPLE_DURATION) # L = number of samples in a training example before padding\n\ndef random_slice(samples):\n    begin = np.random.randint(0, len(samples) - L)\n    return samples[begin : begin+L]\n\ndef pad(samples):\n    if len(samples) >= L: \n        return samples\n    else: \n        return np.pad(samples, pad_width=(L - len(samples), 0), mode='constant', constant_values=(0, 0))\n#===========================================================================================================================================\n# TRAINING DATA\n# splits each sample into 3-second chunks, and uses all of them for training data\n\ntraining_data = []\nprogress = 0\nfor file_name in os.listdir('../input/freesound-audio-tagging-2019/train_curated/'):\n    \n    # read data array indicated by 'file_name' and pick a random 1s slice within it\n    file_data, _ = librosa.core.load('../input/freesound-audio-tagging-2019/train_curated/' + file_name, sr=SAMPLE_RATE)\n    duration = librosa.core.get_duration(file_data, sr=SAMPLE_RATE)\n    \n    # split array into slices of duration SAMPLE_DURATION\n    for i in range(math.ceil(duration/SAMPLE_DURATION)):\n        begin = i*L\n        file_data_i = file_data[begin:begin+L]\n        if len(file_data_i) < L:\n            file_data_i = pad(file_data_i)\n        \n        file_array_i = librosa.feature.mfcc(file_data_i, \n                                            sr=SAMPLE_RATE, \n                                            n_fft=2560, \n                                            hop_length=347*int(SAMPLE_DURATION), \n                                            n_mels=128, \n                                            n_mfcc=64)\n        training_data.append((file_name+'_'+str(i), file_array_i, train_labels_dict[file_name]))\n    \n    # progress tracker\n    if progress%100 == 0:\n        print('processed', progress, 'files')\n    progress += 1\nprint('train data done')\n\n# shuffle order of data\nrandom.Random(1001).shuffle(training_data)\n#===========================================================================================================================================\n# do the same for test data: no need to do this for model selection\ntest = pd.read_csv(\"../input/freesound-audio-tagging-2019/sample_submission.csv\")\ntest_data = []\nprogress = 0\nfor file_name in test['fname'].tolist():\n    \n    # read data array indicated by 'file_name' and pick a random 1s slice within it\n    file_data, _ = librosa.core.load('../input/freesound-audio-tagging-2019/test/' + file_name, sr=SAMPLE_RATE)\n    duration = librosa.core.get_duration(file_data, sr=SAMPLE_RATE)\n    \n    # split array into slices of duration SAMPLE_DURATION\n    for i in range(math.ceil(duration/SAMPLE_DURATION)):\n        begin = i*L\n        file_data_i = file_data[begin:begin+L]\n        if len(file_data_i) < L:\n            file_data_i = pad(file_data_i)\n        \n        file_array_i = librosa.feature.mfcc(file_data_i, \n                                            sr=SAMPLE_RATE, \n                                            n_fft=2560, \n                                            hop_length=347*int(SAMPLE_DURATION), \n                                            n_mels=128, \n                                            n_mfcc=64)\n        test_data.append((file_name+'_'+str(i), file_array_i))\n    \n    # progress tracker\n    if progress%100 == 0:\n        print('processed', progress, 'files')\n    progress += 1\nprint('test data done')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utilities"},{"metadata":{"trusted":true},"cell_type":"code","source":"# utilities to get X, y, and arithmetic and geometric averaging.\n# consult function descriptions for clarification.\n#===========================================================================================================================================\n# get train data\ndef get_train(train_files):\n    \n    '''\n    Constructs the X_train and y_train matrices of [children files], given an array/list of [parent filenames] for training.\n    \n    Input(s) : 'train_files': an array/list of [parent filenames] for training (len N_train_parents), obtained from StratifiedKFold.\n    \n    Output(s): 'X_train': the training matrix, shape (N_train_children, MFCC_array.shape[0], MFCC_array.shape[1])\n               'y_train': the training labels, shape (N_train_children, C)\n    '''\n    \n    # get the data in one list\n    # format: [(file_name, file_array, file_label), ...]\n    datalist = []\n    for file_name, file_array, file_label in training_data:\n        if file_name.split('_')[0] in train_files:\n            datalist.append((file_name, file_array, file_label))\n        else:\n            continue\n    \n    # shuffle\n    random.Random(1001).shuffle(datalist)\n    \n    # get X and y\n    X_train = np.empty((len(datalist), datalist[0][1].shape[0], datalist[0][1].shape[1]))\n    y_train = np.empty((len(datalist), len(mlb.classes_)))\n    i = 0\n    for file_name, file_array, file_label in datalist:\n        X_train[i] = file_array\n        y_train[i] = file_label\n        i += 1\n\n    return X_train, y_train\n    \n# get val data\ndef get_val(val_files):\n    \n    '''\n    Constructs the X_val and y_val matrices of [children files], given an array/list of [parent filenames] for validation.\n    \n    Input(s) : 'val_files': an array/list of [parent filenames] for validation (len N_val_parents), obtained from StratifiedKFold.\n    \n    Output(s): 'X_val': the validation matrix, shape (N_val_children, MFCC_array.shape[0], MFCC_array.shape[1])\n               'y_val': the validation labels, shape (N_val_children, C)\n               'filenames_val': 1D array, shape (N_val_children,), stores children filenames for geometric averaging. \n                                Must be in the same order as X_val.\n    '''\n    \n    # get the data in one list\n    # format: [(file_name, file_array, file_label), ...]\n    datalist = []\n    for file_name, file_array, file_label in training_data:\n        if file_name.split('_')[0] in val_files:\n            datalist.append((file_name, file_array, file_label))\n        else:\n            continue\n    \n    # get filenames, X, and y\n    filenames_val = np.empty((len(datalist)), dtype=object)\n    X_val = np.empty((len(datalist), datalist[0][1].shape[0], datalist[0][1].shape[1]))\n    y_val = np.empty((len(datalist), len(mlb.classes_)))\n    i = 0\n    for file_name, file_array, file_label in datalist:\n        filenames_val[i] = file_name\n        X_val[i] = file_array\n        y_val[i] = file_label\n        i += 1\n\n    return filenames_val, X_val, y_val\n\n# get test data\ndef get_test():\n    \n    '''\n    Constructs the X_train matrix of [children files], given an array/list of [parent filenames] for test data.\n    \n    Input(s) : None, takes data from 'test_data'\n    \n    Output(s): 'X_test': the test data matrix, shape (N_test_children, MFCC_array.shape[0], MFCC_array.shape[1])\n               'filenames_test': 1D array, shape (N_test_children,), stores children filenames for geometric averaging. \n                                Must be in the same order as X_test.\n    '''\n    \n    # get filenames and X\n    filenames_test = np.empty((len(test_data)), dtype=object)\n    X_test = np.empty((len(test_data), test_data[0][1].shape[0], test_data[0][1].shape[1]))\n    i = 0\n    for file_name, file_array in test_data:\n        filenames_test[i] = file_name\n        X_test[i] = file_array\n        i += 1\n\n    return filenames_test, X_test\n\ndef arithmetic_average_val_scores(filenames_val, y_pred, y_true):\n    \n    '''\n    Calculates the arithmetic average of the [prediction of a parent file], given a [collection of predictions of children files]\n    \n    Input(s): 'filenames_val': 1D array, shape (N_val_children,), stores children filenames.\n                               Must be in the same order as y_true and y_pred.\n                               Output from get_val().\n              'y_pred': 2D array, shape (N_val_children, C), stores predictions of each child file.\n              'y_true': 2D array, shape (N_val_children, C), stores ground truth labels of each child file.\n              \n    Output(s): 'y_pred_parents': 2D array, shape (N_val_parents, C), stores predictions of each parent file by averaging from children.\n               'y_true_parents': 2D array, shape (N_val_parents, C), stores ground truth labels of each parent file.\n    '''\n    \n    # format:\n    # filename: (sum_array, label, num_children)\n    sum_scores = {}\n    for i, filename in enumerate(filenames_val):\n        \n        split_filename = filename.split('_')[0]\n        \n        if split_filename not in sum_scores.keys():\n            sum_scores[split_filename] = [y_pred[i], y_true[i], 1]\n        else:\n            sum_scores[split_filename][0] += y_pred[i]\n            sum_scores[split_filename][2] += 1\n    \n    # average the scores\n    y_pred_parents = np.empty((len(sum_scores.keys()), len(mlb.classes_)))\n    y_true_parents = np.empty((len(sum_scores.keys()), len(mlb.classes_)))\n    i = 0\n    for filename, values in sum_scores.items():\n        y_pred_parents[i] = values[0]/values[2]\n        y_true_parents[i] = values[1]\n        i += 1\n    \n    # return averaged scores\n    return y_pred_parents, y_true_parents\n\ndef geometric_average_val_scores(filenames_val, y_pred, y_true):\n    \n    '''\n    Calculates the geometric average of the [prediction of a parent file], given a [collection of predictions of children files]\n    \n    Input(s): 'filenames_val': 1D array, shape (N_val_children,), stores children filenames\n                               Must be in the same order as y_true and y_pred.\n                               Output from get_val().\n              'y_pred': 2D array, shape (N_val_children, C), stores predictions of each child file.\n              'y_true': 2D array, shape (N_val_children, C), stores ground truth labels of each child file.\n              \n    Output(s): 'y_pred_parents': 2D array, shape (N_val_parents, C), stores predictions of each parent file by averaging from children.\n               'y_true_parents': 2D array, shape (N_val_parents, C), stores ground truth labels of each parent file.\n               'filenames_parents': list, len(N_val_parents), stores the names of the parents in the same order as y_pred_parents\n    '''\n    \n    # format:\n    # filename: (sum_array, label, num_children)\n    sum_scores = {}\n    for i, filename in enumerate(filenames_val):\n        \n        split_filename = filename.split('_')[0]\n        \n        if split_filename not in sum_scores.keys():\n            sum_scores[split_filename] = [y_pred[i], y_true[i], 1]\n        else:\n            sum_scores[split_filename][0] *= y_pred[i]\n            sum_scores[split_filename][2] += 1\n    \n    # average the scores\n    filenames_parents = []\n    y_pred_parents = np.empty((len(sum_scores.keys()), len(mlb.classes_)))\n    y_true_parents = np.empty((len(sum_scores.keys()), len(mlb.classes_)))\n    i = 0\n    for filename, values in sum_scores.items():\n        filenames_parents.append(filename)\n        y_pred_parents[i] = values[0]**(1.0/values[2])\n        y_true_parents[i] = values[1]\n        i += 1\n    \n    # return averaged scores\n    return filenames_parents, y_pred_parents, y_true_parents","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training, single fold"},{"metadata":{"trusted":true},"cell_type":"code","source":"# learning rate scheduler\ndef step_decay(epoch):\n    \n    initial_lrate = 4e-4\n    drop = 0.5\n    epoch_cutoff = 7\n    epochs_drop = 3.0\n    \n    if epoch<epoch_cutoff:\n        return initial_lrate\n    else:\n        return initial_lrate * math.pow(drop, math.floor((1+epoch-epoch_cutoff)/epochs_drop))\n\n# get lists of train and val filenames\ntrain_files = all_files\n\n# get data matrices\nX_train, y_train = get_train(train_files)\nX_train = np.expand_dims(X_train, axis=3)\nfilenames_test, X_test = get_test()\nX_test = np.expand_dims(X_test, axis=3)\n\n# build, summarize, and fit\nmodel = cnn_6(lr=4e-4, do=0.5, l2reg=0)\nmodel.summary()\nlrate = LearningRateScheduler(step_decay, verbose=1)\nmodel_history = model.fit(X_train, y_train, batch_size=128, epochs=20, verbose=1, callbacks=[lrate])\n\n# evaluate model on training data\ny_train_pred = model.predict(X_train, batch_size=128)\ntrain_lwlwrap = calculate_overall_lwlrap_sklearn(y_train, y_train_pred)\nprint('train lwlwrap score:', train_lwlwrap)\n\n# create predictions for test data\ny_test_pred   = model.predict(X_test, batch_size=128) # children format\nfilnames_test_parents, y_test_pred, _ = geometric_average_val_scores(filenames_test, y_test_pred, y_test_pred)\n\nprint('training complete')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create predictions file"},{"metadata":{"trusted":true},"cell_type":"code","source":"# construct dataframe from array and insert 'fname' column\npredictions = pd.DataFrame(data=y_test_pred) \npredictions.insert(loc=0, column='fname', value=filnames_test_parents) \n\n# rename columns: ['0', '1', '2', ...] -> ['fname', 'Accelerating_and_revving_and_vroom', 'Accordion', ...]\ncolumns = ['fname']\ncolumns.extend(mlb.classes_.tolist())\npredictions.columns = columns\n\n# save as output csv\npredictions.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}