{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Speech Recognition using Keras\n---\n\nThe model used in this notebook comes from https://www.kaggle.com/alexozerin/end-to-end-baseline-tf-estimator-lb-0-72"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport re\nfrom glob import glob\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"First let's look at the data that we have. It turns out that we have two text files that contains the file name for validation and test set, and a directory `'audio'` that contains the wav files."},{"metadata":{"trusted":true},"cell_type":"code","source":"ls ../input/train","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here we can see all the labels we have in our dataset. And for now let's only deal with 10 labels:\n\n**yes no up down left right on off stop go**"},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = []\nfor i in glob('../input/train/audio/*/*.wav'):\n    labels += [i.split('/')[-2]]\nlabels = np.unique(np.array(labels), return_counts=True)\n\nd = {}\nfor i in range(len(labels[0])):\n    d[labels[0][i]] = labels[1][i]\nprint(d)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"POSSIBLE_LABELS = d.keys()\n#POSSIBLE_LABELS = 'yes no up down left right on off stop go unknown silence'.split()\nid2name = {i: name for i, name in enumerate(POSSIBLE_LABELS)}\nname2id = {name: i for i, name in id2name.items()}\nprint(id2name)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's load our data and their file names into a dataframe."},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_data(data_dir):\n    \"\"\" Return three dataframes for train,  validation, and test data\n    \"\"\"\n    # Just a simple regexp for paths with three groups:\n    # prefix, label, user_id\n    pattern = re.compile(\"(.+\\/)?(\\w+)\\/([^_]+)_.+wav\")\n    all_files = glob(os.path.join(data_dir, 'train/audio/*/*wav'))\n\n    validation_files = np.loadtxt(os.path.join(data_dir, 'train/validation_list.txt'), dtype='str')\n    print(\"Example in 'validation_list.txt': \", validation_files[0])\n    valset = set()\n    for entry in validation_files:\n        r = re.match(pattern, entry)\n        if r:\n            valset.add(r.group(3))\n            \n    testing_files = np.loadtxt(os.path.join(data_dir, 'train/testing_list.txt'), dtype='str')\n    testset = set()\n    for entry in testing_files:\n        r = re.match(pattern, entry)\n        if r:\n            testset.add(r.group(3))\n\n    possible = set(POSSIBLE_LABELS)\n    train, val, test = [], [], []\n    for entry in all_files:\n        r = re.match(pattern, entry)\n        if r:\n            label, uid = r.group(2), r.group(3)\n            #if label == '_background_noise_':\n            #    label = 'silence'\n            #elif label not in possible:\n            #    label = 'unknown'\n\n            label_id = name2id[label]\n\n            sample = (label, label_id, uid, entry)\n            if uid in valset:\n                val.append(sample)\n            elif uid in testset:\n                test.append(sample)\n            else:\n                train.append(sample)\n\n    print('There are {} train, {} val, and {} test samples'.format(len(train), len(val), len(test)))\n    \n    columns_list = ['label', 'label_id', 'user_id', 'wav_file']\n    \n    train_df = pd.DataFrame(train, columns = columns_list)\n    valid_df = pd.DataFrame(val, columns = columns_list)\n    test_df = pd.DataFrame(test, columns = columns_list)\n    \n    return train_df, valid_df, test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df, valid_df, test_df = load_data('../input/')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(6, 6))\nvals = np.array([train_df.shape[0], valid_df.shape[0], test_df.shape[0]])\n\ncmap = plt.get_cmap(\"tab20c\")\nouter_colors = cmap(np.arange(3)*4)\nwedges, a, a = plt.pie(vals, radius=0.9, autopct=\"%.2f%%\", colors=outer_colors, pctdistance=0.75, \n       wedgeprops=dict(width=0.5, edgecolor='w'), textprops={'fontsize': 13, 'color': 'w'})\nplt.legend(wedges, ['train', 'validation', 'test'], fontsize=13, loc=\"center left\", bbox_to_anchor=(0.9, 0, 0, 1))\n\nplt.title('Data Preview', fontsize=16)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"silence_files = train_df[train_df.label == '_background_noise_']\ntrain_df      = train_df[train_df.label != '_background_noise_']\nPOSSIBLE_LABELS = train_df.label.unique().tolist()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's start by reading the wav files into numpy array using `scipy.io.wavfile`."},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.io import wavfile\n\ndef read_wav_file(fname):\n    _, wav = wavfile.read(fname)\n    wav = wav.astype(np.float32) / np.iinfo(np.int16).max\n    return wav","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"silence_data = np.concatenate([read_wav_file(x) for x in silence_files.wav_file.values])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.signal import stft\n\ndef process_wav_file(fname):\n    wav = read_wav_file(fname)\n    \n    L = 16000  # 1 sec\n    \n    if len(wav) > L:\n        i = np.random.randint(0, len(wav) - L)\n        wav = wav[i:(i+L)]\n    elif len(wav) < L:\n        rem_len = L - len(wav)\n        i = np.random.randint(0, len(silence_data) - rem_len)\n        silence_part = silence_data[i:(i+L)]\n        j = np.random.randint(0, rem_len)\n        silence_part_left  = silence_part[0:j]\n        silence_part_right = silence_part[j:rem_len]\n        wav = np.concatenate([silence_part_left, wav, silence_part_right])\n    \n    specgram = stft(wav, 16000, nperseg = 400, noverlap = 240, nfft = 512, padded = False, boundary = None)\n    phase = np.angle(specgram[2]) / np.pi\n    amp = np.log1p(np.abs(specgram[2]))\n    \n    return np.stack([phase, amp], axis = 2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's look at one example of the wav file, and see how its spectrum looks like.\n\nCodes comes from https://www.kaggle.com/davids1992/speech-representation-and-data-exploration"},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy import signal\n\ndef log_specgram(audio, sample_rate, window_size=20,\n                 step_size=10, eps=1e-10):\n    nperseg = int(round(window_size * sample_rate / 1e3))\n    noverlap = int(round(step_size * sample_rate / 1e3))\n    freqs, times, spec = signal.spectrogram(audio,\n                                    fs=sample_rate,\n                                    window='hann',\n                                    nperseg=nperseg,\n                                    noverlap=noverlap,\n                                    detrend=False)\n    return freqs, times, np.log(spec.T.astype(np.float32) + eps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_rate, samples = wavfile.read(train_df.wav_file[1])\nfreqs, times, spectrogram = log_specgram(samples, sample_rate)\n\nfig = plt.figure(figsize=(14, 8))\nax1 = fig.add_subplot(211)\nax1.set_title('Raw wave of ' + train_df.wav_file[0])\nax1.set_ylabel('Amplitude')\nax1.plot(np.linspace(0, sample_rate/len(samples), sample_rate), samples)\n\nax2 = fig.add_subplot(212)\nax2.imshow(spectrogram.T, aspect='auto', origin='lower', \n           extent=[times.min(), times.max(), freqs.min(), freqs.max()])\nax2.set_yticks(freqs[::16])\nax2.set_xticks(times[::16])\nax2.set_title('Spectrogram of ' + train_df.wav_file[0])\nax2.set_ylabel('Freqs in Hz')\nax2.set_xlabel('Seconds')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here is the **Mel power spectrogram** and **MFCC** graph. We actually don't use them in our model, but just keep in mind these are also good input types for audio problems."},{"metadata":{"trusted":true},"cell_type":"code","source":"import librosa\n\nsample_rate, samples = wavfile.read(train_df.wav_file[0])\nS = librosa.feature.melspectrogram(samples.astype(np.float32), sr=sample_rate, n_mels=128)\nlog_S = librosa.power_to_db(S, ref=np.max)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import librosa.display\n\nplt.figure(figsize=(12, 4))\nlibrosa.display.specshow(log_S, sr=sample_rate, x_axis='time', y_axis='mel')\nplt.title('Mel power spectrogram ')\nplt.colorbar(format='%+02.0f dB')\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mfcc = librosa.feature.mfcc(S=log_S, n_mfcc=20)\n\ndelta2_mfcc = librosa.feature.delta(mfcc, order=2)\n\nplt.figure(figsize=(12, 4))\nlibrosa.display.specshow(delta2_mfcc)\nplt.ylabel('MFCC coeffs')\nplt.xlabel('Time')\nplt.title('MFCC')\nplt.colorbar()\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can also listen to this audio using `Ipython.display`."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.wav_file[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import IPython.display as ipd\n\nsample_rates = []\nsamples = []\nfor i in range(10):\n    sample_rate, sample = wavfile.read(train_df.wav_file[i])\n    sample_rates += [sample_rate]\n    samples += [sample]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ipd.Audio(samples[0], rate=sample_rates[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n\n## Building models using Keras"},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nimport tensorflow as tf\nfrom tensorflow.python.keras.models import Model\nfrom tensorflow.python.keras.layers import Input, Reshape, Conv2D, MaxPooling2D, AveragePooling2D, Activation, BatchNormalization\nfrom tensorflow.python.keras.layers import GlobalAveragePooling2D, GlobalMaxPool2D, concatenate, Dense, Dropout, CuDNNGRU, ELU\nfrom tensorflow.python.keras.optimizers import RMSprop\nfrom tensorflow.python.keras.utils import to_categorical\nfrom sklearn.preprocessing import label_binarize","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Build data loaders for the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_generator(train_batch_size, sample_number):\n    while True:\n        replace = False\n        if sample_number > 1340:\n            replace = True\n        this_train = train_df.groupby('label_id').apply(lambda x: x.sample(n = sample_number, replace=replace))\n        shuffled_ids = random.sample(range(this_train.shape[0]), this_train.shape[0])\n        for start in range(0, len(shuffled_ids), train_batch_size):\n            x_batch = []\n            y_batch = []\n            end = min(start + train_batch_size, len(shuffled_ids))\n            i_train_batch = shuffled_ids[start:end]\n            for i in i_train_batch:\n                x_batch.append(process_wav_file(this_train.wav_file.values[i]))\n                y_batch.append(this_train.label_id.values[i])\n            x_batch = np.array(x_batch)\n            #y_batch = to_categorical(y_batch, num_classes=len(POSSIBLE_LABELS))\n            y_batch = label_binarize(y_batch, classes=range(len(POSSIBLE_LABELS)))\n            yield x_batch, y_batch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def valid_generator(val_batch_size):\n    while True:\n        ids = list(range(valid_df.shape[0]))\n        for start in range(0, len(ids), val_batch_size):\n            x_batch = []\n            y_batch = []\n            end = min(start + val_batch_size, len(ids))\n            i_val_batch = ids[start:end]\n            for i in i_val_batch:\n                x_batch.append(process_wav_file(valid_df.wav_file.values[i]))\n                y_batch.append(valid_df.label_id.values[i])\n            x_batch = np.array(x_batch)\n            #y_batch = to_categorical(y_batch, num_classes=len(POSSIBLE_LABELS))\n            y_batch = label_binarize(y_batch, classes=range(len(POSSIBLE_LABELS)))\n            yield x_batch, y_batch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Model structure:"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K\nfrom keras.layers import Input, Dense\nfrom keras.models import Model\nfrom keras.layers import Dense, Dropout, Reshape, Permute\nfrom keras.layers import Conv2D, MaxPooling2D, ZeroPadding2D\nfrom keras.layers import BatchNormalization\nfrom keras.layers import ELU\nfrom keras.layers import CuDNNGRU\nfrom keras.optimizers import SGD\n\ninput_shape = (257, 98, 2)\nchannel_axis = 3\nfreq_axis = 2\ntime_axis = 1\n\nmelgram_input = Input(shape=input_shape)\n#x = ZeroPadding2D(padding=(0, 37))(melgram_input)\nx = BatchNormalization(axis=freq_axis, name='bn_0_freq')(melgram_input)\n\n# Conv block 1\nx_1 = Conv2D(64, (3, 3), name=\"conv1\", padding=\"same\")(x)\nx_1 = BatchNormalization(axis=channel_axis, name=\"bn1\")(x_1)\nx_1 = ELU()(x_1)\nx_1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool1')(x_1)\nx_1 = Dropout(0.1, name='dropout1')(x_1)\n\n# Conv block 2\nx_2 = Conv2D(128, (3, 3), name=\"conv2\", padding=\"same\")(x_1)\nx_2 = BatchNormalization(axis=channel_axis, name=\"bn2\")(x_2)\nx_2 = ELU()(x_2)\nx_2 = MaxPooling2D(pool_size=(3, 3), strides=(3, 3), name='pool2')(x_2)\nx_2 = Dropout(0.1, name='dropout2')(x_2)\n\n# Conv block 3\nx_3 = Conv2D(128, (3, 3), name=\"conv3\", padding=\"same\")(x_2)\nx_3 = BatchNormalization(axis=channel_axis, name=\"bn3\")(x_3)\nx_3 = ELU()(x_3)\nx_3 = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool3')(x_3)\nx_3 = Dropout(0.1, name='dropout3')(x_3)\n\n# Conv block 4\nx_4 = Conv2D(128, (3, 3), name=\"conv4\", padding=\"same\")(x_3)\nx_4 = BatchNormalization(axis=channel_axis, name=\"bn4\")(x_4)\nx_4 = ELU()(x_4)\nx_4 = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool4')(x_4)\nx_4 = Dropout(0.1, name='dropout4')(x_4)\n\n# reshaping\nx = Reshape((2, 128))(x_4)\n\n# GRU block 1, 2, output\nx_gru = CuDNNGRU(32, return_sequences=True, name='gru1')(x)\nx_gru = CuDNNGRU(32, return_sequences=False, name='gru2')(x_gru)\nx_gru = Dropout(0.3)(x_gru)\noutput = Dense(11, activation='sigmoid', name='output')(x_gru)\n\nmodel = Model(inputs = melgram_input, outputs = output)\nmodel.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_in = Input(shape = (257,98,2)) # Input layer\nx = BatchNormalization()(x_in)\nfor i in range(4): # Four convolutional layer modules.\n    x = Conv2D(16*(2 ** i), (3,3))(x)\n    x = BatchNormalization()(x)\n    x = ELU()(x)\n    x = MaxPooling2D((2,2))(x)\n    x = Dropout(0.1)(x)\nx = Conv2D(128, (1,1))(x)\nx_branch_1 = AveragePooling2D()(x)\nx_branch_2 = MaxPooling2D()(x)\nx = concatenate([x_branch_1, x_branch_2]) # Take use of both pooling by concate\nx = Reshape((7, 512))(x)\nx_gru = CuDNNGRU(32, return_sequences=True)(x)\nx_gru = CuDNNGRU(32, return_sequences=False)(x_gru)\nx_gru = Dropout(0.3)(x_gru)\nx = Dense(256, activation = 'relu')(x_gru) # Linear layer\nx = Dropout(0.5)(x)\nx = Dense(len(POSSIBLE_LABELS), activation = 'softmax')(x) # Output layer\n\nmodel = Model(inputs = x_in, outputs = x)\nmodel.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.python.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n\ncallbacks = [EarlyStopping(monitor='val_acc', patience=10, verbose=1, min_delta=0.01, mode='max'),\n             ReduceLROnPlateau(monitor='val_acc', factor=0.1, patience=3, verbose=1, epsilon=0.01, mode='max'),\n             ModelCheckpoint(monitor='val_acc', filepath='starter.hdf5', verbose=1, save_best_only=True, save_weights_only=True, mode='max')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit_generator(generator=train_generator(256), steps_per_epoch=151, epochs=50, verbose=1, callbacks=callbacks,\n                              validation_data=valid_generator(256), validation_steps=151)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size=256\nsample_size=1700 # upsample/downsample all categories to the same size\nhistory = model.fit_generator(generator=train_generator(batch_size, sample_size),\n                              steps_per_epoch=sample_size*len(POSSIBLE_LABELS)//batch_size, epochs=100, verbose=1, callbacks=callbacks,\n                              validation_data=valid_generator(batch_size),\n                              validation_steps=sample_size*len(POSSIBLE_LABELS)//batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights('starter.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = []\nfor i in test_df.wav_file.values:\n    X_test += [process_wav_file(i)]\nX_test = np.array(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes = np.argmax(predictions, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n\n## Results analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, roc_curve, auc, roc_auc_score\nfrom itertools import cycle\nfrom scipy import interp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nplt.plot(history.history['acc'])\nplt.plot(history.history['val_acc'])\nplt.title('Model accuracy', fontsize=16)\nplt.xlabel('Epoch', fontsize=16)\nplt.ylabel('Accuracy', fontsize=16)\nplt.grid(axis='y')\nplt.xticks(fontsize=13)\nplt.yticks(fontsize=13)\nplt.legend(['Train', 'Validation'], loc='upper left', fontsize=13)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('Model loss', fontsize=16)\nplt.xlabel('Epoch', fontsize=16)\nplt.ylabel('Loss', fontsize=16)\nplt.grid(axis='y')\nplt.xticks(fontsize=13)\nplt.yticks(fontsize=13)\nplt.legend(['Train', 'Validation'], loc='upper left', fontsize=13)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test_class = test_df.label_id.values\npred = model.predict(X_test)\ncm = confusion_matrix(y_test_class, np.argmax(pred, axis=1))\nprint('Test set accuracy: ', str(round(np.sum(y_test_class == np.argmax(pred, axis=1))/test_df.shape[0], 4)*100)+'%')\nnum_classes = cm.shape[0]\ncount = np.unique(y_test_class, return_counts=True)[1].reshape(num_classes, 1)\n\nfig = plt.figure(figsize=(15, 15))\nax = plt.subplot(111)\nim = ax.imshow(cm/count, cmap='YlGnBu')\nim.set_clim(0, 1)\ncbar = ax.figure.colorbar(im, ax=ax)\nax.set_xticks(np.arange(num_classes))\nax.set_yticks(np.arange(num_classes))\nplt.yticks(range(num_classes), list(name2id.keys())[1:], fontsize=13)\nplt.xticks(range(num_classes), list(name2id.keys())[1:], fontsize=13)\nfor i in range(num_classes):\n    for j in range(num_classes):\n        text = ax.text(i, j, cm[j][i], ha=\"center\", va=\"center\", color=\"w\" if (cm/count)[j, i] > 0.5 else \"black\", fontsize=13)\nax.set_ylabel('True Label', fontsize=16)\nax.set_xlabel('Predicted Label', fontsize=16)\nax.set_title('Confusion Matrix', fontsize=16, fontweight='bold')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import label_binarize\n\ny_test = label_binarize(y_test_class, classes=range(num_classes))\n\nfpr = {}\ntpr = {}\nroc_auc = {}\nfor i in range(num_classes):\n    fpr[i], tpr[i], _ = roc_curve(y_test[:, i], pred[:, i])\n    roc_auc[i] = auc(fpr[i], tpr[i])\n    \nfpr['micro'], tpr['micro'], _ = roc_curve(y_test.ravel(), pred.ravel())\nroc_auc['micro'] = auc(fpr['micro'], tpr['micro'])\n\n# Compute macro-average ROC curve and AUC\n# Aggregate all false positive rates\nall_fpr = np.unique(np.concatenate([fpr[i] for i in range(num_classes)]))\n# Interpolate all ROC curves at this points\nmean_tpr = np.zeros_like(all_fpr)\nfor i in range(num_classes):\n    mean_tpr += interp(all_fpr, fpr[i], tpr[i])\n# Average and compute AUC\nmean_tpr /= num_classes\n\nfpr['macro'] = all_fpr\ntpr['macro'] = mean_tpr\nroc_auc['macro'] = auc(fpr['macro'], tpr['macro'])\n\n# Plot all ROC curves\nplt.figure(figsize=(10, 10))\n    \nfor i in range(num_classes):\n        plt.plot(fpr[i], tpr[i], alpha=0.2,\n                 label='ROC curve of class {0} (area = {1:0.4f})'\n                 ''.format(id2name[i+1], roc_auc[i]))\n\nplt.plot(fpr['micro'], tpr['micro'],\n         label='micro-average ROC curve (area = {0:0.4f})'\n         ''.format(roc_auc['micro']),\n         color='orangered', linestyle=':', linewidth=3)\n\nplt.plot(fpr['macro'], tpr['macro'],\n         label='macro-average ROC curve (area = {0:0.4f})'\n         ''.format(roc_auc['macro']),\n         color='navy', linestyle=':', linewidth=3)\n\nplt.plot([0, 1], [0, 1], 'k--')\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.05])\nplt.xticks(fontsize=13)\nplt.xticks(fontsize=13)\nplt.title('ROC Curves', fontsize=16)\nplt.xlabel('False Positive Rate', fontsize=16)\nplt.ylabel('True Positive Rate', fontsize=16)\nplt.legend(loc=4)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}