{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Spectrogram-based CNN for the Tensorflow Speech Recognition Challenge (~ 90 % accuracy)\n\n\nThe goal of this challenge is to classify 65,000 one-second audio clips for 30 short words, building an algorithm that understands simple spoken command.\n\nThis notebooks shows all the steps needed to create a speech model using short audio tracks in a tidy way.\n\n\n## <ins>Notes</ins>\n\n- The notebook is using the CNN architecture published at:\nhttps://blogs.rstudio.com/ai/posts/2018-06-06-simple-audio-classification-keras/\n\n- The current pipeline is using batch processing and online augmentation, randomly modifying the audio clips before being used to train the network.\n\n\n## <ins>Methods</ins>\n\n- The audio clips are analyzed as images computing Mel-frequency cepstral normalized coefficients (MFCCs). MFCCs are an alternative representation of the Mel-frequency spectrogram. The MFCCs can be obtained applying the discrete cosine transform (DCT) to a Mel-frequency spectrogram (take a look to this article: https://towardsdatascience.com/getting-to-know-the-mel-spectrogram-31bca3e2d9d0). Additional information can be found in any digital signal processing textbook (concepts such as Fourier transform and Short Time Fourier Transform are fundamental).\n\n- To improve the robustness of the algorithm, some online data augmentation methods are sequentially applied to the raw audio clips before computing MFCCs: addition of background noise, audio shifting and pitch modification. Each audio was edited independently with these filters with a chance of 70% in each sequential step.","metadata":{}},{"cell_type":"code","source":"#%env JOBLIB_TEMP_FOLDER=/tmp","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:29.819888Z","iopub.execute_input":"2021-09-22T04:34:29.820264Z","iopub.status.idle":"2021-09-22T04:34:29.824690Z","shell.execute_reply.started":"2021-09-22T04:34:29.820220Z","shell.execute_reply":"2021-09-22T04:34:29.823505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%capture\n\n!pip install pyunpack\n!pip install patool\n\nimport os\nos.environ[\"CUDA_VISIBLE_DEVICES\"]='0,1'\nos.system('apt-get install p7zip')\n\nimport glob\nfrom pyunpack import Archive\nfrom collections import Counter\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers,activations\nfrom tensorflow.keras.utils import Sequence\nfrom tensorflow.keras.models import load_model\nimport numpy as np \nimport pandas as pd \nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport IPython.display as ipd\nimport librosa, librosa.display\nfrom sklearn.preprocessing import StandardScaler,MinMaxScaler,LabelBinarizer\nfrom sklearn.metrics import classification_report, confusion_matrix, roc_auc_score\nimport math\nimport shutil\nimport pickle\nimport multiprocessing\nimport gc\n\ntf.random.set_seed(9)\n#%load_ext tensorboard   ","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2021-09-22T04:34:29.826580Z","iopub.execute_input":"2021-09-22T04:34:29.827173Z","iopub.status.idle":"2021-09-22T04:34:45.265991Z","shell.execute_reply.started":"2021-09-22T04:34:29.827065Z","shell.execute_reply":"2021-09-22T04:34:45.265073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.__version__","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:45.267427Z","iopub.execute_input":"2021-09-22T04:34:45.267787Z","iopub.status.idle":"2021-09-22T04:34:45.278621Z","shell.execute_reply.started":"2021-09-22T04:34:45.267748Z","shell.execute_reply":"2021-09-22T04:34:45.277467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.python.client import device_lib\n\ndevice_lib.list_local_devices()","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:45.281992Z","iopub.execute_input":"2021-09-22T04:34:45.282352Z","iopub.status.idle":"2021-09-22T04:34:45.857910Z","shell.execute_reply.started":"2021-09-22T04:34:45.282316Z","shell.execute_reply":"2021-09-22T04:34:45.856694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_path = \"/kaggle\"","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:45.861535Z","iopub.execute_input":"2021-09-22T04:34:45.862342Z","iopub.status.idle":"2021-09-22T04:34:45.867244Z","shell.execute_reply.started":"2021-09-22T04:34:45.862299Z","shell.execute_reply":"2021-09-22T04:34:45.866228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Copy the train data into the \"output\" directory:","metadata":{}},{"cell_type":"code","source":"\nif not os.path.exists(root_path + '/working/train/'):\n    os.makedirs(root_path + '/working/train/')\n    Archive(root_path + '/input/tensorflow-speech-recognition-challenge/train.7z').extractall(root_path + '/working')\n    \n\ntrain_path = root_path + '/working/train/audio/'","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:45.869173Z","iopub.execute_input":"2021-09-22T04:34:45.870055Z","iopub.status.idle":"2021-09-22T04:34:45.877054Z","shell.execute_reply.started":"2021-09-22T04:34:45.869957Z","shell.execute_reply":"2021-09-22T04:34:45.875977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualization of the Mel spectrogram of a sample audio:","metadata":{}},{"cell_type":"code","source":"train_audio_sample = os.path.join(train_path, \"yes/0a7c2a8d_nohash_0.wav\")\nx,sr = librosa.load(train_audio_sample, sr = 16000)\n\nipd.Audio(x, rate=sr)","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2021-09-22T04:34:45.878863Z","iopub.execute_input":"2021-09-22T04:34:45.879749Z","iopub.status.idle":"2021-09-22T04:34:45.896047Z","shell.execute_reply.started":"2021-09-22T04:34:45.879711Z","shell.execute_reply":"2021-09-22T04:34:45.894866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hop_length = 256\nS = librosa.feature.melspectrogram(x, sr=sr, n_fft=4096, hop_length=hop_length)\nlogS = librosa.power_to_db(abs(S))\n\nplt.figure(figsize=(14, 9))\n\nplt.figure(1)\n\nplt.subplot(211)\nplt.title('Spectrogram')\nlibrosa.display.specshow(logS, sr=sr, hop_length=hop_length, x_axis= None, y_axis='mel')\n#plt.colorbar(format='%+2.0f dB')\n\nplt.subplot(212)\nplt.title('Audioform')\nlibrosa.display.waveplot(x, sr=sr)\n","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:45.897824Z","iopub.execute_input":"2021-09-22T04:34:45.898286Z","iopub.status.idle":"2021-09-22T04:34:46.178434Z","shell.execute_reply.started":"2021-09-22T04:34:45.898229Z","shell.execute_reply":"2021-09-22T04:34:46.177577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualization of MFCCs","metadata":{}},{"cell_type":"code","source":"mfccs = librosa.feature.mfcc(x, sr=sr,  n_mfcc=40) \nscaler = StandardScaler()\nee= scaler.fit_transform(mfccs.T)\n\nplt.figure(figsize=(14, 5))\nlibrosa.display.specshow(ee.T)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:46.179562Z","iopub.execute_input":"2021-09-22T04:34:46.179944Z","iopub.status.idle":"2021-09-22T04:34:46.244097Z","shell.execute_reply.started":"2021-09-22T04:34:46.179903Z","shell.execute_reply":"2021-09-22T04:34:46.243241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Functions we need for this pipeline","metadata":{}},{"cell_type":"code","source":"def pad_audio(samples, L):\n    if len(samples) >= L: \n        return samples\n    else: \n        return np.pad(samples, pad_width=(L - len(samples), 0), mode='constant', constant_values=(0, 0))\n    \n    \ndef chop_audio(samples, L=16000):\n    while True:\n        beg = np.random.randint(0, len(samples) - L)\n        yield samples[beg: beg + L]\n\n\ndef choose_background_generator(sound, backgrounds, max_alpha = 0.7):\n    if backgrounds is None:\n        return sound\n    my_gen = backgrounds[np.random.randint(len(backgrounds))]\n    background = next(my_gen) * np.random.uniform(0, max_alpha)\n    augmented_data = sound + background\n    augmented_data = augmented_data.astype(type(sound[0]))\n    return augmented_data \n\n\ndef random_shift(sound, shift_max = 0.2, sampling_rate = 16000):\n    shift = np.random.randint(sampling_rate * shift_max)\n    out = np.roll(sound, shift)\n    # time shift\n    if shift > 0:\n        out[:shift] = 0\n    else:\n        out[shift:] = 0\n    return out\n\n\ndef random_change_pitch(x, sr=16000):\n    pitch_factor = np.random.randint(1, 4)\n    out = librosa.effects.pitch_shift(x, sr, pitch_factor)\n    return out\n\n\ndef random_speed_up(x):\n    where = [\"start\", \"end\"][np.random.randint(0, 1)]\n    speed_factor = np.random.uniform(0, 0.5)\n    up = librosa.effects.time_stretch(x, 1 + speed_factor)\n    up_len = up.shape[0]\n    if where == \"end\":\n        up = np.concatenate((up, np.zeros((x.shape[0] - up_len,))))\n    else:\n        up = np.concatenate((np.zeros((x.shape[0] - up_len,)), up))\n    return up\n\n\ndef get_image_list(train_audio_path):\n    classes = os.listdir(train_audio_path)\n    classes = [thisclass for thisclass in classes if thisclass != '_background_noise_']\n    index = [i for i,j in enumerate(classes)]\n    outlist = []\n    labels = []\n    for thisindex,thisclass in zip(index, classes):\n        filelist = [f for f in os.listdir(os.path.join(train_audio_path, thisclass)) if f.endswith('.wav')]\n        filelist = [os.path.join(train_audio_path, thisclass, x) for x in filelist]\n        outlist.append(filelist)\n        labels.append(np.full(len(filelist), fill_value= thisindex))   \n    return outlist,labels,dict(zip(classes,index))\n\n\ndef split_train_test_stratified_shuffle(images_list, labels, train_size = 0.9):\n    classes_size = [len(x) for x in images_list]\n    classes_vector = [np.arange(x) for x in classes_size]\n    total = np.sum(classes_size)\n    total_train = [int(train_size * total * x) for x in classes_size / total]\n    train_index = [np.random.choice(x,y,replace=False) for x,y in zip(classes_size,total_train)]\n    validation_index = [np.setdiff1d(i,j) for i,j in zip(classes_vector,train_index)]\n\n    train_set = [np.array(x)[idx] for x,idx in zip(images_list,train_index)]\n    validation_set = [np.array(x)[idx] for x,idx in zip(images_list,validation_index)]\n    train_labels = [np.array(x)[idx] for x,idx in zip(labels,train_index)]\n    validation_labels = [np.array(x)[idx] for x,idx in zip(labels,validation_index)]\n\n    train_set = np.array([element for array in train_set for element in array])\n    validation_set = np.array([element for array in validation_set for element in array])\n    train_labels = np.array([element for array in train_labels for element in array])\n    validation_labels = np.array([element for array in validation_labels for element in array])\n\n    train_shuffle = np.random.permutation(len(train_set))\n    validation_shuffle =  np.random.permutation(len(validation_set))\n\n    train_set = train_set[train_shuffle]\n    validation_set = validation_set[validation_shuffle]\n    train_labels = train_labels[train_shuffle]\n    validation_labels = validation_labels[validation_shuffle]\n    \n    return train_set,train_labels,validation_set,validation_labels\n\n        \ndef preprocess_data(file, background_generator, target_sr = 16000, n_mfcc = 40, threshold = 0.7):\n    # downsample to 16 kHz\n    x,sr = librosa.load(file, sr = target_sr)\n    x = pad_audio(x, sr)\n    if np.random.uniform(0, 1) > threshold:\n        x = choose_background_generator(x, background_generator) # add noinse to 30% of data\n    if np.random.uniform(0, 1) > threshold:\n        x = random_shift(x) \n    if np.random.uniform(0, 1) > threshold: \n        x = random_change_pitch(x) \n    if np.random.uniform(0, 1) > threshold:\n        x = random_speed_up(x) \n    mfccs = librosa.feature.mfcc(x, sr=sr,  n_mfcc=n_mfcc) # transpose for sklearn\n    mfccs = np.moveaxis(mfccs, 1, 0)\n    #scaler = MinMaxScaler() \n    scaler = StandardScaler() \n    mfccs_scaled = scaler.fit_transform(mfccs)\n    return mfccs_scaled.reshape(mfccs_scaled.shape[0], mfccs_scaled.shape[1], 1) # channels last\n\n\nclass data_generator(Sequence):\n    def __init__(self, x_set, y_set, batch_size, background_generator):\n        self.x, self.y = x_set, y_set\n        self.batch_size = batch_size\n        self.background_generator = background_generator\n        #self.on_epoch_end()\n\n    def __len__(self):\n        return  math.ceil(len(self.x) / self.batch_size)\n\n    def __getitem__(self, idx):\n        idx_from = idx * self.batch_size\n        idx_to = (idx + 1) * self.batch_size\n        batch_x = self.x[idx_from:idx_to]\n        batch_y = self.y[idx_from:idx_to]\n        x = [preprocess_data(elem, self.background_generator) for elem in batch_x] \n        y = batch_y\n        return np.array(x), np.array(y)\n    \n    #def on_epoch_end(self):\n    #    print(\"Index epoch: %s, total samples %s\" %(self.idx, (self.idx + 1) * self.batch_size))\n        \n\ndef build_model(n_classes, input_shape):\n    model_input = keras.Input(shape=input_shape)\n    img_1 = layers.Convolution2D(filters = 32, kernel_size = (3,3), padding = \"same\", activation=activations.relu)(model_input)\n    img_1 = layers.MaxPooling2D(pool_size=(2, 2))(img_1)\n    img_1 = layers.Convolution2D(filters = 64, kernel_size = (3,3), padding = \"same\", activation=activations.relu)(img_1)\n    img_1 = layers.MaxPooling2D(pool_size=(2, 2))(img_1)\n    img_1 = layers.Convolution2D(filters = 128, kernel_size = (3,3), padding = \"same\", activation=activations.relu)(img_1)\n    img_1 = layers.MaxPooling2D(pool_size=(2, 2))(img_1)\n    img_1 = layers.Convolution2D(filters = 256, kernel_size = (3,3), padding = \"same\", activation=activations.relu)(img_1)\n    img_1 = layers.MaxPooling2D(pool_size=(2, 2))(img_1)\n    img_1 = layers.Dropout(rate=0.25)(img_1)\n    img_1 = layers.Flatten()(img_1)\n    img_1 = layers.Dense(128, activation=activations.relu)(img_1)\n    img_1 = layers.Dropout(rate=0.5)(img_1)\n    model_output = layers.Dense(n_classes, activation=activations.softmax)(img_1)\n    model = keras.Model(model_input, model_output)\n    return model\n\n\ndef multiclass_roc(y_test, y_pred, average=\"macro\"):\n    lb = LabelBinarizer()\n    lb.fit(y_test)\n    y_test = lb.transform(y_test)\n    y_pred = lb.transform(y_pred)\n    all_labels = np.unique(y_test)\n\n    for (idx, c_label) in enumerate(all_labels):\n        fpr, tpr, thresholds = roc_curve(y_test[:,idx].astype(int), y_pred[:,idx])\n        c_ax.plot(fpr, tpr, label = '%s (AUC:%0.2f)'  % (c_label, auc(fpr, tpr)))\n    c_ax.plot(fpr, fpr, 'b-', label = 'Random Guessing')\n    return roc_auc_score(y_test, y_pred, average=average)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:46.245813Z","iopub.execute_input":"2021-09-22T04:34:46.246166Z","iopub.status.idle":"2021-09-22T04:34:46.313337Z","shell.execute_reply.started":"2021-09-22T04:34:46.246129Z","shell.execute_reply":"2021-09-22T04:34:46.312468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Loading data","metadata":{}},{"cell_type":"markdown","source":"## Background data paths","metadata":{}},{"cell_type":"code","source":"# Load data with backgrounds\n\nwavfiles = glob.glob(os.path.join(train_path, \"_background_noise_/*wav\"))\nwavfiles = [librosa.load(elem, sr = 16000)[0] for elem in wavfiles]\nbackground_generator = [chop_audio(x) for x in wavfiles]","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:46.314957Z","iopub.execute_input":"2021-09-22T04:34:46.315541Z","iopub.status.idle":"2021-09-22T04:34:46.352060Z","shell.execute_reply.started":"2021-09-22T04:34:46.315496Z","shell.execute_reply":"2021-09-22T04:34:46.351327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training data paths, split train-test via stratified sampling, call a data generator for Keras","metadata":{}},{"cell_type":"code","source":"# load train\n\nimages_list,labels,classes_map =  get_image_list(train_path)\n\ntrain_set,train_labels,validation_set,validation_labels = split_train_test_stratified_shuffle(images_list,labels)\ntrain_datagen = data_generator(train_set, train_labels, 40, background_generator)\nvalidation_datagen = data_generator(validation_set, validation_labels,40,  None)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:46.353423Z","iopub.execute_input":"2021-09-22T04:34:46.353925Z","iopub.status.idle":"2021-09-22T04:34:46.706701Z","shell.execute_reply.started":"2021-09-22T04:34:46.353886Z","shell.execute_reply":"2021-09-22T04:34:46.705705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Preliminar testing","metadata":{}},{"cell_type":"markdown","source":"## Preprocessor works","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(14, 5))\nlibrosa.display.specshow(preprocess_data(train_audio_sample, None).reshape(32, 40).T)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:46.708160Z","iopub.execute_input":"2021-09-22T04:34:46.708567Z","iopub.status.idle":"2021-09-22T04:34:47.399809Z","shell.execute_reply.started":"2021-09-22T04:34:46.708520Z","shell.execute_reply":"2021-09-22T04:34:47.398823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(14, 5))\nlibrosa.display.specshow(preprocess_data(train_audio_sample, background_generator).reshape(32, 40).T)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:47.401628Z","iopub.execute_input":"2021-09-22T04:34:47.401995Z","iopub.status.idle":"2021-09-22T04:34:47.490403Z","shell.execute_reply.started":"2021-09-22T04:34:47.401956Z","shell.execute_reply":"2021-09-22T04:34:47.489347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data augmentation works","metadata":{}},{"cell_type":"markdown","source":"## Random shift","metadata":{}},{"cell_type":"code","source":"start_random = random_shift(x)\nipd.Audio(start_random , rate=sr)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:47.491971Z","iopub.execute_input":"2021-09-22T04:34:47.492368Z","iopub.status.idle":"2021-09-22T04:34:47.501470Z","shell.execute_reply.started":"2021-09-22T04:34:47.492327Z","shell.execute_reply":"2021-09-22T04:34:47.500540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Increase speed","metadata":{}},{"cell_type":"code","source":"higher_speed = random_speed_up(x)\nipd.Audio(higher_speed , rate=sr)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:47.503457Z","iopub.execute_input":"2021-09-22T04:34:47.503952Z","iopub.status.idle":"2021-09-22T04:34:47.525234Z","shell.execute_reply.started":"2021-09-22T04:34:47.503913Z","shell.execute_reply":"2021-09-22T04:34:47.524372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Change pitch","metadata":{}},{"cell_type":"code","source":"pitch_changed = random_change_pitch(x)\nipd.Audio(pitch_changed, rate=sr)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:47.526775Z","iopub.execute_input":"2021-09-22T04:34:47.527108Z","iopub.status.idle":"2021-09-22T04:34:47.566276Z","shell.execute_reply.started":"2021-09-22T04:34:47.527072Z","shell.execute_reply":"2021-09-22T04:34:47.565467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train not in validation","metadata":{}},{"cell_type":"code","source":"inv_map =  {v: k for k, v in classes_map.items()}\nany_present=[i in validation_set for i in train_set]\nnp.any(any_present)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:47.567920Z","iopub.execute_input":"2021-09-22T04:34:47.568301Z","iopub.status.idle":"2021-09-22T04:34:58.258358Z","shell.execute_reply.started":"2021-09-22T04:34:47.568264Z","shell.execute_reply":"2021-09-22T04:34:58.257144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Same order in data and labels","metadata":{}},{"cell_type":"code","source":"# Same order in \n\ntest1 = np.random.randint(10, 100, 10)\n\ntrain_set[test1],[inv_map[int(i)] for i in train_labels[test1]]","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:58.268158Z","iopub.execute_input":"2021-09-22T04:34:58.270693Z","iopub.status.idle":"2021-09-22T04:34:58.287803Z","shell.execute_reply.started":"2021-09-22T04:34:58.270642Z","shell.execute_reply":"2021-09-22T04:34:58.286671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test1 = np.random.randint(10, 100, 10)\nvalidation_set[test1],[inv_map[int(i)] for i in validation_labels[test1]]","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:58.292016Z","iopub.execute_input":"2021-09-22T04:34:58.292583Z","iopub.status.idle":"2021-09-22T04:34:58.322424Z","shell.execute_reply.started":"2021-09-22T04:34:58.292544Z","shell.execute_reply":"2021-09-22T04:34:58.321173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Stratified sampling works","metadata":{}},{"cell_type":"code","source":"unique, counts = np.unique(validation_labels, return_counts=True)\nx=dict(zip(unique, counts))\nout = pd.DataFrame(sorted(x.items(), key=lambda kv: kv[0]))\nout.drop(0, inplace = True, axis = 1)\nout = out.apply(lambda x: 100 * x/sum(x))\n\ntotal_labels = [y for x in labels for y in x]\nunique, counts = np.unique(total_labels, return_counts=True)\ny=dict(zip(unique, counts))\nout2 = pd.DataFrame(sorted(y.items(), key=lambda kv: kv[0]))\nout2.drop(0, inplace = True, axis = 1)\n\nout2 = out2.apply(lambda x: 100 * x/sum(x))","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:58.324041Z","iopub.execute_input":"2021-09-22T04:34:58.324539Z","iopub.status.idle":"2021-09-22T04:34:58.378257Z","shell.execute_reply.started":"2021-09-22T04:34:58.324490Z","shell.execute_reply":"2021-09-22T04:34:58.377464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out2.join(out, lsuffix='_left', rsuffix='_right')[:5]","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:58.379595Z","iopub.execute_input":"2021-09-22T04:34:58.380182Z","iopub.status.idle":"2021-09-22T04:34:58.397248Z","shell.execute_reply.started":"2021-09-22T04:34:58.380145Z","shell.execute_reply":"2021-09-22T04:34:58.396406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.allclose(out.iloc[:,0].values, out2.iloc[:,0].values,  atol=0.01)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:58.398539Z","iopub.execute_input":"2021-09-22T04:34:58.399077Z","iopub.status.idle":"2021-09-22T04:34:58.407328Z","shell.execute_reply.started":"2021-09-22T04:34:58.399042Z","shell.execute_reply":"2021-09-22T04:34:58.406385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Build and train the model","metadata":{}},{"cell_type":"code","source":"# check format, channel last, (x_train.shape[0], rows, cols, 1)\nprint(keras.backend.image_data_format())","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:58.409007Z","iopub.execute_input":"2021-09-22T04:34:58.409844Z","iopub.status.idle":"2021-09-22T04:34:58.416573Z","shell.execute_reply.started":"2021-09-22T04:34:58.409806Z","shell.execute_reply":"2021-09-22T04:34:58.414599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build the model","metadata":{}},{"cell_type":"code","source":"rows = 32\ncolumns = 40\nbatch_size = 100\nepochs = 50\nbase_path = root_path + \"/working/models\"\n\nif not os.path.exists(base_path):\n    os.makedirs(base_path)\n\ntrain_size = train_set.shape[0]\nvalidation_size = validation_set.shape[0]\nsteps_per_epoch = train_size//batch_size\n\nlr = 1e-3\n#epoch_decay = 8 # decay each epoch_decay epochs\n#decay_steps = epoch_decay * train_size//batch_size\n#lr_schedule = keras.optimizers.schedules.ExponentialDecay(\n#    lr, decay_steps = decay_steps, decay_rate=0.96, staircase=True\n#)\ntensorboard_dir=base_path + \"/logs\"\ntensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=tensorboard_dir)\n\ncheckpoint_filepath = os.path.join(base_path, 'cp-{epoch:04d}.ckpt')\n    \n    \ncheckpoint_callback = keras.callbacks.ModelCheckpoint(\n        filepath= checkpoint_filepath,\n        save_best_only=True,\n        save_weights_only=True,\n        monitor='val_acc',\n        mode='max',\n        verbose=1)\n\nreduce_lr_callback = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2,\n                                                       patience=3, min_lr=1e-5, vebose=1)\n\n    \nearlystop_callback = keras.callbacks.EarlyStopping(\n        monitor=\"val_loss\",\n        min_delta=1e-3,\n        patience=5,\n        verbose=1)\n    \noptimizer = keras.optimizers.Adam(learning_rate = lr)\nloss_fn = keras.losses.SparseCategoricalCrossentropy()\nacc_metric = keras.metrics.SparseCategoricalAccuracy()\n    \nmodel = build_model(len(classes_map), (rows, columns, 1))\nmodel.compile(optimizer = optimizer, loss = loss_fn, metrics= [acc_metric])   \nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:58.419621Z","iopub.execute_input":"2021-09-22T04:34:58.421631Z","iopub.status.idle":"2021-09-22T04:34:59.028753Z","shell.execute_reply.started":"2021-09-22T04:34:58.421591Z","shell.execute_reply":"2021-09-22T04:34:59.027638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fit the model","metadata":{}},{"cell_type":"code","source":"#!tensorboard --logdir=/home/user/Documentos/kaggle/working/model/logs","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:59.031219Z","iopub.execute_input":"2021-09-22T04:34:59.037681Z","iopub.status.idle":"2021-09-22T04:34:59.044749Z","shell.execute_reply.started":"2021-09-22T04:34:59.037635Z","shell.execute_reply":"2021-09-22T04:34:59.043792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the extension and start TensorBoard\n\nhistory = model.fit(train_datagen,\n                    steps_per_epoch= steps_per_epoch,\n                    epochs = epochs,\n                    validation_data = validation_datagen,\n                    validation_steps = validation_size//batch_size,\n                    callbacks=[earlystop_callback, reduce_lr_callback, checkpoint_callback, tensorboard_callback],\n                    use_multiprocessing=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T04:34:59.047722Z","iopub.execute_input":"2021-09-22T04:34:59.048620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.rmtree(train_path)\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize history\n# Plot history: Loss\nplt.plot(history.history['val_loss'], label = \"val_loss\")\nplt.plot(history.history['loss'], label = \"loss\")\nplt.title('Loss history')\nplt.ylabel('Loss value')\nplt.xlabel('No. epoch')\nplt.show()\n\n# Plot history: Accuracy\nplt.plot(history.history['sparse_categorical_accuracy'], label = \"accuracy\")\nplt.plot(history.history['val_sparse_categorical_accuracy'], label = \"val_accuracy\")\nplt.title('Accuracy history')\nplt.ylabel('Accuracy value (%)')\nplt.xlabel('No. epoch')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.exists(root_path + '/working/test/'):\n    os.makedirs(root_path + '/working/test/')\n    Archive(root_path + '/input/tensorflow-speech-recognition-challenge/test.7z').extractall(root_path + '/working')\n\ntest_path = root_path + '/working/test'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data,test_labels,_ = get_image_list(test_path) # single folder\ntest_data = test_data[0]\ntest_labels = test_labels[0]\n\ntest_datagen = data_generator(test_data, test_labels, batch_size,  None)\ntest_size = len(test_data)\ntest_steps = np.ceil(test_size / (batch_size))  # steps same than train; https://github.com/keras-team/keras/issues/3477             \ny_pred = model.predict_generator(test_datagen, steps = test_steps, verbose=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_labs = np.argmax(y_pred, axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking some predictions","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes_map","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inv_map = inv_map = {v: k for k, v in classes_map.items()}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_audio_sample =  test_data[16]\nx,sr = librosa.load(train_audio_sample, sr = 16000)\n\nipd.Audio(x, rate=sr)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inv_map[y_labs[16]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_audio_sample =  test_data[100]\nx,sr = librosa.load(train_audio_sample, sr = 16000)\n\nipd.Audio(x, rate=sr)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inv_map[y_labs[100]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_audio_sample =  test_data[1001]\nx,sr = librosa.load(train_audio_sample, sr = 16000)\n\nipd.Audio(x, rate=sr)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inv_map[y_labs[1001]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.rmtree(test_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_submission = pd.DataFrame({'fname':  test_data, 'label': [inv_map[x] for x in y_labs]})\nmy_submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}