{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"В этом ноутбуке используется датасет Google Speech Commands(GSC). Для аугментации данных на чистые голосовые команды накладывается шум.","metadata":{}},{"cell_type":"code","source":"!pip install python_speech_features\n!pip install pydub","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-03T12:00:59.369227Z","iopub.execute_input":"2022-03-03T12:00:59.369519Z","iopub.status.idle":"2022-03-03T12:01:13.950940Z","shell.execute_reply.started":"2022-03-03T12:00:59.369487Z","shell.execute_reply":"2022-03-03T12:01:13.950117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !cp /content/drive/MyDrive/noises.zip noises.zip \n# !unzip noises.zip","metadata":{"execution":{"iopub.status.busy":"2022-03-03T12:01:13.953851Z","iopub.execute_input":"2022-03-03T12:01:13.954328Z","iopub.status.idle":"2022-03-03T12:01:13.958038Z","shell.execute_reply.started":"2022-03-03T12:01:13.954286Z","shell.execute_reply":"2022-03-03T12:01:13.957264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Download clear data","metadata":{}},{"cell_type":"code","source":"# !wget http://download.tensorflow.org/data/speech_commands_v0.02.tar.gz\n# !tar -x -f /content/speech_commands_v0.02.tar.gz","metadata":{"execution":{"iopub.status.busy":"2022-03-03T12:01:13.959516Z","iopub.execute_input":"2022-03-03T12:01:13.960002Z","iopub.status.idle":"2022-03-03T12:01:13.968632Z","shell.execute_reply.started":"2022-03-03T12:01:13.959965Z","shell.execute_reply":"2022-03-03T12:01:13.967899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add noise to clear data","metadata":{}},{"cell_type":"code","source":"from python_speech_features import mfcc\nimport scipy.io.wavfile as wav\nimport numpy as np\nimport os\nfrom tqdm import tqdm\n\nfrom pydub import AudioSegment as am","metadata":{"execution":{"iopub.status.busy":"2022-03-03T12:01:13.970982Z","iopub.execute_input":"2022-03-03T12:01:13.971299Z","iopub.status.idle":"2022-03-03T12:01:13.979946Z","shell.execute_reply.started":"2022-03-03T12:01:13.971273Z","shell.execute_reply":"2022-03-03T12:01:13.979174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom sklearn.model_selection import train_test_split\nfrom random import randint\nimport scipy.io.wavfile as wav\nimport numpy as np\nfrom keras import backend as K\nfrom keras.layers import Activation\nfrom keras.layers import Input, Lambda, Dense, Dropout, Convolution2D, MaxPooling2D, Flatten, AveragePooling2D, BatchNormalization\nfrom keras.models import Sequential, Model, load_model\nfrom keras.callbacks import ModelCheckpoint","metadata":{"execution":{"iopub.status.busy":"2022-03-03T12:01:13.982633Z","iopub.execute_input":"2022-03-03T12:01:13.983516Z","iopub.status.idle":"2022-03-03T12:01:13.990627Z","shell.execute_reply.started":"2022-03-03T12:01:13.983477Z","shell.execute_reply":"2022-03-03T12:01:13.989862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# noises = []\n# for filename in os.listdir('noises'):\n#     sound = am.from_file('noises'+\"/\"+filename, format='wav')\n#     sound = sound.set_frame_rate(16000)\n#     sound = sound.set_channels(1)\n#     sound.export('file_trim_5s.wav', format='wav')  \n    \n#     (rate,sig) = wav.read('file_trim_5s.wav')\n#     noises.append(sig)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T12:01:13.991881Z","iopub.execute_input":"2022-03-03T12:01:13.992533Z","iopub.status.idle":"2022-03-03T12:01:13.998897Z","shell.execute_reply.started":"2022-03-03T12:01:13.992496Z","shell.execute_reply":"2022-03-03T12:01:13.998092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# types = ['down', 'go', 'left', 'no', 'off', 'on', 'right', 'stop', 'up', 'yes']\n# datasize = 38546\n\n# dataX = np.zeros( (datasize, 16000), dtype=\"int16\") \n# dataY = np.zeros( (datasize), dtype=\"int8\") \n\n# cnt = 0\n\n# pbar = tqdm(total=datasize)\n\n# for i, word in enumerate(types):\n#     for filename in os.listdir(word):\n#         (rate,sig) = wav.read(word+\"/\"+filename)\n#         l = len(sig)\n#         if l > 16000:\n#             sig = sig[:16000]\n#         elif l<16000:\n#             sig = np.pad(sig, (0, 16000-l), 'constant')\n\n#         dataX[cnt] = sig\n#         dataY[cnt] = i\n#         cnt+=1\n#         pbar.update(1)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T12:01:14.000304Z","iopub.execute_input":"2022-03-03T12:01:14.001120Z","iopub.status.idle":"2022-03-03T12:01:14.007745Z","shell.execute_reply.started":"2022-03-03T12:01:14.001034Z","shell.execute_reply":"2022-03-03T12:01:14.006965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def make_some_noise(clean):\n#     max_amp = 1 - np.random.random() / 3\n#     noise = noises[np.random.randint(0, 17)]\n#     noise_amp = np.random.rand() * max_amp\n#     max_start = len(noise) - 16000\n#     start = np.random.randint(0, max_start + 1)\n#     noise_part = noise[start:start+16000]\n#     coef = np.abs(noise_part).max()\n#     noise_mult = 1\n#     if coef != 0:\n#       noise_mult = np.abs(clean.max()) / np.abs(noise_part).max() * noise_amp\n#     return (clean + noise_part * noise_mult) / (1 + noise_amp)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T12:01:14.008959Z","iopub.execute_input":"2022-03-03T12:01:14.009668Z","iopub.status.idle":"2022-03-03T12:01:14.019861Z","shell.execute_reply.started":"2022-03-03T12:01:14.009632Z","shell.execute_reply":"2022-03-03T12:01:14.018973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class DataGenerator(keras.utils.Sequence):\n#     def __init__(self, bcount, batch_size=256, dim=(32,32,32), n_channels=1,\n#                  n_classes=10, shuffle=False):\n      \n#         self.dim = dim\n#         self.batch_size = batch_size\n#         self.bcount = bcount\n#         self.n_channels = n_channels\n#         self.n_classes = n_classes\n#         self.shuffle = shuffle\n#         self.on_epoch_end()\n\n#     def __len__(self):\n#         return self.bcount\n\n#     def __getitem__(self, index):\n#         X, y = self.__data_generation()\n\n#         return X, y\n\n#     def on_epoch_end(self):\n#         if self.shuffle == True:\n#             np.random.shuffle(self.indexes)\n\n#     def __data_generation(self):\n      \n#         # ВОТ ЭТУ ***** НУЖНО УСКОРИТЬ\n\n#         X = np.empty( (256, 99, 13), dtype=\"float32\") \n#         Y = np.empty( (256), dtype=\"int8\") \n\n#         for i in range(256):\n#           ind = np.random.randint(0, 38546)\n#           msk = make_some_noise(dataX[ind]).astype('int16')\n#           cur = mfcc(msk,16000)\n          \n#           X[i] = cur\n#           Y[i] = dataY[ind]\n          \n\n#         return X, keras.utils.to_categorical(Y, num_classes=self.n_classes)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T12:01:14.021177Z","iopub.execute_input":"2022-03-03T12:01:14.021925Z","iopub.status.idle":"2022-03-03T12:01:14.029122Z","shell.execute_reply.started":"2022-03-03T12:01:14.021888Z","shell.execute_reply":"2022-03-03T12:01:14.028185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train model","metadata":{}},{"cell_type":"code","source":"num_classes = 10\ninput_shape = (99, 13, 1)\n\nmodel = keras.Sequential(\n    [\n        keras.Input(shape=input_shape),\n        layers.BatchNormalization(),\n        layers.Conv2D(64, kernel_size=(3, 3), activation=\"relu\", padding = 'same'),\n        layers.Conv2D(64, kernel_size=(3, 3), activation=\"relu\", padding = 'same', strides = (2,2)),\n        layers.Conv2D(128, kernel_size=(3, 3), activation=\"relu\", padding = 'same'),\n        layers.Conv2D(128, kernel_size=(3, 3), activation=\"relu\", padding = 'same', strides = (2,2)),\n        layers.Conv2D(256, kernel_size=(3, 3), activation=\"relu\", padding = 'same'),\n        layers.Conv2D(256, kernel_size=(3, 3), activation=\"relu\", padding = 'same', strides = (2,2)),\n        layers.Conv2D(512, kernel_size=(3, 3), activation=\"relu\", padding = 'same'),\n        layers.Conv2D(256, kernel_size=(3, 3), activation=\"relu\", padding = 'same'),\n        layers.Conv2D(128, kernel_size=(3, 3), activation=\"relu\", padding = 'same'),\n        layers.Flatten(),\n        layers.BatchNormalization(),\n        layers.Dense(256,activation=\"relu\"),\n        layers.Dropout(0.1),\n        layers.Dense(num_classes, activation=\"softmax\"),\n    ]\n)\n","metadata":{"execution":{"iopub.status.busy":"2022-03-03T12:01:14.030090Z","iopub.execute_input":"2022-03-03T12:01:14.033324Z","iopub.status.idle":"2022-03-03T12:01:14.151158Z","shell.execute_reply.started":"2022-03-03T12:01:14.033286Z","shell.execute_reply":"2022-03-03T12:01:14.150447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# training_generator = DataGenerator(100)\n# validation_generator = DataGenerator(10)\n\n\n# model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n\n# checkpoint_filepath = './'\n# mcp_save = ModelCheckpoint(filepath=checkpoint_filepath, save_best_only=True, monitor='val_accuracy', mode='max')\n\n# # Train model on dataset\n# history = model.fit_generator(generator=training_generator,\n#                     validation_data=validation_generator, callbacks = [mcp_save],  epochs=1000)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T12:01:14.152498Z","iopub.execute_input":"2022-03-03T12:01:14.152749Z","iopub.status.idle":"2022-03-03T12:01:14.156922Z","shell.execute_reply.started":"2022-03-03T12:01:14.152710Z","shell.execute_reply":"2022-03-03T12:01:14.155873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Read test data","metadata":{}},{"cell_type":"code","source":"test_path = '../input/classification-of-short-noisy-audio-speech/hackaton_ds/test/'\ntest = []\nfile_names = []\n\npbar = tqdm(total=29620, position=0, leave=True)\n\nfor file_name in os.listdir(test_path):\n    (rate,sig) = wav.read(test_path+file_name)\n    file_names.append(file_name.split('.')[0])\n\n    file_mel = mfcc(sig,rate)\n\n    tmp = np.zeros((99, 13))\n    tmp[:file_mel.shape[0],:file_mel.shape[1]] = file_mel\n\n    test.append(tmp)\n    pbar.update(1)\n    pbar.set_description(f\"\\Read {len(test)} phrazes\")\n\ntest = np.array(test)\ntest = test.reshape((len(test), 99, 13, 1))","metadata":{"execution":{"iopub.status.busy":"2022-03-03T12:01:14.159390Z","iopub.execute_input":"2022-03-03T12:01:14.159660Z","iopub.status.idle":"2022-03-03T12:04:04.110931Z","shell.execute_reply.started":"2022-03-03T12:01:14.159626Z","shell.execute_reply":"2022-03-03T12:04:04.110176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Make submition","metadata":{}},{"cell_type":"code","source":"model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\nmodel.load_weights('../input/weights/weights.028-0.9898.h5')","metadata":{"execution":{"iopub.status.busy":"2022-03-03T12:04:04.113150Z","iopub.execute_input":"2022-03-03T12:04:04.113565Z","iopub.status.idle":"2022-03-03T12:04:04.555452Z","shell.execute_reply.started":"2022-03-03T12:04:04.113516Z","shell.execute_reply":"2022-03-03T12:04:04.554670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(test)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T12:04:04.558152Z","iopub.execute_input":"2022-03-03T12:04:04.558361Z","iopub.status.idle":"2022-03-03T12:04:10.108893Z","shell.execute_reply.started":"2022-03-03T12:04:04.558335Z","shell.execute_reply":"2022-03-03T12:04:10.107978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"types = ['down', 'go', 'left', 'no', 'off', 'on', 'right', 'stop', 'up', 'yes']\nanswer = []\nfor i in np.argmax(predictions, axis = -1):\n    answer.append(types[i])","metadata":{"execution":{"iopub.status.busy":"2022-03-03T12:04:10.110342Z","iopub.execute_input":"2022-03-03T12:04:10.110620Z","iopub.status.idle":"2022-03-03T12:04:10.125779Z","shell.execute_reply.started":"2022-03-03T12:04:10.110581Z","shell.execute_reply":"2022-03-03T12:04:10.125024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({'id':file_names, 'category':answer})\ndf.to_csv('./submition.csv',index=False, sep=',', encoding='utf-8')","metadata":{"execution":{"iopub.status.busy":"2022-03-03T12:04:10.128869Z","iopub.execute_input":"2022-03-03T12:04:10.129575Z","iopub.status.idle":"2022-03-03T12:04:10.199721Z","shell.execute_reply.started":"2022-03-03T12:04:10.129528Z","shell.execute_reply":"2022-03-03T12:04:10.199097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}