{"cells":[{"metadata":{"_uuid":"d9596d80cb6445d4214dda15e40d777cadbd4669","_cell_guid":"8fd82027-7be0-4a4e-a921-b8acacaaf077","trusted":true,"collapsed":true},"cell_type":"code","source":"import os\nfrom os.path import isdir, join\nfrom pathlib import Path\nimport pandas as pd\nimport random\n# Math\nimport numpy as np\nfrom scipy.fftpack import fft\nfrom scipy import signal\nfrom scipy.io import wavfile\nimport librosa\n\nfrom sklearn.decomposition import PCA\n\n# Visualization\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport IPython.display as ipd\nimport librosa.display\n\nimport pandas as pd\n\nimport csv\nimport h5py\nimport math","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom pydub import AudioSegment","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"%%time\n#!cp ../input/tensorflow-speech-recognition-challenge/train.7z .\n#!cp ../input/tensorflow-speech-recognition-challenge/sample*.7z .\n#!cp ../input/train.7z .\n#!cp ../input/test.7z .\n!apt -y install libsndfile1\n!pip3 install pydub\n# ON-TPU\n#!apt install p7zip\n#!/usr/bin/p7zip -h\n#!p7zip -d train.7z\n#!p7zip -d sample*.7z","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"#!ls train/audio/\n!rm train.7z\n#!rm -rf train/audio/_back*\n!find train/audio/. | grep \"\\.wav\" > trainwav.txt\n#!ls train/\n#!df . -h","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"class Preprocess:\t\n\n\tdef __init__(self):\t\t\n\t\tself.pathlist = \"kaggle.txt\"\n\t\tself.base_path = \"dataset/train/audio/\"\n\t\tself.frame_length = 0.025\n\t\tself.frame_stride = 0.010\n\t\tself.sample_rate = 16000\n\t\tself.input_nfft = int(round(self.sample_rate * self.frame_length))\n\t\tself.input_stride = int(round(self.sample_rate * self.frame_stride))\n\t\tself.labels = {'yes':0, 'no':1, 'up':2, 'down':3, 'left':4, 'right':5, 'on':6, 'off':7, 'stop':8, 'go':9, 'silence':10, 'unknown':11}\n\n\n\tdef getwavelist(self):\n\t\tf = open(self.base_path + self.pathlist,'r')\n\t\tself.filelist = f.readlines()\n\t\trandom.shuffle(self.filelist)\n\n\t\tself.numofdata = len(self.filelist)\n\t\tprint(self.numofdata, \"is num of data\")\n\n\n\tdef split_silence(self):\n\t\tbasic_path = \"dataset/train/audio/_background_noise_/\"\n\t\tdest_path = \"dataset/train/audio/silence/\"\n\n\t\tsil_fl = os.listdir(basic_path)\n\t\tsil_fl.remove(\"README.md\")\n\n\t\tnumof_silfile = 0\n\t\tfor fidx, fl in enumerate(sil_fl):    \n\t\t\ts, sr = librosa.load(basic_path + fl, sr=16000)\n\t\t\tfilelen = int(len(s)/sr)\n\n\t\t\taudio = AudioSegment.from_wav(basic_path + fl)\n\n\t\t\tfor sidx in range(0, (filelen-1)*1000, 250):\n\t\t\t\tnewAudio = audio[sidx:sidx+1000]\n\t\t\t\tnewAudio.export(dest_path + '%d_%d.wav' % (fidx, sidx) , format=\"wav\")\n\t\t\t\tnumof_silfile += 1\n\n\t\tprint(\"%d silence file is created.\" % numof_silfile)\n\n\n\tdef preprocessing(self):\n\t\ttrain_data = np.zeros([self.numofdata, 40, 101], dtype=float)\n\t\ttrain_label = np.zeros([self.numofdata], dtype=int)\n\n\t\tunknown_cnt = 0\n\t\ttotal_cnt = 0\n\t\tfor fl in self.filelist:\n\t\t    fl = fl[:-1]\n\t\t    lab = self.labels.get(fl.split(\"/\")[1])    \n\t\t    \n\t\t    if str(lab) == \"None\":\n\t\t        if unknown_cnt >= 4000:\n\t\t            continue\n\t\t            \n\t\t        train_label[total_cnt] = 11\n\t\t        unknown_cnt += 1        \n\t\t    else:        \n\t\t        train_label[total_cnt] = lab\n\t\t        \n\t\t    samples, sample_rate = librosa.load(self.base_path + fl, sr=16000)\n\t\t    S = librosa.feature.melspectrogram(y=samples, n_mels=40, n_fft=self.input_nfft, hop_length=self.input_stride)\n\t\t    if S.shape[1] != 101:\n\t\t        S = librosa.util.fix_length(S, 101, axis=1) # zero-paddings\n\t\t        \n\t\t    train_data[total_cnt] = S\n\t\t    total_cnt += 1\n\t\t    if total_cnt%2000 == 0 :\n\t\t        print(total_cnt)\n\n\t\tself.numofdata = total_cnt\n\t\tprint(\"%d file processed. Done.\" % self.numofdata)\n\n\t\ttrain_data = train_data[:self.numofdata]\n\t\tt_data  = train_data.reshape(self.numofdata, 40, 101, 1)\n\t\tt_label = keras.utils.to_categorical(train_label, 12)\n\n\t\treturn t_data, t_label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"class Modeling:\n\n\tdef __init(self):\n\t\tpass\n\n\n\tdef build_model(self):  \n\t\tself.model = keras.Sequential([\n\t\t\tlayers.Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=[40,101,1]),\n\t\t    layers.Conv2D(64, kernel_size=(3, 3), activation='relu'),\n\t\t    layers.Dropout(0.2),\n\t\t    layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2)),\n\t\t    layers.Conv2D(128, kernel_size=(3, 3), activation='relu'),\n\t\t    layers.Dropout(0.3),\n\t\t    layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2)),\n\t\t    layers.Flatten(),\n\t\t    layers.Dense(256, activation='relu'),\n\t\t    layers.Dropout(0.5),\n\t\t    layers.Dense(12, activation='softmax')\n\t\t])\n  \t\t\n\t\toptimizer = tf.keras.optimizers.Adam()\n\t\tself.model.compile(loss='categorical_crossentropy',\n                \toptimizer=optimizer,\n                \tmetrics=['acc'])\n\n\t\tprint(self.model.summary())\n\n\n\tdef load_model(self, weight):\n\t\tself.model.load_weights(weight)\n\n\n\tdef save_model(self, filename):\n\t\tself.model.save_weights(filename)\n\n\n\tdef train_model(self, train_data, train_label, bs=512, epoch=5, v_split=0.1):\n\t\thistory = self.model.fit(train_data, train_label, shuffle=True, batch_size=bs,\n\t\t\t\t\t\t\t\t epochs=epoch, validation_split=v_split, verbose=1)\n\t\treturn history\n\n\n\tdef predict_model(self, test_data, bs=512):\n\t\tpredict_result = self.model.predict(test_data, batch_size=bs, verbose=1)\n\t\tpred_res = np.argmax(predict_result, axis=-1)\t\t\n\t\treturn pred_res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"class Postprocess:\n\n\tdef __init__(self):\n\t\tself.pathlist = \"kaggle.txt\"\n\t\tself.base_path = \"dataset/test/audio/\"\n\t\tself.submission_file = \"hj_submission.csv\"\n\t\tself.frame_length = 0.025\n\t\tself.frame_stride = 0.010\n\t\tself.sample_rate = 16000\n\t\tself.input_nfft = int(round(self.sample_rate * self.frame_length))\n\t\tself.input_stride = int(round(self.sample_rate * self.frame_stride))\n\t\tself.labels = ['yes', 'no', 'up', 'down', 'left', 'right', 'on', 'off', 'stop', 'go', 'silence', 'unknown']\n\n\t\tIdLookupTable = pd.read_csv(\"dataset/sample_submission.csv\", na_values = \"?\", comment='\\t', sep=\",\", skipinitialspace=True)\n\t\tself.Fname = IdLookupTable[\"fname\"]\n\t\tself.numofdata = len(self.Fname)\t\t\n\n\n\tdef makehdf5(self, f_towrite):\t\t\n\t\tf = open(self.base_path + self.pathlist, 'r')\n\t\tfilelist = f.readlines()\n\t\ttest_data = np.zeros([self.numofdata, 40, 101], dtype=float)\n\n\t\tfor idx, fl in enumerate(filelist):\n\t\t    fl = fl[:-1]\n\n\t\t    samples, sample_rate = librosa.load(self.base_path + fl, sr=self.sample_rate)\n\t\t    S = librosa.feature.melspectrogram(y=samples, n_mels=40, n_fft=self.input_nfft, hop_length=self.input_stride)\n\t\t    if S.shape[1] != 101:\n\t\t        S = librosa.util.fix_length(S, 101, axis=1)  # zero-paddings\n\n\t\t    test_data[idx] = S\n\t\t    if idx % 2000 == 0:\n\t\t        print(idx)\n\n\t\twith h5py.File(f_towrite, 'w') as hf:\n\t\t    dy_str = h5py.special_dtype(vlen=str)\n\t\t    fn_data = hf.create_dataset('feature', [self.numofdata, 40, 101], dtype=float)\n\t\t    fn_data[:self.numofdata] = test_data[:self.numofdata]\n\n\t\tf.close()\n\n\tdef loadhdf5(self, f_toread):\n\t\twith h5py.File(f_toread, 'r') as hf:\n\t\t\tfor i in range(math.ceil(self.numofdata / 10000)):\n\t\t\t\tif (i+1)*10000 > self.numofdata:\n\t\t\t\t\tfeature = hf.get('feature')[i*10000 : self.numofdata]\n\t\t\t\t\tyield feature.reshape(self.numofdata - i*10000, 40, 101, 1)\n\t\t\t\telse:\n\t\t  \t\t\tfeature = hf.get('feature')[i*10000 : (i+1)*10000]\n\t\t  \t\t\tyield feature.reshape(10000, 40, 101, 1)\n\n\n\tdef postprocessing(self, predict_res):\n\t\tf = open(self.submission_file, 'w', newline='')\t\t\n\t\twr = csv.writer(f)\n\t\twr.writerow(['fname','label'])\n\n\t\tfor i in range(self.numofdata):\n\t\t    pred_label = self.labels[int(predict_res[i])]\n\t\t    wr.writerow([self.Fname[i], pred_label]) \n\n\t\tf.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"def main():\n\n\t# PREPROCESSING\n\tpre_process = Preprocess()\n\t#pre_process.split_silence() # 1568 silence file is created.\n\t#pre_process.getwavelist() # 66289 is num of data\n\t#t_data, t_label = pre_process.preprocessing() # but only 29500 file is used in here\n\n\t# MODELING AND TRAINING\n\tmodel = Modeling()\n\tmodel.build_model()\n\t#history = model.train_model(t_data, t_label, epoch=30)\n\n\t# save weights\n\t#model.save_model(\"dataset/train/\" + \"hj_weights.h5\")\n\tmodel.load_model(\"dataset/train/hj_weights.h5\")\n\n\t#result = model.predict_model(t_data[:512])\n\t#print(result)\n\n\t# POSTPROCESSING\n\tpost_process = Postprocess()\n\t#post_process.makehdf5(\"dataset/test/postprocess.hdf5\") # only once is enough\n\n\tpost_feature = post_process.loadhdf5(\"dataset/test/postprocess.hdf5\")\n\ttest_predict = np.zeros([post_process.numofdata], dtype=int)\n\tfor idx, pf in enumerate(post_feature):\n\t\tpred_res = model.predict_model(pf)\t\t\n\t\ttest_predict[idx*10000:idx*10000+len(pred_res)] = pred_res\n\t\n\tpost_process.postprocessing(test_predict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"model = Modeling()\nmodel.build_model()\n#model.load_model(\"weights/hj_weights.h5\")\n!ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"if __name__ == \"__main__\":    \n    pass\n    #main()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}