{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfrom os.path import isdir, join\nimport shutil\nfrom tqdm import tqdm\n# TODO: create train and test dataset\n\nf = open('../input/train/testing_list.txt', 'r')\ntest_list = f.readlines()\nf.close()\n\naudio_path = \"../input/train/audio\"\n\ntest_path_list = []\nfor n in test_list:\n test_path_list.append(n[:-1])\n\ntest_path = \"../test/\"\nos.makedirs(test_path)\ni=0\nfor file in test_path_list:\n #print((audio_path+\"/\"+ file))\n shutil.copy((audio_path+\"/\"+ file), \"../test/\"+str(i)+\".wav\")\n # if isdir((audio_path +\"/\" + file)):\n i+=1\n\n\n\ndirs = [f for f in os.listdir(audio_path) if isdir(join(audio_path, f))]\ndirs.sort()\nrecordings = []\nfor direct in dirs:\n waves = [f for f in os.listdir(join(audio_path, direct)) if f.endswith('.wav')]\n for wave in waves:\n recordings.append(str(direct)+\"/\"+wave)\n\nprint(\"total data available is {}\".format(len(recordings)))\n\ntrain_path = \"../train/\"\nfor cat in dirs:\n path = train_path + str(cat)\n os.makedirs(path)\n\n\nprint(\"creating test and train dataset...\")\n\nfor direct in tqdm(dirs):\n for f in os.listdir(join(audio_path, direct)):\n if f.endswith('.wav'):\n # print(str(direct)+\"/\"+f)\n if (str(direct)+\"/\"+f) not in test_path_list:\n \n \n shutil.copy(str(audio_path)+\"/\"+str(direct)+\"/\"+f, '../train/'+str(direct))\n\nprint(\"number of test points {}\".format(i))\nprint(\"counting the train dataset...\")\ntrain_recordings = []\nfor direct in tqdm(dirs):\n waves = [f for f in os.listdir(join(train_path, direct)) if f.endswith('.wav')]\n for wave in waves:\n train_recordings.append(str(direct)+\"/\"+wave)\nprint(\"total number of files in train is {}\".format(len(train_recordings)))","execution_count":null,"outputs":[{"output_type":"stream","text":"\r 0%| | 0/31 [00:00 0 else result\n\n def get_one_word_wav(self, idx, speed_rate=None):\n wav = librosa.load(self.wav_list[idx], sr=self.sr)[0]\n if speed_rate:\n wav = librosa.effects.time_stretch(wav, speed_rate)\n if len(wav) < self.sr:\n wav = np.pad(wav, (0, self.sr - len(wav)), 'constant')\n return wav[:self.sr]\n\n def get_silent_wav(self, num_noise=1, max_ratio = 0.5):\n return self.get_mix_noises(num_noise=num_noise, max_ratio=max_ratio)\n\n def timeshift(self, wav, ms = 100):\n shift = (self.sr * ms) // 1000\n shift = random.randint(-shift, shift)\n a = -min(0, shift)\n b = max(0, shift)\n data = np.pad(wav, (a, b), \"constant\")\n return data[:len(data) - a] if a else data[b:]\n\n def get_noisy_wav(self, idx):\n scale = random.uniform(0.75, 1.25)\n num_noise = random.choice([1,2])\n max_ratio = random.choice([0.1, 0.5, 1, 2])\n mix_noise_proba = 0.25\n shift_range = random.randint(80, 120)\n if random.random() < mix_noise_proba:\n return scale * (self.timeshift(self.get_one_word_wav(idx), shift_range) + self.get_mix_noises(\n num_noise, max_ratio))\n else:\n return scale * self.timeshift(self.get_one_word_wav(idx), shift_range)\n\n def __len__(self):\n return len(self.wav_list)\n\n def __getitem__(self, idx):\n \"\"\"reads one sample\"\"\"\n wav_numpy = self.preprocess_fun(self.get_noisy_wav(idx), **self.preprocess_param)\n if self.resize_shape:\n wav_numpy = resize(wav_numpy, (self.resize_shape, self.resize_shape), preserve_range=True)\n wav_tensor = torch.from_numpy(wav_numpy).float()\n if not self.is_1d:\n wav_tensor = wav_tensor.unsqueeze(0)\n\n label_word = self.semi_dict[self.wav_list[idx].split('/')[-1]]\n if label_word == \"unknown\":\n label = 10\n elif label_word == 'silence':\n label = 11\n else:\n label = self.label_words_dict[label_word]\n\n return {'spec': wav_tensor, 'id': self.wav_list[idx], 'label': label}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport numpy as np\nfrom torch.utils.data import Dataset\nimport librosa\nfrom glob import glob\nimport random\nfrom skimage.transform import resize\nimport pandas as pd\nfrom random import sample\n\nSR=16000\n\nclass SpeechDataset(Dataset):\n def __init__(self, mode, label_words_dict, wav_list, add_noise, preprocess_fun, preprocess_param = {}, sr=SR, resize_shape=None, is_1d=False):\n \"\"\"Args:\n mode: train or evaluate or test\n label_words_dict: a dict of words for labels\n wav_list: a list of wav file paths\n add_noise: boolean. if background noise should be added\n preprocess_fun: function to load/process wav file\n preprocess_param: params for preprocess_fun\n sr: default 16000\n resize_shape: None. only for 2d cnn.\n is_1d: boolean. if it is going to be 1d cnn or 2d cnn\n \"\"\"\n self.mode = mode\n self.label_words_dict = label_words_dict\n self.wav_list = wav_list\n self.add_noise = add_noise\n self.sr = sr\n self.n_silence = int(len(wav_list) * 0.09)\n self.preprocess_fun = preprocess_fun\n self.preprocess_param = preprocess_param\n\n # read all background noise here\n self.background_noises = [librosa.load(x, sr=self.sr)[0] for x in glob(\"../train/_background_noise_/*.wav\")]\n self.resize_shape = resize_shape\n self.is_1d = is_1d\n\n def get_one_noise(self):\n \"\"\"generates one single noise clip\"\"\"\n selected_noise = self.background_noises[random.randint(0, len(self.background_noises) - 1)]\n # only takes out 16000\n start_idx = random.randint(0, len(selected_noise) - 1 - self.sr)\n return selected_noise[start_idx:(start_idx + self.sr)]\n\n def get_mix_noises(self, num_noise=1, max_ratio=0.1):\n result = np.zeros(self.sr)\n for _ in range(num_noise):\n result += random.random() * max_ratio * self.get_one_noise()\n return result / num_noise if num_noise > 0 else result\n\n def get_one_word_wav(self, idx):\n wav = librosa.load(self.wav_list[idx], sr=self.sr)[0]\n if len(wav) < self.sr:\n wav = np.pad(wav, (0, self.sr - len(wav)), 'constant')\n return wav[:self.sr]\n\n def get_silent_wav(self, num_noise=1, max_ratio=0.5):\n return self.get_mix_noises(num_noise=num_noise, max_ratio=max_ratio)\n\n def timeshift(self, wav, ms=100):\n shift = (self.sr * ms) // 1000\n shift = random.randint(-shift, shift)\n a = -min(0, shift)\n b = max(0, shift)\n data = np.pad(wav, (a, b), \"constant\")\n return data[:len(data) - a] if a else data[b:]\n\n def get_noisy_wav(self, idx):\n scale = random.uniform(0.75, 1.25)\n num_noise = random.choice([1, 2])\n max_ratio = random.choice([0.1, 0.5, 1, 1.5])\n mix_noise_proba = random.choice([0.1, 0.3])\n shift_range = random.randint(80, 120)\n one_word_wav = self.get_one_word_wav(idx)\n if random.random() < mix_noise_proba:\n return scale * (self.timeshift(one_word_wav, shift_range) + self.get_mix_noises(\n num_noise, max_ratio))\n else:\n return one_word_wav\n\n def __len__(self):\n if self.mode == 'test':\n return len(self.wav_list)\n else:\n return len(self.wav_list) + self.n_silence\n\n def __getitem__(self, idx):\n \"\"\"reads one sample\"\"\"\n if idx < len(self.wav_list):\n wav_numpy = self.preprocess_fun(\n self.get_one_word_wav(idx) if self.mode != 'train' else self.get_noisy_wav(idx),\n **self.preprocess_param)\n if self.resize_shape:\n wav_numpy = resize(wav_numpy, (self.resize_shape, self.resize_shape), preserve_range=True)\n wav_tensor = torch.from_numpy(wav_numpy).float()\n if not self.is_1d:\n wav_tensor = wav_tensor.unsqueeze(0)\n if self.mode == 'test':\n return {'spec': wav_tensor, 'id': self.wav_list[idx]}\n\n label = self.label_words_dict[self.wav_list[idx].split(\"/\")[-2]] if self.wav_list[idx].split(\n \"/\")[-2] in self.label_words_dict else len(self.label_words_dict)\n\n return {'spec': wav_tensor, 'id': self.wav_list[idx], 'label': label}\n\n else:\n \"\"\"generates silence here\"\"\"\n wav_numpy = self.preprocess_fun(self.get_silent_wav(\n num_noise=random.choice([0, 1, 2, 3]),\n max_ratio=random.choice([x / 10. for x in range(20)])), **self.preprocess_param)\n if self.resize_shape:\n wav_numpy = resize(wav_numpy, (self.resize_shape, self.resize_shape), preserve_range=True)\n wav_tensor = torch.from_numpy(wav_numpy).float()\n if not self.is_1d:\n wav_tensor = wav_tensor.unsqueeze(0)\n return {'spec': wav_tensor, 'id': 'silence', 'label': len(self.label_words_dict) + 1}\n\n\ndef get_label_dict():\n words = ['yes', 'no', 'up', 'down', 'left', 'right', 'on', 'off', 'stop', 'go']\n label_to_int = dict(zip(words, range(len(words))))\n int_to_label = dict(zip(range(len(words)), words))\n int_to_label.update({len(words): 'unknown', len(words) + 1: 'silence'})\n return label_to_int, int_to_label\n\n\ndef get_wav_list(words, unknown_ratio=0.2):\n full_train_list = glob(\"../train/*/*.wav\")\n full_test_list = glob(\"../test/*.wav\")\n\n # sample full train list\n sampled_train_list = []\n for w in full_train_list:\n l = w.split(\"/\")[-2]\n if l not in words:\n if random.random() < unknown_ratio:\n sampled_train_list.append(w)\n else:\n sampled_train_list.append(w)\n\n return sampled_train_list, full_test_list\n\n\ndef get_sub_list(num, sub_path):\n lst = []\n df = pd.read_csv(sub_path)\n words = ['yes', 'no', 'up', 'down', 'left', 'right', 'on', 'off', 'stop', 'go', 'silence', 'unknown']\n each_num = int(num * 0.085)\n for w in words:\n tmp = df['fname'][df['label'] == w].sample(each_num).tolist()\n lst += [\"../test/\" + x for x in tmp]\n return lst\n\n\ndef get_semi_list(words, sub_path, unknown_ratio=0.2, test_ratio=0.2):\n train_list, _ = get_wav_list(words=words, unknown_ratio=unknown_ratio)\n test_list = get_sub_list(num=int(len(train_list) * test_ratio), sub_path=sub_path)\n lst = train_list + test_list\n return sample(lst, len(lst))\n\n\ndef preprocess_mfcc(wave):\n\n spectrogram = librosa.feature.melspectrogram(wave, sr=SR, n_mels=40, hop_length=160, n_fft=480, fmin=20, fmax=4000)\n idx = [spectrogram > 0]\n spectrogram[idx] = np.log(spectrogram[idx])\n\n dct_filters = librosa.filters.dct(n_filters=40, n_input=40)\n mfcc = [np.matmul(dct_filters, x) for x in np.split(spectrogram, spectrogram.shape[1], axis=1)]\n mfcc = np.hstack(mfcc)\n mfcc = mfcc.astype(np.float32)\n return mfcc\n\n\ndef preprocess_mel(data, n_mels=40, normalization=False):\n spectrogram = librosa.feature.melspectrogram(data, sr=SR, n_mels=n_mels, hop_length=160, n_fft=480, fmin=20, fmax=4000)\n spectrogram = librosa.power_to_db(spectrogram)\n spectrogram = spectrogram.astype(np.float32)\n if normalization:\n spectrogram = spectrogram.spectrogram()\n spectrogram -= spectrogram\n return spectrogram\n\n\ndef preprocess_wav(wav, normalization=True):\n data = wav.reshape(1, -1)\n if normalization:\n mean = data.mean()\n data -= mean\n return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nmodel trainer\n\"\"\"\nfrom torch.autograd import Variable\nfrom torch.utils.data import DataLoader\nimport torch\nfrom time import time\nfrom torch.nn import Softmax\nimport numpy as np\nimport pandas as pd\nimport os\nfrom random import choice\n\n\ndef train_model(model_class, preprocess_fun, is_1d, reshape_size, BATCH_SIZE, epochs, CODER, preprocess_param={},\n bagging_num=1, semi_train_path=None, pretrained=None, pretraining=False, MGPU=False):\n \"\"\"\n :param model_class: model class. e.g. vgg, resnet, senet\n :param preprocess_fun: preprocess function. e.g. mel, mfcc, raw wave\n :param is_1d: boolean. True for conv1d models and false for conv2d\n :param reshape_size: int. only for conv2d, reshape the image size\n :param BATCH_SIZE: batch size.\n :param epochs: number of epochs\n :param CODER: string for saving and loading model/files\n :param preprocess_param: parameters for preprocessing function\n :param bagging_num: number of training per model, aka bagging models\n :param semi_train_path: path to semi supervised learning file.\n :param pretrained: path to pretrained model\n :param pretraining: boolean. if this is pretraining\n :param MGPU: whether using multiple gpus\n \"\"\"\n\n def get_model(model=model_class, m=MGPU, pretrained=pretrained):\n mdl = torch.nn.DataParallel(model()) if m else model()\n if not pretrained:\n return mdl\n else:\n print(\"load pretrained model here...\")\n mdl.load_state_dict(torch.load(pretrained))\n if 'vgg' in pretrained:\n fixed_layers = list(mdl.features)\n for l in fixed_layers:\n for p in l.parameters():\n p.requires_grad = False\n return mdl\n\n label_to_int, int_to_label = get_label_dict()\n for b in range(bagging_num):\n print(\"training model # \", b)\n\n loss_fn = torch.nn.CrossEntropyLoss()\n\n speechmodel = get_model()\n speechmodel = speechmodel.cuda()\n\n total_correct = 0\n num_labels = 0\n start_time = time()\n\n for e in range(epochs):\n print(\"training epoch \", e)\n learning_rate = 0.01 if e < 10 else 0.001\n optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, speechmodel.parameters()), lr=learning_rate, momentum=0.9, weight_decay=0.00001)\n speechmodel.train()\n if semi_train_path:\n train_list = get_semi_list(words=label_to_int.keys(), sub_path=semi_train_path,\n test_ratio=choice([0.2, 0.25, 0.3, 0.35]))\n print(\"semi training list length: \", len(train_list))\n else:\n train_list, _ = get_wav_list(words=label_to_int.keys())\n\n if pretraining:\n traindataset = PreDataset(label_words_dict=label_to_int,\n add_noise=True, preprocess_fun=preprocess_fun, preprocess_param=preprocess_param,\n resize_shape=reshape_size, is_1d=is_1d)\n else:\n traindataset = SpeechDataset(mode='train', label_words_dict=label_to_int, wav_list=train_list,\n add_noise=True, preprocess_fun=preprocess_fun, preprocess_param=preprocess_param,\n resize_shape=reshape_size, is_1d=is_1d)\n trainloader = DataLoader(traindataset, BATCH_SIZE, shuffle=True)\n for batch_idx, batch_data in enumerate(trainloader):\n spec = batch_data['spec']\n label = batch_data['label']\n spec, label = Variable(spec.cuda()), Variable(label.cuda())\n y_pred = speechmodel(spec)\n _, pred_labels = torch.max(y_pred.data, 1)\n correct = (pred_labels == label.data).sum()\n loss = loss_fn(y_pred, label)\n\n total_correct += correct\n num_labels += len(label)\n\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n print(\"training loss:\", 100. * total_correct / num_labels, time()-start_time)\n\n # save model\n create_directory(\"model\")\n torch.save(speechmodel.state_dict(), \"model/model_%s_%s.pth\" % (CODER, b))\n\n if not pretraining:\n print(\"doing prediction...\")\n softmax = Softmax()\n\n trained_models = [\"model/model_%s_%s.pth\" % (CODER, b) for b in range(bagging_num)]\n\n # prediction\n _, test_list = get_wav_list(words=label_to_int.keys())\n testdataset = SpeechDataset(mode='test', label_words_dict=label_to_int, wav_list=test_list,\n add_noise=False, preprocess_fun=preprocess_fun, preprocess_param=preprocess_param,\n resize_shape=reshape_size, is_1d=is_1d)\n testloader = DataLoader(testdataset, BATCH_SIZE, shuffle=False)\n\n for e, m in enumerate(trained_models):\n print(\"predicting \", m)\n speechmodel = get_model(m=MGPU)\n speechmodel.load_state_dict(torch.load(m))\n speechmodel = speechmodel.cuda()\n speechmodel.eval()\n\n test_fnames, test_labels = [], []\n pred_scores = []\n # do prediction and make a submission file\n for batch_idx, batch_data in enumerate(testloader):\n spec = Variable(batch_data['spec'].cuda())\n fname = batch_data['id']\n y_pred = softmax(speechmodel(spec))\n pred_scores.append(y_pred.data.cpu().numpy())\n test_fnames += fname\n\n if e == 0:\n final_pred = np.vstack(pred_scores)\n final_test_fnames = test_fnames\n else:\n final_pred += np.vstack(pred_scores)\n assert final_test_fnames == test_fnames\n\n final_pred /= len(trained_models)\n final_labels = [int_to_label[x] for x in np.argmax(final_pred, 1)]\n\n test_fnames = [x.split(\"/\")[-1] for x in final_test_fnames]\n\n labels = ['yes', 'no', 'up', 'down', 'left', 'right', 'on', 'off', 'stop', 'go', 'unknown', 'silence']\n pred_scores = pd.DataFrame(np.vstack(final_pred), columns=labels)\n pred_scores['fname'] = test_fnames\n\n create_directory(\"pred_scores\")\n pred_scores.to_csv(\"pred_scores/%s.csv\" % CODER, index=False)\n\n create_directory(\"sub\")\n pd.DataFrame({'fname': test_fnames,\n 'label': final_labels}).to_csv(\"sub/%s.csv\" % CODER, index=False)\n\n\ndef create_directory(dir):\n if not os.path.exists(dir):\n os.makedirs(dir)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list_2d = [('mfcc', preprocess_mfcc)]\nBAGGING_NUM=1\n\ndef train_and_predict(cfg_dict, preprocess_list):\n for p, preprocess_fun in preprocess_list:\n cfg = cfg_dict.copy()\n cfg['preprocess_fun'] = preprocess_fun\n cfg['CODER'] += '_%s' %p\n cfg['bagging_num'] = BAGGING_NUM\n print(\"training \", cfg['CODER'])\n train_model(**cfg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport math\n\nclass VGG(torch.nn.Module):\n\n def __init__(self, features):\n super(VGG, self).__init__()\n n_labels = 12\n self.features = features\n self.classifier = nn.Sequential(\n nn.Linear(256 * 2, 1024),\n nn.ReLU(True),\n nn.Dropout(),\n nn.Linear(1024, 256),\n nn.ReLU(True),\n nn.Dropout(0.25),\n nn.Linear(256, n_labels),\n )\n\n def forward(self, x):\n x = self.features(x)\n max_x = F.adaptive_max_pool2d(x, (1, 1))\n avg_x = F.adaptive_avg_pool2d(x, (1, 1))\n x = torch.cat([max_x, avg_x], 1)\n x = x.view(x.size(0), -1)\n x = self.classifier(x)\n return x\n\n\ndef make_layers(cfg, batch_norm=True, ks=3, kp=1):\n layers = []\n in_channels = 1\n for v in cfg:\n if v == 'M':\n layers += [nn.MaxPool2d(kernel_size=2, stride=2)]\n else:\n conv2d = nn.Conv2d(in_channels, v, kernel_size=ks, padding=kp)\n if batch_norm:\n layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)]\n else:\n layers += [conv2d, nn.ReLU(inplace=True)]\n in_channels = v\n return nn.Sequential(*layers)\n\n\ndef vgg2d():\n \"\"\"VGG 16-layer model with batch normalization\"\"\"\n return VGG(make_layers([64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 256, 256, 256, 'M']))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vgg2d_config = {\n 'model_class': vgg2d,\n 'is_1d': False,\n 'reshape_size': 128,\n 'BATCH_SIZE': 32,\n 'epochs': 100,\n 'CODER': 'vgg2d'\n}\n\nprint(\"train vgg2d...........\")\ntrain_and_predict(vgg2d_config, list_2d)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls sub/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"op = pd.read_csv(\"sub/vgg2d_mel.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"op.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}