{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Configuration"},{"metadata":{},"cell_type":"markdown","source":"* Usual Conv Net\n* noisy all\n* noisy pitch"},{"metadata":{},"cell_type":"markdown","source":"## imports"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport sys\nimport time\nimport pickle\nimport random\nimport logging\nimport datetime as dt\n\nimport librosa\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom PIL import Image\nfrom collections import Counter, defaultdict\nfrom functools import partial\nfrom pathlib import Path\nfrom psutil import cpu_count\n\nfrom fastprogress import master_bar, progress_bar\nfrom torch.optim import Adam, SGD\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.transforms import transforms\n\nfrom sklearn.model_selection import train_test_split, KFold\n\nfrom tqdm import tqdm_notebook","execution_count":1,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_logger(name=\"Main\", tag=\"exp\", log_dir=\"log/\"):\n log_path = Path(log_dir)\n path = log_path / tag\n path.mkdir(exist_ok=True, parents=True)\n\n logger = logging.getLogger(name)\n logger.setLevel(logging.INFO)\n\n fh = logging.FileHandler(\n path / (dt.datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\") + \".log\"))\n sh = logging.StreamHandler(sys.stdout)\n formatter = logging.Formatter(\n \"%(asctime)s %(name)s %(levelname)s %(message)s\")\n\n fh.setFormatter(formatter)\n sh.setFormatter(formatter)\n logger.addHandler(fh)\n logger.addHandler(sh)\n return logger","execution_count":2,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n random.seed(seed)\n os.environ['PYTHONHASHSEED'] = str(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.deterministic = True\n\nSEED = 520\nseed_everything(SEED)","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from official code https://colab.research.google.com/drive/1AgPdhSp7ttY18O3fEoHOQKlt_3HJDLi8#scrollTo=cRCaCIb9oguU\ndef _one_sample_positive_class_precisions(scores, truth):\n \"\"\"Calculate precisions for each true class for a single sample.\n\n Args:\n scores: np.array of (num_classes,) giving the individual classifier scores.\n truth: np.array of (num_classes,) bools indicating which classes are true.\n\n Returns:\n pos_class_indices: np.array of indices of the true classes for this sample.\n pos_class_precisions: np.array of precisions corresponding to each of those\n classes.\n \"\"\"\n num_classes = scores.shape[0]\n pos_class_indices = np.flatnonzero(truth > 0)\n # Only calculate precisions if there are some true classes.\n if not len(pos_class_indices):\n return pos_class_indices, np.zeros(0)\n # Retrieval list of classes for this sample.\n retrieved_classes = np.argsort(scores)[::-1]\n # class_rankings[top_scoring_class_index] == 0 etc.\n class_rankings = np.zeros(num_classes, dtype=np.int)\n class_rankings[retrieved_classes] = range(num_classes)\n # Which of these is a true label?\n retrieved_class_true = np.zeros(num_classes, dtype=np.bool)\n retrieved_class_true[class_rankings[pos_class_indices]] = True\n # Num hits for every truncated retrieval list.\n retrieved_cumulative_hits = np.cumsum(retrieved_class_true)\n # Precision of retrieval list truncated at each hit, in order of pos_labels.\n precision_at_hits = (\n retrieved_cumulative_hits[class_rankings[pos_class_indices]] /\n (1 + class_rankings[pos_class_indices].astype(np.float)))\n return pos_class_indices, precision_at_hits\n\n\ndef calculate_per_class_lwlrap(truth, scores):\n \"\"\"Calculate label-weighted label-ranking average precision.\n\n Arguments:\n truth: np.array of (num_samples, num_classes) giving boolean ground-truth\n of presence of that class in that sample.\n scores: np.array of (num_samples, num_classes) giving the classifier-under-\n test's real-valued score for each class for each sample.\n\n Returns:\n per_class_lwlrap: np.array of (num_classes,) giving the lwlrap for each\n class.\n weight_per_class: np.array of (num_classes,) giving the prior of each\n class within the truth labels. Then the overall unbalanced lwlrap is\n simply np.sum(per_class_lwlrap * weight_per_class)\n \"\"\"\n assert truth.shape == scores.shape\n num_samples, num_classes = scores.shape\n # Space to store a distinct precision value for each class on each sample.\n # Only the classes that are true for each sample will be filled in.\n precisions_for_samples_by_classes = np.zeros((num_samples, num_classes))\n for sample_num in range(num_samples):\n pos_class_indices, precision_at_hits = (\n _one_sample_positive_class_precisions(scores[sample_num, :],\n truth[sample_num, :]))\n precisions_for_samples_by_classes[sample_num, pos_class_indices] = (\n precision_at_hits)\n labels_per_class = np.sum(truth > 0, axis=0)\n weight_per_class = labels_per_class / float(np.sum(labels_per_class))\n # Form average of each column, i.e. all the precisions assigned to labels in\n # a particular class.\n per_class_lwlrap = (np.sum(precisions_for_samples_by_classes, axis=0) /\n np.maximum(1, labels_per_class))\n # overall_lwlrap = simple average of all the actual per-class, per-sample precisions\n # = np.sum(precisions_for_samples_by_classes) / np.sum(precisions_for_samples_by_classes > 0)\n # also = weighted mean of per-class lwlraps, weighted by class label prior across samples\n # = np.sum(per_class_lwlrap * weight_per_class)\n return per_class_lwlrap, weight_per_class","execution_count":4,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"dataset_dir = Path('../input/freesound-audio-tagging-2019')\npreprocessed_dir = Path('../input/freesound-normal-log-mel-features-1-channel/')\naugmented_dir = Path(\"../input/freesound-sox-noisy-pitch-dataset/\")","execution_count":5,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"csvs = {\n 'train_curated': dataset_dir / 'train_curated.csv',\n 'train_noisy': dataset_dir / 'train_noisy.csv',\n 'sample_submission': dataset_dir / 'sample_submission.csv',\n \"pitch\": augmented_dir / \"pitch.csv\"\n}\n\ndataset = {\n 'train_curated': dataset_dir / 'train_curated',\n 'train_noisy': dataset_dir / 'train_noisy',\n 'test': dataset_dir / 'test',\n}\n\nmels = {\n 'train_curated': preprocessed_dir / 'mels_train_curated.pkl',\n 'test': preprocessed_dir / 'mels_test.pkl', # NOTE: this data doesn't work at 2nd stage\n 'train_noisy': preprocessed_dir / 'mels_train_noisy.pkl',\n \"pitch\": augmented_dir / \"mel_pitch.pkl\"\n}","execution_count":6,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_noisy = pd.read_csv(csvs['train_noisy'])\ntrain_noisy.head()","execution_count":9,"outputs":[{"output_type":"execute_result","execution_count":9,"data":{"text/plain":" fname labels\n0 00097e21.wav Bathtub_(filling_or_washing)\n1 000b6cfb.wav Motorcycle\n2 00116cd2.wav Marimba_and_xylophone,Glockenspiel\n3 00127d14.wav Water_tap_and_faucet,Sink_(filling_or_washing)\n4 0019adae.wav Raindrop","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
fnamelabels
000097e21.wavBathtub_(filling_or_washing)
1000b6cfb.wavMotorcycle
200116cd2.wavMarimba_and_xylophone,Glockenspiel
300127d14.wavWater_tap_and_faucet,Sink_(filling_or_washing)
40019adae.wavRaindrop
\n
"},"metadata":{}}]},{"metadata":{},"cell_type":"markdown","source":"## Train Test Split"},{"metadata":{"trusted":true},"cell_type":"code","source":"sampled = train_noisy.sample(frac=0.2, replace=False, random_state=SEED)\nval_idx = sampled.index.values\n\ntrain_idx = []\nfor i in range(len(train_noisy)):\n if i not in val_idx:\n train_idx.append(i)\n \ntrain_idx = np.array(train_idx)\n\nlen(train_idx), len(val_idx), len(train_noisy)","execution_count":10,"outputs":[{"output_type":"execute_result","execution_count":10,"data":{"text/plain":"(15852, 3963, 19815)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_noisy.loc[train_idx, :].reset_index(drop=True)\nval_df = train_noisy.loc[val_idx, :].reset_index(drop=True)","execution_count":12,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get melspectrogram"},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(mels[\"train_noisy\"], \"rb\") as f:\n x_noisy = pickle.load(f)\n \nwith open(mels[\"pitch\"], \"rb\") as f:\n x_pitch = pickle.load(f)\n \nlen(x_noisy), len(x_pitch)","execution_count":13,"outputs":[{"output_type":"execute_result","execution_count":13,"data":{"text/plain":"(19815, 19815)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = []\nx_val = []\n\nfor i in val_idx:\n x_val.append(x_noisy[i])\n \nfor i in train_idx:\n x_train.append(x_noisy[i])\n x_train.append(x_pitch[i])\n \nlen(x_train), len(x_val)","execution_count":16,"outputs":[{"output_type":"execute_result","execution_count":16,"data":{"text/plain":"(31704, 3963)"},"metadata":{}}]},{"metadata":{},"cell_type":"markdown","source":"## Create target"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(csvs['sample_submission'])\nlabels = test_df.columns[1:].tolist()\nnum_classes = len(labels)\nprint(num_classes)","execution_count":17,"outputs":[{"output_type":"stream","text":"80\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = np.zeros((len(train_idx) * 2, num_classes), dtype=np.int)\nfor i, row in train_df.iterrows():\n label_list = row.labels.split(\",\")\n for label in label_list:\n idx = labels.index(label)\n y_train[2 * i, idx] = 1\n y_train[2 * i + 1, idx] = 1\n\n\ny_val = np.zeros((len(val_idx), num_classes)).astype(int)\nfor i, row in val_df.iterrows():\n label_list = row.labels.split(\",\")\n for label in label_list:\n idx = labels.index(label)\n y_val[i, idx] = 1\n \ny_train.shape, y_val.shape","execution_count":22,"outputs":[{"output_type":"execute_result","execution_count":22,"data":{"text/plain":"((31704, 80), (3963, 80))"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":23,"outputs":[{"output_type":"execute_result","execution_count":23,"data":{"text/plain":"30"},"metadata":{}}]},{"metadata":{},"cell_type":"markdown","source":"## Data Transformation"},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_dict = {\n 'train': transforms.Compose([\n transforms.RandomHorizontalFlip(0.5),\n transforms.ToTensor(),\n ]),\n 'test': transforms.Compose([\n transforms.RandomHorizontalFlip(0.5),\n transforms.ToTensor(),\n ]),\n}","execution_count":24,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FATTrainDataset(Dataset):\n def __init__(self, mels, labels, transforms):\n super().__init__()\n self.mels = mels\n self.labels = labels\n self.transforms = transforms\n \n def __len__(self):\n return len(self.mels)\n \n def __getitem__(self, idx):\n # crop 1sec\n image = Image.fromarray(self.mels[idx], mode='L') \n time_dim, base_dim = image.size\n crop = random.randint(0, time_dim - base_dim)\n image = image.crop([crop, 0, crop + base_dim, base_dim])\n image = self.transforms(image).div_(255)[0, :, :]\n \n label = self.labels[idx]\n label = torch.from_numpy(label).float()\n \n return image, label","execution_count":25,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def init_layer(layer, nonlinearity=\"leaky_relu\"):\n nn.init.kaiming_uniform_(layer.weight, nonlinearity=nonlinearity)\n \n if hasattr(layer, \"bias\"):\n if layer.bias is not None:\n layer.bias.data.fill_(0.0)\n \n \ndef init_bn(bn):\n bn.bias.data.fill_(0.0)\n bn.running_mean.data.fill_(0.0)\n bn.weight.data.fill_(1.0)\n bn.running_var.data.fill_(1.0)\n \n \nclass SpatialAttention2d(nn.Module):\n def __init__(self, channel):\n super(SpatialAttention2d, self).__init__()\n self.squeeze = nn.Conv2d(channel, 1, kernel_size=1, bias=False)\n self.sigmoid = nn.Sigmoid()\n\n def forward(self, x):\n z = self.squeeze(x)\n z = self.sigmoid(z)\n return x * z\n\n\nclass GAB(nn.Module):\n def __init__(self, input_dim, reduction=4):\n super(GAB, self).__init__()\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n self.conv1 = nn.Conv2d(input_dim, input_dim // reduction, kernel_size=1, stride=1)\n self.conv2 = nn.Conv2d(input_dim // reduction, input_dim, kernel_size=1, stride=1)\n self.relu = nn.ReLU(inplace=True)\n self.sigmoid = nn.Sigmoid()\n\n def forward(self, x):\n z = self.global_avgpool(x)\n z = self.relu(self.conv1(z))\n z = self.sigmoid(self.conv2(z))\n return x * z\n\n \nclass SCse(nn.Module):\n def __init__(self, dim):\n super(SCse, self).__init__()\n self.satt = SpatialAttention2d(dim)\n self.catt = GAB(dim)\n\n def forward(self, x):\n return self.satt(x) + self.catt(x)\n \n \nclass SELayer(nn.Module):\n def __init__(self, channel, reduction=16):\n super(SELayer, self).__init__()\n self.avg_pool = nn.AdaptiveAvgPool2d(1)\n self.linear1 = nn.Linear(channel, channel // reduction, bias=False)\n self.relu = nn.ReLU(inplace=True)\n self.linear2 = nn.Linear(channel // reduction, channel, bias=False)\n self.sigmoid = nn.Sigmoid()\n \n def forward(self, x):\n b, c, _, _ = x.size()\n y = self.avg_pool(x).view(b, c)\n y = self.sigmoid(\n self.linear2(\n self.relu(\n self.linear1(y))))\n y = y.view(b, c, 1, 1)\n return x * y.expand_as(x)\n \n \n \nclass ConvBlock(nn.Module):\n def __init__(self, in_channels, out_channels, reduction=16):\n super(ConvBlock, self).__init__()\n \n self.conv1 = nn.Conv2d(in_channels=in_channels,\n out_channels=out_channels,\n kernel_size=(2, 2),\n stride=(1, 1),\n padding=(1, 1),\n bias=False)\n self.conv2 = nn.Conv2d(in_channels=out_channels,\n out_channels=out_channels,\n kernel_size=(2, 2),\n stride=(1, 1),\n padding=(1, 1),\n bias=False)\n self.bn1 = nn.BatchNorm2d(out_channels)\n self.bn2 = nn.BatchNorm2d(out_channels)\n self.scse = SCse(out_channels)\n # self.se = SELayer(out_channels)\n \n self.init_weights()\n \n def init_weights(self):\n init_layer(self.conv1)\n init_layer(self.conv2)\n init_bn(self.bn1)\n init_bn(self.bn2)\n \n def forward(self, inp, pool_size=(2, 2), pool_type=\"avg\"):\n x = inp\n x = F.relu_(self.bn1(self.conv1(x)))\n x = F.relu_(self.scse(self.bn2(self.conv2(x))))\n # x = F.relu_(self.se(self.bn2(self.conv2(x))))\n if pool_type == \"max\":\n x = F.max_pool2d(x, kernel_size=pool_size)\n elif pool_type == \"avg\":\n x = F.avg_pool2d(x, kernel_size=pool_size)\n elif pool_type == \"both\":\n x1 = F.max_pool2d(x, kernel_size=pool_size)\n x2 = F.avg_pool2d(x, kernel_size=pool_size)\n x = x1 + x2\n else:\n import pdb\n pdb.set_trace()\n return x\n \n \nclass ConvNet(nn.Module):\n def __init__(self, n_classes=80):\n super(ConvNet, self).__init__()\n self.conv1 = ConvBlock(1, 32)\n self.conv2 = ConvBlock(32, 64)\n self.conv3 = ConvBlock(64, 128)\n self.conv4 = ConvBlock(128, 256)\n self.conv5 = ConvBlock(256, 512)\n \n self.bn1 = nn.BatchNorm1d((1 + 4 + 20) * 512)\n self.drop1 = nn.Dropout(0.4)\n self.fc1 = nn.Linear((1 + 4 + 20) * 512, 512)\n self.prelu = nn.PReLU()\n self.bn2 = nn.BatchNorm1d(512)\n self.drop2 = nn.Dropout(0.2)\n self.fc2 = nn.Linear(512, n_classes)\n \n def init_weight(self):\n init_layer(self.fc1)\n init_layer(self.fc2)\n init_bn(self.bn1)\n init_bn(self.bn2)\n \n def forward(self, x):\n x = x.view(x.size(0), 1, x.size(1), x.size(2))\n x = self.conv1(x, pool_size=(1, 1), pool_type=\"both\")\n x = self.conv2(x, pool_size=(4, 1), pool_type=\"both\")\n x = self.conv3(x, pool_size=(1, 3), pool_type=\"both\")\n x = self.conv4(x, pool_size=(4, 1), pool_type=\"both\")\n x = self.conv5(x, pool_size=(1, 3), pool_type=\"both\")\n \n x1_max = F.max_pool2d(x, (5, 8))\n x1_mean = F.avg_pool2d(x, (5, 8))\n x1 = (x1_max + x1_mean).reshape(x.size(0), -1)\n \n x2_max = F.max_pool2d(x, (2, 4))\n x2_mean = F.avg_pool2d(x, (2, 4))\n x2 = (x2_max + x2_mean).reshape(x.size(0), -1)\n \n x = torch.mean(x, dim=3)\n x, _ = torch.max(x, dim=2)\n \n x = torch.cat([x, x1, x2], dim=1)\n x = self.drop1(self.bn1(x))\n x = self.prelu(self.fc1(x))\n x = self.drop2(self.bn2(x))\n x = self.fc2(x)\n \n return x","execution_count":26,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## train"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(x_train, y_train, x_val, y_val, train_transforms):\n num_epochs = 60\n batch_size = 128\n test_batch_size = 128\n lr = 1e-3\n eta_min = 1e-5\n t_max = 10\n \n num_classes = y_train.shape[1]\n \n train_dataset = FATTrainDataset(x_train, y_train, train_transforms)\n valid_dataset = FATTrainDataset(x_val, y_val, train_transforms)\n\n train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n valid_loader = DataLoader(valid_dataset, batch_size=test_batch_size, shuffle=False)\n\n model = ConvNet(n_classes=80).cuda()\n criterion1 = nn.BCEWithLogitsLoss().cuda()\n\n optimizer = Adam(params=model.parameters(), lr=lr, amsgrad=False)\n scheduler = CosineAnnealingLR(optimizer, T_max=t_max, eta_min=eta_min)\n\n best_epoch = -1\n best_lwlrap = 0.\n mb = master_bar(range(num_epochs))\n torch.cuda.empty_cache()\n\n for epoch in mb:\n start_time = time.time()\n model.train()\n avg_loss = 0.\n\n for x_batch, y_batch in progress_bar(train_loader, parent=mb):\n preds = model(x_batch.cuda())\n loss = criterion1(preds, y_batch.cuda())\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n\n avg_loss += loss.item() / len(train_loader)\n\n model.eval()\n valid_preds = np.zeros((len(x_val), num_classes))\n avg_val_loss = 0.\n\n for i, (x_batch, y_batch) in enumerate(valid_loader):\n with torch.no_grad():\n preds = model(x_batch.cuda()).detach()\n loss = criterion1(preds, y_batch.cuda())\n preds = torch.sigmoid(preds)\n valid_preds[i * test_batch_size: (i+1) * test_batch_size] = preds.cpu().numpy()\n\n avg_val_loss += loss.item() / len(valid_loader)\n \n score, weight = calculate_per_class_lwlrap(y_val, valid_preds)\n lwlrap = (score * weight).sum()\n \n scheduler.step()\n\n if (epoch + 1) % 1 == 0:\n elapsed = time.time() - start_time\n mb.write(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f} avg_val_loss: {avg_val_loss:.4f} val_lwlrap: {lwlrap:.6f} time: {elapsed:.0f}s')\n \n if lwlrap > best_lwlrap:\n best_epoch = epoch + 1\n best_lwlrap = lwlrap\n torch.save(model.state_dict(), 'weight_best.pt')\n \n return {\n 'best_epoch': best_epoch,\n 'best_lwlrap': best_lwlrap,\n }","execution_count":27,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result = train_model(x_train, y_train, x_val, y_val, transforms_dict[\"train\"])","execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":"","text/html":"\n
\n \n \n 1.67% [1/60 02:47<2:45:10]\n
\n \nEpoch 1 - avg_train_loss: 0.2011 avg_val_loss: 0.0689 val_lwlrap: 0.251212 time: 168s

\n\n

\n \n \n 45.97% [114/248 01:13<01:26]\n
\n "},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.cuda.empty_cache()\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}