{"cells":[{"metadata":{},"cell_type":"markdown","source":"**This kernel is used for training our models. In the end, we went with an ensemble of 15 models similar to the one trained here.**\n\n**Their CV are all above 0.86 and 5-folds LB varies between 0.71 and 0.72**\n\nFor our solution writeup, please check : https://www.kaggle.com/c/freesound-audio-tagging-2019/discussion/95409#latest-551352"},{"metadata":{},"cell_type":"markdown","source":"# Initialization"},{"metadata":{},"cell_type":"markdown","source":"## Imports"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import re\nimport gc\nimport os\nimport time\nimport pywt\nimport pickle\nimport random\nimport operator\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport tensorflow as tf\nimport IPython.display as ipd\nimport matplotlib.pyplot as plt\n\nfrom PIL import Image\nfrom sklearn.metrics import *\nfrom collections import Counter\nfrom sklearn.model_selection import *\nfrom tqdm import tqdm_notebook as tqdm\nfrom keras.utils import to_categorical\nfrom multiprocessing import Pool, cpu_count\nfrom collections import OrderedDict\n\nimport torch\nimport torch.nn as nn\nimport torch.utils.data\nimport torch.nn.functional as F\nimport torchvision.models as models\nfrom torch.autograd import Variable\nfrom torch.optim.lr_scheduler import * \nfrom torchvision.transforms import transforms\nfrom torch.utils.data import Dataset, DataLoader\n\nimport librosa\nimport librosa.display\n\n\nbegin = time.time()\nsns.set_style('white')\nwarnings.simplefilter(action='ignore', category=FutureWarning) \nwarnings.simplefilter(action='ignore', category=UserWarning)\nwarnings.simplefilter(action='ignore', category=RuntimeWarning)\n\nprint(\"Number of available cpu cores: {}\".format(cpu_count()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Seeding"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    tf.set_random_seed(seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed = 2019\nseed_everything(seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"N_JOBS = cpu_count()\nos.environ['MKL_NUM_THREADS'] = str(N_JOBS)\nos.environ['OMP_NUM_THREADS'] = str(N_JOBS)\nDataLoader = partial(DataLoader, num_workers=N_JOBS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_PATH = '../input/freesound-audio-tagging-2019/'\nTRAIN_CURATED_PATH = DATA_PATH + 'train_curated/'\nTRAIN_NOISY_PATH = DATA_PATH + 'train_noisy/'\nTEST_PATH = DATA_PATH + 'test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(DATA_PATH + 'train_curated.csv')\ndf_noisy = pd.read_csv(DATA_PATH + 'train_noisy.csv')\ndf_test = pd.read_csv(DATA_PATH + 'sample_submission.csv')\n\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = list(df_test.columns[1:])\nnum_classes = len(labels)\n\nprint(\"Number of classes :\", num_classes)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"*Removing \"corrupted\" files*"},{"metadata":{"trusted":true},"cell_type":"code","source":"to_remove = [\"f76181c4.wav\", \"77b925c2.wav\", \"6a1f682a.wav\", \"c7db12aa.wav\", \"7752cc8a.wav\", \"1d44b0bd.wav\"]\n\nfor i in df_train.copy().index:\n    if df_train['fname'][i] in to_remove:\n        df_train.drop(i, inplace=True)\n        \ndf_train = df_train.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"*Restraining noisy data to samples with only one label*"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_noisy[\"nb_labels\"] = df_noisy[\"labels\"].apply(lambda x: len(x.split(',')))\ndf_noisy = df_noisy[df_noisy[\"nb_labels\"] == 1].copy().reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Signal Processing"},{"metadata":{},"cell_type":"markdown","source":"## Config"},{"metadata":{"trusted":true},"cell_type":"code","source":"class config:\n    sampling_rate = 44100  # 44.1 kHz\n    duration = 4 #2 # Minimum length for short samples (seconds)\n    samples = sampling_rate * duration # Minimum sample size\n    \n    top_db = 60 # Noise filtering, default = 60\n    \n    # Frequencies kept in spectrograms\n    fmin = 20\n    fmax =  sampling_rate // 2  # Shannon theorem\n\n    # Spectrogram parameters\n    n_mels = 64 # = spec_height\n    n_fft = n_mels * 30 # Size of fft window - smooths the spectrogram\n    spec_min_width = 256 #128\n    x_mean,x_std = -35.7, 21.6\n    hop_length = duration * sampling_rate // spec_min_width + 1 # Number of samples between each frame - impacts y size of spectrogram","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Read Audio"},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_audio(pathname, conf, trim_long_data):\n    y, sr = librosa.load(pathname, sr=conf.sampling_rate)\n    # trim silence\n    if len(y) > 0: # workaround: 0 length causes error\n        y, _ = librosa.effects.trim(y) # trim, top_db=default(60)\n    # make it unified length to conf.samples\n    if len(y) > conf.samples: # long enough\n        if trim_long_data: y = y[0:0+conf.samples]\n    else: # pad blank\n        padding = conf.samples - len(y)    # add padding at both ends\n        offset = padding // 2\n        y = np.pad(y, (offset, conf.samples - len(y) - offset), 'constant')\n    return y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## MEL Spectrogram\nThe three chanel mode adds first and second or delta to the data. It did not improve the results."},{"metadata":{"trusted":true},"cell_type":"code","source":"def audio_to_melspectrogram(audio, config, three_chanels=False):\n    spectrogram = librosa.feature.melspectrogram(audio, \n                                                 sr=config.sampling_rate,\n                                                 n_mels=config.n_mels,\n                                                 hop_length=config.hop_length,\n                                                 n_fft=config.n_fft,\n                                                 fmin=config.fmin,\n                                                 fmax=config.fmax)\n    logmel = librosa.power_to_db(spectrogram).astype(np.float32)\n    \n    if three_chanels:\n        return np.array([logmel, librosa.feature.delta(logmel), librosa.feature.delta(logmel, order=2)])\n    else:\n        return logmel","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Normalize\nWe can either use individual image statistics or statistics computed on train data."},{"metadata":{"trusted":true},"cell_type":"code","source":"def normalize(X, mean=None, std=None):\n    mean = mean or X.mean()\n    std = std or (X-X.mean()).std()\n    return ((X - mean)/std).astype(np.float16)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_labels(df, labels):\n    mapping = {label:i for i, label in enumerate(labels)}\n    y = np.zeros((len(df), len(labels)))\n    all_labels = df['labels'].apply(lambda x: x.split(','))\n    for i, label in enumerate(all_labels):\n        for l in label:\n            y[i, mapping[l]] = 1\n    return y.astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = get_labels(df_train, labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Select best noisy samples\n\nWe load predictions that were computed using a model that was trained only on curated data. The idea is to keep the ones that were the most correctly predicted, using the log loss as metric. As some class were better predicted than others, we restrict the number of added samples to 50 per class."},{"metadata":{"trusted":true},"cell_type":"code","source":"scores_noisy = np.load(\"../input/fat-cp/scores.npy\")\ny_noisy = get_labels(df_noisy, labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def sort_by_loss(y_noisy, scores_noisy):\n    losses_dic = {i : log_loss(y_noisy[i], scores_noisy[i]) for i in range(scores_noisy.shape[0])}\n    sorted_dic = sorted(losses_dic.items(), key=operator.itemgetter(1))\n    return sorted_dic","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sorted_by_loss = sort_by_loss(y_noisy, scores_noisy)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def select_noisy(sorted_by_loss, nb_noisy, max_per_class=50):\n    selected = np.array(sorted_by_loss)[:nb_noisy, 0].astype(int)\n    to_keep = []\n    counts = {}\n    for s in selected:\n        l = df_noisy[\"labels\"][s]\n        try:\n            counts[l] += 1\n        except:\n            counts[l] = 0\n        if counts[l] < max_per_class:\n            to_keep.append(s)\n            \n        \n    return df_noisy.iloc[to_keep].reset_index(drop=True)\n    \n    df_noisy_selected","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nb_noisy = 5000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_noisy_selected = select_noisy(sorted_by_loss, nb_noisy, max_per_class=50)\ny_noisy = get_labels(df_noisy_selected, labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Input data"},{"metadata":{"trusted":true},"cell_type":"code","source":"three_chanels = False\ncrop = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def process(df, path, config, crop=False, three_chanels=False):\n    X = []\n    for i in df.index:\n        signal = read_audio(path + df['fname'][i], config, crop)\n        X.append(normalize(audio_to_melspectrogram(signal, config), config.x_mean, config.x_std)) #normalize based on global statistics\n        #X.append(normalize(audio_to_melspectrogram(signal, config))) #normalize based on individual statistics\n    return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df_train = df_train.head(100)\n# df_test = df_test.head(100)\n# df_noisy_selected = df_noisy_selected.head(100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nX_train = process(df_train, TRAIN_CURATED_PATH, config, crop=crop, three_chanels=three_chanels)\nX_noisy = process(df_noisy_selected, TRAIN_NOISY_PATH, config, crop=crop, three_chanels=three_chanels)\nX_test = process(df_test, TEST_PATH, config, three_chanels=three_chanels)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plt.matshow(X_train[0].astype(np.float32))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Datasets"},{"metadata":{},"cell_type":"markdown","source":"### Augmentation\nWe augment spectrograms by hiding random time & frequency intervals"},{"metadata":{"trusted":true},"cell_type":"code","source":"def spec_augment(spec: np.ndarray, num_mask=2, freq_masking_max_percentage=0.15, time_masking_max_percentage=0.3):\n    spec = spec.copy()\n    for i in range(num_mask):\n        num_freqs, num_frames = spec.shape\n        freq_percentage = random.uniform(0.0, freq_masking_max_percentage)\n        time_percentage = random.uniform(0.0, time_masking_max_percentage)\n        \n        num_freqs_to_mask = int(freq_percentage * num_freqs)\n        num_frames_to_mask = int(time_percentage * num_frames)\n        \n        t0 = int(np.random.uniform(low=0.0, high=num_frames - num_frames_to_mask))\n        f0 = int(np.random.uniform(low=0.0, high=num_freqs - num_freqs_to_mask))\n        \n        spec[:, t0:t0 + num_frames_to_mask] = 0      \n        spec[f0:f0 + num_freqs_to_mask, :] = 0 \n        \n    return spec","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.matshow(spec_augment(X_train[0].astype(np.float32)))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Usual transforms\nOnly the conversion to tensor is kept, we also tried flipping and resized cropping."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"transforms_dict = {\n    'train': transforms.Compose([\n        transforms.ToTensor(),\n    ]),\n    'test': transforms.Compose([\n        transforms.ToTensor(),\n    ]),\n    'viz': transforms.Compose([\n    ]),\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"class FATDatasetTrain(Dataset):\n    def __init__(self, mels, transforms, y, apply_spec_aug=False):\n        super().__init__()\n        self.mels = mels\n        self.labels = y\n        self.transforms = transforms\n        self.apply_spec_aug = apply_spec_aug\n        \n    def __len__(self):\n        return len(self.mels)\n    \n    def __getitem__(self, idx):\n        data = self.mels[idx].astype(np.float32)\n        base_dim,time_dim = data.shape\n        crop = random.randint(0, max(time_dim - config.spec_min_width,0))\n        data = data[0:base_dim,crop:crop + config.spec_min_width]\n        \n        if self.apply_spec_aug: \n            data = spec_augment(data)\n\n        data = np.expand_dims(data, axis=2)\n        data = self.transforms(data)\n        \n        label = self.labels[idx]\n        label = torch.from_numpy(label).float()\n        return data, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Test + TTA\nTTA helped to fight instability, it worked well for us."},{"metadata":{"trusted":true},"cell_type":"code","source":"class FATDatasetTest(Dataset):\n    def __init__(self, mels, transforms, y=None, nb_tta=5):\n        super().__init__()\n        self.mels = mels\n        self.transforms = transforms\n        self.tta = nb_tta\n        if y is None:\n            self.y = np.zeros(len(self.mels))\n        else:\n            self.y = y\n        \n    def __len__(self):\n        return len(self.mels) * self.tta\n    \n    def __getitem__(self, idx):\n        new_idx = idx % len(self.mels)\n        \n        data = self.mels[new_idx].astype(np.float32)#/255.0\n        base_dim,time_dim = data.shape\n        crop = random.randint(0, max(time_dim - config.spec_min_width,0))\n        data = data[0:base_dim,crop:crop + config.spec_min_width]\n        data = np.expand_dims(data, axis=2)\n        data = self.transforms(data)\n        \n        return data, self.y[new_idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"dataset = FATDatasetTrain(X_train[:10], transforms_dict['viz'], y_train[:10], apply_spec_aug=True)\nprint('Input shape : ', dataset[0][0].squeeze().shape)\nplt.matshow(dataset[0][0].squeeze())\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modeling"},{"metadata":{},"cell_type":"markdown","source":"## Tools"},{"metadata":{},"cell_type":"markdown","source":"### Adaptative Pooling"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def adaptive_concat_pool2d(x, sz=(1,1)):\n    out1 = F.adaptive_avg_pool2d(x, sz).view(x.size(0), -1)\n    out2 = F.adaptive_max_pool2d(x, sz).view(x.size(0), -1)\n    return torch.cat([out1, out2], 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### ConvBlock\nAdapted from https://www.kaggle.com/mhiro2/simple-2d-cnn-classifier-with-pytorch"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class ConvBlock(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size=3, pool=True):\n        super().__init__()\n        \n        padding = kernel_size // 2\n        self.pool = pool\n        \n        self.conv1 = nn.Sequential(\n            nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, stride=1, padding=padding),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(),\n        )\n        self.conv2 = nn.Sequential(\n            nn.Conv2d(out_channels + in_channels, out_channels, kernel_size=kernel_size, stride=1, padding=padding),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(),\n        )\n\n        self._init_weights()\n        \n    def _init_weights(self):\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight)\n                if m.bias is not None:\n                    nn.init.zeros_(m.bias)\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.constant_(m.weight, 1)\n                nn.init.zeros_(m.bias)\n        \n    def forward(self, x): # x.shape = [batch_size, in_channels, a, b]\n        x1 = self.conv1(x)\n        x = self.conv2(torch.cat([x, x1],1))\n        if(self.pool): x = F.avg_pool2d(x, 2)\n        return x   # x.shape = [batch_size, out_channels, a//2, b//2]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Models"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Classifier_M1(nn.Module):\n    def __init__(self, num_classes=num_classes):\n        super().__init__()\n        self.conv = nn.Sequential(\n            ConvBlock(in_channels=1, out_channels=64),\n            ConvBlock(in_channels=64, out_channels=128),\n            ConvBlock(in_channels=128, out_channels=256),\n            ConvBlock(in_channels=256, out_channels=512),\n        )\n        \n        self.fc = nn.Sequential(\n            nn.BatchNorm1d(512*2),\n            nn.Linear(512*2, 128),\n            nn.PReLU(),\n            nn.BatchNorm1d(128),\n            nn.Linear(128, num_classes),\n        )\n\n    def forward(self, x): # batch_size, 3, a, b\n        x = self.conv(x)  # batch_size, 512, a//16, b//16\n        x = self.fc(adaptive_concat_pool2d(x))\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Classifier_M2(nn.Module):\n    def __init__(self, num_classes=num_classes):\n        super().__init__()\n        self.conv1 = ConvBlock(1,64)\n        self.conv2 = ConvBlock(64,128)\n        self.conv3 = ConvBlock(128,256)\n        self.conv4 = ConvBlock(256,512,pool=False)\n        \n        self.fc = nn.Sequential(\n            nn.BatchNorm1d(1792),\n            nn.Linear(1792, 256),\n            nn.PReLU(),\n            nn.BatchNorm1d(256),\n            nn.Linear(256, num_classes),\n        )\n\n    def forward(self, x): # batch_size, 3, a, b\n        x1 = self.conv1(x)\n        x2 = self.conv2(x1)\n        x3 = self.conv3(x2)\n        x4 = self.conv4(x3)\n        #pyramid pooling\n        x = torch.cat([adaptive_concat_pool2d(x2), adaptive_concat_pool2d(x3),\n                       adaptive_concat_pool2d(x4)], 1)\n        x = self.fc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Classifier_M3(nn.Module):\n    def __init__(self, num_classes=num_classes):\n        super().__init__()\n        self.conv1 = ConvBlock(1,64)\n        self.conv2 = ConvBlock(64,128)\n        self.conv3 = ConvBlock(128,256)\n        self.conv4 = ConvBlock(256,512)\n        self.conv5 = ConvBlock(512,1024,pool=False)\n        \n        self.fc = nn.Sequential(\n            nn.BatchNorm1d(3840),\n            nn.Linear(3840, 256),\n            nn.PReLU(),\n            nn.BatchNorm1d(256),\n            nn.Linear(256, num_classes),\n        )\n\n    def forward(self, x): # batch_size, 3, a, b\n        x1 = self.conv1(x)\n        x2 = self.conv2(x1)\n        x3 = self.conv3(x2)\n        x4 = self.conv4(x3)\n        x5 = self.conv5(x4)\n        #pyramid pooling\n        x = torch.cat([adaptive_concat_pool2d(x2), adaptive_concat_pool2d(x3),\n                       adaptive_concat_pool2d(x4),adaptive_concat_pool2d(x5)], 1)\n        x = self.fc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Tools"},{"metadata":{},"cell_type":"markdown","source":"## LwLRAP"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def _one_sample_positive_class_precisions(scores, truth):\n    num_classes = scores.shape[0]\n    pos_class_indices = np.flatnonzero(truth > 0)\n\n    if not len(pos_class_indices):\n        return pos_class_indices, np.zeros(0)\n\n    retrieved_classes = np.argsort(scores)[::-1]\n\n    class_rankings = np.zeros(num_classes, dtype=np.int)\n    class_rankings[retrieved_classes] = range(num_classes)\n\n    retrieved_class_true = np.zeros(num_classes, dtype=np.bool)\n    retrieved_class_true[class_rankings[pos_class_indices]] = True\n\n    retrieved_cumulative_hits = np.cumsum(retrieved_class_true)\n\n    precision_at_hits = (\n            retrieved_cumulative_hits[class_rankings[pos_class_indices]] /\n            (1 + class_rankings[pos_class_indices].astype(np.float)))\n    return pos_class_indices, precision_at_hits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def lwlrap(truth, scores):\n    assert truth.shape == scores.shape\n    num_samples, num_classes = scores.shape\n    precisions_for_samples_by_classes = np.zeros((num_samples, num_classes))\n    for sample_num in range(num_samples):\n        pos_class_indices, precision_at_hits = _one_sample_positive_class_precisions(scores[sample_num, :], truth[sample_num, :])\n        precisions_for_samples_by_classes[sample_num, pos_class_indices] = precision_at_hits\n        \n    labels_per_class = np.sum(truth > 0, axis=0)\n    weight_per_class = labels_per_class / float(np.sum(labels_per_class))\n\n    per_class_lwlrap = (np.sum(precisions_for_samples_by_classes, axis=0) /\n                        np.maximum(1, labels_per_class))\n    return per_class_lwlrap, weight_per_class","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Mixup\nFacebook's implementation is used for Mixup. With its default parameters."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"alpha_ = 0.4\nsns.distplot(np.random.beta(alpha_, alpha_, 100000))\nplt.xlim(0, 1)\nplt.title(f'Distribution used for Mixup (alpha = {alpha_})', size=15)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def mixup_data(x, y, alpha=alpha_, use_cuda=True):\n    if alpha > 0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1\n\n    batch_size = x.size()[0]\n    if use_cuda:\n        index = torch.randperm(batch_size).cuda()\n    else:\n        index = torch.randperm(batch_size)\n\n    mixed_x = lam * x + (1 - lam) * x[index, :]\n    y_a, y_b = y, y[index]\n    return mixed_x, y_a, y_b, lam","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def mixup_criterion(criterion, pred, y_a, y_b, lam):\n    return lam * criterion(pred.float().cuda(), y_a.float().cuda()) + (1 - lam) * criterion(pred.float().cuda(), y_b.float().cuda())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Label Smoothing\nWe tried this as well, without any success. I do believe that clever use of label smoothing & noisy data can improve results"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def smooth(y, eps=0.4):\n    a = 1 - eps * (1 + 1/y.shape[1])\n    b = eps / y.shape[1]\n    return a * y + b","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Learning Rate\nI reimplemented a custom Scheduler, that does Cosine Annealing and warmup. We found out that warmup played a quite important role"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class WarmupCosineAnnealingLR(CosineAnnealingLR):\n    def __init__(self, optimizer, warmup_prop, T_max, eta_min=0, last_epoch=-1):\n        self.warmup_prop = warmup_prop\n        super(CosineAnnealingLR, self).__init__(optimizer, T_max, eta_min, last_epoch)\n\n    def get_lr(self):\n        if self.last_epoch < T_max * warmup_prop:\n            return [self.base_lrs * self.last_epoch/T_max\n                    for base_lr, group in zip(self.base_lrs, self.optimizer.param_groups)]\n        else:\n            return super(CosineAnnealingLR, self).step()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"lr_max = 0.001\nlr_min = 1e-5\nwarmup_prop = 0.1\nnb_epochs = 80\nepochs = [i for i in range(nb_epochs)]\nlrs_warmup = [lr_max * i / (warmup_prop * nb_epochs) for i in range(1, int(warmup_prop * nb_epochs) + 1)]\nlrs_scheduler = [lr_min + 0.5 * (lr_max - lr_min) * (1 + np.cos(np.pi * i / ((1 - warmup_prop) * nb_epochs))) for i in range(int((1 - warmup_prop) * nb_epochs))]\n\nplt.figure(figsize=(15, 5))\nplt.axhline(lr_min, label='lr_min', c='skyblue', linestyle='--')\nplt.axhline(lr_max, label='lr_max', c='orangered', linestyle='--')\nplt.plot(epochs, lrs_warmup + lrs_scheduler[:len(epochs)])\nplt.grid(True)\nplt.legend()\nplt.title('Learning Rate Scheduling Policy', size=15)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Sigmoid"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Save & Load"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def save_model_weights(model, filename, verbose=1):\n    if verbose:\n        print(f'Saving weights to {filename}\\n')\n    torch.save(model.state_dict(), filename)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def load_model_weights(model, filename, verbose=1):\n    if verbose:\n        print(f'Loading weights from {filename}\\n')\n    model.load_state_dict(torch.load(filename))\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predict"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def predict(dataset, model, batch_size=64):\n    model.eval()\n    y = np.array([[]] * num_classes).T\n    loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=False)\n    with torch.no_grad():\n        for x, target in loader:\n            y = np.concatenate((y, sigmoid(model(x.cuda()).view(-1, num_classes).detach().cpu().numpy())))\n    return y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fit\nCode is a bit ugly, you might not want to dive into it. It's PyTorch. Here are the main points :\n\n- Weight decay (0.001)\n- Gradient clipping (1)\n- Binary Cross Entropy loss\n- Adam is used here, Adabound was also tried\n- Custom lr scheduler (see above), with a warmup proportion 0.1 usually.\n- Apply mixup with a probability *mixup* (chosen at 1)\n- Checkpointing is used \n"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def fit(model, train_dataset, val_dataset, epochs=50, batch_size=128, nb_tta=5, mixup=False, warmup_prop=0.1,\n        verbose=1, cp=False, model_name='model', lr=0.001):\n    avg_val_loss = 1000\n    best_score = 0\n    clip_value = 1.0\n    \n    model.cuda()\n    \n    opt_params = [\n        {'params': [p for n, p in list(model.named_parameters()) if not any(nd in n for nd in ['bias', 'LayerNorm.bias', 'LayerNorm.weight'])], 'weight_decay': 0.01},\n        {'params': [p for n, p in list(model.named_parameters()) if any(nd in n for nd in ['bias', 'LayerNorm.bias', 'LayerNorm.weight'])], 'weight_decay': 0.0}\n    ]\n\n    for p in model.parameters():\n        p.register_hook(lambda grad: torch.clamp(grad, -clip_value, clip_value))\n    \n    optimizer = torch.optim.Adam(opt_params, lr=lr)\n    scheduler = CosineAnnealingLR(optimizer, T_max=epochs, eta_min=1e-7)\n    loss_fn = nn.BCEWithLogitsLoss(reduction='sum')\n    \n    train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n    val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n        \n\n    if warmup_prop >= 1: \n        warmup_prop = warmup_prop / epochs\n\n    for epoch in range(epochs):\n        model.train()\n        avg_loss = 0\n        start_time = time.time()  \n        \n        if epoch < epochs * warmup_prop:\n            lr = 5e-5 + 0.001 * epoch / (epochs * warmup_prop)\n            for param_group in optimizer.param_groups:\n                param_group['lr'] = lr\n        else:\n            scheduler.step()\n            lr = np.mean([param_group['lr'] for param_group in optimizer.param_groups])\n        \n        for x, y_batch in train_loader:\n            optimizer.zero_grad()\n            \n            dice = np.random.random() < mixup\n            if dice:\n                x, y_a, y_b, lam = mixup_data(x, y_batch.float())\n                x, y_a, y_b = map(Variable, (x, y_a, y_b))\n    \n\n            y_pred = model(x.cuda()).view(-1, num_classes)\n            if dice:\n                loss = mixup_criterion(loss_fn, y_pred, y_a, y_b, lam)\n            else:\n                loss = loss_fn(y_pred.float(), y_batch.float().cuda())\n                \n            loss.backward()\n            avg_loss += loss.item() / len(train_dataset)\n            optimizer.step()\n    \n        model.eval()\n        avg_val_loss = 0.\n        pred_val = np.array([[]]*num_classes).T\n        \n        for x, y_batch in val_loader:\n            y_pred = model(x.cuda()).view(-1, num_classes).detach()\n            avg_val_loss += loss_fn(y_pred.float(), y_batch.float().cuda()).item() / len(val_dataset)\n            pred_val = np.concatenate((pred_val, sigmoid(y_pred.cpu().numpy())))\n            \n        pred_val = np.mean(pred_val.reshape((nb_tta, -1, num_classes,)), axis=0)\n        score_class, weight = lwlrap(val_dataset.y, pred_val)\n        score = (score_class * weight).sum()  \n                \n        elapsed_time = time.time() - start_time\n        \n        if cp:\n            if score > best_score:\n                save_model_weights(model, f\"{model_name}.pt\", verbose=0)\n                best_score = score\n        \n        if (epoch + 1) % verbose == 0:\n            elapsed_time = elapsed_time * verbose\n            print(f'Epoch {epoch+1}/{epochs}     lr={lr:.1e}     lwlrap={score:.5f}     ', end='')\n            print(f'loss={avg_loss:.2f}     val_loss={avg_val_loss:.2f}     t={elapsed_time:.0f}s')\n            \n    return best_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Refit\n- Same thing as the previous function, but:\n - We change the scheduler to reduce_lr_on_plateau, starting from 0.001\n - Used with curated data only"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def refit(model, train_dataset, val_dataset, epochs=10, batch_size=128, nb_tta=5, mixup=False,\n          verbose=1, cp=False, model_name='model', best_score=0, lr=0.001):\n    avg_val_loss = 1000\n    clip_value = 1.0\n    \n    model.cuda()\n    \n    opt_params = [\n        {'params': [p for n, p in list(model.named_parameters()) if not any(nd in n for nd in ['bias', 'LayerNorm.bias', 'LayerNorm.weight'])], 'weight_decay': 0.01},\n        {'params': [p for n, p in list(model.named_parameters()) if any(nd in n for nd in ['bias', 'LayerNorm.bias', 'LayerNorm.weight'])], 'weight_decay': 0.0}\n    ]\n    \n    for p in model.parameters():\n        p.register_hook(lambda grad: torch.clamp(grad, -clip_value, clip_value))\n    \n    optimizer = torch.optim.Adam(opt_params, lr)\n    scheduler = ReduceLROnPlateau(optimizer, mode='min', patience=3, factor=0.1, verbose=0)\n    loss_fn = nn.BCEWithLogitsLoss(reduction='sum')\n    \n    train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n    val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n\n    for epoch in range(epochs):\n        model.train()\n        avg_loss = 0\n        start_time = time.time()  \n\n        scheduler.step(avg_val_loss)\n        lr = np.mean([param_group['lr'] for param_group in optimizer.param_groups])\n        \n        for x, y_batch in train_loader:\n            optimizer.zero_grad()\n            \n            dice = np.random.random() < mixup\n            if dice:\n                x, y_a, y_b, lam = mixup_data(x, y_batch.float())\n                x, y_a, y_b = map(Variable, (x, y_a, y_b))\n    \n\n            y_pred = model(x.cuda()).view(-1, num_classes)\n            if dice:\n                loss = mixup_criterion(loss_fn, y_pred, y_a, y_b, lam)\n            else:\n                loss = loss_fn(y_pred.float(), y_batch.float().cuda())\n                \n            loss.backward()\n            avg_loss += loss.item() / len(train_dataset)\n            optimizer.step()\n    \n        model.eval()\n        avg_val_loss = 0.\n        pred_val = np.array([[]]*num_classes).T\n        \n        for x, y_batch in val_loader:\n            y_pred = model(x.cuda()).view(-1, num_classes).detach()\n            avg_val_loss += loss_fn(y_pred.float(), y_batch.float().cuda()).item() / len(val_dataset)\n            pred_val = np.concatenate((pred_val, sigmoid(y_pred.cpu().numpy())))\n            \n        pred_val = np.mean(pred_val.reshape((nb_tta, -1, num_classes,)), axis=0)\n        score_class, weight = lwlrap(val_dataset.y, pred_val)\n        score = (score_class * weight).sum()  \n                \n        elapsed_time = time.time() - start_time\n        \n        if cp:\n            if score > best_score:\n                save_model_weights(model, f\"{model_name}.pt\", verbose=0)\n                best_score = score\n        \n        if (epoch + 1) % verbose == 0:\n            elapsed_time = elapsed_time * verbose\n            print(f'Epoch {epoch+1}/{epochs}     lr={lr:.1e}     lwlrap={score:.5f}     ', end='')\n            print(f'loss={avg_loss:.2f}     val_loss={avg_val_loss:.2f}     t={elapsed_time:.0f}s')\n    \n        if lr <= 1e-6:\n            break\n    return best_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## $k$-folds\n- 5 folds Stratified, using one of the labels if there are more. Only curated data is kept.\n- Fit for 65 (approx.) epochs on curated + selected noisy (optional)\n- Refit twice for a few epochs on curated only"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def k_fold(model_class, X, y, X_test, transform_dic, X_noisy=[], y_noisy=np.array([]),\n           k=5, batch_size=32, epochs=5, seed=2019, nb_tta=5, tta_eval=25, mixup=False,\n           verbose=1, save=True, cp=False, warmup_prop=0.1, model_name=\"model\", pretrained_path=''):\n    \n    splits = list(StratifiedKFold(n_splits=k, shuffle=True, random_state=seed).split(X, np.argmax(y, axis=1)))\n        \n    pred_test = np.zeros((len(X_test), num_classes))\n    pred_oof = np.zeros((y.shape[0], num_classes))\n\n    for i, (train_idx, val_idx) in enumerate(splits):\n        print(f\"-------------   Fold {i+1}  -------------\")\n        seed_everything(seed + i)\n        start_time = time.time()\n        \n        model = model_class(num_classes=num_classes)\n        \n        if len(pretrained_path):\n            load_model_weights(model, pretrained_path)\n        \n        train_dataset = FATDatasetTrain([X[i] for i in train_idx] + X_noisy, transforms=transform_dic['train'], \n                                        y=np.concatenate((y[train_idx], y_noisy)),apply_spec_aug=True)\n        val_dataset = FATDatasetTest([X[i] for i in val_idx], transforms=transform_dic['test'], y=y[val_idx], nb_tta=nb_tta)\n        test_dataset = FATDatasetTest(X_test, transforms=transform_dic['test'], nb_tta=tta_eval)\n        \n        print('\\n - Fitting \\n')\n        \n        best_score = fit(model, train_dataset, val_dataset, epochs=epochs, batch_size=batch_size, nb_tta=nb_tta, mixup=mixup, \n                         warmup_prop=warmup_prop, verbose=verbose, cp=cp, model_name=f'{model_name}_{i+1}', lr=1e-3)\n        \n        print('\\n - Re-fitting with curated data only (1/2)\\n')\n        \n        retrain_dataset = FATDatasetTrain([X[i] for i in train_idx], transforms=transform_dic['train'], y=y[train_idx])\n        \n        best_score = refit(model, retrain_dataset, val_dataset, epochs=epochs, batch_size=batch_size, nb_tta=nb_tta, mixup=mixup, \n                           verbose=verbose, cp=cp, model_name=f'{model_name}_{i+1}', best_score=best_score, lr=1e-4)\n        \n        if cp:\n            load_model_weights(model, f\"{model_name}_{i+1}.pt\", verbose=1)\n        elif save:\n            save_model_weights(model, f\"{model_name}_{i+1}.pt\", verbose=1)\n        \n        print('\\n - Re-fitting with curated data only (2/2)\\n')\n        \n        best_score = refit(model, retrain_dataset, val_dataset, epochs=epochs, batch_size=batch_size, nb_tta=nb_tta, mixup=mixup, \n                           verbose=verbose, cp=cp, model_name=f'{model_name}_{i+1}', best_score=best_score, lr=1e-4)\n        \n        if cp:\n            load_model_weights(model, f\"{model_name}_{i+1}.pt\", verbose=1)\n        elif save:\n            save_model_weights(model, f\"{model_name}_{i+1}.pt\", verbose=1)\n            \n        print(f'\\n - Predicting with {tta_eval} TTA \\n')\n        \n        val_dataset = FATDatasetTest([X[i] for i in val_idx], transforms=transform_dic['test'], y=y[val_idx], nb_tta=tta_eval)\n        pred_val = np.mean(predict(val_dataset, model).reshape((tta_eval, -1, num_classes,)), axis=0)\n        pred_oof[val_idx, :] = pred_val\n        \n        score_class, weight = lwlrap(y[val_idx], pred_val)\n        score = (score_class * weight).sum()\n        \n        pred_test += np.mean(predict(test_dataset, model).reshape((tta_eval, -1, num_classes,)), axis=0)\n\n        \n        print(f\"\\n lwlrap : Scored {score :.4f} on validation data\")\n        print(f\"\\n    Done in {(time.time() - start_time) / 60 :.1f} minutes \\n\")\n        \n    return pred_test, pred_oof","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{},"cell_type":"markdown","source":"### Parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_name = \"model\"\nk = 5\nepochs = 65\nbatch_size = 64\nnb_tta = 5\nmixup = 1\ncp = True\nwarmup_prop = 0.1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"model = Classifier_M3\ntransforms_ = transforms_dict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Weights pretrained on noisy data\nIn case we want to use weights pretrained on noisy, we specify weights to load."},{"metadata":{"trusted":true},"cell_type":"code","source":"# cp_path = \"../input/fat-cp/new_default_65.pt\"\ncp_path = ''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Fitting"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"pred_test, pred_oof = k_fold(model, X_train, y_train, X_test, transforms_,\n                             X_noisy=X_noisy, y_noisy=y_noisy, \n                             k=k, batch_size=batch_size, epochs=epochs, \n                             nb_tta=nb_tta, cp=cp, warmup_prop=warmup_prop,\n                             mixup=mixup, seed=seed, \n                             model_name=model_name, pretrained_path=cp_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score_class, weight = lwlrap(y_train, pred_oof)\nscore = (score_class * weight).sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Local CV score is : {score:.4f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"score_df = pd.DataFrame(np.array([labels, np.round(score_class, 5), np.round(weight, 5)]).T, columns=['Label', 'Score', 'Weight']).set_index('Label').sort_values('Score')\nscore_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test[df_test.columns[1:]] = pred_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}