{"cells":[{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"!pip install pytorch_lightning efficientnet_pytorch colorednoise torchlibrosa audiomentations","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"scrolled":false},"cell_type":"code","source":"import sys\nimport random\nimport os\nimport math\nimport time\n\nimport numpy as np\nimport pandas as pd\nimport librosa\nimport librosa.display\nfrom collections import defaultdict\nfrom tqdm import tqdm\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.metrics import label_ranking_average_precision_score\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom pytorch_lightning.core import LightningModule\nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nfrom pytorch_lightning.callbacks.early_stopping import EarlyStopping\nfrom pytorch_lightning.loggers import TensorBoardLogger\nfrom audiomentations import Compose, AddGaussianNoise, Gain, TimeStretch, PitchShift","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"def set_random_seeds(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"BASE_INPUT_DIR = '/kaggle/input/'\nTRAIN_INPUT_DIR = os.path.join(BASE_INPUT_DIR, 'rfcx-species-audio-detection/train')\nTEST_INPUT_DIR = os.path.join(BASE_INPUT_DIR, 'rfcx-species-audio-detection/test')\nOUTPUT_DIR = './output'\n\ntrain_tp = pd.read_csv(os.path.join(BASE_INPUT_DIR, 'rfcx-species-audio-detection/train_tp.csv'))\nsubmission = pd.read_csv(os.path.join(BASE_INPUT_DIR, 'rfcx-species-audio-detection/sample_submission.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"sys.path.append(os.path.join(BASE_INPUT_DIR, 'panns-sed'))\n\nfrom models import Cnn14_DecisionLevelAtt, AttBlock, init_layer\nfrom pytorch_utils import move_data_to_device, do_mixup","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"PROJECT_ID = 'kaggle-moa-296003'\nfrom google.cloud import storage\nstorage_client = storage.Client(project=PROJECT_ID)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"bucket = storage_client.get_bucket('tn-kaggle-data')\nblob = bucket.blob('panns_weights.zip')\nblob.download_to_filename('./panns_weights.zip')\n!unzip ./panns_weights.zip\n!rm panns_weights.zip","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# LRAP. Instance-level average\n# Assume float preds [BxC], labels [BxC] of 0 or 1\ndef LRAP(preds, labels):\n    # Ranks of the predictions\n    ranked_classes = torch.argsort(preds, dim=-1, descending=True)\n    # i, j corresponds to rank of prediction in row i\n    class_ranks = torch.zeros_like(ranked_classes)\n    for i in range(ranked_classes.size(0)):\n        for j in range(ranked_classes.size(1)):\n            class_ranks[i, ranked_classes[i][j]] = j + 1\n    # Mask out to only use the ranks of relevant GT labels\n    ground_truth_ranks = class_ranks * labels + (1e6) * (1 - labels)\n    # All the GT ranks are in front now\n    sorted_ground_truth_ranks, _ = torch.sort(ground_truth_ranks, dim=-1, descending=False)\n    pos_matrix = torch.tensor(np.array([i+1 for i in range(labels.size(-1))])).unsqueeze(0)\n    score_matrix = pos_matrix / sorted_ground_truth_ranks\n    score_mask_matrix, _ = torch.sort(labels, dim=-1, descending=True)\n    scores = score_matrix * score_mask_matrix\n    score = (scores.sum(-1) / labels.sum(-1)).mean()\n    return score.item()\n\n\n# label-level average\n# Assume float preds [BxC], labels [BxC] of 0 or 1\ndef LWLRAP(labels, preds):\n    # Ranks of the predictions\n    ranked_classes = torch.argsort(preds, dim=-1, descending=True)\n    # i, j corresponds to rank of prediction in row i\n    class_ranks = torch.zeros_like(ranked_classes)\n    for i in range(ranked_classes.size(0)):\n        for j in range(ranked_classes.size(1)):\n            class_ranks[i, ranked_classes[i][j]] = j + 1\n    # Mask out to only use the ranks of relevant GT labels\n    ground_truth_ranks = class_ranks * labels + (1e6) * (1 - labels)\n    # All the GT ranks are in front now\n    sorted_ground_truth_ranks, _ = torch.sort(ground_truth_ranks, dim=-1, descending=False)\n    # Number of GT labels per instance\n    num_labels = labels.sum(-1)\n    pos_matrix = torch.tensor(np.array([i+1 for i in range(labels.size(-1))]), device=DEVICE).unsqueeze(0)\n    score_matrix = pos_matrix / sorted_ground_truth_ranks\n    score_mask_matrix, _ = torch.sort(labels, dim=-1, descending=True)\n    scores = score_matrix * score_mask_matrix\n    score = scores.sum() / labels.sum()\n    return score.item()\n\n\nclass ImprovedPANNsLoss(nn.Module):\n\n    def __init__(self):\n        super().__init__()\n        self.normal_loss = nn.BCELoss()\n        self.bce = nn.BCELoss()\n\n    def forward(self, output_dict, target):\n        return self.bce(output_dict['clipwise_output'], target)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Constants & Config"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"DEVICE= \"cuda:0\"\n\nNUM_SPECIES = 24\n\nIMG_SIZE = (224, 512, 3)\nIMG_HEIGHT = IMG_SIZE[0]\nIMG_WIDTH = IMG_SIZE[1]\n\nFMIN = 40.0\nFMAX = 24000.0\n\nSR = 48000\nFRAME_SIZE = 1024\nHOP_LENGTH = 320\nN_MELS = 64\n\nCLIP_DURATION = 60\nSEGMENT_DURATION = 10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"cfg = {\n    'preprocess': {\n        'frame_size': FRAME_SIZE,\n        'hop_length': HOP_LENGTH,\n        'sub_segment_duration': 6,\n\n        'do_mixup': True,\n        'mixup_alpha': 0.2\n    },\n    'training': {\n        'n_folds': 5,\n        'batch_size': 24,\n        'epochs': 100,\n        'max_lr': 1e-4,\n    },\n    'model': {\n        'base_model': 'cnn14_att',\n    },\n    'inference': {\n        'segment_stride': 6\n    }\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocessing"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"class RFCXDataset(torch.utils.data.Dataset):\n\n    def __init__(self, samples, configs, is_train):\n        self.samples = samples\n        self.cfg = configs\n        self.is_train = is_train\n        self.augment = Compose([\n            AddGaussianNoise(min_amplitude=0.001, max_amplitude=0.015, p=0.5),\n            Gain(min_gain_in_db=-12, max_gain_in_db=12, p=0.5),\n        ])\n        \n    def _load_audio(self, recording_id):\n        filepath = os.path.join(TRAIN_INPUT_DIR, recording_id + '.flac')\n        data, _ = librosa.load(filepath, sr=SR)\n        return data\n    \n    def _cut_wav(self, audio_data, sample):\n        tmin = sample['t_min']\n        tmax = sample['t_max']\n        sub_segment_duration = cfg['preprocess']['sub_segment_duration']\n\n        if self.is_train:\n            if tmax - tmin < sub_segment_duration:\n                min_left = max(0.0, tmax - sub_segment_duration)\n                max_left = min(tmin, CLIP_DURATION - sub_segment_duration)\n            else:\n                shrinkage = (tmax - tmin) - sub_segment_duration\n                min_left = tmin\n                max_left = tmin + shrinkage\n            left_cut = np.random.uniform(low=min_left, high=max_left)\n        else:\n            if tmax - tmin < sub_segment_duration:\n                extension = max(0.0, sub_segment_duration - (tmax - tmin))/2\n                left_extend = extension\n                if tmax + extension > CLIP_DURATION:\n                    left_extend += tmax + extension - CLIP_DURATION\n                left_cut = max(0.0, tmin - left_extend)\n            else:\n                shrinkage = (tmax - tmin) - sub_segment_duration\n                left_cut = tmin + shrinkage/2\n\n        left_cut_sample = int(np.floor(left_cut * SR))\n        right_cut_sample = left_cut_sample + sub_segment_duration*SR\n        cut = audio_data[left_cut_sample:right_cut_sample]\n        assert len(cut) == sub_segment_duration*SR\n        return cut\n    \n    def _one_hot(self, idx):\n        target = np.zeros(NUM_SPECIES, dtype=np.float32)\n        sparse_label = self.samples.species_id.iloc[idx]\n        target[sparse_label] = 1\n        return target\n\n    def __len__(self):\n        return self.samples.shape[0]\n\n    def __getitem__(self, idx: int):\n        sample = self.samples.iloc[idx, :]\n        data = self._load_audio(sample.recording_id)\n        wav, target = self._cut_wav(data, sample), self._one_hot(idx)\n        if self.is_train:\n            wav = self.augment(samples=wav, sample_rate=SR)\n        return wav, target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Mixup(object):\n\n    def __init__(self, mixup_alpha, random_seed=1234):\n        \"\"\"Mixup coefficient generator.\n        \"\"\"\n        self.mixup_alpha = mixup_alpha\n        self.random_state = np.random.RandomState(random_seed)\n\n    def get_lambda(self, batch_size):\n        \"\"\"Get mixup random coefficients.\n        Args:\n          batch_size: int\n        Returns:\n          mixup_lambdas: (batch_size,)\n        \"\"\"\n        mixup_lambdas = []\n        for n in range(0, batch_size, 2):\n            lam = self.random_state.beta(self.mixup_alpha, self.mixup_alpha, 1)[0]\n            mixup_lambdas.append(lam)\n            mixup_lambdas.append(1. - lam)\n\n        return np.array(mixup_lambdas)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def create_model(load_pretrained=True):\n    with_mixup = cfg['preprocess']['do_mixup']\n    args = {\n        'sample_rate': SR,\n        'window_size': FRAME_SIZE,\n        'hop_size': HOP_LENGTH,\n        'mel_bins': N_MELS,\n        'fmin': FMIN, \n        'fmax': FMAX,\n        'classes_num': 527,\n    }\n    base_model = Cnn14_DecisionLevelAtt(**args)\n    if load_pretrained:\n        checkpoint = torch.load(os.path.join(BASE_INPUT_DIR, 'panns-sed/Cnn14_DecisionLevelAtt_mAP=0.425.pth'))\n        base_model.load_state_dict(checkpoint['model'])\n\n    base_model.att_block = AttBlock(2048, NUM_SPECIES, activation=\"sigmoid\")\n    base_model.att_block.init_weights()\n    init_layer(base_model.fc1)\n    return base_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"class LightModel(LightningModule):\n\n    def __init__(self, model, train_samples, val_samples):\n        super().__init__()\n        self.model = model\n        self.train_samples = train_samples\n        self.val_samples = val_samples\n        self.loss_fn = ImprovedPANNsLoss()\n        self.mixup_augmenter = Mixup(cfg['preprocess']['mixup_alpha'])\n\n    def forward(self, batch, mixup_lambda=None):\n        return self.model(batch, mixup_lambda=mixup_lambda)\n    \n    def train_dataloader(self):\n        batch_size = cfg['training']['batch_size']\n        if cfg['preprocess']['do_mixup']:\n            batch_size *= 2\n        return DataLoader(\n            RFCXDataset(self.train_samples, cfg, True),\n            batch_size=batch_size,\n            shuffle=True,\n            pin_memory=True,\n            drop_last=True\n        )\n\n    def val_dataloader(self):\n        return DataLoader(\n            RFCXDataset(self.val_samples, cfg, False), \n            batch_size=cfg['training']['batch_size'],\n            shuffle=False,\n            pin_memory=True,\n            drop_last=False\n        )\n\n    def training_step(self, batch, batch_idx):\n        mixup_lambda = None\n        if cfg['preprocess']['do_mixup']:\n            mixup_lambda = self.mixup_augmenter.get_lambda(batch_size=len(batch))\n            mixup_lambda = move_data_to_device(mixup_lambda, DEVICE)\n        y, target = [x.to(DEVICE) for x in batch] \n        target = do_mixup(target, mixup_lambda)\n        output = model(y, mixup_lambda=mixup_lambda)\n        bceLoss = self.loss_fn(output, target)\n        loss = bceLoss\n        self.log('train_loss', loss, on_epoch=True)\n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        y, target = [x.to(self.device) for x in batch]\n        output = model(y)\n        bceLoss = self.loss_fn(output, target)\n        loss = bceLoss\n        self.log('val_loss', loss, on_epoch=True)\n        lwap = LWLRAP(target, output['clipwise_output'])\n        self.log(\"LwAP\", lwap, on_epoch=True)\n\n    def configure_optimizers(self):\n        optimizer = torch.optim.Adam(model.parameters(), lr=cfg['training']['max_lr'])\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)\n        lr_scheduler = {\"scheduler\": scheduler }\n        return [optimizer], [lr_scheduler]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"class RFCXTestDataset(torch.utils.data.Dataset):\n\n    def __init__(self, samples, segmented=False):\n        self.samples = samples\n        self.segmented = segmented\n        \n    def load_test_data(self, recording_id):\n        filepath = os.path.join(TEST_INPUT_DIR, recording_id + '.flac')\n        data, _ = librosa.load(filepath, sr=SR)\n        all_segments = []\n        if self.segmented:\n            for i in range(0, 55, cfg['inference']['segment_stride']):\n                sample_min = i * SR\n                sample_max = sample_min + cfg['preprocess']['sub_segment_duration']*SR\n                segment = data[sample_min:sample_max]\n                assert len(segment) == cfg['preprocess']['sub_segment_duration']*SR\n                all_segments.append(segment)\n        else:\n            all_segments.append(data)\n        return torch.tensor(all_segments)\n\n    def __len__(self):\n        return self.samples.shape[0]\n\n    def __getitem__(self, idx: int):\n        sample = self.samples.iloc[idx, :]\n        return sample.recording_id, self.load_test_data(sample.recording_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"test_dataset = RFCXTestDataset(submission)\ndata_loader = DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4, pin_memory=True, drop_last=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models_path = './output'\nmodel_files = os.listdir(models_path)\nfile_by_fold = []\nfor i in range(cfg['training']['n_folds']):\n    for f in model_files:\n        if f.startswith(f'weights-{i}'):\n            file_by_fold.append(f)\n            break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"submission_folds = []\n\nfor fold in range(5):\n    fold_start = time.time()\n    checkpoint_path = os.path.join(models_path, file_by_fold[fold])\n    model = LightModel(create_model(load_pretrained=False), train_tp, train_tp).to(DEVICE)\n    model.load_state_dict(torch.load(checkpoint_path, map_location=DEVICE)[\"state_dict\"])\n    model.eval()\n    fold_submission = submission.copy()\n    fold_preds = []\n    print(f'Load Fold {fold}. Elapsed:', round(time.time() - fold_start, 2))\n\n    with torch.no_grad():\n        start = time.time()\n        for i, batch in enumerate(data_loader):\n            for recording_id, inputs in zip(batch[0], batch[1]):\n                inputs = inputs.to(DEVICE)\n                segment_preds = model(inputs)[\"clipwise_output\"]\n                rec_pred = torch.max(segment_preds, dim=0).values.detach().cpu().numpy()\n                fold_preds.append(rec_pred)\n                print(recording_id, 'elapsed:', round(time.time() - start, 2))\n\n    fold_submission.iloc[:, 1:] = np.stack(fold_preds)\n    fold_submission.to_csv(f'fold_{fold}.csv', index=False)\n    submission_folds.append(fold_submission)\n    print('Fold', fold, 'Elapsed:', round(time.time() - fold_start, 2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"from scipy.stats.mstats import gmean\n\nprobs = np.stack([sf.iloc[:, 1:].values for sf in submission_folds])\nsubmission.iloc[:, 1:] = gmean(probs, axis=0)\nsubmission.to_csv('./submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}