{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install resnest > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport sys\nimport csv\nimport glob\nimport random\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch import optim\nfrom torchvision import models\nimport torch.utils.data as data\nfrom torch.utils.data import DataLoader\n\nfrom pytorch_lightning.core import LightningModule\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nfrom pytorch_lightning.loggers import TensorBoardLogger\nfrom pytorch_lightning import Trainer, seed_everything\nfrom resnest.torch import resnest50\n\nfrom scipy.io import wavfile\nfrom skimage.transform import resize\nimport librosa\nimport cv2\nfrom pathlib import Path\nimport warnings\n\nimport soundfile as sf\nfrom sklearn.model_selection import StratifiedKFold","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"data_dir = '../input/rfcx-species-audio-detection/'\n\ntrain_tp_df = pd.read_csv(os.path.join(data_dir, 'train_tp.csv'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_folds(df, k=5):\n    df[\"kfold\"] = -1\n    df = df.sample(frac=1).reset_index(drop=True)\n    y = df.species_id.values\n    kf = StratifiedKFold(n_splits=k, shuffle=True)\n    for fold_, (t, v_) in enumerate(kf.split(X=df, y=y)):\n        df.loc[v_, \"kfold\"] = fold_\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelSpecSlicesDataset(data.Dataset):\n    def __init__(\n        self,\n        data_dir=None,\n        df=None,\n        num_classes=24,\n        sample_rate=48000,\n        sample_size=48000 * 10,\n        melspectrogram_parameters={},\n        img_height=224,\n        img_width=512,\n        is_train=False,\n        is_validation=False,\n        is_testset=False,\n        testset_recording_ids=[]\n    ):\n        assert sample_rate * 10 == sample_size\n        \n        self.data_dir = data_dir\n        self.df = df\n        self.num_classes = num_classes\n        self.sample_rate = sample_rate\n        self.sample_size = sample_size\n        self.melspectrogram_parameters = melspectrogram_parameters\n        self.img_height = img_height\n        self.img_width = img_width\n        self.is_train = is_train\n        self.is_validation = is_validation\n        self.is_testset = is_testset\n        \n        if self.is_testset:\n            self.testset_recording_ids = testset_recording_ids\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        if self.is_testset:\n            return self.get_testset_batch(self.testset_recording_ids[idx])\n        \n        recording_id = self.df.recording_id.values[idx]\n        t_min = self.df.t_min.values[idx]\n        t_max = self.df.t_max.values[idx]\n        label = self.df.species_id.values[idx]\n\n        wav_slice, label = self.get_wav_slice(recording_id, t_min, t_max)\n        mel_spec = self.get_mel_spec(wav_slice)\n\n        return mel_spec, label\n    \n    def get_labels(self, recording_id, beginning_time, ending_time):\n        beginning_time = beginning_time / self.sample_rate\n        ending_time = ending_time / self.sample_rate\n        \n        assert beginning_time >= 0 and beginning_time <= 60\n        assert ending_time >= 0 and ending_time <= 60\n        \n        query_string = f\"recording_id == '{recording_id}' & \"\n        query_string += f\"t_min < {ending_time} & t_max > {beginning_time}\"\n        all_tp_events = self.df.query(query_string)\n\n        label = np.zeros(24, dtype=np.float32)\n        for species_id in all_tp_events[\"species_id\"].unique():\n            label[int(species_id)] = 1.0\n        \n        return label\n\n    def load_wav(self, recording_id):\n        if self.is_testset:\n            file_path = os.path.join(self.data_dir, \"test\", recording_id)\n        else:\n            file_path = os.path.join(self.data_dir, \"train\", recording_id + \".flac\")\n            \n        wav, _ = librosa.load(\n            file_path,\n            sr=self.sample_rate\n        )\n        return wav\n\n    def get_wav_slice(self, recording_id, t_min, t_max):\n        wav = self.load_wav(recording_id)\n\n        t_min = t_min * self.sample_rate\n        t_max = t_max * self.sample_rate\n\n        center = np.round((t_min + t_max) / 2)\n        beginning = center - self.sample_size / 2\n        if beginning < 0:\n            beginning = 0\n        \n        if not self.is_validation:\n            beginning = np.random.randint( beginning , center)\n\n        ending = beginning + self.sample_size\n        if ending > len(wav):\n            ending = len(wav)\n            beginning = ending - self.sample_size\n\n        wav_slice = wav[int(beginning) : int(ending)]\n        \n        labels = self.get_labels(recording_id, beginning, ending)\n        return wav_slice, labels\n\n    def get_mel_spec(self, wav_slice):\n        mel_spec = librosa.feature.melspectrogram(\n            wav_slice, sr=self.sample_rate, **self.melspectrogram_parameters\n        )\n        mel_spec = resize(mel_spec, (self.img_height, self.img_width))\n\n        # Normalize to 0...1 - this is what goes into neural net\n        mel_spec = mel_spec - np.min(mel_spec)\n        mel_spec = mel_spec / np.max(mel_spec)\n\n        mel_spec = np.stack((mel_spec, mel_spec, mel_spec))\n        return mel_spec\n    \n    def get_testset_batch(self, recording_id):\n        wav = self.load_wav(recording_id)\n\n        # Split into n chunks\n        return (\n            list(map(self.get_mel_spec, np.array_split(wav, 6))),\n            recording_id.replace(\".flac\", \"\"),\n        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Run some tests on the dataset configured for processing test set\ntest_dataset = MelSpecSlicesDataset(\n    data_dir=\"../input/rfcx-species-audio-detection\",\n    testset_recording_ids=os.listdir(\"../input/rfcx-species-audio-detection/test\"),\n    df=None,\n    num_classes=24,\n    sample_rate=48000,\n    sample_size=48000 * 10,\n    melspectrogram_parameters = {\n        \"n_fft\": 2048,\n        \"hop_length\": 512,\n        \"n_mels\": 256,\n        \"fmin\": 24,\n        \"fmax\": 24000,\n        \"power\": 2,\n    },\n    img_height=224,\n    img_width=512,\n    is_validation=False,\n    is_testset=True,\n)\n\ntest_batch, recording_id = test_dataset[0]\nprint(len(test_batch))\nprint(test_batch[0].shape)\nprint(recording_id)\n\n# Each file should be split into 6 chunks\nassert len(test_batch) == 6\n\n# The image input dimensions\nassert test_batch[0].shape == (3, 224, 512)\n\n# Test that the first chunk is equal to itself\nassert np.array_equal(test_batch[0], test_batch[0])\n\n# Test that other chunks are different from the first\nassert not np.array_equal(test_batch[0], test_batch[1])\nassert not np.array_equal(test_batch[0], test_batch[5])\ndel test_dataset, test_batch, recording_id","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Metrics"},{"metadata":{"trusted":true},"cell_type":"code","source":"def _one_sample_positive_class_precisions(scores, truth):\n    num_classes = scores.shape[0]\n    pos_class_indices = np.flatnonzero(truth > 0)\n\n    if not len(pos_class_indices):\n        return pos_class_indices, np.zeros(0)\n\n    retrieved_classes = np.argsort(scores)[::-1]\n\n    class_rankings = np.zeros(num_classes, dtype=np.int)\n    class_rankings[retrieved_classes] = range(num_classes)\n\n    retrieved_class_true = np.zeros(num_classes, dtype=np.bool)\n    retrieved_class_true[class_rankings[pos_class_indices]] = True\n\n    retrieved_cumulative_hits = np.cumsum(retrieved_class_true)\n\n    precision_at_hits = retrieved_cumulative_hits[class_rankings[pos_class_indices]] / (\n        1 + class_rankings[pos_class_indices].astype(np.float)\n    )\n    return pos_class_indices, precision_at_hits\n\n\ndef lwlrap(truth, scores):\n    assert truth.shape == scores.shape\n    num_samples, num_classes = scores.shape\n    precisions_for_samples_by_classes = np.zeros((num_samples, num_classes))\n    for sample_num in range(num_samples):\n        pos_class_indices, precision_at_hits = _one_sample_positive_class_precisions(\n            scores[sample_num, :], truth[sample_num, :]\n        )\n        precisions_for_samples_by_classes[\n            sample_num, pos_class_indices\n        ] = precision_at_hits\n\n    labels_per_class = np.sum(truth > 0, axis=0)\n    weight_per_class = labels_per_class / float(np.sum(labels_per_class))\n\n    per_class_lwlrap = np.sum(precisions_for_samples_by_classes, axis=0) / np.maximum(\n        1, labels_per_class\n    )\n    return per_class_lwlrap, weight_per_class","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loss Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, gamma=2):\n        super().__init__()\n        self.gamma = gamma\n\n    def forward(self, logit, target):\n        target = target.float()\n        max_val = (-logit).clamp(min=0)\n        loss = logit - logit * target + max_val + \\\n            ((-max_val).exp() + (-logit - max_val).exp()).log()\n\n        invprobs = F.logsigmoid(-logit * (target * 2.0 - 1.0))\n        loss = (invprobs * self.gamma).exp() * loss\n        if len(loss.size()) == 2:\n            loss = loss.sum(dim=1)\n        return loss.mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RainforestModelPL(LightningModule):\n\n    def __init__(\n        self,\n        random_seed: int = 1234,\n        fold: int = 0,\n        lr: float = 0.001,\n        num_classes: int = 24,\n        batch_size: int = 16,\n        num_workers: int = 6,\n        n_mels: int = 224,\n        fmin: int = 40,\n        fmax: int = 24000,\n        img_height: int = 224,\n        img_width: int = 512,\n        sample_rate: int = 48000,\n        sample_size: int = 48000 * 10,\n        train_folds_df = None\n    ):\n        super().__init__()\n        \n        assert sample_rate * 10 == sample_size\n        \n        self.save_hyperparameters()\n        \n        self.melspectrogram_parameters = {\n            \"n_fft\": 2048,\n            \"hop_length\": 512,\n            \"n_mels\": n_mels,\n            \"fmin\": fmin,\n            \"fmax\": fmax,\n            \"power\": 2,\n        }\n\n        model = resnest50(pretrained=True)\n\n        model.fc = nn.Sequential(\n            nn.Linear(2048, 1024),\n            nn.ReLU(),\n            nn.Dropout(p=0.2),\n            nn.Linear(1024, 1024),\n            nn.ReLU(),\n            nn.Dropout(p=0.2),\n            nn.Linear(1024, num_classes),\n        )\n\n        self.net = model\n        self.criterion = FocalLoss()\n\n    def forward(self, x):\n        return self.net(x)\n\n    def training_step(self, batch, batch_idx):\n        data, target = batch\n        output = self(data)\n        loss = self.criterion(output, target)\n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        data, target = batch\n        output = self(data)\n        val_loss = self.criterion(output, target)\n\n        p = torch.sigmoid(output)\n        \n        score_class, weight = lwlrap(target.cpu().numpy(), p.cpu().numpy())\n        score = (score_class * weight).sum()\n\n        self.log(\"val_lrap\", score, on_step=False, on_epoch=True, prog_bar=True)\n        self.log('val_loss', val_loss, on_step=False, on_epoch=True, prog_bar=True)\n        \n        \n    def configure_optimizers(self):\n        \n        # Paramters from https://www.kaggle.com/fffrrt/all-in-one-rfcx-baseline-for-beginners/notebook\n        # Try finding better params using something like Optuna \n        # https://github.com/optuna/optuna/blob/master/examples/pytorch_lightning_simple.py\n        optimizer = torch.optim.SGD(self.parameters(), lr=self.hparams.lr, weight_decay=0.0001, momentum=0.9)\n        scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.4)\n\n        lr_scheduler = {\"scheduler\": scheduler }\n        return [optimizer], [lr_scheduler]\n\n    def setup(self, stage):\n        train_folds_df = self.hparams.train_folds_df\n        \n        train_df = train_folds_df[\n            train_folds_df.kfold != self.hparams.fold\n        ].reset_index(drop=True)\n        val_df = train_folds_df[train_folds_df.kfold == self.hparams.fold].reset_index(\n            drop=True\n        )\n\n        self.train_dataset = MelSpecSlicesDataset(\n            data_dir=data_dir,\n            df=train_df,\n            num_classes=self.hparams.num_classes,\n            sample_rate=self.hparams.sample_rate,\n            sample_size=self.hparams.sample_size,\n            img_height=self.hparams.img_height,\n            img_width=self.hparams.img_width,\n            melspectrogram_parameters=self.melspectrogram_parameters,\n        )\n\n        self.validation_dataset = MelSpecSlicesDataset(\n            data_dir=data_dir,\n            df=val_df,\n            num_classes=self.hparams.num_classes,\n            sample_rate=self.hparams.sample_rate,\n            sample_size=self.hparams.sample_size,\n            img_height=self.hparams.img_height,\n            img_width=self.hparams.img_width,\n            melspectrogram_parameters=self.melspectrogram_parameters,\n            is_validation=True,\n        )\n\n    def train_dataloader(self):\n        return DataLoader(\n            self.train_dataset,\n            batch_size=self.hparams.batch_size,\n            num_workers=self.hparams.num_workers,\n            shuffle=False,\n            pin_memory=False,\n            \n            # Drop the last batch if it's smaller than\n            # the set batch size. Some implementations\n            # of batch norm throw an error if there is \n            # only one sample in the batch.\n            drop_last=True,\n        )\n\n    def val_dataloader(self):\n        return DataLoader(\n            self.validation_dataset,\n            batch_size=self.hparams.batch_size,\n            num_workers=self.hparams.num_workers,\n            shuffle=False,\n            pin_memory=False,\n            drop_last=True,\n        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Seed for reproducibility\nrandom_seed = 1234\nseed_everything(seed=random_seed)\nrandom.seed(random_seed)\nnp.random.seed(random_seed)\nos.environ[\"PYTHONHASHSEED\"] = str(random_seed)\ntorch.manual_seed(random_seed)\ntorch.cuda.manual_seed(random_seed)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_fold(experiment_version, fold, max_epochs, train_folds_df):\n    log_dir = \"./lightning_logs\"\n    \n    full_log_dir_path = os.path.join(\n        log_dir, \"melspec\", \"version_\" + experiment_version, \"fold_\" + str(fold)\n    )\n    checkpoint_callback = ModelCheckpoint(\n        monitor=\"val_lrap\",\n        dirpath=full_log_dir_path,\n        filename=\"{epoch:02d}-{vr:.3f}-{vp:.3f}-{val_lrap:.3f}\",\n        mode=\"max\",\n    )\n\n    logger = TensorBoardLogger(\n        \"lightning_logs\", name=\"melspec\", version=\"version_\" + experiment_version\n    )\n\n\n    trainer = Trainer(\n        gpus=1,\n        max_epochs=max_epochs,\n        callbacks=[checkpoint_callback],\n        logger=logger,\n    )\n    \n    trainer.fit(RainforestModelPL(random_seed=random_seed, fold=0, lr=0.01, train_folds_df=train_folds_df))\n\n    print(\n        \"Completed training for experiment version: \",\n        \"version_\" + experiment_version,\n        \"fold: \" + str(fold),\n    )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Folds"},{"metadata":{"trusted":true},"cell_type":"code","source":"experiment_id = 'resnest04'\nmax_epochs = 1\ntrain_folds_df = create_folds(train_tp_df)\ntrain_fold(experiment_id, 0, max_epochs, train_folds_df)\n# train_fold(experiment_id, 1, max_epochs, train_folds_df)\n# train_fold(experiment_id, 2, max_epochs, train_folds_df)\n# train_fold(experiment_id, 3, max_epochs, train_folds_df)\n# train_fold(experiment_id, 4, max_epochs, train_folds_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test Set Inference & Create Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir(\"./lightning_logs/melspec/version_resnest03\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"glob.glob(\"./lightning_logs/version_resnest03/fold_*/*.ckpt\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_submission(experiment):\n    print('Creating submission file for experiment: ', experiment)\n    \n    sub_df = pd.read_csv(\"../input/rfcx-species-audio-detection/sample_submission.csv\")\n    pred_cols = sub_df.columns.tolist()[1:]\n    sub_df[pred_cols] = None\n    \n    experiment_dir = os.path.join('./lightning_logs/melspec', experiment)\n    subs_output_dir = os.path.join('./lightning_logs/melspec', experiment, 'subs')\n    \n    def _load_model(ckpt_fold_path):\n        rainforest_model = RainforestModelPL.load_from_checkpoint(\n            ckpt_fold_path, hparams_file=os.path.join(experiment_dir, \"hparams.yaml\")\n        )\n        rainforest_model.cuda()\n        rainforest_model.eval()\n        return rainforest_model\n    \n    \n    checkpoints_paths = glob.glob(experiment_dir + \"/fold_*/*.ckpt\")\n    nets = list(map(_load_model, checkpoints_paths))\n    \n    test_dataset = MelSpecSlicesDataset(\n        data_dir=\"../input/rfcx-species-audio-detection\",\n        testset_recording_ids=os.listdir(\"../input/rfcx-species-audio-detection/test\"),\n        df=None,\n        num_classes=nets[0].hparams.num_classes,\n        sample_rate=nets[0].hparams.sample_rate,\n        sample_size=nets[0].hparams.sample_size,\n        melspectrogram_parameters = {\n            \"n_fft\": 2048,\n            \"hop_length\": 512,\n            \"n_mels\": nets[0].hparams.n_mels,\n            \"fmin\": nets[0].hparams.fmin,\n            \"fmax\": nets[0].hparams.fmax,\n            \"power\": 2,\n        },\n        img_height=nets[0].hparams.img_height,\n        img_width=nets[0].hparams.img_width,\n        is_validation=False,\n        is_testset=True,\n    )\n    \n    for index, (chunks, recording_id) in tqdm(enumerate(test_dataset)):\n        with torch.no_grad():\n\n            nets_preds = []\n\n            # Aggreate predictions for all nets\n            for net in nets:\n                output = net(torch.tensor(chunks).cuda())\n                nets_preds.append(\n                    torch.max(net(torch.tensor(chunks).cuda()), dim=0)\n                    .values.detach()\n                    .cpu()\n                    .numpy()\n                )\n            sub_df.loc[sub_df.recording_id == recording_id, pred_cols] = np.sum(\n                np.stack(nets_preds), axis=0\n            )\n\n    print(sub_df.head())\n    if not os.path.exists(subs_output_dir):\n        os.makedirs(subs_output_dir)\n    sub_df.to_csv(experiment + \"__agg_avg_score_\" + avg_score + \".csv\", index=False)\n    print(\"Created agg submission for experiment: \", experiment)\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"create_submission(\"version_\" + experiment_id)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### References\n- https://www.kaggle.com/fffrrt/all-in-one-rfcx-baseline-for-beginners/notebook\n- https://www.kaggle.com/c/rfcx-species-audio-detection/discussion/198418"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}