{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"My partner will be describing the psuedo labeling generation procedure, performance of different model architectures and\nloss functions in more detail. Here are some points that i found to be important.\n1. Catastrophic forgetting in neural networks\nThere is imbalance in the distribution of bird species, for ex: species 3 occurs very frequently.\nDuring training I found that initially model learns to classify species 3 and as the training proceeds\nit starts \"forgetting\". The confidence for species 3 goes on decreasing which negatively impacts the lb score.\nSo, we need to make sure that other species are learnt without forgetting species 3. \nI found that recall rate for species 3 can be improved by setting pos_weight in BCELoss.\nYou may find this paper interesting if you are more curious: https://arxiv.org/pdf/1612.00796.pdf (especially section 2.1)\n2. Augmenting other datasets\\\nNot all parts of the audio are occupied by bird species. I replaced these unoccupied parts with bird\nsongs from cornell. \n3. Misc\n    - Validation scheme should be similar to test scheme.\n    For ex: If you feed 5s chunks during test and then take max, the same thing should\n    be done during validation also.\n    - I found Click Noise Augmentation to be very useful (https://librosa.org/doc/0.8.0/generated/librosa.clicks.html)\n    - Using pretrained weights (imagenet/cornell) can help to converge much faster.\n    - Model Averaging seems to always lead to better generalization.\n    - 5s crops seems to perform slightly better than 10s crops","metadata":{}},{"cell_type":"code","source":"!pip install resnest > /dev/null\n!pip install colorednoise > /dev/null","metadata":{"papermill":{"duration":24.928028,"end_time":"2021-02-18T08:19:09.87789","exception":false,"start_time":"2021-02-18T08:18:44.949862","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations as A\nfrom resnest.torch.resnet import ResNet, Bottleneck\nimport random\nfrom glob import glob\nfrom collections import OrderedDict\nimport os.path as osp\nimport os\nfrom pytorch_lightning.loggers import TensorBoardLogger\nfrom pytorch_lightning.callbacks.early_stopping import EarlyStopping\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nfrom pytorch_lightning import LightningModule\nfrom pytorch_lightning import Trainer\nfrom skimage.transform import resize\nfrom torchvision.models import resnet18, resnet34, resnet50\nfrom resnest.torch import resnest50\nfrom tqdm.auto import tqdm\nimport colorednoise as cn\nimport librosa\nimport torchaudio\nimport torch.nn.functional as F\nfrom torch.utils.data import WeightedRandomSampler\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision\nimport torch\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score, confusion_matrix\nimport matplotlib\nmatplotlib.use('Agg')","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":5.913696,"end_time":"2021-02-18T08:19:15.808227","exception":false,"start_time":"2021-02-18T08:19:09.894531","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=42):\n    print(f'setting everything to seed {seed}')\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.cuda.empty_cache()\n    \nseed_everything(42)","metadata":{"papermill":{"duration":0.029148,"end_time":"2021-02-18T08:19:15.852955","exception":false,"start_time":"2021-02-18T08:19:15.823807","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/c/rfcx-species-audio-detection/discussion/198418\n# label-level average\n# Assume float preds [BxC], labels [BxC] of 0 or 1\ndef LWLRAP(preds, labels):\n    # Ranks of the predictions\n    ranked_classes = torch.argsort(preds, dim=-1, descending=True)\n    # i, j corresponds to rank of prediction in row i\n    class_ranks = torch.zeros_like(ranked_classes).to(preds.device)\n    for i in range(ranked_classes.size(0)):\n        for j in range(ranked_classes.size(1)):\n            class_ranks[i, ranked_classes[i][j]] = j + 1\n    # Mask out to only use the ranks of relevant GT labels\n    ground_truth_ranks = class_ranks * labels + (1e6) * (1 - labels)\n    # All the GT ranks are in front now\n    sorted_ground_truth_ranks, _ = torch.sort(\n        ground_truth_ranks, dim=-1, descending=False)\n    # Number of GT labels per instance\n    num_labels = labels.sum(-1)\n    pos_matrix = torch.tensor(\n        np.array([i+1 for i in range(labels.size(-1))])).unsqueeze(0).to(preds.device)\n    score_matrix = pos_matrix / sorted_ground_truth_ranks\n    score_mask_matrix, _ = torch.sort(labels, dim=-1, descending=True)\n    scores = score_matrix * score_mask_matrix\n    score = scores.sum() / labels.sum()\n    return score.item()\n","metadata":{"papermill":{"duration":0.030495,"end_time":"2021-02-18T08:19:15.899709","exception":false,"start_time":"2021-02-18T08:19:15.869214","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    batch_size = 8\n    weight_decay = 1e-8\n    lr = 1e-3\n    num_workers = 4\n    epochs = 6\n    num_classes = 24\n    sr = 32_000\n    duration = 5\n    total_duration = 60\n    nmels = 128\n    EXTRAS_DIR = \"../input/rfcxextras\"\n    ROOT = \"../input/rfcx-species-audio-detection\"\n    TRAIN_AUDIO_ROOT = osp.join(ROOT, \"train\")\n    TEST_AUDIO_ROOT = osp.join(ROOT, \"test\")\n    loss_fn = torch.nn.BCEWithLogitsLoss()","metadata":{"papermill":{"duration":0.02452,"end_time":"2021-02-18T08:19:15.939854","exception":false,"start_time":"2021-02-18T08:19:15.915334","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Audio Augmentations","metadata":{}},{"cell_type":"code","source":"# Mostly taken from https://www.kaggle.com/hidehisaarai1213/rfcx-audio-data-augmentation-japanese-english\nclass AudioTransform:\n    def __init__(self, always_apply=False, p=0.5):\n        self.always_apply = always_apply\n        self.p = p\n\n    def __call__(self, y: np.ndarray):\n        if self.always_apply:\n            return self.apply(y)\n        else:\n            if np.random.rand() < self.p:\n                return self.apply(y)\n            else:\n                return y\n\n    def apply(self, y: np.ndarray):\n        raise NotImplementedError\n\n\nclass Compose:\n    def __init__(self, transforms: list):\n        self.transforms = transforms\n\n    def __call__(self, y: np.ndarray):\n        for trns in self.transforms:\n            y = trns(y)\n        return y\n\n\nclass OneOf:\n    def __init__(self, transforms: list):\n        self.transforms = transforms\n\n    def __call__(self, y: np.ndarray):\n        n_trns = len(self.transforms)\n        trns_idx = np.random.choice(n_trns)\n        trns = self.transforms[trns_idx]\n        return trns(y)\n\n\nclass GaussianNoiseSNR(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, min_snr=5.0, max_snr=20.0, **kwargs):\n        super().__init__(always_apply, p)\n\n        self.min_snr = min_snr\n        self.max_snr = max_snr\n\n    def apply(self, y: np.ndarray, **params):\n        snr = np.random.uniform(self.min_snr, self.max_snr)\n        a_signal = np.sqrt(y ** 2).max()\n        a_noise = a_signal / (10 ** (snr / 20))\n\n        white_noise = np.random.randn(len(y))\n        a_white = np.sqrt(white_noise ** 2).max()\n        augmented = (y + white_noise * 1 / a_white * a_noise).astype(y.dtype)\n        return augmented\n\n\nclass PinkNoiseSNR(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, min_snr=5.0, max_snr=20.0, **kwargs):\n        super().__init__(always_apply, p)\n\n        self.min_snr = min_snr\n        self.max_snr = max_snr\n\n    def apply(self, y: np.ndarray, **params):\n        snr = np.random.uniform(self.min_snr, self.max_snr)\n        a_signal = np.sqrt(y ** 2).max()\n        a_noise = a_signal / (10 ** (snr / 20))\n\n        pink_noise = cn.powerlaw_psd_gaussian(1, len(y))\n        a_pink = np.sqrt(pink_noise ** 2).max()\n        augmented = (y + pink_noise * 1 / a_pink * a_noise).astype(y.dtype)\n        return augmented\n\n\nclass TimeShift(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, max_shift_second=2, sr=32000, padding_mode=\"zero\"):\n        super().__init__(always_apply, p)\n\n        assert padding_mode in [\n            \"replace\", \"zero\"], \"`padding_mode` must be either 'replace' or 'zero'\"\n        self.max_shift_second = max_shift_second\n        self.sr = sr\n        self.padding_mode = padding_mode\n\n    def apply(self, y: np.ndarray, **params):\n        shift = np.random.randint(-self.sr * self.max_shift_second,\n                                  self.sr * self.max_shift_second)\n        augmented = np.roll(y, shift)\n        # if self.padding_mode == \"zero\":\n        #     if shift > 0:\n        #         augmented[:shift] = 0\n        #     else:\n        #         augmented[shift:] = 0\n        return augmented\n\n\nclass VolumeControl(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, db_limit=10, mode=\"uniform\"):\n        super().__init__(always_apply, p)\n\n        assert mode in [\"uniform\", \"fade\", \"fade\", \"cosine\", \"sine\"], \\\n            \"`mode` must be one of 'uniform', 'fade', 'cosine', 'sine'\"\n\n        self.db_limit = db_limit\n        self.mode = mode\n\n    def apply(self, y: np.ndarray, **params):\n        db = np.random.uniform(-self.db_limit, self.db_limit)\n        if self.mode == \"uniform\":\n            db_translated = 10 ** (db / 20)\n        elif self.mode == \"fade\":\n            lin = np.arange(len(y))[::-1] / (len(y) - 1)\n            db_translated = 10 ** (db * lin / 20)\n        elif self.mode == \"cosine\":\n            cosine = np.cos(np.arange(len(y)) / len(y) * np.pi * 2)\n            db_translated = 10 ** (db * cosine / 20)\n        else:\n            sine = np.sin(np.arange(len(y)) / len(y) * np.pi * 2)\n            db_translated = 10 ** (db * sine / 20)\n        augmented = y * db_translated\n        return augmented","metadata":{"papermill":{"duration":0.052492,"end_time":"2021-02-18T08:19:16.007707","exception":false,"start_time":"2021-02-18T08:19:15.955215","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mono_to_color(X, eps=1e-6, mean=None, std=None):\n    X = np.stack([X, X, X], axis=-1)\n\n    # Standardize\n    mean = mean or X.mean()\n    std = std or X.std()\n    X = (X - mean) / (std + eps)\n\n    # Normalize to [0, 255]\n    _min, _max = X.min(), X.max()\n\n    if (_max - _min) > eps:\n        V = np.clip(X, _min, _max)\n        V = 255 * (V - _min) / (_max - _min)\n        V = V.astype(np.uint8)\n    else:\n        V = np.zeros_like(X, dtype=np.uint8)\n\n    return V\n\n\ndef normalize(image, mean=None, std=None):\n    image = image / 255.0\n    if mean is not None and std is not None:\n        image = (image - mean) / std\n    return image.astype(np.float32)\n","metadata":{"papermill":{"duration":0.028209,"end_time":"2021-02-18T08:19:16.051552","exception":false,"start_time":"2021-02-18T08:19:16.023343","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RFCDataset:\n    def __init__(self, tp, fp=None, config=None,\n                 mode='train', inv_counts=None):\n        self.tp = tp\n        self.fp = pd.read_csv(\"../input/rfcxextras/cornell-train.csv\")\n        self.fp = self.fp[self.fp.ebird_code<'c'].reset_index(drop=True)\n        self.fp_root = \"../input/birdsong-resampled-train-audio-00/\"        \n        self.inv_counts = inv_counts\n        self.config = config\n        self.sr = self.config.sr\n        self.total_duration = self.config.total_duration\n        self.duration = self.config.duration\n        self.data_root = self.config.TRAIN_AUDIO_ROOT\n        self.nmels = self.config.nmels\n        self.fmin, self.fmax = 84, self.sr//2\n        self.mode = mode\n        self.num_classes = self.config.num_classes\n        self.resampler = torchaudio.transforms.Resample(\n            orig_freq=48_000, new_freq=self.sr)\n        self.mel = torchaudio.transforms.MelSpectrogram(sample_rate=self.sr, n_mels=self.nmels,\n                                                        f_min=self.fmin, f_max=self.fmax,\n                                                        n_fft=2048)\n        self.transform = Compose([\n            OneOf([\n                GaussianNoiseSNR(min_snr=10),\n                PinkNoiseSNR(min_snr=10)\n            ]),\n            TimeShift(sr=self.sr),\n            VolumeControl(p=0.5)\n        ])\n        self.img_transform = A.Compose([\n            A.OneOf([\n                A.Cutout(max_h_size=5, max_w_size=20),\n                A.CoarseDropout(max_holes=4),\n                A.RandomBrightness(p=0.25),\n            ], p=0.5)])\n        self.num_splits = self.config.total_duration//self.duration\n        assert self.config.total_duration == self.duration * \\\n            self.num_splits, \"not a multiple\"\n\n    def __len__(self):\n        return len(self.tp)\n\n    def __getitem__(self, idx):\n        labels = np.zeros((self.num_classes,), dtype=np.float32)\n\n        recording_id = self.tp.loc[idx, 'recording_id']\n        df = self.tp.loc[self.tp.recording_id == recording_id]\n        maybe_labels = df.species_id.unique()\n        np.put(labels, maybe_labels, 0.2)\n\n        df = df.sample(weights=df.species_id.apply(\n            lambda x: self.inv_counts[x]))\n        fn = osp.join(self.data_root, f\"{recording_id}.flac\")\n        df = df.squeeze()\n        t0 = max(df['t_min'], 0)\n        t1 = max(df['t_max'], 0)\n        t0 = np.random.uniform(t0, t1)\n        t0 = max(t0, 0)\n        t0 = min(t0, self.total_duration-self.duration)\n        t1 = t0 + self.duration\n        valid_df = self.tp[self.tp.recording_id == recording_id]\n        valid_df = valid_df[(valid_df.t_min < t1) & (valid_df.t_max > t0)]\n        y, _ = librosa.load(fn, sr=None, offset=t0,\n                            duration=self.duration)\n        if len(valid_df):\n            np.put(labels, valid_df.species_id.unique(), 1)\n        np.put(labels, df.species_id, 1)\n\n        if random.random()<0.5:\n            end_idx = int((valid_df.t_max.max() - t0)*self.sr)\n            rem_len = max(0, len(y) - end_idx)\n            idx = np.random.randint(0, len(self.fp))\n            \n            fn = osp.join(self.fp_root, self.fp.ebird_code[idx],self.fp.filename[idx])\n            fn = fn.replace('mp3', 'wav')\n            y_other, _ = librosa.load(fn, sr=self.sr,\n                                    duration=None, mono=True,\n                                    res_type='kaiser_fast')\n            aug_len = min(len(y_other), rem_len)\n            y[end_idx:end_idx+aug_len] = y_other[:aug_len]\n\n        y = self.resampler(torch.from_numpy(y).float()).numpy()\n        # do augmentation\n        y = self.transform(y)\n        if random.random() < 0.25:\n            tempo, beats = librosa.beat.beat_track(y=y, sr=self.sr)\n            y = librosa.clicks(frames=beats, sr=self.sr, length=len(y))\n\n        melspec = librosa.feature.melspectrogram(\n            y, sr=self.sr, n_mels=self.nmels, fmin=self.fmin, fmax=self.fmax,\n        )\n        melspec = librosa.power_to_db(melspec)\n        melspec = mono_to_color(melspec)\n        melspec = normalize(melspec, mean=None, std=None)\n        melspec = self.img_transform(image=melspec)['image']\n        melspec = np.moveaxis(melspec, 2, 0)\n        return melspec, labels","metadata":{"papermill":{"duration":0.051034,"end_time":"2021-02-18T08:19:16.117954","exception":false,"start_time":"2021-02-18T08:19:16.06692","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RFCTestDataset:\n    def __init__(self, tp, fp=None, config=None,\n                 mode='test'):\n        self.tp = tp\n        self.fp = fp\n        self.config = config\n        self.sr = self.config.sr\n        self.duration = self.config.duration\n        if mode == 'val':\n            self.data_root = self.config.TRAIN_AUDIO_ROOT\n        else:\n            self.data_root = self.config.TEST_AUDIO_ROOT\n\n        self.nmels = self.config.nmels\n        self.fmin, self.fmax = 84, self.sr//2\n        self.mode = mode\n        self.resampler = torchaudio.transforms.Resample(\n            orig_freq=48_000, new_freq=self.sr)\n        self.num_classes = self.config.num_classes\n        self.num_splits = self.config.total_duration//self.duration\n        assert self.config.total_duration == self.duration * \\\n            self.num_splits, \"not a multiple\"\n\n    def __len__(self):\n        return len(self.tp.recording_id.unique())\n\n    def __getitem__(self, idx):\n        recording_id = self.tp.loc[idx, 'recording_id']\n        df = self.tp.loc[self.tp.recording_id == recording_id]\n        if self.mode == 'val':\n            fn = f\"{self.config.EXTRAS_DIR}/train_melspec32k_10s/train_melspec32k_10s/{recording_id}.npy\"\n        else:\n            fn = f\"{self.config.EXTRAS_DIR}/test_melspec32k_10s/test_melspec32k_10s/{recording_id}.npy\"\n        try:\n            melspec_stacked = np.load(fn)\n        except:\n            audio_fn = osp.join(self.data_root, f\"{recording_id}.flac\")\n            y, _ = librosa.load(audio_fn, sr=None,\n                                duration=self.config.total_duration)\n            # split into n arrays\n            y_stacked = np.stack(np.split(y, self.num_splits), 0)\n            melspec_stacked = []\n            for y in y_stacked:\n                y = self.resampler(torch.from_numpy(y).float()).numpy()\n                melspec = librosa.feature.melspectrogram(\n                    y, sr=self.sr, n_mels=self.nmels, fmin=self.fmin, fmax=self.fmax,\n                )\n                melspec = librosa.power_to_db(melspec)\n                melspec = mono_to_color(melspec)\n                melspec = normalize(melspec, mean=None, std=None)\n                melspec = np.moveaxis(melspec, 2, 0)\n                melspec_stacked.append(melspec)\n\n            melspec_stacked = np.stack(melspec_stacked)\n            np.save(fn, melspec_stacked)\n\n        if self.mode == 'val':\n            species = df.loc[:, 'species_id'].unique()\n            labels = np.zeros((self.num_classes,))\n            np.put(labels, species, 1)\n\n            return melspec_stacked, labels\n        else:\n            melspec_stacked = np.load(fn)\n            return melspec_stacked","metadata":{"papermill":{"duration":0.03812,"end_time":"2021-02-18T08:19:16.17142","exception":false,"start_time":"2021-02-18T08:19:16.1333","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# resnest 50 trained on cornell \n# https://www.kaggle.com/theoviel/birds-cp-1\nMODEL_CONFIGS = {\n    \"resnest50_fast_1s1x64d\":\n    {\n        \"num_classes\": 264,\n        \"block\": Bottleneck,\n        \"layers\": [3, 4, 6, 3],\n        \"radix\": 1,\n        \"groups\": 1,\n        \"bottleneck_width\": 64,\n        \"deep_stem\": True,\n        \"stem_width\": 32,\n        \"avg_down\": True,\n        \"avd\": True,\n        \"avd_first\": True\n    }\n}\n\n\ndef get_model(pretrained=True, n_class=24):\n    # model = torchvision.models.resnext50_32x4d(pretrained=False)\n    # model = torchvision.models.resnext101_32x8d(pretrained=False)\n    model = ResNet(**MODEL_CONFIGS[\"resnest50_fast_1s1x64d\"])\n    n_features = model.fc.in_features\n    model.fc = nn.Linear(n_features, 264)\n    # model.load_state_dict(torch.load('resnext50_32x4d_extra_2.pt'))\n    # model.load_state_dict(torch.load('resnext101_32x8d_wsl_extra_4.pt'))\n    fn = '../input/birds-cp-1/resnest50_fast_1s1x64d_conf_1.pt'\n    model.load_state_dict(torch.load(fn, map_location='cpu'))\n    model.fc = nn.Linear(n_features, n_class)\n    return model\n","metadata":{"papermill":{"duration":0.025266,"end_time":"2021-02-18T08:19:16.211841","exception":false,"start_time":"2021-02-18T08:19:16.186575","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def worker_init_fn(worker_id):                                                          \n    np.random.seed(np.random.get_state()[1][0] + worker_id)\n    \nclass BaseNet(LightningModule):\n    def __init__(self, config, train_recid, val_recid):\n        super().__init__()\n        self.config = config\n        self.batch_size = self.config.batch_size\n        self.num_workers = self.config.num_workers\n        self.lr = self.config.lr\n        self.epochs = self.config.epochs\n\n        self.weight_decay = self.config.weight_decay\n        # to improve species 3 recall rate\n        pos_weight = torch.ones((24,))\n        pos_weight[3] = 4\n        self.loss_fn = torch.nn.BCEWithLogitsLoss(pos_weight=pos_weight)\n        self.sr = self.config.sr\n        self.train_recid = train_recid\n        self.val_recid = val_recid\n\n    def train_dataloader(self):\n        tp = train_tp[train_tp.recording_id.isin(\n            self.train_recid)].reset_index(drop=True)\n        self.train_recid = tp.recording_id.unique()\n        inv_counts = dict(1/tp.species_id.value_counts())\n        weights = tp.species_id.apply(lambda x: inv_counts[x])\n        tp_aug = new_labels[new_labels.recording_id.isin(tp.recording_id)]\n        tp = pd.concat([tp, tp_aug], ignore_index=True)\n        train_dataset = RFCDataset(tp, train_fp,\n                                   config=self.config,\n                                   mode='train',\n                                   inv_counts=inv_counts)\n        train_sampler = WeightedRandomSampler(weights, num_samples=len(train_dataset),\n                                              replacement=True)\n\n        train_loader = DataLoader(train_dataset, batch_size=self.batch_size,\n                                  num_workers=self.num_workers,\n                                  sampler=train_sampler,\n                                  worker_init_fn=worker_init_fn,\n                                  drop_last=True,\n                                  pin_memory=True)\n        return train_loader\n\n    def val_dataloader(self):\n        val_tp = train_tp[train_tp.recording_id.isin(\n            self.val_recid)].reset_index(drop=True)\n        val_recid = val_tp.recording_id.unique()\n        overlap = set(val_recid).intersection(set(self.train_recid))\n#         print('overlapped ids', overlap)\n        val_tp = val_tp[~val_tp.recording_id.isin(overlap)]\n        val_tp_aug = new_labels[new_labels.recording_id.isin(\n            val_tp.recording_id)]\n        val_tp = pd.concat([val_tp, val_tp_aug], ignore_index=True)\n        val_dataset = RFCTestDataset(val_tp, train_fp,\n                                     config=self.config,\n                                     mode='val')\n        val_loader = DataLoader(val_dataset, batch_size=self.batch_size,\n                                num_workers=self.num_workers, shuffle=False,\n                                pin_memory=True)\n        return val_loader\n\n    def configure_optimizers(self):\n        optim = torch.optim.AdamW(self.parameters(), lr=self.config.lr,\n                                  weight_decay=self.config.weight_decay)\n        scheduler = {\n            'scheduler': torch.optim.lr_scheduler.ReduceLROnPlateau(optim,\n                                                                    mode='min',\n                                                                    factor=0.5,\n                                                                    patience=2,\n                                                                    verbose=True),\n            'monitor': 'val_loss',\n            'interval': 'epoch',\n            'frequency': 1,\n            'strict': True,\n        }\n\n        self.optimizer = optim\n        self.scheduler = scheduler\n\n        return [optim], [scheduler]\n\n","metadata":{"papermill":{"duration":0.037803,"end_time":"2021-02-18T08:19:16.264724","exception":false,"start_time":"2021-02-18T08:19:16.226921","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RFCNet(BaseNet):\n    def __init__(self, **kwargs):\n        super().__init__(**kwargs)\n        n_class = self.config.num_classes\n        self.model = get_model(\n            pretrained=True, n_class=n_class)\n        self.cnf_matrix = np.zeros((n_class, n_class))\n\n    def forward(self, x):\n        return self.model(x)\n\n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        preds = self(x)\n        loss = self.loss_fn(preds, y)\n        with torch.no_grad():\n            lwlrap = LWLRAP(preds, y)\n        metrics = {\"train_loss\": loss.item(), \"train_lwlrap\": lwlrap}\n        self.log_dict(metrics,\n                      on_epoch=True, on_step=True)\n\n        return loss\n\n    @torch.no_grad()\n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        for i, x_partial in enumerate(torch.split(x, 1, dim=1)):\n            x_partial = x_partial.squeeze(1)\n            if i == 0:\n                preds = self(x_partial)\n            else:\n                # take max over predictions\n                preds = torch.max(preds, self(x_partial))\n        val_loss = self.loss_fn(preds, y).item()\n        val_lwlrap = LWLRAP(preds, y)\n        # loss is tensor. The Checkpoint Callback is monitoring 'checkpoint_on'\n        metrics = {\"val_loss\": val_loss, \"val_lwlrap\": val_lwlrap}\n        self.log_dict(metrics, prog_bar=True,\n                      on_epoch=True, on_step=True)","metadata":{"papermill":{"duration":0.029993,"end_time":"2021-02-18T08:19:16.310021","exception":false,"start_time":"2021-02-18T08:19:16.280028","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Average model weights","metadata":{}},{"cell_type":"code","source":"def average_model(paths):\n    weights = np.ones((len(paths),))\n    weights = weights/weights.sum()\n    for i, p in enumerate(paths):\n        m = torch.load(p)['state_dict']\n        if i == 0:\n            averaged_w = OrderedDict()\n            for k in m.keys():\n                if 'pos' in k: continue\n                # remove pl prefix in state dict\n                knew = k.replace('model.', '')\n                averaged_w[knew] = weights[i]*m[k]\n        else:\n            for k in m.keys():\n                if 'pos' in k: continue\n                knew = k.replace('model.', '')\n                averaged_w[knew] = averaged_w[knew] + weights[i]*m[k]\n    return averaged_w","metadata":{"papermill":{"duration":0.026478,"end_time":"2021-02-18T08:19:16.351879","exception":false,"start_time":"2021-02-18T08:19:16.325401","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model training","metadata":{}},{"cell_type":"code","source":"config = Config()\ntrain_tp = pd.read_csv(osp.join(config.ROOT, 'train_tp.csv'))\n\nfold_df = pd.read_csv(\n    osp.join(config.EXTRAS_DIR, 'preprocessed_rainforest_dataset.csv'))\nfn = \"../input/extra-labels-for-rcfx-competition-data/extra_labels_v71.csv\"\nprint(fn)\nnew_labels = pd.read_csv(fn)\nnew_labels['t_diff'] = new_labels['t_max'] - new_labels['t_min']\nidx = np.where(new_labels['t_diff'] < 0)[0]\nnew_labels = new_labels.drop(idx, axis=0).reset_index(drop=True)\nnum_folds = len(fold_df.fold.unique())\ntrain_fp = pd.read_csv(osp.join(config.ROOT, 'train_fp.csv'))\nfor fold in range(num_folds):\n    print('\\n\\nTraining fold', fold)\n    print('*' * 40)\n\n    train_recid = fold_df[fold_df.fold != fold].recording_id\n    val_recid = fold_df[fold_df.fold == fold].recording_id\n    model = RFCNet(config=config, train_recid=train_recid,\n                   val_recid=val_recid)\n    checkpoint_callback = ModelCheckpoint(\n        monitor='val_lwlrap_epoch',\n        filename='{epoch:02d}-{val_loss_epoch:.2f}-{val_lwlrap_epoch:.2}',\n        mode='max',\n        save_top_k=5,\n        save_weights_only=True,\n    )\n    early_stopping = EarlyStopping(monitor='val_lwlrap_epoch', mode='max', patience=5,\n                                   verbose=True)\n    trainer = Trainer(gpus=1,\n                      max_epochs=config.epochs,\n                      progress_bar_refresh_rate=1,                      \n                      #   gradient_clip_val=2,\n                      accumulate_grad_batches=4,\n                      num_sanity_val_steps=0,\n                      callbacks=[checkpoint_callback, early_stopping])\n\n    trainer.fit(model)","metadata":{"papermill":{"duration":12702.148966,"end_time":"2021-02-18T11:50:58.515949","exception":false,"start_time":"2021-02-18T08:19:16.366983","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Validation","metadata":{}},{"cell_type":"code","source":"def get_one_hot(targets, nb_classes=24):\n    res = np.eye(nb_classes)[np.array(targets).reshape(-1)]\n    return res.reshape(list(targets.shape)+[nb_classes])\n\n\nsub = pd.read_csv(osp.join(config.ROOT, 'sample_submission.csv'))\nspecies_cols = list(sub.columns)\nspecies_cols.remove('recording_id')\n\ncv_preds = pd.DataFrame(columns=species_cols)\ncv_preds['recording_id'] = train_tp['recording_id'].drop_duplicates()\ncv_preds = cv_preds.set_index('recording_id')\n\nlabel_df = pd.DataFrame(columns=species_cols)\nlabel_df['recording_id'] = train_tp['recording_id'].drop_duplicates()\nlabel_df = label_df.set_index('recording_id')\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = get_model(pretrained=False)\nmodel.to(device)\nfor fold in range(num_folds):\n    paths = glob(f\"./lightning_logs/version_{fold}/checkpoints/*.ckpt\")\n    print(paths)\n    averaged_w = average_model(paths)\n    model.load_state_dict(averaged_w)\n    model.eval()\n    train_recid = fold_df[fold_df.fold!=fold].recording_id\n    val_recid = fold_df[fold_df.fold==fold].recording_id\n\n    val_tp = train_tp[train_tp.recording_id.isin(val_recid)].reset_index(drop=True)\n    val_recid = val_tp.recording_id.unique()\n    overlap = set(val_recid).intersection(set(train_recid))\n    val_tp = val_tp[~val_tp.recording_id.isin(overlap)]\n    val_tp_aug = new_labels[new_labels.recording_id.isin(val_tp.recording_id)]\n    val_tp = pd.concat([val_tp, val_tp_aug], ignore_index=True)\n\n    dataset = RFCTestDataset(val_tp, config=config, mode='val')\n    test_loader = DataLoader(dataset, batch_size=config.batch_size,\n                             num_workers=config.num_workers,\n                             shuffle=False, drop_last=False)\n    tk = test_loader\n    with torch.no_grad():\n        fold_preds, labels = [], []\n        for i, (im, l) in enumerate(tk):\n            # continue\n            im = im.to(device)\n            for j, x_partial in enumerate(torch.split(im, 1, dim=1)):\n                x_partial = x_partial.squeeze(1)\n                if j == 0:\n                    preds = model(x_partial)\n                else:\n                    preds = torch.max(preds, model(x_partial))\n\n\n            o = preds.sigmoid().cpu().numpy()\n            # o = preds.cpu().numpy()\n            fold_preds.extend(o)\n            labels.extend(l.cpu().numpy())\n        # continue\n        p = torch.from_numpy(np.array(fold_preds)) \n        t = torch.from_numpy(np.array(labels))\n        print(f\"lwlrap: {LWLRAP(p, t):.6}\")\n        cv_preds.loc[val_recid, species_cols] = fold_preds\n        label_df.loc[val_recid, species_cols] = labels\n\n# print(cv_preds.head())\ncv_preds.to_csv('cv_preds.csv')\n\nrecid = train_tp['recording_id'].values\ncv_preds = cv_preds.loc[recid].values.astype(np.float32)\ncv_preds = torch.from_numpy(cv_preds)\n\nlabels = label_df.loc[recid].values.astype(np.float32)\nlabels = torch.from_numpy(labels)\n\nprint(f\"lwlrap: {LWLRAP(cv_preds, labels):.6}\")\n","metadata":{"execution":{"iopub.execute_input":"2021-02-18T11:50:58.635919Z","iopub.status.busy":"2021-02-18T11:50:58.634752Z","iopub.status.idle":"2021-02-18T11:52:04.059236Z","shell.execute_reply":"2021-02-18T11:52:04.059999Z"},"papermill":{"duration":65.499569,"end_time":"2021-02-18T11:52:04.060246","exception":false,"start_time":"2021-02-18T11:50:58.560677","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test predictions","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv(osp.join(config.ROOT, 'sample_submission.csv'))\nspecies_cols = list(sub.columns)\nspecies_cols.remove('recording_id')\n# initialize to zero.\nsub.loc[:, species_cols] = 0\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = get_model(pretrained=False)\nmodel.to(device)\nfor fold in range(num_folds):\n    paths = glob(f\"./lightning_logs/version_{fold}/checkpoints/*.ckpt\")\n    print(paths)\n    averaged_w = average_model(paths)\n    model.load_state_dict(averaged_w)\n    model.eval()\n    dataset = RFCTestDataset(sub, config=config, mode='test')\n    test_loader = DataLoader(dataset, batch_size=config.batch_size,\n                             num_workers=4,\n                             shuffle=False, drop_last=False)\n    tk = tqdm(test_loader, total=len(test_loader))\n    sub_index = 0\n    with torch.no_grad():\n        for i, im in enumerate(tk):\n            im = im.to(device)\n            for i, x_partial in enumerate(torch.split(im, 1, dim=1)):\n                x_partial = x_partial.squeeze(1)\n                if i == 0:\n                    preds = model(x_partial)\n                else:\n                    # take max over predictions\n                    preds = torch.max(preds, model(x_partial))\n\n            o = preds.sigmoid().cpu().numpy()\n            # o = preds.cpu().numpy()\n            for val in o:\n                sub.loc[sub_index, species_cols] += val\n                sub_index += 1\n\n# # take average of predictions\nsub.loc[:, species_cols] /= num_folds\nsub.to_csv('submission.csv', index=False)\nprint(sub.head())\nprint(sub.max(1).head())\n","metadata":{"execution":{"iopub.execute_input":"2021-02-18T11:52:04.251738Z","iopub.status.busy":"2021-02-18T11:52:04.241814Z","iopub.status.idle":"2021-02-18T11:58:30.214808Z","shell.execute_reply":"2021-02-18T11:58:30.21422Z"},"papermill":{"duration":386.070309,"end_time":"2021-02-18T11:58:30.215028","exception":false,"start_time":"2021-02-18T11:52:04.144719","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.iloc[:, 1:].describe()","metadata":{"papermill":{"duration":0.045327,"end_time":"2021-02-18T11:58:30.585836","exception":false,"start_time":"2021-02-18T11:58:30.540509","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}