{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":21669,"databundleVersionId":1692278,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install resnest > /dev/null\n!pip install colorednoise > /dev/null","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"import albumentations as A\nfrom resnest.torch.resnet import ResNet, Bottleneck\nimport random\nfrom glob import glob\nfrom collections import OrderedDict\nimport os.path as osp\nimport os\nfrom pytorch_lightning.loggers import TensorBoardLogger\nfrom pytorch_lightning.callbacks.early_stopping import EarlyStopping\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nfrom pytorch_lightning import LightningModule\nfrom pytorch_lightning import Trainer\nfrom skimage.transform import resize\nfrom torchvision.models import resnet18, resnet34, resnet50\nfrom resnest.torch import resnest50\nfrom tqdm.auto import tqdm\nimport colorednoise as cn\nimport librosa\nimport torchaudio\nimport torch.nn.functional as F\nfrom torch.utils.data import WeightedRandomSampler\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision\nimport torch\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score, confusion_matrix\nimport matplotlib\nmatplotlib.use('Agg')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"def seed_everything(seed=42):\n    print(f'setting everything to seed {seed}')\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.cuda.empty_cache()\n    \nseed_everything(42)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/c/rfcx-species-audio-detection/discussion/198418\n# label-level average\n# Assume float preds [BxC], labels [BxC] of 0 or 1\ndef LWLRAP(preds, labels):\n    # Ranks of the predictions\n    ranked_classes = torch.argsort(preds, dim=-1, descending=True)\n    # i, j corresponds to rank of prediction in row i\n    class_ranks = torch.zeros_like(ranked_classes).to(preds.device)\n    for i in range(ranked_classes.size(0)):\n        for j in range(ranked_classes.size(1)):\n            class_ranks[i, ranked_classes[i][j]] = j + 1\n    # Mask out to only use the ranks of relevant GT labels\n    ground_truth_ranks = class_ranks * labels + (1e6) * (1 - labels)\n    # All the GT ranks are in front now\n    sorted_ground_truth_ranks, _ = torch.sort(\n        ground_truth_ranks, dim=-1, descending=False)\n    # Number of GT labels per instance\n    num_labels = labels.sum(-1)\n    pos_matrix = torch.tensor(\n        np.array([i+1 for i in range(labels.size(-1))])).unsqueeze(0).to(preds.device)\n    score_matrix = pos_matrix / sorted_ground_truth_ranks\n    score_mask_matrix, _ = torch.sort(labels, dim=-1, descending=True)\n    scores = score_matrix * score_mask_matrix\n    score = scores.sum() / labels.sum()\n    return score.item()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    batch_size = 8\n    weight_decay = 1e-8\n    lr = 1e-3\n    num_workers = 4\n    epochs = 5\n    num_classes = 24\n    sr = 32_000\n    duration = 5\n    total_duration = 60\n    nmels = 128\n    EXTRAS_DIR = \"../input/rfcxextras\"\n    ROOT = \"../input/rfcx-species-audio-detection\"\n    TRAIN_AUDIO_ROOT = osp.join(ROOT, \"train\")\n    TEST_AUDIO_ROOT = osp.join(ROOT, \"test\")\n    loss_fn = torch.nn.BCEWithLogitsLoss()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Mostly taken from https://www.kaggle.com/hidehisaarai1213/rfcx-audio-data-augmentation-japanese-english\nclass AudioTransform:\n    def __init__(self, always_apply=False, p=0.5):\n        self.always_apply = always_apply\n        self.p = p\n\n    def __call__(self, y: np.ndarray):\n        if self.always_apply:\n            return self.apply(y)\n        else:\n            if np.random.rand() < self.p:\n                return self.apply(y)\n            else:\n                return y\n\n    def apply(self, y: np.ndarray):\n        raise NotImplementedError\n\n\nclass Compose:\n    def __init__(self, transforms: list):\n        self.transforms = transforms\n\n    def __call__(self, y: np.ndarray):\n        for trns in self.transforms:\n            y = trns(y)\n        return y\n\n\nclass OneOf:\n    def __init__(self, transforms: list):\n        self.transforms = transforms\n\n    def __call__(self, y: np.ndarray):\n        n_trns = len(self.transforms)\n        trns_idx = np.random.choice(n_trns)\n        trns = self.transforms[trns_idx]\n        return trns(y)\n\n\nclass GaussianNoiseSNR(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, min_snr=5.0, max_snr=20.0, **kwargs):\n        super().__init__(always_apply, p)\n\n        self.min_snr = min_snr\n        self.max_snr = max_snr\n\n    def apply(self, y: np.ndarray, **params):\n        snr = np.random.uniform(self.min_snr, self.max_snr)\n        a_signal = np.sqrt(y ** 2).max()\n        a_noise = a_signal / (10 ** (snr / 20))\n\n        white_noise = np.random.randn(len(y))\n        a_white = np.sqrt(white_noise ** 2).max()\n        augmented = (y + white_noise * 1 / a_white * a_noise).astype(y.dtype)\n        return augmented\n\n\nclass PinkNoiseSNR(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, min_snr=5.0, max_snr=20.0, **kwargs):\n        super().__init__(always_apply, p)\n\n        self.min_snr = min_snr\n        self.max_snr = max_snr\n\n    def apply(self, y: np.ndarray, **params):\n        snr = np.random.uniform(self.min_snr, self.max_snr)\n        a_signal = np.sqrt(y ** 2).max()\n        a_noise = a_signal / (10 ** (snr / 20))\n\n        pink_noise = cn.powerlaw_psd_gaussian(1, len(y))\n        a_pink = np.sqrt(pink_noise ** 2).max()\n        augmented = (y + pink_noise * 1 / a_pink * a_noise).astype(y.dtype)\n        return augmented\n\n\nclass TimeShift(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, max_shift_second=2, sr=32000, padding_mode=\"zero\"):\n        super().__init__(always_apply, p)\n\n        assert padding_mode in [\n            \"replace\", \"zero\"], \"`padding_mode` must be either 'replace' or 'zero'\"\n        self.max_shift_second = max_shift_second\n        self.sr = sr\n        self.padding_mode = padding_mode\n\n    def apply(self, y: np.ndarray, **params):\n        shift = np.random.randint(-self.sr * self.max_shift_second,\n                                  self.sr * self.max_shift_second)\n        augmented = np.roll(y, shift)\n        # if self.padding_mode == \"zero\":\n        #     if shift > 0:\n        #         augmented[:shift] = 0\n        #     else:\n        #         augmented[shift:] = 0\n        return augmented\n\n\nclass VolumeControl(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, db_limit=10, mode=\"uniform\"):\n        super().__init__(always_apply, p)\n\n        assert mode in [\"uniform\", \"fade\", \"fade\", \"cosine\", \"sine\"], \\\n            \"`mode` must be one of 'uniform', 'fade', 'cosine', 'sine'\"\n\n        self.db_limit = db_limit\n        self.mode = mode\n\n    def apply(self, y: np.ndarray, **params):\n        db = np.random.uniform(-self.db_limit, self.db_limit)\n        if self.mode == \"uniform\":\n            db_translated = 10 ** (db / 20)\n        elif self.mode == \"fade\":\n            lin = np.arange(len(y))[::-1] / (len(y) - 1)\n            db_translated = 10 ** (db * lin / 20)\n        elif self.mode == \"cosine\":\n            cosine = np.cos(np.arange(len(y)) / len(y) * np.pi * 2)\n            db_translated = 10 ** (db * cosine / 20)\n        else:\n            sine = np.sin(np.arange(len(y)) / len(y) * np.pi * 2)\n            db_translated = 10 ** (db * sine / 20)\n        augmented = y * db_translated\n        return augmented","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mono_to_color(X, eps=1e-6, mean=None, std=None):\n    X = np.stack([X, X, X], axis=-1)\n\n    # Standardize\n    mean = mean or X.mean()\n    std = std or X.std()\n    X = (X - mean) / (std + eps)\n\n    # Normalize to [0, 255]\n    _min, _max = X.min(), X.max()\n\n    if (_max - _min) > eps:\n        V = np.clip(X, _min, _max)\n        V = 255 * (V - _min) / (_max - _min)\n        V = V.astype(np.uint8)\n    else:\n        V = np.zeros_like(X, dtype=np.uint8)\n\n    return V\n\n\ndef normalize(image, mean=None, std=None):\n    image = image / 255.0\n    if mean is not None and std is not None:\n        image = (image - mean) / std\n    return image.astype(np.float32)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RFCDataset:\n    def __init__(self, tp, fp=None, config=None,\n                 mode='train', inv_counts=None):\n        self.tp = tp\n        self.fp = pd.read_csv(\"../input/rfcxextras/cornell-train.csv\")\n        self.fp = self.fp[self.fp.ebird_code<'c'].reset_index(drop=True)\n        self.fp_root = \"../input/birdsong-resampled-train-audio-00/\"        \n        self.inv_counts = inv_counts\n        self.config = config\n        self.sr = self.config.sr\n        self.total_duration = self.config.total_duration\n        self.duration = self.config.duration\n        self.data_root = self.config.TRAIN_AUDIO_ROOT\n        self.nmels = self.config.nmels\n        self.fmin, self.fmax = 84, self.sr//2\n        self.mode = mode\n        self.num_classes = self.config.num_classes\n        self.resampler = torchaudio.transforms.Resample(\n            orig_freq=48_000, new_freq=self.sr)\n        self.mel = torchaudio.transforms.MelSpectrogram(sample_rate=self.sr, n_mels=self.nmels,\n                                                        f_min=self.fmin, f_max=self.fmax,\n                                                        n_fft=2048)\n        self.transform = Compose([\n            OneOf([\n                GaussianNoiseSNR(min_snr=10),\n                PinkNoiseSNR(min_snr=10)\n            ]),\n            TimeShift(sr=self.sr),\n            VolumeControl(p=0.5)\n        ])\n        self.img_transform = A.Compose([\n            A.OneOf([\n                A.Cutout(max_h_size=5, max_w_size=20),\n                A.CoarseDropout(max_holes=4),\n                A.RandomBrightness(p=0.25),\n            ], p=0.5)])\n        self.num_splits = self.config.total_duration//self.duration\n        assert self.config.total_duration == self.duration * \\\n            self.num_splits, \"not a multiple\"\n\n    def __len__(self):\n        return len(self.tp)\n\n    def __getitem__(self, idx):\n        labels = np.zeros((self.num_classes,), dtype=np.float32)\n\n        recording_id = self.tp.loc[idx, 'recording_id']\n        df = self.tp.loc[self.tp.recording_id == recording_id]\n        maybe_labels = df.species_id.unique()\n        np.put(labels, maybe_labels, 0.2)\n\n        df = df.sample(weights=df.species_id.apply(\n            lambda x: self.inv_counts[x]))\n        fn = osp.join(self.data_root, f\"{recording_id}.flac\")\n        df = df.squeeze()\n        t0 = max(df['t_min'], 0)\n        t1 = max(df['t_max'], 0)\n        t0 = np.random.uniform(t0, t1)\n        t0 = max(t0, 0)\n        t0 = min(t0, self.total_duration-self.duration)\n        t1 = t0 + self.duration\n        valid_df = self.tp[self.tp.recording_id == recording_id]\n        valid_df = valid_df[(valid_df.t_min < t1) & (valid_df.t_max > t0)]\n        y, _ = librosa.load(fn, sr=None, offset=t0,\n                            duration=self.duration)\n        if len(valid_df):\n            np.put(labels, valid_df.species_id.unique(), 1)\n        np.put(labels, df.species_id, 1)\n\n        if random.random()<0.5:\n            end_idx = int((valid_df.t_max.max() - t0)*self.sr)\n            rem_len = max(0, len(y) - end_idx)\n            idx = np.random.randint(0, len(self.fp))\n            \n            fn = osp.join(self.fp_root, self.fp.ebird_code[idx],self.fp.filename[idx])\n            fn = fn.replace('mp3', 'wav')\n            y_other, _ = librosa.load(fn, sr=self.sr,\n                                    duration=None, mono=True,\n                                    res_type='kaiser_fast')\n            aug_len = min(len(y_other), rem_len)\n            y[end_idx:end_idx+aug_len] = y_other[:aug_len]\n\n        y = self.resampler(torch.from_numpy(y).float()).numpy()\n        # do augmentation\n        y = self.transform(y)\n        if random.random() < 0.25:\n            tempo, beats = librosa.beat.beat_track(y=y, sr=self.sr)\n            y = librosa.clicks(frames=beats, sr=self.sr, length=len(y))\n\n        melspec = librosa.feature.melspectrogram(\n            y, sr=self.sr, n_mels=self.nmels, fmin=self.fmin, fmax=self.fmax,\n        )\n        melspec = librosa.power_to_db(melspec)\n        melspec = mono_to_color(melspec)\n        melspec = normalize(melspec, mean=None, std=None)\n        melspec = self.img_transform(image=melspec)['image']\n        melspec = np.moveaxis(melspec, 2, 0)\n        return melspec, labels","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RFCTestDataset:\n    def __init__(self, tp, fp=None, config=None,\n                 mode='test'):\n        self.tp = tp\n        self.fp = fp\n        self.config = config\n        self.sr = self.config.sr\n        self.duration = self.config.duration\n        if mode == 'val':\n            self.data_root = self.config.TRAIN_AUDIO_ROOT\n        else:\n            self.data_root = self.config.TEST_AUDIO_ROOT\n\n        self.nmels = self.config.nmels\n        self.fmin, self.fmax = 84, self.sr//2\n        self.mode = mode\n        self.resampler = torchaudio.transforms.Resample(\n            orig_freq=48_000, new_freq=self.sr)\n        self.num_classes = self.config.num_classes\n        self.num_splits = self.config.total_duration//self.duration\n        assert self.config.total_duration == self.duration * \\\n            self.num_splits, \"not a multiple\"\n\n    def __len__(self):\n        return len(self.tp.recording_id.unique())\n\n    def __getitem__(self, idx):\n        recording_id = self.tp.loc[idx, 'recording_id']\n        df = self.tp.loc[self.tp.recording_id == recording_id]\n        if self.mode == 'val':\n            fn = f\"{self.config.EXTRAS_DIR}/train_melspec32k_10s/train_melspec32k_10s/{recording_id}.npy\"\n        else:\n            fn = f\"{self.config.EXTRAS_DIR}/test_melspec32k_10s/test_melspec32k_10s/{recording_id}.npy\"\n        try:\n            melspec_stacked = np.load(fn)\n        except:\n            audio_fn = osp.join(self.data_root, f\"{recording_id}.flac\")\n            y, _ = librosa.load(audio_fn, sr=None,\n                                duration=self.config.total_duration)\n            # split into n arrays\n            y_stacked = np.stack(np.split(y, self.num_splits), 0)\n            melspec_stacked = []\n            for y in y_stacked:\n                y = self.resampler(torch.from_numpy(y).float()).numpy()\n                melspec = librosa.feature.melspectrogram(\n                    y, sr=self.sr, n_mels=self.nmels, fmin=self.fmin, fmax=self.fmax,\n                )\n                melspec = librosa.power_to_db(melspec)\n                melspec = mono_to_color(melspec)\n                melspec = normalize(melspec, mean=None, std=None)\n                melspec = np.moveaxis(melspec, 2, 0)\n                melspec_stacked.append(melspec)\n\n            melspec_stacked = np.stack(melspec_stacked)\n            np.save(fn, melspec_stacked)\n\n        if self.mode == 'val':\n            species = df.loc[:, 'species_id'].unique()\n            labels = np.zeros((self.num_classes,))\n            np.put(labels, species, 1)\n\n            return melspec_stacked, labels\n        else:\n            melspec_stacked = np.load(fn)\n            return melspec_stacked","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# resnest 50 trained on cornell \n# https://www.kaggle.com/theoviel/birds-cp-1\nMODEL_CONFIGS = {\n    \"resnest50_fast_1s1x64d\":\n    {\n        \"num_classes\": 264,\n        \"block\": Bottleneck,\n        \"layers\": [3, 4, 6, 3],\n        \"radix\": 1,\n        \"groups\": 1,\n        \"bottleneck_width\": 64,\n        \"deep_stem\": True,\n        \"stem_width\": 32,\n        \"avg_down\": True,\n        \"avd\": True,\n        \"avd_first\": True\n    }\n}\n\n\ndef get_model(pretrained=True, n_class=24):\n    # model = torchvision.models.resnext50_32x4d(pretrained=False)\n    # model = torchvision.models.resnext101_32x8d(pretrained=False)\n    model = ResNet(**MODEL_CONFIGS[\"resnest50_fast_1s1x64d\"])\n    n_features = model.fc.in_features\n    model.fc = nn.Linear(n_features, 264)\n    # model.load_state_dict(torch.load('resnext50_32x4d_extra_2.pt'))\n    # model.load_state_dict(torch.load('resnext101_32x8d_wsl_extra_4.pt'))\n    fn = '../input/birds-cp-1/resnest50_fast_1s1x64d_conf_1.pt'\n    model.load_state_dict(torch.load(fn, map_location='cpu'))\n    model.fc = nn.Linear(n_features, n_class)\n    return model\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install efficientnet_pytorch","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet\nfrom torchvision.models import mobilenet_v2\nfrom torchvision.models import resnet50\n\ndef get_model(pretrained=True, n_class=24):\n    # Load pre-trained models\n    efficient_net = EfficientNet.from_pretrained('efficientnet-b0')  # Change model type as needed\n    mobilenet = mobilenet_v2(pretrained=True)\n    resnet = resnet50(pretrained=True)\n\n    # Modify classification heads\n    efficient_net._fc = nn.Linear(efficient_net._fc.in_features, n_class)\n    mobilenet.classifier[-1] = nn.Linear(mobilenet.classifier[-1].in_features, n_class)\n    resnet.fc = nn.Linear(resnet.fc.in_features, n_class)\n\n    # Create ensemble model\n    class EnsembleModel(nn.Module):\n        def __init__(self, efficient_net, mobilenet, resnet, n_class):\n            super(EnsembleModel, self).__init__()\n            self.efficient_net = efficient_net\n            self.mobilenet = mobilenet\n            self.resnet = resnet\n            self.output_layer = nn.Linear(3 * n_class, n_class)\n\n        def forward(self, x):\n            output_efficient = self.efficient_net(x)\n            output_mobilenet = self.mobilenet(x)\n            output_resnet = self.resnet(x)\n            # Combine outputs, for example, by concatenating\n            combined_output = torch.cat((output_efficient, output_mobilenet, output_resnet), dim=1)\n            output = self.output_layer(combined_output)\n            return output\n\n    # Create and return ensemble model\n    ensemble_model = EnsembleModel(efficient_net, mobilenet, resnet, n_class)\n    return ensemble_model\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def worker_init_fn(worker_id):                                                          \n    np.random.seed(np.random.get_state()[1][0] + worker_id)\n    \nclass BaseNet(LightningModule):\n    def __init__(self, config, train_recid, val_recid):\n        super().__init__()\n        self.config = config\n        self.batch_size = self.config.batch_size\n        self.num_workers = self.config.num_workers\n        self.lr = self.config.lr\n        self.epochs = self.config.epochs\n\n        self.weight_decay = self.config.weight_decay\n        # to improve species 3 recall rate\n        pos_weight = torch.ones((24,))\n        pos_weight[3] = 4\n        self.loss_fn = torch.nn.BCEWithLogitsLoss(pos_weight=pos_weight)\n        self.sr = self.config.sr\n        self.train_recid = train_recid\n        self.val_recid = val_recid\n\n    def train_dataloader(self):\n        tp = train_tp[train_tp.recording_id.isin(\n            self.train_recid)].reset_index(drop=True)\n        self.train_recid = tp.recording_id.unique()\n        inv_counts = dict(1/tp.species_id.value_counts())\n        weights = tp.species_id.apply(lambda x: inv_counts[x])\n        tp_aug = new_labels[new_labels.recording_id.isin(tp.recording_id)]\n        tp = pd.concat([tp, tp_aug], ignore_index=True)\n        train_dataset = RFCDataset(tp, train_fp,\n                                   config=self.config,\n                                   mode='train',\n                                   inv_counts=inv_counts)\n        train_sampler = WeightedRandomSampler(weights, num_samples=len(train_dataset),\n                                              replacement=True)\n\n        train_loader = DataLoader(train_dataset, batch_size=self.batch_size,\n                                  num_workers=self.num_workers,\n                                  sampler=train_sampler,\n                                  worker_init_fn=worker_init_fn,\n                                  drop_last=True,\n                                  pin_memory=True)\n        return train_loader\n\n    def val_dataloader(self):\n        val_tp = train_tp[train_tp.recording_id.isin(\n            self.val_recid)].reset_index(drop=True)\n        val_recid = val_tp.recording_id.unique()\n        overlap = set(val_recid).intersection(set(self.train_recid))\n#         print('overlapped ids', overlap)\n        val_tp = val_tp[~val_tp.recording_id.isin(overlap)]\n        val_tp_aug = new_labels[new_labels.recording_id.isin(\n            val_tp.recording_id)]\n        val_tp = pd.concat([val_tp, val_tp_aug], ignore_index=True)\n        val_dataset = RFCTestDataset(val_tp, train_fp,\n                                     config=self.config,\n                                     mode='val')\n        val_loader = DataLoader(val_dataset, batch_size=self.batch_size,\n                                num_workers=self.num_workers, shuffle=False,\n                                pin_memory=True)\n        return val_loader\n\n    def configure_optimizers(self):\n        optim = torch.optim.AdamW(self.parameters(), lr=self.config.lr,\n                                  weight_decay=self.config.weight_decay)\n        scheduler = {\n            'scheduler': torch.optim.lr_scheduler.ReduceLROnPlateau(optim,\n                                                                    mode='min',\n                                                                    factor=0.5,\n                                                                    patience=2,\n                                                                    verbose=True),\n            'monitor': 'val_loss',\n            'interval': 'epoch',\n            'frequency': 1,\n            'strict': True,\n        }\n\n        self.optimizer = optim\n        self.scheduler = scheduler\n\n        return [optim], [scheduler]\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RFCNet(BaseNet):\n    def __init__(self, **kwargs):\n        super().__init__(**kwargs)\n        n_class = self.config.num_classes\n        self.model = get_model(\n            pretrained=True, n_class=n_class)\n        self.cnf_matrix = np.zeros((n_class, n_class))\n\n    def forward(self, x):\n        return self.model(x)\n\n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        preds = self(x)\n        loss = self.loss_fn(preds, y)\n        with torch.no_grad():\n            lwlrap = LWLRAP(preds, y)\n        metrics = {\"train_loss\": loss.item(), \"train_lwlrap\": lwlrap}\n        self.log_dict(metrics,\n                      on_epoch=True, on_step=True)\n\n        return loss\n\n    @torch.no_grad()\n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        for i, x_partial in enumerate(torch.split(x, 1, dim=1)):\n            x_partial = x_partial.squeeze(1)\n            if i == 0:\n                preds = self(x_partial)\n            else:\n                # take max over predictions\n                preds = torch.max(preds, self(x_partial))\n        val_loss = self.loss_fn(preds, y).item()\n        val_lwlrap = LWLRAP(preds, y)\n        # loss is tensor. The Checkpoint Callback is monitoring 'checkpoint_on'\n        metrics = {\"val_loss\": val_loss, \"val_lwlrap\": val_lwlrap}\n        self.log_dict(metrics, prog_bar=True,\n                      on_epoch=True, on_step=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def average_model(paths):\n    weights = np.ones((len(paths),))\n    weights = weights/weights.sum()\n    for i, p in enumerate(paths):\n        m = torch.load(p)['state_dict']\n        if i == 0:\n            averaged_w = OrderedDict()\n            for k in m.keys():\n                if 'pos' in k: continue\n                # remove pl prefix in state dict\n                knew = k.replace('model.', '')\n                averaged_w[knew] = weights[i]*m[k]\n        else:\n            for k in m.keys():\n                if 'pos' in k: continue\n                knew = k.replace('model.', '')\n                averaged_w[knew] = averaged_w[knew] + weights[i]*m[k]\n    return averaged_w","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = Config()\ntrain_tp = pd.read_csv(osp.join(config.ROOT, 'train_tp.csv'))\n\nfold_df = pd.read_csv(\n    osp.join(config.EXTRAS_DIR, 'preprocessed_rainforest_dataset.csv'))\nfn = \"../input/extra-labels-for-rcfx-competition-data/extra_labels_v71.csv\"\nprint(fn)\nnew_labels = pd.read_csv(fn)\nnew_labels['t_diff'] = new_labels['t_max'] - new_labels['t_min']\nidx = np.where(new_labels['t_diff'] < 0)[0]\nnew_labels = new_labels.drop(idx, axis=0).reset_index(drop=True)\nnum_folds = len(fold_df.fold.unique())\ntrain_fp = pd.read_csv(osp.join(config.ROOT, 'train_fp.csv'))\nfor fold in range(num_folds):\n    print('\\n\\nTraining fold', fold)\n    print('*' * 40)\n\n    train_recid = fold_df[fold_df.fold != fold].recording_id\n    val_recid = fold_df[fold_df.fold == fold].recording_id\n    model = RFCNet(config=config, train_recid=train_recid,\n                   val_recid=val_recid)\n    checkpoint_callback = ModelCheckpoint(\n        monitor='val_lwlrap_epoch',\n        filename='{epoch:02d}-{val_loss_epoch:.2f}-{val_lwlrap_epoch:.2}',\n        mode='max',\n        save_top_k=5,\n        save_weights_only=True,\n    )\n    early_stopping = EarlyStopping(monitor='val_lwlrap_epoch', mode='max', patience=5,\n                                   verbose=True)\n    trainer = Trainer(gpus=1,\n                      max_epochs=config.epochs,\n                      progress_bar_refresh_rate=1,                      \n                      #   gradient_clip_val=2,\n                      accumulate_grad_batches=4,\n                      num_sanity_val_steps=0,\n                      callbacks=[checkpoint_callback, early_stopping])\n\n    # Fine-tune the model\n    for param in model.model.parameters():\n        param.requires_grad = True\n\n    # Define optimizer for fine-tuning\n    optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)  # Adjust learning rate if needed\n\n    trainer.fit(model)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_one_hot(targets, nb_classes=24):\n    res = np.eye(nb_classes)[np.array(targets).reshape(-1)]\n    return res.reshape(list(targets.shape)+[nb_classes])\n\n\nsub = pd.read_csv(osp.join(config.ROOT, 'sample_submission.csv'))\nspecies_cols = list(sub.columns)\nspecies_cols.remove('recording_id')\n\ncv_preds = pd.DataFrame(columns=species_cols)\ncv_preds['recording_id'] = train_tp['recording_id'].drop_duplicates()\ncv_preds = cv_preds.set_index('recording_id')\n\nlabel_df = pd.DataFrame(columns=species_cols)\nlabel_df['recording_id'] = train_tp['recording_id'].drop_duplicates()\nlabel_df = label_df.set_index('recording_id')\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = get_model(pretrained=False)\nmodel.to(device)\nfor fold in range(5):\n    paths = glob(f\"./lightning_logs/version_{fold}/checkpoints/*.ckpt\")\n    print(paths)\n    averaged_w = average_model(paths)\n    model.load_state_dict(averaged_w)\n    model.eval()\n    train_recid = fold_df[fold_df.fold!=fold].recording_id\n    val_recid = fold_df[fold_df.fold==fold].recording_id\n\n    val_tp = train_tp[train_tp.recording_id.isin(val_recid)].reset_index(drop=True)\n    val_recid = val_tp.recording_id.unique()\n    overlap = set(val_recid).intersection(set(train_recid))\n    val_tp = val_tp[~val_tp.recording_id.isin(overlap)]\n    val_tp_aug = new_labels[new_labels.recording_id.isin(val_tp.recording_id)]\n    val_tp = pd.concat([val_tp, val_tp_aug], ignore_index=True)\n\n    dataset = RFCTestDataset(val_tp, config=config, mode='val')\n    test_loader = DataLoader(dataset, batch_size=config.batch_size,\n                             num_workers=config.num_workers,\n                             shuffle=False, drop_last=False)\n    tk = test_loader\n    with torch.no_grad():\n        fold_preds, labels = [], []\n        for i, (im, l) in enumerate(tk):\n            # continue\n            im = im.to(device)\n            for j, x_partial in enumerate(torch.split(im, 1, dim=1)):\n                x_partial = x_partial.squeeze(1)\n                if j == 0:\n                    preds = model(x_partial)\n                else:\n                    preds = torch.max(preds, model(x_partial))\n\n\n            o = preds.sigmoid().cpu().numpy()\n            # o = preds.cpu().numpy()\n            fold_preds.extend(o)\n            labels.extend(l.cpu().numpy())\n        # continue\n        p = torch.from_numpy(np.array(fold_preds)) \n        t = torch.from_numpy(np.array(labels))\n        print(f\"lwlrap: {LWLRAP(p, t):.6}\")\n        cv_preds.loc[val_recid, species_cols] = fold_preds\n        label_df.loc[val_recid, species_cols] = labels\n\n# print(cv_preds.head())\ncv_preds.to_csv('cv_preds.csv')\n\nrecid = train_tp['recording_id'].values\ncv_preds = cv_preds.loc[recid].values.astype(np.float32)\ncv_preds = torch.from_numpy(cv_preds)\n\nlabels = label_df.loc[recid].values.astype(np.float32)\nlabels = torch.from_numpy(labels)\n\nprint(f\"lwlrap: {LWLRAP(cv_preds, labels):.6}\")\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv(osp.join(config.ROOT, 'sample_submission.csv'))\nspecies_cols = list(sub.columns)\nspecies_cols.remove('recording_id')\n# initialize to zero.\nsub.loc[:, species_cols] = 0\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = get_model(pretrained=False)\nmodel.to(device)\nfor fold in range(num_folds):\n    paths = glob(f\"./lightning_logs/version_{fold}/checkpoints/*.ckpt\")\n    print(paths)\n    averaged_w = average_model(paths)\n    model.load_state_dict(averaged_w)\n    model.eval()\n    dataset = RFCTestDataset(sub, config=config, mode='test')\n    test_loader = DataLoader(dataset, batch_size=config.batch_size,\n                             num_workers=4,\n                             shuffle=False, drop_last=False)\n    tk = tqdm(test_loader, total=len(test_loader))\n    sub_index = 0\n    with torch.no_grad():\n        for i, im in enumerate(tk):\n            im = im.to(device)\n            for i, x_partial in enumerate(torch.split(im, 1, dim=1)):\n                x_partial = x_partial.squeeze(1)\n                if i == 0:\n                    preds = model(x_partial)\n                else:\n                    # take max over predictions\n                    preds = torch.max(preds, model(x_partial))\n\n            o = preds.sigmoid().cpu().numpy()\n            # o = preds.cpu().numpy()\n            for val in o:\n                sub.loc[sub_index, species_cols] += val\n                sub_index += 1\n\n# # take average of predictions\nsub.loc[:, species_cols] /= num_folds\nsub.to_csv('submission.csv', index=False)\nprint(sub.head())\nprint(sub.max(1).head())\n","metadata":{},"execution_count":null,"outputs":[]}]}