{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":21669,"databundleVersionId":1692278,"sourceType":"competition"},{"sourceId":1305829,"sourceType":"datasetVersion","datasetId":755946},{"sourceId":1306896,"sourceType":"datasetVersion","datasetId":756642},{"sourceId":1307127,"sourceType":"datasetVersion","datasetId":756745},{"sourceId":1307546,"sourceType":"datasetVersion","datasetId":756953},{"sourceId":1484211,"sourceType":"datasetVersion","datasetId":836163},{"sourceId":1956112,"sourceType":"datasetVersion","datasetId":1167564},{"sourceId":1956203,"sourceType":"datasetVersion","datasetId":1109784}],"dockerImageVersionId":30043,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install resnest > /dev/null\n!pip install colorednoise > /dev/null","metadata":{"papermill":{"duration":24.928028,"end_time":"2021-02-18T08:19:09.87789","exception":false,"start_time":"2021-02-18T08:18:44.949862","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-04-20T10:53:44.912266Z","iopub.execute_input":"2024-04-20T10:53:44.912664Z","iopub.status.idle":"2024-04-20T10:54:03.270488Z","shell.execute_reply.started":"2024-04-20T10:53:44.912629Z","shell.execute_reply":"2024-04-20T10:54:03.269520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations as A\nfrom resnest.torch.resnet import ResNet, Bottleneck\nimport random\nfrom glob import glob\nfrom collections import OrderedDict\nimport os.path as osp\nimport os\nfrom pytorch_lightning.loggers import TensorBoardLogger\nfrom pytorch_lightning.callbacks.early_stopping import EarlyStopping\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nfrom pytorch_lightning import LightningModule\nfrom pytorch_lightning import Trainer\nfrom skimage.transform import resize\nfrom torchvision.models import resnet18, resnet34, resnet50\nfrom resnest.torch import resnest50\nfrom tqdm.auto import tqdm\nimport colorednoise as cn\nimport librosa\nimport torchaudio\nimport torch.nn.functional as F\nfrom torch.utils.data import WeightedRandomSampler\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision\nimport torch\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score, confusion_matrix\nimport matplotlib\nmatplotlib.use('Agg')","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":5.913696,"end_time":"2021-02-18T08:19:15.808227","exception":false,"start_time":"2021-02-18T08:19:09.894531","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-04-20T10:54:03.273231Z","iopub.execute_input":"2024-04-20T10:54:03.273793Z","iopub.status.idle":"2024-04-20T10:54:09.275050Z","shell.execute_reply.started":"2024-04-20T10:54:03.273744Z","shell.execute_reply":"2024-04-20T10:54:09.274123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=42):\n    print(f'setting everything to seed {seed}')\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.cuda.empty_cache()\n    \nseed_everything(42)","metadata":{"papermill":{"duration":0.029148,"end_time":"2021-02-18T08:19:15.852955","exception":false,"start_time":"2021-02-18T08:19:15.823807","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-04-20T10:54:12.886674Z","iopub.execute_input":"2024-04-20T10:54:12.887032Z","iopub.status.idle":"2024-04-20T10:54:12.895879Z","shell.execute_reply.started":"2024-04-20T10:54:12.886997Z","shell.execute_reply":"2024-04-20T10:54:12.895074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/c/rfcx-species-audio-detection/discussion/198418\n# label-level average\n# Assume float preds [BxC], labels [BxC] of 0 or 1\ndef LWLRAP(preds, labels):\n    # Ranks of the predictions\n    ranked_classes = torch.argsort(preds, dim=-1, descending=True)\n    # i, j corresponds to rank of prediction in row i\n    class_ranks = torch.zeros_like(ranked_classes).to(preds.device)\n    for i in range(ranked_classes.size(0)):\n        for j in range(ranked_classes.size(1)):\n            class_ranks[i, ranked_classes[i][j]] = j + 1\n    # Mask out to only use the ranks of relevant GT labels\n    ground_truth_ranks = class_ranks * labels + (1e6) * (1 - labels)\n    # All the GT ranks are in front now\n    sorted_ground_truth_ranks, _ = torch.sort(\n        ground_truth_ranks, dim=-1, descending=False)\n    # Number of GT labels per instance\n    num_labels = labels.sum(-1)\n    pos_matrix = torch.tensor(\n        np.array([i+1 for i in range(labels.size(-1))])).unsqueeze(0).to(preds.device)\n    score_matrix = pos_matrix / sorted_ground_truth_ranks\n    score_mask_matrix, _ = torch.sort(labels, dim=-1, descending=True)\n    scores = score_matrix * score_mask_matrix\n    score = scores.sum() / labels.sum()\n    return score.item()\n","metadata":{"papermill":{"duration":0.030495,"end_time":"2021-02-18T08:19:15.899709","exception":false,"start_time":"2021-02-18T08:19:15.869214","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-04-20T10:54:15.263416Z","iopub.execute_input":"2024-04-20T10:54:15.263768Z","iopub.status.idle":"2024-04-20T10:54:15.275312Z","shell.execute_reply.started":"2024-04-20T10:54:15.263738Z","shell.execute_reply":"2024-04-20T10:54:15.274342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    batch_size = 8\n    weight_decay = 1e-8\n    lr = 1e-3\n    num_workers = 4\n    epochs = 5\n    num_classes = 24\n    sr = 32_000\n    duration = 5\n    total_duration = 60\n    nmels = 128\n    EXTRAS_DIR = \"../input/rfcxextras\"\n    ROOT = \"../input/rfcx-species-audio-detection\"\n    TRAIN_AUDIO_ROOT = osp.join(ROOT, \"train\")\n    TEST_AUDIO_ROOT = osp.join(ROOT, \"test\")\n    loss_fn = torch.nn.BCEWithLogitsLoss()","metadata":{"papermill":{"duration":0.02452,"end_time":"2021-02-18T08:19:15.939854","exception":false,"start_time":"2021-02-18T08:19:15.915334","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-20T10:54:22.588107Z","iopub.execute_input":"2024-04-20T10:54:22.588493Z","iopub.status.idle":"2024-04-20T10:54:22.595248Z","shell.execute_reply.started":"2024-04-20T10:54:22.588456Z","shell.execute_reply":"2024-04-20T10:54:22.594416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Audio Augmentations","metadata":{}},{"cell_type":"code","source":"# Mostly taken from https://www.kaggle.com/hidehisaarai1213/rfcx-audio-data-augmentation-japanese-english\nclass AudioTransform:\n    def __init__(self, always_apply=False, p=0.5):\n        self.always_apply = always_apply\n        self.p = p\n\n    def __call__(self, y: np.ndarray):\n        if self.always_apply:\n            return self.apply(y)\n        else:\n            if np.random.rand() < self.p:\n                return self.apply(y)\n            else:\n                return y\n\n    def apply(self, y: np.ndarray):\n        raise NotImplementedError\n\n\nclass Compose:\n    def __init__(self, transforms: list):\n        self.transforms = transforms\n\n    def __call__(self, y: np.ndarray):\n        for trns in self.transforms:\n            y = trns(y)\n        return y\n\n\nclass OneOf:\n    def __init__(self, transforms: list):\n        self.transforms = transforms\n\n    def __call__(self, y: np.ndarray):\n        n_trns = len(self.transforms)\n        trns_idx = np.random.choice(n_trns)\n        trns = self.transforms[trns_idx]\n        return trns(y)\n\n\nclass GaussianNoiseSNR(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, min_snr=5.0, max_snr=20.0, **kwargs):\n        super().__init__(always_apply, p)\n\n        self.min_snr = min_snr\n        self.max_snr = max_snr\n\n    def apply(self, y: np.ndarray, **params):\n        snr = np.random.uniform(self.min_snr, self.max_snr)\n        a_signal = np.sqrt(y ** 2).max()\n        a_noise = a_signal / (10 ** (snr / 20))\n\n        white_noise = np.random.randn(len(y))\n        a_white = np.sqrt(white_noise ** 2).max()\n        augmented = (y + white_noise * 1 / a_white * a_noise).astype(y.dtype)\n        return augmented\n\n\nclass PinkNoiseSNR(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, min_snr=5.0, max_snr=20.0, **kwargs):\n        super().__init__(always_apply, p)\n\n        self.min_snr = min_snr\n        self.max_snr = max_snr\n\n    def apply(self, y: np.ndarray, **params):\n        snr = np.random.uniform(self.min_snr, self.max_snr)\n        a_signal = np.sqrt(y ** 2).max()\n        a_noise = a_signal / (10 ** (snr / 20))\n\n        pink_noise = cn.powerlaw_psd_gaussian(1, len(y))\n        a_pink = np.sqrt(pink_noise ** 2).max()\n        augmented = (y + pink_noise * 1 / a_pink * a_noise).astype(y.dtype)\n        return augmented\n\n\nclass TimeShift(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, max_shift_second=2, sr=32000, padding_mode=\"zero\"):\n        super().__init__(always_apply, p)\n\n        assert padding_mode in [\n            \"replace\", \"zero\"], \"`padding_mode` must be either 'replace' or 'zero'\"\n        self.max_shift_second = max_shift_second\n        self.sr = sr\n        self.padding_mode = padding_mode\n\n    def apply(self, y: np.ndarray, **params):\n        shift = np.random.randint(-self.sr * self.max_shift_second,\n                                  self.sr * self.max_shift_second)\n        augmented = np.roll(y, shift)\n        # if self.padding_mode == \"zero\":\n        #     if shift > 0:\n        #         augmented[:shift] = 0\n        #     else:\n        #         augmented[shift:] = 0\n        return augmented\n\n\nclass VolumeControl(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, db_limit=10, mode=\"uniform\"):\n        super().__init__(always_apply, p)\n\n        assert mode in [\"uniform\", \"fade\", \"fade\", \"cosine\", \"sine\"], \\\n            \"`mode` must be one of 'uniform', 'fade', 'cosine', 'sine'\"\n\n        self.db_limit = db_limit\n        self.mode = mode\n\n    def apply(self, y: np.ndarray, **params):\n        db = np.random.uniform(-self.db_limit, self.db_limit)\n        if self.mode == \"uniform\":\n            db_translated = 10 ** (db / 20)\n        elif self.mode == \"fade\":\n            lin = np.arange(len(y))[::-1] / (len(y) - 1)\n            db_translated = 10 ** (db * lin / 20)\n        elif self.mode == \"cosine\":\n            cosine = np.cos(np.arange(len(y)) / len(y) * np.pi * 2)\n            db_translated = 10 ** (db * cosine / 20)\n        else:\n            sine = np.sin(np.arange(len(y)) / len(y) * np.pi * 2)\n            db_translated = 10 ** (db * sine / 20)\n        augmented = y * db_translated\n        return augmented","metadata":{"papermill":{"duration":0.052492,"end_time":"2021-02-18T08:19:16.007707","exception":false,"start_time":"2021-02-18T08:19:15.955215","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-20T10:54:24.824118Z","iopub.execute_input":"2024-04-20T10:54:24.824507Z","iopub.status.idle":"2024-04-20T10:54:24.863004Z","shell.execute_reply.started":"2024-04-20T10:54:24.824468Z","shell.execute_reply":"2024-04-20T10:54:24.862094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mono_to_color(X, eps=1e-6, mean=None, std=None):\n    X = np.stack([X, X, X], axis=-1)\n\n    # Standardize\n    mean = mean or X.mean()\n    std = std or X.std()\n    X = (X - mean) / (std + eps)\n\n    # Normalize to [0, 255]\n    _min, _max = X.min(), X.max()\n\n    if (_max - _min) > eps:\n        V = np.clip(X, _min, _max)\n        V = 255 * (V - _min) / (_max - _min)\n        V = V.astype(np.uint8)\n    else:\n        V = np.zeros_like(X, dtype=np.uint8)\n\n    return V\n\n\ndef normalize(image, mean=None, std=None):\n    image = image / 255.0\n    if mean is not None and std is not None:\n        image = (image - mean) / std\n    return image.astype(np.float32)\n","metadata":{"papermill":{"duration":0.028209,"end_time":"2021-02-18T08:19:16.051552","exception":false,"start_time":"2021-02-18T08:19:16.023343","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RFCDataset:\n    def __init__(self, tp, fp=None, config=None,\n                 mode='train', inv_counts=None):\n        self.tp = tp\n        self.fp = pd.read_csv(\"../input/rfcxextras/cornell-train.csv\")\n        self.fp = self.fp[self.fp.ebird_code<'c'].reset_index(drop=True)\n        self.fp_root = \"../input/birdsong-resampled-train-audio-00/\"        \n        self.inv_counts = inv_counts\n        self.config = config\n        self.sr = self.config.sr\n        self.total_duration = self.config.total_duration\n        self.duration = self.config.duration\n        self.data_root = self.config.TRAIN_AUDIO_ROOT\n        self.nmels = self.config.nmels\n        self.fmin, self.fmax = 84, self.sr//2\n        self.mode = mode\n        self.num_classes = self.config.num_classes\n        self.resampler = torchaudio.transforms.Resample(\n            orig_freq=48_000, new_freq=self.sr)\n        self.mel = torchaudio.transforms.MelSpectrogram(sample_rate=self.sr, n_mels=self.nmels,\n                                                        f_min=self.fmin, f_max=self.fmax,\n                                                        n_fft=2048)\n        self.transform = Compose([\n            OneOf([\n                GaussianNoiseSNR(min_snr=10),\n                PinkNoiseSNR(min_snr=10)\n            ]),\n            TimeShift(sr=self.sr),\n            VolumeControl(p=0.5)\n        ])\n        self.img_transform = A.Compose([\n            A.OneOf([\n                A.Cutout(max_h_size=5, max_w_size=20),\n                A.CoarseDropout(max_holes=4),\n                A.RandomBrightness(p=0.25),\n            ], p=0.5)])\n        self.num_splits = self.config.total_duration//self.duration\n        assert self.config.total_duration == self.duration * \\\n            self.num_splits, \"not a multiple\"\n\n    def __len__(self):\n        return len(self.tp)\n\n    def __getitem__(self, idx):\n        labels = np.zeros((self.num_classes,), dtype=np.float32)\n\n        recording_id = self.tp.loc[idx, 'recording_id']\n        df = self.tp.loc[self.tp.recording_id == recording_id]\n        maybe_labels = df.species_id.unique()\n        np.put(labels, maybe_labels, 0.2)\n\n        df = df.sample(weights=df.species_id.apply(\n            lambda x: self.inv_counts[x]))\n        fn = osp.join(self.data_root, f\"{recording_id}.flac\")\n        df = df.squeeze()\n        t0 = max(df['t_min'], 0)\n        t1 = max(df['t_max'], 0)\n        t0 = np.random.uniform(t0, t1)\n        t0 = max(t0, 0)\n        t0 = min(t0, self.total_duration-self.duration)\n        t1 = t0 + self.duration\n        valid_df = self.tp[self.tp.recording_id == recording_id]\n        valid_df = valid_df[(valid_df.t_min < t1) & (valid_df.t_max > t0)]\n        y, _ = librosa.load(fn, sr=None, offset=t0,\n                            duration=self.duration)\n        if len(valid_df):\n            np.put(labels, valid_df.species_id.unique(), 1)\n        np.put(labels, df.species_id, 1)\n\n        if random.random()<0.5:\n            end_idx = int((valid_df.t_max.max() - t0)*self.sr)\n            rem_len = max(0, len(y) - end_idx)\n            idx = np.random.randint(0, len(self.fp))\n            \n            fn = osp.join(self.fp_root, self.fp.ebird_code[idx],self.fp.filename[idx])\n            fn = fn.replace('mp3', 'wav')\n            y_other, _ = librosa.load(fn, sr=self.sr,\n                                    duration=None, mono=True,\n                                    res_type='kaiser_fast')\n            aug_len = min(len(y_other), rem_len)\n            y[end_idx:end_idx+aug_len] = y_other[:aug_len]\n\n        y = self.resampler(torch.from_numpy(y).float()).numpy()\n        # do augmentation\n        y = self.transform(y)\n        if random.random() < 0.25:\n            tempo, beats = librosa.beat.beat_track(y=y, sr=self.sr)\n            y = librosa.clicks(frames=beats, sr=self.sr, length=len(y))\n\n        melspec = librosa.feature.melspectrogram(\n            y, sr=self.sr, n_mels=self.nmels, fmin=self.fmin, fmax=self.fmax,\n        )\n        melspec = librosa.power_to_db(melspec)\n        melspec = mono_to_color(melspec)\n        melspec = normalize(melspec, mean=None, std=None)\n        melspec = self.img_transform(image=melspec)['image']\n        melspec = np.moveaxis(melspec, 2, 0)\n        return melspec, labels","metadata":{"papermill":{"duration":0.051034,"end_time":"2021-02-18T08:19:16.117954","exception":false,"start_time":"2021-02-18T08:19:16.06692","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# resnest 50 trained on cornell \n# https://www.kaggle.com/theoviel/birds-cp-1\nMODEL_CONFIGS = {\n    \"resnest50_fast_1s1x64d\":\n    {\n        \"num_classes\": 264,\n        \"block\": Bottleneck,\n        \"layers\": [3, 4, 6, 3],\n        \"radix\": 1,\n        \"groups\": 1,\n        \"bottleneck_width\": 64,\n        \"deep_stem\": True,\n        \"stem_width\": 32,\n        \"avg_down\": True,\n        \"avd\": True,\n        \"avd_first\": True\n    }\n}\n\n\ndef get_model(pretrained=True, n_class=24):\n    # model = torchvision.models.resnext50_32x4d(pretrained=False)\n    # model = torchvision.models.resnext101_32x8d(pretrained=False)\n    model = ResNet(**MODEL_CONFIGS[\"resnest50_fast_1s1x64d\"])\n    n_features = model.fc.in_features\n    model.fc = nn.Linear(n_features, 264)\n    # model.load_state_dict(torch.load('resnext50_32x4d_extra_2.pt'))\n    # model.load_state_dict(torch.load('resnext101_32x8d_wsl_extra_4.pt'))\n    fn = '../input/birds-cp-1/resnest50_fast_1s1x64d_conf_1.pt'\n    model.load_state_dict(torch.load(fn, map_location='cpu'))\n    model.fc = nn.Linear(n_features, n_class)\n    return model\n","metadata":{"papermill":{"duration":0.025266,"end_time":"2021-02-18T08:19:16.211841","exception":false,"start_time":"2021-02-18T08:19:16.186575","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-04-20T10:54:37.320827Z","iopub.execute_input":"2024-04-20T10:54:37.321170Z","iopub.status.idle":"2024-04-20T10:54:37.330248Z","shell.execute_reply.started":"2024-04-20T10:54:37.321140Z","shell.execute_reply":"2024-04-20T10:54:37.329349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RFCTestDataset:\n    def __init__(self, tp, fp=None, config=None,\n                 mode='test'):\n        self.tp = tp\n        self.fp = fp\n        self.config = config\n        self.sr = self.config.sr\n        self.duration = self.config.duration\n        if mode == 'val':\n            self.data_root = self.config.TRAIN_AUDIO_ROOT\n        else:\n            self.data_root = self.config.TEST_AUDIO_ROOT\n\n        self.nmels = self.config.nmels\n        self.fmin, self.fmax = 84, self.sr//2\n        self.mode = mode\n        self.resampler = torchaudio.transforms.Resample(\n            orig_freq=48_000, new_freq=self.sr)\n        self.num_classes = self.config.num_classes\n        self.num_splits = self.config.total_duration//self.duration\n        assert self.config.total_duration == self.duration * \\\n            self.num_splits, \"not a multiple\"\n\n    def __len__(self):\n        return len(self.tp.recording_id.unique())\n\n    def __getitem__(self, idx):\n        recording_id = self.tp.loc[idx, 'recording_id']\n        df = self.tp.loc[self.tp.recording_id == recording_id]\n        if self.mode == 'val':\n            fn = f\"{self.config.EXTRAS_DIR}/train_melspec32k_10s/train_melspec32k_10s/{recording_id}.npy\"\n        else:\n            fn = f\"{self.config.EXTRAS_DIR}/test_melspec32k_10s/test_melspec32k_10s/{recording_id}.npy\"\n        try:\n            melspec_stacked = np.load(fn)\n        except:\n            audio_fn = osp.join(self.data_root, f\"{recording_id}.flac\")\n            y, _ = librosa.load(audio_fn, sr=None,\n                                duration=self.config.total_duration)\n            # split into n arrays\n            y_stacked = np.stack(np.split(y, self.num_splits), 0)\n            melspec_stacked = []\n            for y in y_stacked:\n                y = self.resampler(torch.from_numpy(y).float()).numpy()\n                melspec = librosa.feature.melspectrogram(\n                    y, sr=self.sr, n_mels=self.nmels, fmin=self.fmin, fmax=self.fmax,\n                )\n                melspec = librosa.power_to_db(melspec)\n                melspec = mono_to_color(melspec)\n                melspec = normalize(melspec, mean=None, std=None)\n                melspec = np.moveaxis(melspec, 2, 0)\n                melspec_stacked.append(melspec)\n\n            melspec_stacked = np.stack(melspec_stacked)\n            np.save(fn, melspec_stacked)\n\n        if self.mode == 'val':\n            species = df.loc[:, 'species_id'].unique()\n            labels = np.zeros((self.num_classes,))\n            np.put(labels, species, 1)\n\n            return melspec_stacked, labels\n        else:\n            melspec_stacked = np.load(fn)\n            return melspec_stacked","metadata":{"papermill":{"duration":0.03812,"end_time":"2021-02-18T08:19:16.17142","exception":false,"start_time":"2021-02-18T08:19:16.1333","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-20T10:54:39.841820Z","iopub.execute_input":"2024-04-20T10:54:39.842209Z","iopub.status.idle":"2024-04-20T10:54:39.865643Z","shell.execute_reply.started":"2024-04-20T10:54:39.842179Z","shell.execute_reply":"2024-04-20T10:54:39.864461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install efficientnet_pytorch\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# For ensemble model run this (default)","metadata":{}},{"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet\nfrom torchvision.models import mobilenet_v2\nfrom torchvision.models import resnet50\n\ndef get_model(pretrained=True, n_class=24):\n    # Load pre-trained models\n    efficient_net = EfficientNet.from_pretrained('efficientnet-b0')  # Change model type as needed\n    mobilenet = mobilenet_v2(pretrained=True)\n    resnet = resnet50(pretrained=True)\n\n    # Modify classification heads\n    efficient_net._fc = nn.Linear(efficient_net._fc.in_features, n_class)\n    mobilenet.classifier[-1] = nn.Linear(mobilenet.classifier[-1].in_features, n_class)\n    resnet.fc = nn.Linear(resnet.fc.in_features, n_class)\n\n    # Create ensemble model\n    class EnsembleModel(nn.Module):\n        def __init__(self, efficient_net, mobilenet, resnet, n_class):\n            super(EnsembleModel, self).__init__()\n            self.efficient_net = efficient_net\n            self.mobilenet = mobilenet\n            self.resnet = resnet\n            self.output_layer = nn.Linear(3 * n_class, n_class)\n\n        def forward(self, x):\n            output_efficient = self.efficient_net(x)\n            output_mobilenet = self.mobilenet(x)\n            output_resnet = self.resnet(x)\n            # Combine outputs, for example, by concatenating\n            combined_output = torch.cat((output_efficient, output_mobilenet, output_resnet), dim=1)\n            output = self.output_layer(combined_output)\n            return output\n\n    # Create and return ensemble model\n    ensemble_model = EnsembleModel(efficient_net, mobilenet, resnet, n_class)\n    return ensemble_model\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# For efficient model run this ","metadata":{}},{"cell_type":"code","source":"# from efficientnet_pytorch import EfficientNet\n# import torch.nn as nn\n\n# def get_model(pretrained=True, n_class=24):\n#     efficient_net = EfficientNet.from_pretrained('efficientnet-b0' if pretrained else None)\n#     if pretrained:\n#         num_ftrs = efficient_net._fc.in_features\n#         efficient_net._fc = nn.Linear(num_ftrs, n_class)\n#     return efficient_net\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# For mobilenetv2 run this","metadata":{}},{"cell_type":"code","source":"# from torchvision.models import mobilenet_v2\n# import torch.nn as nn\n\n# def get_model(pretrained=True, n_class=24):\n#     mobilenet = mobilenet_v2(pretrained=pretrained)\n#     if pretrained:\n#         num_ftrs = mobilenet.classifier[-1].in_features\n#         mobilenet.classifier[-1] = nn.Linear(num_ftrs, n_class)\n#     return mobilenet\n","metadata":{"execution":{"iopub.status.busy":"2024-04-20T11:03:45.742911Z","iopub.execute_input":"2024-04-20T11:03:45.743304Z","iopub.status.idle":"2024-04-20T11:03:45.750027Z","shell.execute_reply.started":"2024-04-20T11:03:45.743264Z","shell.execute_reply":"2024-04-20T11:03:45.748810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# For resnet50 model run this ","metadata":{}},{"cell_type":"code","source":"# from torchvision.models import resnet50\n# import torch.nn as nn\n\n# def get_model(pretrained=True, n_class=24):\n#     resnet = resnet50(pretrained=pretrained)\n#     if pretrained:\n#         num_ftrs = resnet.fc.in_features\n#         resnet.fc = nn.Linear(num_ftrs, n_class)\n#     return resnet\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def worker_init_fn(worker_id):                                                          \n    np.random.seed(np.random.get_state()[1][0] + worker_id)\n    \nclass BaseNet(LightningModule):\n    def __init__(self, config, train_recid, val_recid):\n        super().__init__()\n        self.config = config\n        self.batch_size = self.config.batch_size\n        self.num_workers = self.config.num_workers\n        self.lr = self.config.lr\n        self.epochs = self.config.epochs\n\n        self.weight_decay = self.config.weight_decay\n        # to improve species 3 recall rate\n        pos_weight = torch.ones((24,))\n        pos_weight[3] = 4\n        self.loss_fn = torch.nn.BCEWithLogitsLoss(pos_weight=pos_weight)\n        self.sr = self.config.sr\n        self.train_recid = train_recid\n        self.val_recid = val_recid\n\n    def train_dataloader(self):\n        tp = train_tp[train_tp.recording_id.isin(\n            self.train_recid)].reset_index(drop=True)\n        self.train_recid = tp.recording_id.unique()\n        inv_counts = dict(1/tp.species_id.value_counts())\n        weights = tp.species_id.apply(lambda x: inv_counts[x])\n        tp_aug = new_labels[new_labels.recording_id.isin(tp.recording_id)]\n        tp = pd.concat([tp, tp_aug], ignore_index=True)\n        train_dataset = RFCDataset(tp, train_fp,\n                                   config=self.config,\n                                   mode='train',\n                                   inv_counts=inv_counts)\n        train_sampler = WeightedRandomSampler(weights, num_samples=len(train_dataset),\n                                              replacement=True)\n\n        train_loader = DataLoader(train_dataset, batch_size=self.batch_size,\n                                  num_workers=self.num_workers,\n                                  sampler=train_sampler,\n                                  worker_init_fn=worker_init_fn,\n                                  drop_last=True,\n                                  pin_memory=True)\n        return train_loader\n\n    def val_dataloader(self):\n        val_tp = train_tp[train_tp.recording_id.isin(\n            self.val_recid)].reset_index(drop=True)\n        val_recid = val_tp.recording_id.unique()\n        overlap = set(val_recid).intersection(set(self.train_recid))\n#         print('overlapped ids', overlap)\n        val_tp = val_tp[~val_tp.recording_id.isin(overlap)]\n        val_tp_aug = new_labels[new_labels.recording_id.isin(\n            val_tp.recording_id)]\n        val_tp = pd.concat([val_tp, val_tp_aug], ignore_index=True)\n        val_dataset = RFCTestDataset(val_tp, train_fp,\n                                     config=self.config,\n                                     mode='val')\n        val_loader = DataLoader(val_dataset, batch_size=self.batch_size,\n                                num_workers=self.num_workers, shuffle=False,\n                                pin_memory=True)\n        return val_loader\n\n    def configure_optimizers(self):\n        optim = torch.optim.AdamW(self.parameters(), lr=self.config.lr,\n                                  weight_decay=self.config.weight_decay)\n        scheduler = {\n            'scheduler': torch.optim.lr_scheduler.ReduceLROnPlateau(optim,\n                                                                    mode='min',\n                                                                    factor=0.5,\n                                                                    patience=2,\n                                                                    verbose=True),\n            'monitor': 'val_loss',\n            'interval': 'epoch',\n            'frequency': 1,\n            'strict': True,\n        }\n\n        self.optimizer = optim\n        self.scheduler = scheduler\n\n        return [optim], [scheduler]\n\n","metadata":{"papermill":{"duration":0.037803,"end_time":"2021-02-18T08:19:16.264724","exception":false,"start_time":"2021-02-18T08:19:16.226921","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-20T11:03:50.225871Z","iopub.execute_input":"2024-04-20T11:03:50.226211Z","iopub.status.idle":"2024-04-20T11:03:50.252905Z","shell.execute_reply.started":"2024-04-20T11:03:50.226182Z","shell.execute_reply":"2024-04-20T11:03:50.252059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RFCNet(BaseNet):\n    def __init__(self, **kwargs):\n        super().__init__(**kwargs)\n        n_class = self.config.num_classes\n        self.model = get_model(\n            pretrained=True, n_class=n_class)\n        self.cnf_matrix = np.zeros((n_class, n_class))\n\n    def forward(self, x):\n        return self.model(x)\n\n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        preds = self(x)\n        loss = self.loss_fn(preds, y)\n        with torch.no_grad():\n            lwlrap = LWLRAP(preds, y)\n        metrics = {\"train_loss\": loss.item(), \"train_lwlrap\": lwlrap}\n        self.log_dict(metrics,\n                      on_epoch=True, on_step=True)\n\n        return loss\n\n    @torch.no_grad()\n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        for i, x_partial in enumerate(torch.split(x, 1, dim=1)):\n            x_partial = x_partial.squeeze(1)\n            if i == 0:\n                preds = self(x_partial)\n            else:\n                # take max over predictions\n                preds = torch.max(preds, self(x_partial))\n        val_loss = self.loss_fn(preds, y).item()\n        val_lwlrap = LWLRAP(preds, y)\n        # loss is tensor. The Checkpoint Callback is monitoring 'checkpoint_on'\n        metrics = {\"val_loss\": val_loss, \"val_lwlrap\": val_lwlrap}\n        self.log_dict(metrics, prog_bar=True,\n                      on_epoch=True, on_step=True)","metadata":{"papermill":{"duration":0.029993,"end_time":"2021-02-18T08:19:16.310021","exception":false,"start_time":"2021-02-18T08:19:16.280028","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-20T11:03:53.536861Z","iopub.execute_input":"2024-04-20T11:03:53.537228Z","iopub.status.idle":"2024-04-20T11:03:53.552887Z","shell.execute_reply.started":"2024-04-20T11:03:53.537193Z","shell.execute_reply":"2024-04-20T11:03:53.551769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Average model weights","metadata":{}},{"cell_type":"code","source":"def average_model(paths):\n    weights = np.ones((len(paths),))\n    weights = weights/weights.sum()\n    for i, p in enumerate(paths):\n        m = torch.load(p)['state_dict']\n        if i == 0:\n            averaged_w = OrderedDict()\n            for k in m.keys():\n                if 'pos' in k: continue\n                # remove pl prefix in state dict\n                knew = k.replace('model.', '')\n                averaged_w[knew] = weights[i]*m[k]\n        else:\n            for k in m.keys():\n                if 'pos' in k: continue\n                knew = k.replace('model.', '')\n                averaged_w[knew] = averaged_w[knew] + weights[i]*m[k]\n    return averaged_w","metadata":{"papermill":{"duration":0.026478,"end_time":"2021-02-18T08:19:16.351879","exception":false,"start_time":"2021-02-18T08:19:16.325401","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-20T11:03:56.114935Z","iopub.execute_input":"2024-04-20T11:03:56.115297Z","iopub.status.idle":"2024-04-20T11:03:56.124946Z","shell.execute_reply.started":"2024-04-20T11:03:56.115264Z","shell.execute_reply":"2024-04-20T11:03:56.124004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model training","metadata":{}},{"cell_type":"code","source":"config = Config()\ntrain_tp = pd.read_csv(osp.join(config.ROOT, 'train_tp.csv'))\n\nfold_df = pd.read_csv(\n    osp.join(config.EXTRAS_DIR, 'preprocessed_rainforest_dataset.csv'))\nfn = \"../input/extra-labels-for-rcfx-competition-data/extra_labels_v71.csv\"\nprint(fn)\nnew_labels = pd.read_csv(fn)\nnew_labels['t_diff'] = new_labels['t_max'] - new_labels['t_min']\nidx = np.where(new_labels['t_diff'] < 0)[0]\nnew_labels = new_labels.drop(idx, axis=0).reset_index(drop=True)\nnum_folds = len(fold_df.fold.unique())\ntrain_fp = pd.read_csv(osp.join(config.ROOT, 'train_fp.csv'))\nfor fold in range(num_folds):\n    print('\\n\\nTraining fold', fold)\n    print('*' * 40)\n\n    train_recid = fold_df[fold_df.fold != fold].recording_id\n    val_recid = fold_df[fold_df.fold == fold].recording_id\n    model = RFCNet(config=config, train_recid=train_recid,\n                   val_recid=val_recid)\n    checkpoint_callback = ModelCheckpoint(\n        monitor='val_lwlrap_epoch',\n        filename='{epoch:02d}-{val_loss_epoch:.2f}-{val_lwlrap_epoch:.2}',\n        mode='max',\n        save_top_k=5,\n        save_weights_only=True,\n    )\n    early_stopping = EarlyStopping(monitor='val_lwlrap_epoch', mode='max', patience=5,\n                                   verbose=True)\n    trainer = Trainer(gpus=1,\n                      max_epochs=config.epochs,\n                      progress_bar_refresh_rate=1,                      \n                      #   gradient_clip_val=2,\n                      accumulate_grad_batches=4,\n                      num_sanity_val_steps=0,\n                      callbacks=[checkpoint_callback, early_stopping])\n\n    # Fine-tune the model\n    for param in model.model.parameters():\n        param.requires_grad = True\n\n    # Define optimizer for fine-tuning\n    optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)  # Adjust learning rate if needed\n\n    trainer.fit(model)\n","metadata":{"papermill":{"duration":12702.148966,"end_time":"2021-02-18T11:50:58.515949","exception":false,"start_time":"2021-02-18T08:19:16.366983","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-20T11:35:42.865770Z","iopub.execute_input":"2024-04-20T11:35:42.866137Z","iopub.status.idle":"2024-04-20T11:35:42.918743Z","shell.execute_reply.started":"2024-04-20T11:35:42.866105Z","shell.execute_reply":"2024-04-20T11:35:42.917508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Validation and predictions","metadata":{}},{"cell_type":"code","source":"def get_one_hot(targets, nb_classes=24):\n    res = np.eye(nb_classes)[np.array(targets).reshape(-1)]\n    return res.reshape(list(targets.shape)+[nb_classes])\n\n\nsub = pd.read_csv(osp.join(config.ROOT, 'sample_submission.csv'))\nspecies_cols = list(sub.columns)\nspecies_cols.remove('recording_id')\n\ncv_preds = pd.DataFrame(columns=species_cols)\ncv_preds['recording_id'] = train_tp['recording_id'].drop_duplicates()\ncv_preds = cv_preds.set_index('recording_id')\n\nlabel_df = pd.DataFrame(columns=species_cols)\nlabel_df['recording_id'] = train_tp['recording_id'].drop_duplicates()\nlabel_df = label_df.set_index('recording_id')\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = get_model(pretrained=False)\nmodel.to(device)\nfor fold in range(5):\n    paths = glob(f\"./lightning_logs/version_{fold}/checkpoints/*.ckpt\")\n    print(paths)\n    averaged_w = average_model(paths)\n    model.load_state_dict(averaged_w)\n    model.eval()\n    train_recid = fold_df[fold_df.fold!=fold].recording_id\n    val_recid = fold_df[fold_df.fold==fold].recording_id\n\n    val_tp = train_tp[train_tp.recording_id.isin(val_recid)].reset_index(drop=True)\n    val_recid = val_tp.recording_id.unique()\n    overlap = set(val_recid).intersection(set(train_recid))\n    val_tp = val_tp[~val_tp.recording_id.isin(overlap)]\n    val_tp_aug = new_labels[new_labels.recording_id.isin(val_tp.recording_id)]\n    val_tp = pd.concat([val_tp, val_tp_aug], ignore_index=True)\n\n    dataset = RFCTestDataset(val_tp, config=config, mode='val')\n    test_loader = DataLoader(dataset, batch_size=config.batch_size,\n                             num_workers=config.num_workers,\n                             shuffle=False, drop_last=False)\n    tk = test_loader\n    with torch.no_grad():\n        fold_preds, labels = [], []\n        for i, (im, l) in enumerate(tk):\n            # continue\n            im = im.to(device)\n            for j, x_partial in enumerate(torch.split(im, 1, dim=1)):\n                x_partial = x_partial.squeeze(1)\n                if j == 0:\n                    preds = model(x_partial)\n                else:\n                    preds = torch.max(preds, model(x_partial))\n\n\n            o = preds.sigmoid().cpu().numpy()\n            # o = preds.cpu().numpy()\n            fold_preds.extend(o)\n            labels.extend(l.cpu().numpy())\n        # continue\n        p = torch.from_numpy(np.array(fold_preds)) \n        t = torch.from_numpy(np.array(labels))\n        print(f\"lwlrap: {LWLRAP(p, t):.6}\")\n        cv_preds.loc[val_recid, species_cols] = fold_preds\n        label_df.loc[val_recid, species_cols] = labels\n\n# print(cv_preds.head())\ncv_preds.to_csv('cv_preds.csv')\n\nrecid = train_tp['recording_id'].values\ncv_preds = cv_preds.loc[recid].values.astype(np.float32)\ncv_preds = torch.from_numpy(cv_preds)\n\nlabels = label_df.loc[recid].values.astype(np.float32)\nlabels = torch.from_numpy(labels)\n\nprint(f\"lwlrap: {LWLRAP(cv_preds, labels):.6}\")\n","metadata":{"papermill":{"duration":65.499569,"end_time":"2021-02-18T11:52:04.060246","exception":false,"start_time":"2021-02-18T11:50:58.560677","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-11T08:22:38.567427Z","iopub.execute_input":"2024-04-11T08:22:38.567837Z","iopub.status.idle":"2024-04-11T08:23:57.238911Z","shell.execute_reply.started":"2024-04-11T08:22:38.567793Z","shell.execute_reply":"2024-04-11T08:23:57.237727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test predictions","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv(osp.join(config.ROOT, 'sample_submission.csv'))\nspecies_cols = list(sub.columns)\nspecies_cols.remove('recording_id')\n# initialize to zero.\nsub.loc[:, species_cols] = 0\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = get_model(pretrained=False)\nmodel.to(device)\nfor fold in range(num_folds):\n    paths = glob(f\"./lightning_logs/version_{fold}/checkpoints/*.ckpt\")\n    print(paths)\n    averaged_w = average_model(paths)\n    model.load_state_dict(averaged_w)\n    model.eval()\n    dataset = RFCTestDataset(sub, config=config, mode='test')\n    test_loader = DataLoader(dataset, batch_size=config.batch_size,\n                             num_workers=4,\n                             shuffle=False, drop_last=False)\n    tk = tqdm(test_loader, total=len(test_loader))\n    sub_index = 0\n    with torch.no_grad():\n        for i, im in enumerate(tk):\n            im = im.to(device)\n            for i, x_partial in enumerate(torch.split(im, 1, dim=1)):\n                x_partial = x_partial.squeeze(1)\n                if i == 0:\n                    preds = model(x_partial)\n                else:\n                    # take max over predictions\n                    preds = torch.max(preds, model(x_partial))\n\n            o = preds.sigmoid().cpu().numpy()\n            # o = preds.cpu().numpy()\n            for val in o:\n                sub.loc[sub_index, species_cols] += val\n                sub_index += 1\n\n# # take average of predictions\nsub.loc[:, species_cols] /= num_folds\nsub.to_csv('submission.csv', index=False)\nprint(sub.head())\nprint(sub.max(1).head())\n","metadata":{},"execution_count":null,"outputs":[]}]}