{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ../input/birds-inference-pip-wheels/torchaudio-0.8.1-cp37-cp37m-manylinux1_x86_64.whl ../input/birds-inference-pip-wheels/torch-1.8.1-cp37-cp37m-manylinux1_x86_64.whl -q\n!pip install ../input/birds-inference-pip-wheels/timm-0.4.8.zip --no-index --no-deps -q\n#!pip install ../input/birdclef21trainmeta/timm-0.4.9_23052021/pytorch-image-models-master --no-index --no-deps\n!pip install ../input/birds-inference-pip-wheels/audiomentations-0.16.0-py3-none-any.whl --no-index --no-deps -q\n!pip install ../input/birds-inference-pip-wheels/torchlibrosa-0.0.9-py3-none-any.whl --no-index --no-deps -q\n!pip install pandarallel -q","metadata":{"execution":{"iopub.status.busy":"2023-04-24T09:47:08.177994Z","iopub.execute_input":"2023-04-24T09:47:08.178369Z","iopub.status.idle":"2023-04-24T09:48:13.670696Z","shell.execute_reply.started":"2023-04-24T09:47:08.178251Z","shell.execute_reply":"2023-04-24T09:48:13.669772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import timm\ntimm.__version__","metadata":{"execution":{"iopub.status.busy":"2023-04-24T09:48:13.673333Z","iopub.execute_input":"2023-04-24T09:48:13.673686Z","iopub.status.idle":"2023-04-24T09:48:15.179530Z","shell.execute_reply.started":"2023-04-24T09:48:13.673606Z","shell.execute_reply":"2023-04-24T09:48:15.178697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport timm\nfrom torch import nn\nimport torch\nimport torchaudio as ta\nfrom torch.cuda.amp import autocast\nimport random\n\nfrom torch.nn import functional as F\nfrom torch.distributions import Beta\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import Dataset\n\nimport numpy as np\nimport librosa\nimport ast\n\nimport os\nfrom types import SimpleNamespace\nimport numpy as np\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport json\nimport importlib\nimport sys\nimport random\nfrom tqdm import tqdm\nimport gc\nimport argparse\nimport torch\nfrom torch import optim\nfrom torch.cuda.amp import GradScaler, autocast\nfrom collections import defaultdict\nimport cv2\nfrom copy import copy\nimport os\nfrom transformers import get_cosine_schedule_with_warmup\nfrom torch.utils.data import SequentialSampler, DataLoader\nfrom pandarallel import pandarallel\nimport warnings\n\npandarallel.initialize()\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T09:48:15.181077Z","iopub.execute_input":"2023-04-24T09:48:15.181401Z","iopub.status.idle":"2023-04-24T09:48:22.191198Z","shell.execute_reply.started":"2023-04-24T09:48:15.181365Z","shell.execute_reply":"2023-04-24T09:48:22.190356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=1234):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = False\n    torch.backends.cudnn.benchmark = True","metadata":{"execution":{"iopub.status.busy":"2023-04-24T09:48:22.193210Z","iopub.execute_input":"2023-04-24T09:48:22.193496Z","iopub.status.idle":"2023-04-24T09:48:22.200457Z","shell.execute_reply.started":"2023-04-24T09:48:22.193457Z","shell.execute_reply":"2023-04-24T09:48:22.199601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare Pretrain Data","metadata":{}},{"cell_type":"code","source":"BASE_PATH0 = '/kaggle/input/birdsong-recognition'\nBASE_PATH1 = '/kaggle/input/birdclef-2021'\nBASE_PATH2 = '/kaggle/input/birdclef-2022'\nBASE_PATH3 = '/kaggle/input/birdclef-2023'\nBASE_PATH4 = '/kaggle/input/xeno-canto-bird-recordings-extended-a-m'\nBASE_PATH5 = '/kaggle/input/xeno-canto-bird-recordings-extended-n-z'\n\n\nGCS_PATH0 = BASE_PATH0\nGCS_PATH1 = BASE_PATH1\nGCS_PATH2 = BASE_PATH2\nGCS_PATH3 = BASE_PATH3\nGCS_PATH4 = BASE_PATH4\nGCS_PATH5 = BASE_PATH5","metadata":{"execution":{"iopub.status.busy":"2023-04-24T09:48:22.201760Z","iopub.execute_input":"2023-04-24T09:48:22.202135Z","iopub.status.idle":"2023-04-24T09:48:22.210069Z","shell.execute_reply.started":"2023-04-24T09:48:22.202100Z","shell.execute_reply":"2023-04-24T09:48:22.209313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg = SimpleNamespace()\n\n# paths\ncfg.data_folder = ''\ncfg.name = \"julian\"\ncfg.data_dir = \"../input/birdclef-2023/\"\ncfg.train_data_folder = cfg.data_dir + \"train_audio/\"\ncfg.val_data_folder = cfg.data_dir + \"train_audio/\"\ncfg.output_dir = \"pre_train\"\n\n# dataset\ncfg.dataset = \"base_ds\"\ncfg.min_rating = 0\ncfg.val_df = None\ncfg.batch_size_val = 1\ncfg.train_aug = None\ncfg.val_aug = None\ncfg.test_augs = None\ncfg.wav_len_val = 5  # seconds\ncfg.min_rating = 2.0\ncfg.wav_crop_len = 30  # seconds\n# audio\ncfg.img_height = 256\ncfg.img_weight = 512\ncfg.window_size = 2048\ncfg.sample_rate = 32000\ncfg.fmin = 16\ncfg.fmax = 16386\ncfg.power = 2\ncfg.mel_bins = cfg.img_height\ncfg.hop_size = int(cfg.sample_rate * cfg.wav_crop_len / (cfg.img_weight - 1))\ncfg.top_db = 80.0\n\n# img model\n\ncfg.pretrained = False\ncfg.pretrained_weights = None\ncfg.train = True\ncfg.val = False\ncfg.in_chans = 1\n\ncfg.alpha = 1\ncfg.eval_epochs = 1\ncfg.eval_train_epochs = 1\ncfg.warmup = 0.1\n\ncfg.mel_norm = False\n\ncfg.label_smoothing = 0\n\ncfg.remove_pretrained = []\n\n# training\ncfg.seed = 123\ncfg.save_val_data = True\n\n# ressources\ncfg.mixed_precision = True\ncfg.gpu = 0\ncfg.num_workers = 4 # 18\ncfg.drop_last = True \n\ncfg.mixup2 = 0\n\ncfg.label_smoothing = 0\n\ncfg.mixup_2x = False\n\n\n# Class Labels for BirdCLEF 23\ncfg.class_names = sorted(os.listdir('/kaggle/input/birdclef-2023/train_audio/'))\ncfg.num_classes = len(cfg.class_names)\ncfg.class_labels = list(range(cfg.num_classes))\ncfg.label2name = dict(zip(cfg.class_labels, cfg.class_names))\ncfg.name2label = {v:k for k,v in cfg.label2name.items()}\n\n# Class Labels for BirdCLEF 21 & 22\ncfg.class_names2 = sorted(set(os.listdir('/kaggle/input/birdclef-2021/train_short_audio/')\n                   +os.listdir('/kaggle/input/birdclef-2022/train_audio/')\n                   +os.listdir('/kaggle/input/birdsong-recognition/train_audio/')))\ncfg.num_classes2 = len(cfg.class_names2)\ncfg.class_labels2 = list(range(cfg.num_classes2))\ncfg.label2name2 = dict(zip(cfg.class_labels2,cfg. class_names2))\ncfg.name2label2 = {v:k for k,v in cfg.label2name2.items()}\n\n\n\ncfg.birds = np.array(list(cfg.name2label2.keys()))\n\n\ncfg.n_classes = len(cfg.birds)\n\n\n# training \ncfg.lr = 0.001\ncfg.epochs = 20\ncfg.batch_size = 128\ncfg.batch_size_val = 64\ncfg.backbone = \"resnet34\"\n\n\ncfg.save_val_data = True\ncfg.mixed_precision = True\n\ncfg.mixup = True\ncfg.mix_beta = 1\n\n\ncfg.train_df1 = \"../input/birdclef-2023/train_pre_with_duration.csv\"\n#cfg.train_df2 = \"../input/birdclef-2022-df-train-with-durations/df-with-durations.csv\"\n\n\ncfg.device = 'cuda' if torch.cuda.is_available() else 'cpu'\n\ncfg.tr_collate_fn = None\ncfg.val_collate_fn = None\ncfg.val = False\n\ncfg.dev = False\n\ndebug = False\n\n\n\ncfg","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-04-24T09:48:22.211163Z","iopub.execute_input":"2023-04-24T09:48:22.211426Z","iopub.status.idle":"2023-04-24T09:48:22.478762Z","shell.execute_reply.started":"2023-04-24T09:48:22.211394Z","shell.execute_reply":"2023-04-24T09:48:22.477948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_23 = pd.read_csv(f'{BASE_PATH3}/train_metadata.csv')\ndf_23['filepath'] = GCS_PATH3 + '/train_audio/' + df_23.filename\ndf_23['target'] = df_23.primary_label.map(cfg.name2label)\ndf_23['birdclef'] = '23'\ndf_23['filename'] = df_23.filepath.map(lambda x: x.split('/')[-1])\ndf_23['xc_id'] = df_23.filepath.map(lambda x: x.split('/')[-1].split('.')[0])\n\n# Display rwos\nprint(\"# Samples in BirdCLEF 23: {:,}\".format(len(df_23)))\ndf_23.head(2).style.set_caption(\"BirdCLEF - 23\").set_table_styles([{\n    'selector': 'caption',\n    'props': [\n        ('color', 'blue'),\n        ('font-size', '16px')\n    ]\n}])","metadata":{"execution":{"iopub.status.busy":"2023-04-24T09:48:22.479911Z","iopub.execute_input":"2023-04-24T09:48:22.480771Z","iopub.status.idle":"2023-04-24T09:48:22.701467Z","shell.execute_reply.started":"2023-04-24T09:48:22.480734Z","shell.execute_reply":"2023-04-24T09:48:22.700660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# BirdCLEF-2020\ndf_20 = pd.read_csv('/kaggle/input/birdclef-22-duration/df_20_with_duration.csv')\n\n\n# Xeno-Canto Extend by @vopani\ndf_xc = pd.read_csv('/kaggle/input/birdclef-22-duration/df_xc_with_duration.csv')\n\n# BirdCLEF-2021\ndf_21 = pd.read_csv(f'{BASE_PATH1}/train_metadata.csv')\ndf_21['filepath'] = GCS_PATH1 + '/train_short_audio/' + df_21.primary_label + '/' + df_21.filename\ndf_21['target'] = df_21.primary_label.map(cfg.name2label2)\ndf_21['birdclef'] = '21'\ncorrupt_paths = ['/kaggle/input/birdclef-2021/train_short_audio/houwre/XC590621.ogg',\n                 '/kaggle/input/birdclef-2021/train_short_audio/cogdov/XC579430.ogg']\ndf_21 = df_21[~df_21.filepath.isin(corrupt_paths)] # remove all zero audios\n\ndf_21_with_duration = pd.read_csv('/kaggle/input/birdclef-22-duration/df_21_with_duration.csv')[['filename', 'duration']]\ndf_21 = df_21.merge(df_21_with_duration, on='filename')\n\n# BirdCLEF-2022\ndf_22 = pd.read_csv('/kaggle/input/birdclef-22-duration/df_22_with_duration.csv')\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-04-24T09:48:22.702674Z","iopub.execute_input":"2023-04-24T09:48:22.703010Z","iopub.status.idle":"2023-04-24T09:48:25.184523Z","shell.execute_reply.started":"2023-04-24T09:48:22.702970Z","shell.execute_reply":"2023-04-24T09:48:25.183605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge 2021 and 2022 for pretraining\ndf_pre = pd.concat([df_20, df_21, df_22, df_xc], axis=0, ignore_index=True)\ndf_pre['filename'] = df_pre.filepath.map(lambda x: x.split('/')[-1])\ndf_pre['xc_id'] = df_pre.filepath.map(lambda x: x.split('/')[-1].split('.')[0])\nnodup_idx = df_pre[['xc_id','primary_label','author']].drop_duplicates().index\ndf_pre = df_pre.loc[nodup_idx].reset_index(drop=True)\n\n# # Remove duplicates\ndf_pre = df_pre[~df_pre.xc_id.isin(df_23.xc_id)].reset_index(drop=True)\ncorrupt_mp3s = json.load(open('/kaggle/input/birdclef-corrupt-mp3-files-ds/corrupt_mp3_files.json','r'))\ndf_pre = df_pre[~df_pre.filepath.isin(corrupt_mp3s)]\ndf_pre = df_pre[['filename','filepath','primary_label','secondary_labels',\n                 'rating','author','file_type','xc_id','scientific_name',\n                'common_name','target','birdclef','bird_seen', 'duration']]\n# Display rows\nprint(\"# Samples for Pre-Training: {:,}\".format(len(df_pre)))\ndf_pre.head(2).style.set_caption(\"Pre-Training Data\").set_table_styles([{\n    'selector': 'caption',\n    'props': [\n        ('color', 'blue'),\n        ('font-size', '16px')\n    ]\n}])\n\n# Show distribution\n# plt.figure(figsize=(8, 4))\n# df_pre.birdclef.value_counts().plot.bar()\n# plt.xlabel(\"Dataset\")\n# plt.ylabel(\"Count\")\n# plt.title(\"Dataset distribution for Pre-Training\")\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T09:48:25.185950Z","iopub.execute_input":"2023-04-24T09:48:25.186198Z","iopub.status.idle":"2023-04-24T09:48:26.142686Z","shell.execute_reply.started":"2023-04-24T09:48:25.186163Z","shell.execute_reply":"2023-04-24T09:48:26.141985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def get_file_duration(file_path):\n#     try:\n#         sound, sr = librosa.load(file_path)\n#         return librosa.get_duration(y=sound, sr=sr)\n#     except:\n#         return 0\n\n# df_21['duration'] = df_21.filepath.parallel_apply(get_file_duration)\n# df_21.to_csv('df_21_with_duration.csv', index=None)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T09:48:26.145702Z","iopub.execute_input":"2023-04-24T09:48:26.145934Z","iopub.status.idle":"2023-04-24T09:48:26.149607Z","shell.execute_reply.started":"2023-04-24T09:48:26.145894Z","shell.execute_reply":"2023-04-24T09:48:26.148576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pre","metadata":{"execution":{"iopub.status.busy":"2023-04-24T09:48:26.151419Z","iopub.execute_input":"2023-04-24T09:48:26.151683Z","iopub.status.idle":"2023-04-24T09:48:26.182239Z","shell.execute_reply.started":"2023-04-24T09:48:26.151650Z","shell.execute_reply":"2023-04-24T09:48:26.181472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"def batch_to_device(batch, device):\n    batch_dict = {key: batch[key].to(device) for key in batch}\n    return batch_dict\n\n\n\nclass CustomDataset(Dataset):\n    def __init__(self, df, cfg, aug, mode=\"train\"):\n\n        self.cfg = cfg\n        self.mode = mode\n        self.df = df.copy()\n\n        self.bird2id = {bird: idx for idx, bird in enumerate(cfg.birds)}\n        if self.mode == \"train\":\n            self.data_folder = cfg.train_data_folder\n            self.df = self.df[self.df[\"rating\"] >= self.cfg.min_rating]\n        elif self.mode == \"val\":\n            self.data_folder = cfg.val_data_folder\n        elif self.mode == \"test\":\n            self.data_folder = cfg.test_data_folder\n\n        self.fns = self.df[\"filename\"].unique()\n\n        self.df = self.setup_df()\n\n        self.aug_audio = cfg.train_aug\n\n    def setup_df(self):\n        df = self.df.copy()\n\n        if self.mode == \"train\":\n\n            df[\"weight\"] = np.clip(df[\"rating\"] / df[\"rating\"].max(), 0.1, 1.0)\n            df['target'] = df['primary_label'].apply(self.bird2id.get)\n            labels = np.eye(self.cfg.n_classes)[df[\"target\"].astype(int).values]\n#             label2 = df[\"secondary_labels\"].apply(lambda x: self.secondary2target(x)).values\n#             for i, t in enumerate(label2):\n#                 labels[i, t] = 1\n        else:\n            targets = df[\"birds\"].apply(lambda x: self.birds2target(x)).values\n            labels = np.zeros((df.shape[0], self.cfg.n_classes))\n            for i, t in enumerate(targets):\n                labels[i, t] = 1\n\n        df[[f\"t{i}\" for i in range(self.cfg.n_classes)]] = labels\n\n        if self.mode != \"train\":\n            df = df.groupby(\"filename\")\n\n        return df\n\n    def __getitem__(self, idx):\n\n        if self.mode == \"train\":\n            row = self.df.iloc[idx]\n            fn = row[\"filepath\"]\n            label = row[[f\"t{i}\" for i in range(self.cfg.n_classes)]].values\n            weight = row[\"weight\"]\n            #fold = row[\"fold\"]\n            fold = -1\n\n            #wav_len = row[\"length\"]\n            parts = 1\n        else:\n            fn = self.fns[idx]\n            row = self.df.get_group(fn)\n            label = row[[f\"t{i}\" for i in range(self.cfg.n_classes)]].values\n            wav_len = None\n            parts = label.shape[0]\n            fold = -1\n            weight = 1\n\n        if self.mode == \"train\":\n            #wav_len_sec = wav_len / self.cfg.sample_rate\n            wav_len_sec = row['duration']\n            duration = self.cfg.wav_crop_len\n            max_offset = wav_len_sec - duration\n            max_offset = max(max_offset, 1)\n            offset = np.random.randint(max_offset)\n        else:\n            offset = 0.0\n            duration = None\n\n        wav = self.load_one(fn, offset, duration)\n\n        if wav.shape[0] < (self.cfg.wav_crop_len * self.cfg.sample_rate):\n            pad = self.cfg.wav_crop_len * self.cfg.sample_rate - wav.shape[0]\n            wav = np.pad(wav, (0, pad))\n\n        if self.mode == \"train\":\n            if self.aug_audio:\n                wav = self.aug_audio(samples=wav, sample_rate=self.cfg.sample_rate)\n        else:\n            if self.cfg.val_aug:\n                wav = self.cfg.val_aug(samples=wav, sample_rate=self.cfg.sample_rate)\n\n        wav_tensor = torch.tensor(wav)  # (n_samples)\n        if parts > 1:\n            n_samples = wav_tensor.shape[0]\n            wav_tensor = wav_tensor[: n_samples // parts * parts].reshape(\n                parts, n_samples // parts\n            )\n        \n        #print(wav_tensor.shape)\n        feature_dict = {\n            \"input\": wav_tensor,\n            \"target\": torch.tensor(label.astype(np.float32)),\n            \"weight\": torch.tensor(weight),\n            \"fold\": torch.tensor(fold),\n        }\n        return feature_dict\n\n    def __len__(self):\n        if cfg.dev:\n            return 256\n        return len(self.fns)\n\n    def load_one(self, id_, offset, duration):\n        fp =  id_\n        try:\n            wav, sr = librosa.load(fp, sr=None, offset=offset, duration=duration)\n            if sr != cfg.sample_rate:\n                wav = librosa.resample(wav, orig_sr=sr, target_sr=cfg.sample_rate)\n        except:\n            print(\"FAIL READING rec\", fp)\n\n        return wav\n\n    def birds2target(self, birds):\n        birds = birds.split()\n        target = [self.bird2id.get(item) for item in birds if not item == \"nocall\"]\n        return target\n\n    def secondary2target(self, secondary_label):\n        birds = ast.literal_eval(secondary_label)\n        target = [self.bird2id.get(item) for item in birds if not item == \"nocall\"]\n        return target\n","metadata":{"execution":{"iopub.status.busy":"2023-04-24T09:48:26.183446Z","iopub.execute_input":"2023-04-24T09:48:26.183702Z","iopub.status.idle":"2023-04-24T09:48:26.207989Z","shell.execute_reply.started":"2023-04-24T09:48:26.183669Z","shell.execute_reply":"2023-04-24T09:48:26.207129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset related utils","metadata":{}},{"cell_type":"code","source":"def worker_init_fn(worker_id):\n    np.random.seed(np.random.get_state()[1][0] + worker_id)\n\n\ndef get_train_dataloader(train_ds, cfg):\n    train_dataloader = DataLoader(\n        train_ds,\n        sampler=None,\n        shuffle=True,\n        batch_size=cfg.batch_size,\n        num_workers=cfg.num_workers,\n        pin_memory=False,\n        collate_fn=cfg.tr_collate_fn,\n        drop_last=cfg.drop_last,\n        worker_init_fn=worker_init_fn,\n    )\n    print(f\"train: dataset {len(train_ds)}, dataloader {len(train_dataloader)}\")\n    return train_dataloader\n\n\ndef get_scheduler(cfg, optimizer, total_steps):\n    scheduler = get_cosine_schedule_with_warmup(\n        optimizer,\n        num_warmup_steps=cfg.warmup * (total_steps // cfg.batch_size),\n        num_training_steps=cfg.epochs * (total_steps // cfg.batch_size),\n    )\n    return scheduler\n\ndef get_file_duration(file_path):\n    sound, sr = librosa.load(file_path)\n    \n    return librosa.get_duration(y=sound, sr=sr)\n\ndef load_df(df):\n    train_df1 = df\n    print('*****before filter shape *****')\n    print(train_df1.shape)\n    train_df1 = train_df1[train_df1['duration']!=0]\n    print('*****after filter shape *****')\n    print(train_df1.shape)\n    #train_df1['duration'] = train_df1['filename'].apply(lambda x:get_file_duration(cfg.train_data_folder+x))\n    #train_df2 = pd.read_csv(cfg.train_df2)\n    #train_df = pd.merge(train_df1[['primary_label', 'secondary_labels', 'rating', 'filename']], train_df2[['filename', 'duration']], how='inner', on='filename')\n    return train_df1\n","metadata":{"execution":{"iopub.status.busy":"2023-04-24T09:48:26.209311Z","iopub.execute_input":"2023-04-24T09:48:26.209599Z","iopub.status.idle":"2023-04-24T09:48:26.221553Z","shell.execute_reply.started":"2023-04-24T09:48:26.209563Z","shell.execute_reply":"2023-04-24T09:48:26.220699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GeM and Mix-up","metadata":{}},{"cell_type":"code","source":"def gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1.0 / p)\n\n\nclass GeM(nn.Module):\n    # Generalized mean: https://arxiv.org/abs/1711.02512\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM, self).__init__()\n        self.p = Parameter(torch.ones(1) * p)\n        self.eps = eps\n\n    def forward(self, x):\n        ret = gem(x, p=self.p, eps=self.eps)\n        return ret\n\n    def __repr__(self):\n        return (self.__class__.__name__+ \"(p=\"+ \"{:.4f}\".format(self.p.data.tolist()[0])+ \", eps=\"+ str(self.eps)+ \")\")\n\n\nclass Mixup(nn.Module):\n    def __init__(self, mix_beta):\n\n        super(Mixup, self).__init__()\n        self.beta_distribution = Beta(mix_beta, mix_beta)\n\n    def forward(self, X, Y, weight=None):\n\n        bs = X.shape[0]\n        n_dims = len(X.shape)\n        perm = torch.randperm(bs)\n        coeffs = self.beta_distribution.rsample(torch.Size((bs,))).to(X.device)\n\n        if n_dims == 2:\n            X = coeffs.view(-1, 1) * X + (1 - coeffs.view(-1, 1)) * X[perm]\n        elif n_dims == 3:\n            X = coeffs.view(-1, 1, 1) * X + (1 - coeffs.view(-1, 1, 1)) * X[perm]\n        else:\n            X = coeffs.view(-1, 1, 1, 1) * X + (1 - coeffs.view(-1, 1, 1, 1)) * X[perm]\n\n        Y = coeffs.view(-1, 1) * Y + (1 - coeffs.view(-1, 1)) * Y[perm]\n\n        if weight is None:\n            return X, Y\n        else:\n            weight = coeffs.view(-1) * weight + (1 - coeffs.view(-1)) * weight[perm]\n            return X, Y, weight\n","metadata":{"execution":{"iopub.status.busy":"2023-04-24T09:48:26.222820Z","iopub.execute_input":"2023-04-24T09:48:26.223244Z","iopub.status.idle":"2023-04-24T09:48:26.239262Z","shell.execute_reply.started":"2023-04-24T09:48:26.223209Z","shell.execute_reply":"2023-04-24T09:48:26.238459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self, cfg):\n        super(Net, self).__init__()\n\n        self.cfg = cfg\n\n        self.n_classes = cfg.n_classes\n\n        self.mel_spec = ta.transforms.MelSpectrogram(\n            sample_rate=cfg.sample_rate,\n            n_fft=cfg.window_size,\n            win_length=cfg.window_size,\n            f_min=cfg.fmin,\n            f_max=cfg.fmax,\n            pad=0,\n            n_mels=cfg.mel_bins,\n            hop_length=cfg.hop_size,\n            power=cfg.power,\n            normalized=False,\n        )\n\n        self.amplitude_to_db = ta.transforms.AmplitudeToDB(top_db=cfg.top_db)\n        self.wav2img = torch.nn.Sequential(self.mel_spec, self.amplitude_to_db)\n\n        self.backbone = timm.create_model(\n            cfg.backbone,\n            pretrained=cfg.pretrained,\n            num_classes=0,\n            global_pool=\"\",\n            in_chans=cfg.in_chans,\n        )\n\n        if \"efficientnet\" in cfg.backbone:\n            backbone_out = self.backbone.num_features\n        else:\n            backbone_out = self.backbone.feature_info[-1][\"num_chs\"]\n\n        self.global_pool = GeM()\n\n        self.head = nn.Linear(backbone_out, self.n_classes)\n\n        if cfg.pretrained_weights is not None:\n            sd = torch.load(cfg.pretrained_weights, map_location=\"cpu\")[\"model\"]\n            sd = {k.replace(\"module.\", \"\"): v for k, v in sd.items()}\n            self.load_state_dict(sd, strict=True)\n            print(\"weights loaded from\", cfg.pretrained_weights)\n        self.loss_fn = nn.BCEWithLogitsLoss(reduction=\"none\")\n\n        self.mixup = Mixup(mix_beta=cfg.mix_beta)\n\n        self.factor = int(cfg.wav_crop_len / 5.0)\n\n    def forward(self, batch):\n\n        if not self.training:\n            x = batch[\"input\"]\n            bs, parts, time = x.shape\n            x = x.reshape(parts, time)\n            y = batch[\"target\"]\n            y = y[0]\n        else:\n\n            x = batch[\"input\"]\n            y = batch[\"target\"]\n            bs, time = x.shape\n            x = x.reshape(bs * self.factor, time // self.factor)\n\n        with autocast(enabled=False):\n            x = self.wav2img(x)  # (bs, mel, time)\n            if self.cfg.mel_norm:\n                x = (x + 80) / 80\n\n        x = x.permute(0, 2, 1)\n        x = x[:, None, :, :]\n\n        weight = batch[\"weight\"]\n\n        if self.training:\n            b, c, t, f = x.shape\n            x = x.permute(0, 2, 1, 3)\n            x = x.reshape(b // self.factor, self.factor * t, c, f)\n\n            if self.cfg.mixup:\n                x, y, weight = self.mixup(x, y, weight)\n            if self.cfg.mixup2:\n                x, y, weight = self.mixup(x, y, weight)\n\n            x = x.reshape(b, t, c, f)\n            x = x.permute(0, 2, 1, 3)\n\n        x = self.backbone(x)\n\n        if self.training:\n            b, c, t, f = x.shape\n            x = x.permute(0, 2, 1, 3)\n            x = x.reshape(b // self.factor, self.factor * t, c, f)\n            x = x.permute(0, 2, 1, 3)\n        x = self.global_pool(x)\n        x = x[:, :, 0, 0]\n        logits = self.head(x)\n\n        loss = self.loss_fn(logits, y)\n        loss = (loss.mean(dim=1) * weight) / weight.sum()\n        loss = loss.sum()\n\n        return {\"loss\": loss, \"logits\": logits.sigmoid(), \"logits_raw\": logits, \"target\": y}\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-24T09:48:26.240910Z","iopub.execute_input":"2023-04-24T09:48:26.241169Z","iopub.status.idle":"2023-04-24T09:48:26.260871Z","shell.execute_reply.started":"2023-04-24T09:48:26.241136Z","shell.execute_reply":"2023-04-24T09:48:26.259938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_checkpoint(model, optimizer, epoch, scheduler=None, scaler=None):\n    checkpoint = {\n        \"model\": model.state_dict(),\n        \"optimizer\": optimizer.state_dict(),\n        \"epoch\": epoch,\n    }\n\n    if scheduler is not None:\n        checkpoint[\"scheduler\"] = scheduler.state_dict()\n\n    if scaler is not None:\n        checkpoint[\"scaler\"] = scaler.state_dict()\n    return checkpoint","metadata":{"execution":{"iopub.status.busy":"2023-04-24T09:48:26.262062Z","iopub.execute_input":"2023-04-24T09:48:26.262677Z","iopub.status.idle":"2023-04-24T09:48:26.272537Z","shell.execute_reply.started":"2023-04-24T09:48:26.262640Z","shell.execute_reply":"2023-04-24T09:48:26.271672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nset_seed(cfg.seed)\n\n\n#val_df = pd.read_csv(cfg.val_df)\n#val_dataset = CustomDataset(val_df, cfg, aug=cfg.val_aug, mode=\"val\")\n\ntrain_df = load_df(df_pre)\n\ntrain_dataset = CustomDataset(train_df, cfg, aug=cfg.train_aug, mode=\"train\")\ntrain_dataloader = get_train_dataloader(train_dataset, cfg)\nmodel = Net(cfg)\n\nmodel.to(cfg.device)\n\ntotal_steps = len(train_dataset)\n\nparams = model.parameters()\noptimizer = optim.AdamW(params, lr=cfg.lr, weight_decay=0)\nscheduler = get_scheduler(cfg, optimizer, total_steps)\n\n\ndevice = cfg.device","metadata":{"execution":{"iopub.status.busy":"2023-04-24T09:48:26.274064Z","iopub.execute_input":"2023-04-24T09:48:26.274351Z","iopub.status.idle":"2023-04-24T09:48:30.170870Z","shell.execute_reply.started":"2023-04-24T09:48:26.274315Z","shell.execute_reply":"2023-04-24T09:48:30.170051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!ls -l /root/.cache/torch/hub/checkpoints/","metadata":{"execution":{"iopub.status.busy":"2023-04-24T09:48:30.172266Z","iopub.execute_input":"2023-04-24T09:48:30.172530Z","iopub.status.idle":"2023-04-24T09:48:30.176066Z","shell.execute_reply.started":"2023-04-24T09:48:30.172494Z","shell.execute_reply":"2023-04-24T09:48:30.175366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!cp /root/.cache/torch/hub/checkpoints/resnet34-43635321.pth .","metadata":{"execution":{"iopub.status.busy":"2023-04-24T09:48:30.177358Z","iopub.execute_input":"2023-04-24T09:48:30.177838Z","iopub.status.idle":"2023-04-24T09:48:30.427267Z","shell.execute_reply.started":"2023-04-24T09:48:30.177804Z","shell.execute_reply":"2023-04-24T09:48:30.426497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    os.makedirs(cfg.output_dir)\nexcept:\n    pass\n\nif cfg.mixed_precision:\n    scaler = GradScaler()\nelse:\n    scaler = None\n\n\ncfg.curr_step = 0\ni = 0\nbest_val_loss = np.inf\noptimizer.zero_grad()\nfor epoch in range(cfg.epochs):\n\n    set_seed(cfg.seed + epoch)\n\n    cfg.curr_epoch = epoch\n\n    print(\"EPOCH:\", epoch)\n\n    progress_bar = tqdm(range(len(train_dataloader)))\n    tr_it = iter(train_dataloader)\n\n    losses = []\n\n    gc.collect()\n\n    if cfg.train:\n        # ==== TRAIN LOOP\n        for itr in progress_bar:\n            i += 1\n\n            cfg.curr_step += cfg.batch_size\n\n            data = next(tr_it)\n\n            model.train()\n            torch.set_grad_enabled(True)\n\n            batch = batch_to_device(data, device)\n\n            if cfg.mixed_precision:\n                with autocast():\n                    output_dict = model(batch)\n            else:\n                output_dict = model(batch)\n\n            loss = output_dict[\"loss\"]\n\n            losses.append(loss.item())\n\n            if cfg.mixed_precision:\n                scaler.scale(loss).backward()\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad()\n            else:\n                loss.backward()\n                optimizer.step()\n                optimizer.zero_grad()\n\n            if scheduler is not None:\n                scheduler.step()\n\n            if cfg.curr_step % cfg.batch_size == 0:\n                progress_bar.set_description(f\"loss: {np.mean(losses[-10:]):.4f}\")\n\n    if cfg.val:\n        if (epoch + 1) % cfg.eval_epochs == 0 or (epoch + 1) == cfg.epochs:\n            val_loss = run_eval(model, val_dataloader, cfg)\n        else:\n            val_score = 0\n\n    if cfg.epochs > 0:\n        checkpoint = create_checkpoint(\n            model, optimizer, epoch, scheduler=scheduler, scaler=scaler\n        )\n\n        torch.save(checkpoint, f\"{cfg.output_dir}/checkpoint_last_seed{cfg.seed}_{epoch}.pth\")\n\nif cfg.epochs > 0:\n    checkpoint = create_checkpoint(model, optimizer, epoch, scheduler=scheduler, scaler=scaler)\n\n    torch.save(checkpoint, f\"{cfg.output_dir}/checkpoint_last_seed{cfg.seed}.pth\")","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-04-24T09:48:30.428690Z","iopub.execute_input":"2023-04-24T09:48:30.429128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}