{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!nvidia-smi","metadata":{"id":"C0vy_lvxVQf5","outputId":"0ad3c95b-49a6-4e37-c750-8b7f79e76f57","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q pysndfx SoundFile audiomentations pretrainedmodels efficientnet_pytorch resnest","metadata":{"id":"Yn1Ybf15VAqW","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport librosa as lb\nimport librosa.display as lbd\nimport soundfile as sf\nfrom  soundfile import SoundFile\nimport pandas as pd\nfrom  IPython.display import Audio\nfrom pathlib import Path\n\nimport torch\nfrom torch import nn, optim\nfrom  torch.utils.data import Dataset, DataLoader\n\nfrom resnest.torch import resnest50\n\nfrom matplotlib import pyplot as plt\n\nimport os, random, gc\nimport re, time, json\nfrom  ast import literal_eval\n\n\nfrom IPython.display import Audio\nfrom sklearn.metrics import label_ranking_average_precision_score,f1_score\n\nfrom tqdm.notebook import tqdm\nimport joblib\n\nimport gc\nimport time\nimport datetime\n\nfrom torch.optim import Adam\nfrom transformers import get_linear_schedule_with_warmup\n\nNUM_WORKERS = 4\n\nTODAY = str(datetime.date.today())\nCP_TODAY = f\"/kaggle/working/{TODAY}/\"\n\nif not os.path.exists(CP_TODAY):\n    os.mkdir(CP_TODAY)","metadata":{"id":"2dt7oG43VAqc","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet\nimport pretrainedmodels\nimport resnest.torch as resnest_torch","metadata":{"id":"162Vl9uxe1Mj","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything()","metadata":{"id":"Q39ZsGAhVAqe","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_CLASSES = 397\nSR = 32_000\nDURATION = 7\n\nMAX_READ_SAMPLES = 5 \n\nDATA_ROOT = Path(\"../input/birdclef-2021\")\nMEL_PATHS = sorted(Path(\"../input\").glob(\"kkiller-birdclef-mels-computer-d7-part?/rich_train_metadata.csv\"))\nTRAIN_LABEL_PATHS = sorted(Path(\"../input\").glob(\"kkiller-birdclef-mels-computer-d7-part?/LABEL_IDS.json\"))\n\nMODEL_ROOT = Path(\".\")","metadata":{"id":"2NfkUn9SCWs6","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_BATCH_SIZE = 100\nTRAIN_NUM_WORKERS = 2\n\nVAL_BATCH_SIZE = 128\nVAL_NUM_WORKERS = 2\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nprint(\"Device:\", DEVICE)","metadata":{"id":"Iu56f-7VVAqf","outputId":"0f3fa344-0ed4-47d8-f3c0-218cf3bf5a78","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_df(mel_paths=MEL_PATHS, train_label_paths=TRAIN_LABEL_PATHS):\n  df = None\n  LABEL_IDS = {}\n    \n  for file_path in mel_paths:\n    temp = pd.read_csv(str(file_path), index_col=0)\n    temp[\"impath\"] = temp.apply(lambda row: file_path.parent/\"audio_images/{}/{}.npy\".format(row.primary_label, row.filename), axis=1) \n    df = temp if df is None else df.append(temp)\n    \n  df[\"secondary_labels\"] = df[\"secondary_labels\"].apply(literal_eval)\n\n  for file_path in train_label_paths:\n    with open(str(file_path)) as f:\n      LABEL_IDS.update(json.load(f))\n\n  return LABEL_IDS, df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LABEL_IDS, df = get_df()\ndf[\"primary_label\"].value_counts()\ndf[\"label_id\"].min(), df[\"label_id\"].max()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(name, num_classes=NUM_CLASSES):\n    \"\"\"\n    Loads a pretrained model. \n    Supports ResNest, ResNext-wsl, EfficientNet, ResNext and ResNet.\n\n    Arguments:\n        name {str} -- Name of the model to load\n\n    Keyword Arguments:\n        num_classes {int} -- Number of classes to use (default: {1})\n\n    Returns:\n        torch model -- Pretrained model\n    \"\"\"\n    if \"resnest\" in name:\n        model = getattr(resnest_torch, name)(pretrained=True)\n    elif \"wsl\" in name:\n        model = torch.hub.load(\"facebookresearch/WSL-Images\", name)\n    elif name.startswith(\"resnext\") or  name.startswith(\"resnet\"):\n        model = torch.hub.load(\"pytorch/vision:v0.6.0\", name, pretrained=True)\n    elif name.startswith(\"tf_efficientnet_b\"):\n        model = getattr(timm.models.efficientnet, name)(pretrained=True)\n    elif \"efficientnet-b\" in name:\n        model = EfficientNet.from_pretrained(name)\n    else:\n        model = pretrainedmodels.__dict__[name](pretrained='imagenet')\n\n    if hasattr(model, \"fc\"):\n        nb_ft = model.fc.in_features\n        model.fc = nn.Linear(nb_ft, num_classes)\n    elif hasattr(model, \"_fc\"):\n        nb_ft = model._fc.in_features\n        model._fc = nn.Linear(nb_ft, num_classes)\n    elif hasattr(model, \"classifier\"):\n        nb_ft = model.classifier.in_features\n        model.classifier = nn.Linear(nb_ft, num_classes)\n    elif hasattr(model, \"last_linear\"):\n        nb_ft = model.last_linear.in_features\n        model.last_linear = nn.Linear(nb_ft, num_classes)\n\n    return model","metadata":{"id":"OGPDuihmVAqi","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data(df):\n    def load_row(row):\n        # impath = TRAIN_IMAGES_ROOT/f\"{row.primary_label}/{row.filename}.npy\"\n        return row.filename, np.load(str(row.impath))[:MAX_READ_SAMPLES]\n    pool = joblib.Parallel(4)\n    mapper = joblib.delayed(load_row)\n    tasks = [mapper(row) for row in df.itertuples(False)]\n    res = pool(tqdm(tasks))\n    res = dict(res)\n    return res","metadata":{"id":"7HYQwAyBCWs8","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# audio_image_store = load_data(df)\n# len(audio_image_store)","metadata":{"id":"Vw19bB7mCWs9","outputId":"09a5e374-7e5c-4c92-91e4-b60313cbb1a9","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BirdClefDataset(Dataset):\n\n    def __init__(self, meta, sr=SR, is_train=True, num_classes=NUM_CLASSES, duration=DURATION):\n        \n#         self.audio_image_store = audio_image_store\n        self.meta = meta.copy().reset_index(drop=True)\n        self.sr = sr\n        self.is_train = is_train\n        self.num_classes = num_classes\n        self.duration = duration\n        self.audio_length = self.duration*self.sr\n        self.y = self.meta.label_id.values\n    @staticmethod\n    def normalize(image):\n        image = image.astype(\"float32\", copy=False) / 255.0\n        image = np.stack([image, image, image])\n        return image\n\n    def __len__(self):\n        return len(self.meta)\n    \n    def __getitem__(self, idx):\n        row = self.meta.iloc[idx]\n        image = np.load(str(row.impath))[:MAX_READ_SAMPLES]\n\n        image = image[np.random.choice(len(image))]\n        image = self.normalize(image)\n        \n        \n        t = np.zeros(self.num_classes, dtype=np.float32) + 0.0025 # Label smoothing\n        t[row.label_id] = 0.995\n        \n        return image, t","metadata":{"id":"OWSkCXyhCWs-","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ONE_HOT = np.eye(NUM_CLASSES)\n\ndef f1(truth, pred, threshold=0.5, avg=\"samples\"):\n    \"\"\"\n    The f1 metric for the problem\n    Arguments:\n        truth {np array [N] or [N x C]} -- Ground truths\n        pred {np array [N x C]} -- Predicted probabilites\n    Keyword Arguments:\n        threshold {float} -- Threshold for classification (default: {0.5})\n        avg {str} -- How to perform average in the f1 score (default: {\"samples\"})\n    Returns:\n        float -- f1 score\n    \"\"\"\n\n    if len(truth.shape) == 1:\n        truth = ONE_HOT[truth]\n\n    pred = (pred > threshold).astype(int)\n\n    return f1_score(truth, pred, average=avg)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mixup_data(x, y, alpha=0.4):\n    \"\"\"\n    Applies mixup to a sample\n    Arguments:\n        x {torch tensor} -- Input batch\n        y {torch tensor} -- Labels\n    Keyword Arguments:\n        alpha {float} -- Parameter of the beta distribution (default: {0.4})\n    Returns:\n        torch tensor  -- Mixed input\n        torch tensor  -- Labels of the original batch\n        torch tensor  -- Labels of the shuffle batch\n        float  -- Probability samples by the beta distribution\n    \"\"\"\n    lam = np.random.beta(alpha, alpha) if alpha > 0 else 1\n\n    index = torch.randperm(x.size()[0]).cuda()\n\n    mixed_x = lam * x + (1 - lam) * x[index, :]\n    y_a, y_b = y, y[index]\n\n    return mixed_x, y_a, y_b, lam","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit(\n    model,\n    train_dataset,\n    val_dataset,\n    epochs=50,\n    batch_size=32,\n    val_bs=32,\n    warmup_prop=0.1,\n    lr=1e-3,\n    alpha=0.4,\n    mixup_proba=0.0,\n    verbose=1,\n    verbose_eval=1,\n    epochs_eval_min=0,\n):\n    \"\"\"\n    Usual torch fit function\n    \n    Arguments:\n        model {torch model} -- Model to train\n        train_dataset {torch dataset} -- Dataset to train with\n        val_dataset {torch dataset} -- Dataset to validate with\n    \n    Keyword Arguments:\n        epochs {int} -- Number of epochs (default: {50})\n        batch_size {int} -- Training batch size (default: {32})\n        val_bs {int} -- Validation batch size (default: {32})\n        warmup_prop {float} -- Warmup proportion (default: {0.1})\n        lr {float} -- Start (or maximum) learning rate (default: {1e-3})\n        alpha {float} -- alpha value for mixup (default: {0.4})\n        mixup_proba {float} -- Probability to apply mixup (default: {0.})\n        verbose {int} -- Period (in epochs) to display logs at (default: {1})\n        verbose_eval {int} -- Period (in epochs) to perform evaluation at (default: {1})\n        epochs_eval_min {int} -- Number of epochs to start evaluating from (default: {0})\n    Returns:\n        numpy array -- Predictions at the last epoch\n    \"\"\"\n\n    avg_val_loss = 0\n    avg_loss = 0\n    score = 0\n\n    optimizer = Adam(model.parameters(), lr=lr)\n\n    loss_fct = nn.BCEWithLogitsLoss(reduction=\"mean\").cuda()\n\n    train_loader = DataLoader(\n        train_dataset,\n        batch_size=batch_size,\n        shuffle=True,\n        drop_last=True,\n        num_workers=NUM_WORKERS,\n    )\n    val_loader = DataLoader(\n        val_dataset, batch_size=val_bs, shuffle=False, num_workers=NUM_WORKERS\n    )\n\n    num_warmup_steps = int(warmup_prop * epochs * len(train_loader))\n    num_training_steps = int(epochs * len(train_loader))\n    scheduler = get_linear_schedule_with_warmup(\n        optimizer, num_warmup_steps, num_training_steps\n    )\n\n    for epoch in range(epochs):\n        model.train()\n        start_time = time.time()\n        optimizer.zero_grad()\n\n        avg_loss = 0\n        for step, (x, y_batch) in tqdm(enumerate(train_loader),total=len(train_loader)):\n            if np.random.rand() < mixup_proba:\n                x, y_a, y_b, _ = mixup_data(x.cuda(), y_batch.cuda(), alpha=alpha)\n                y_batch = torch.clamp(y_a + y_b, 0, 1)\n\n            y_pred = model(x.cuda())\n\n            loss = loss_fct(y_pred, y_batch.cuda().float())\n            loss.backward()\n            avg_loss += loss.item() / len(train_loader)\n\n            optimizer.step()\n            optimizer.zero_grad()\n            scheduler.step()\n\n        do_eval = ((epoch + 1) % verbose_eval == 0 and epoch >= epochs_eval_min) or (\n            epoch + 1 == epochs\n        )\n        if do_eval:\n            model.eval()\n\n            avg_val_loss = 0.0\n            with torch.no_grad():\n                preds = np.empty((0, NUM_CLASSES))\n                for x, y_batch in tqdm(val_loader,total=len(val_loader)):\n                    y_pred = model(x.cuda()).detach()\n                    loss = loss_fct(y_pred, y_batch.cuda().float())\n                    avg_val_loss += loss.item() / len(val_loader)\n\n                    preds = np.concatenate([preds, torch.sigmoid(y_pred).cpu().numpy()])\n\n            micro_f1 = f1(val_dataset.y, preds, avg=\"micro\")\n            samples_f1 = f1(val_dataset.y, preds)\n\n        elapsed_time = time.time() - start_time\n        if (epoch + 1) % verbose == 0:\n            elapsed_time = elapsed_time * verbose\n            lr = scheduler.get_lr()[0]\n            print(\n                f\"Epoch {epoch + 1}/{epochs} \\t lr={lr:.1e} \\t t={elapsed_time:.0f}s  \\t loss={avg_loss:.4f} \\t \",\n                end=\"\",\n            )\n            if do_eval:\n                print(\n                    f\"val_loss={avg_val_loss:.4f} \\t micro_f1={micro_f1:.3f} \\t samples_f1={samples_f1:.3f}\"\n                )\n            else:\n                print(\"\")\n\n    torch.cuda.empty_cache()\n    return preds\n\n\ndef predict(model, dataset, batch_size=64):\n    \"\"\"\n    Usual torch predict function\n    Arguments:\n        model {torch model} -- Model to predict with\n        dataset {torch dataset} -- Dataset to predict with on\n    Keyword Arguments:\n        batch_size {int} -- Batch size (default: {32})\n    Returns:\n        numpy array -- Predictions\n    \"\"\"\n    model.eval()\n    preds = np.empty((0, NUM_CLASSES))\n\n    loader = DataLoader(\n        dataset, batch_size=batch_size, shuffle=False, num_workers=NUM_WORKERS\n    )\n    with torch.no_grad():\n        for x, _ in loader:\n            y_pred = model(x.cuda()).detach()\n            preds = np.concatenate([preds, torch.sigmoid(y_pred).cpu().numpy()])\n\n    return preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    selected_model = 'resnest50'\n    SR = 32000\n    DURATION = 7\n    batch_size = 64\n    val_bs = 128\n    lr = 1e-3\n    warmup_prop = 0.05\n    \n    if \"101\" in selected_model or \"b5\" in selected_model or \"b6\" in selected_model:\n        batch_size = batch_size // 2\n        lr = lr / 2\n\n    mixup_proba = 0.5\n    alpha = 5\n    \n    verbose_eval = 1\n    epochs_eval_min = 25\n    epochs = 30\n    save = True\n    \n    name = \"birdcall\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(config,df,fold):\n    \"\"\"\n    Trains and validate a model\n    Arguments:\n        config {Config} -- Parameters\n        df_train {pandas dataframe} -- Training metadata\n        df_val {pandas dataframe} -- Validation metadata\n        fold {int} -- Selected fold\n    Returns:\n        np array -- Validation predictions\n    \"\"\"\n    \n    \n    df_train = df[df.fold != fold].reset_index(drop=True)\n    df_val = df[df.fold == fold].reset_index(drop=True)\n    print(f\"    -> {len(df_train)} training birds\")\n    print(f\"    -> {len(df_val)} validation birds\")\n\n    model = get_model(config.selected_model, num_classes=NUM_CLASSES).cuda()\n    model.zero_grad()\n\n    train_dataset = BirdClefDataset(meta=df_train,sr=config.SR, duration=config.DURATION, is_train=True)\n    val_dataset = BirdClefDataset(meta=df_val,sr=config.SR, duration=config.DURATION, is_train=False)\n\n    n_parameters = count_parameters(model)\n    print(f\"    -> {n_parameters} trainable parameters\\n\")\n\n    pred_val = fit(\n        model,\n        train_dataset,\n        val_dataset,\n        epochs=config.epochs,\n        batch_size=config.batch_size,\n        val_bs=config.val_bs,\n        lr=config.lr,\n        warmup_prop=config.warmup_prop,\n        alpha=config.alpha,\n        mixup_proba=config.mixup_proba,\n        verbose_eval=config.verbose_eval,\n        epochs_eval_min=config.epochs_eval_min,\n    )\n\n    if config.save:\n        save_model_weights(\n            model,\n            f\"{config.selected_model}_{config.name}_{fold}.pt\",\n            cp_folder=CP_TODAY,\n        )\n\n    return pred_val","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_model_weights(model, filename, verbose=1, cp_folder=\"\"):\n    \"\"\"\n    Saves the weights of a PyTorch model\n    \n    Arguments:\n        model {torch module} -- Model to save the weights of\n        filename {str} -- Name of the checkpoint\n    \n    Keyword Arguments:\n        verbose {int} -- Whether to display infos (default: {1})\n        cp_folder {str} -- Folder to save to (default: {''})\n    \"\"\"\n    if verbose:\n        print(f\"\\n -> Saving weights to {os.path.join(cp_folder, filename)}\\n\")\n    torch.save(model.state_dict(), os.path.join(cp_folder, filename))\n\n\ndef load_model_weights(model, filename, verbose=1, cp_folder=\"\"):\n    \"\"\"\n    Loads the weights of a PyTorch model. The exception handles cpu/gpu incompatibilities\n    \n    Arguments:\n        model {torch module} -- Model to load the weights to\n        filename {str} -- Name of the checkpoint\n    \n    Keyword Arguments:\n        verbose {int} -- Whether to display infos (default: {1})\n        cp_folder {str} -- Folder to load from (default: {''})\n    \n    Returns:\n        torch module -- Model with loaded weights\n    \"\"\"\n    if verbose:\n        print(f\"\\n -> Loading weights from {os.path.join(cp_folder,filename)}\\n\")\n    try:\n        model.load_state_dict(os.path.join(cp_folder, filename), strict=strict)\n    except BaseException:\n        model.load_state_dict(\n            torch.load(os.path.join(cp_folder, filename), map_location=\"cpu\"),\n            strict=True,\n        )\n    return model\n\n\ndef count_parameters(model, all=False):\n    \"\"\"\n    Count the parameters of a model\n    \n    Arguments:\n        model {torch module} -- Model to count the parameters of\n    \n    Keyword Arguments:\n        all {bool} -- Whether to include not trainable parameters in the sum (default: {False})\n    \n    Returns:\n        int -- Number of parameters\n    \"\"\"\n    if all:\n        return sum(p.numel() for p in model.parameters())\n    else:\n        return sum(p.numel() for p in model.parameters() if p.requires_grad)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = Config()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train(config,df,0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}