{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook uses some code from a wonderful last-year BirdCLEF competition notebook by kkiller: \nhttps://www.kaggle.com/kneroma/clean-fast-simple-bird-identifier-training-colab\n\n**Please upvote it as well!**\n\nHowever, I'm using PyTorch Lightning here to simplify the code.\n\nAnother note: this notebook trains model on all 152 available classes. However, only 21 classes will be scored, so maybe I'll change the model later.","metadata":{}},{"cell_type":"markdown","source":"## Install and import packages","metadata":{}},{"cell_type":"code","source":"!pip install -q resnest","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:16:36.733202Z","iopub.execute_input":"2022-02-16T11:16:36.733978Z","iopub.status.idle":"2022-02-16T11:16:46.161243Z","shell.execute_reply.started":"2022-02-16T11:16:36.733832Z","shell.execute_reply":"2022-02-16T11:16:46.160369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport json\nimport joblib\nfrom pathlib import Path\nfrom ast import literal_eval\nfrom tqdm.notebook import tqdm\n\nimport torch\nfrom torch import nn, optim\nfrom torch.nn import functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks.model_checkpoint import ModelCheckpoint\nfrom torchmetrics import F1\n\nimport resnest.torch as resnest_torch","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-16T11:16:46.163371Z","iopub.execute_input":"2022-02-16T11:16:46.163648Z","iopub.status.idle":"2022-02-16T11:16:49.543559Z","shell.execute_reply.started":"2022-02-16T11:16:46.163611Z","shell.execute_reply":"2022-02-16T11:16:49.542726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 42\n\nMEL_PATHS = sorted(Path(\"../input\").glob(\"birdclef-2022-melspectrogram-compute/rich_train_metadata.csv\"))\nTRAIN_LABEL_PATHS = sorted(Path(\"../input\").glob(\"birdclef-2022-melspectrogram-compute/LABEL_IDS.json\"))\n\nN_CLASSES = 152\nSR = 32_000\nDURATION = 7\n\nMAX_READ_SAMPLES = 5 \n\nUSE_FOLD = 0\n\nTRAIN_BATCH_SIZE = 100\nTRAIN_NUM_WORKERS = 2\n\nVAL_BATCH_SIZE = 128\nVAL_NUM_WORKERS = 2\n\nEPOCHS = 15","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:16:49.545152Z","iopub.execute_input":"2022-02-16T11:16:49.546159Z","iopub.status.idle":"2022-02-16T11:16:49.559112Z","shell.execute_reply.started":"2022-02-16T11:16:49.546099Z","shell.execute_reply":"2022-02-16T11:16:49.558453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Process dataset","metadata":{}},{"cell_type":"code","source":"def get_df(mel_paths=MEL_PATHS, train_label_paths=TRAIN_LABEL_PATHS):\n    df = None\n    LABEL_IDS = {}\n\n    for file_path in mel_paths:\n        temp = pd.read_csv(str(file_path), index_col=0)\n        temp[\"impath\"] = temp.apply(lambda row: file_path.parent/\"audio_images/{}.npy\".format(row.filename), axis=1) \n        df = temp if df is None else df.append(temp)\n\n        df[\"secondary_labels\"] = df[\"secondary_labels\"].apply(literal_eval)\n\n    for file_path in train_label_paths:\n        with open(str(file_path)) as f:\n            LABEL_IDS.update(json.load(f))\n\n    return LABEL_IDS, df","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:16:49.561341Z","iopub.execute_input":"2022-02-16T11:16:49.561633Z","iopub.status.idle":"2022-02-16T11:16:49.568793Z","shell.execute_reply.started":"2022-02-16T11:16:49.561594Z","shell.execute_reply":"2022-02-16T11:16:49.567861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(name, num_classes=N_CLASSES):\n    \"\"\"\n    Loads a pretrained model. \n    Supports ResNest, ResNext-wsl, EfficientNet, ResNext and ResNet.\n\n    Arguments:\n        name {str} -- Name of the model to load\n\n    Keyword Arguments:\n        num_classes {int} -- Number of classes to use (default: {1})\n\n    Returns:\n        torch model -- Pretrained model\n    \"\"\"\n    if \"resnest\" in name:\n        model = getattr(resnest_torch, name)(pretrained=False) # getting 403 error when trying to download weights\n        model.load_state_dict(torch.load('../input/resnest50/resnest50-528c19ca.pth')) # so let's load them manually\n    elif \"wsl\" in name:\n        model = torch.hub.load(\"facebookresearch/WSL-Images\", name)\n    elif name.startswith(\"resnext\") or  name.startswith(\"resnet\"):\n        model = torch.hub.load(\"pytorch/vision:v0.6.0\", name, pretrained=True)\n    elif name.startswith(\"tf_efficientnet_b\"):\n        model = getattr(timm.models.efficientnet, name)(pretrained=True)\n    elif \"efficientnet-b\" in name:\n        model = EfficientNet.from_pretrained(name)\n    else:\n        model = pretrainedmodels.__dict__[name](pretrained='imagenet')\n\n    if hasattr(model, \"fc\"):\n        nb_ft = model.fc.in_features\n        model.fc = nn.Linear(nb_ft, num_classes)\n    elif hasattr(model, \"_fc\"):\n        nb_ft = model._fc.in_features\n        model._fc = nn.Linear(nb_ft, num_classes)\n    elif hasattr(model, \"classifier\"):\n        nb_ft = model.classifier.in_features\n        model.classifier = nn.Linear(nb_ft, num_classes)\n    elif hasattr(model, \"last_linear\"):\n        nb_ft = model.last_linear.in_features\n        model.last_linear = nn.Linear(nb_ft, num_classes)\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:16:49.570399Z","iopub.execute_input":"2022-02-16T11:16:49.570904Z","iopub.status.idle":"2022-02-16T11:16:49.582762Z","shell.execute_reply.started":"2022-02-16T11:16:49.570868Z","shell.execute_reply":"2022-02-16T11:16:49.581969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data(df):\n    def load_row(row):\n        # impath = TRAIN_IMAGES_ROOT/f\"{row.primary_label}/{row.filename}.npy\"\n        return row.filename, np.load(str(row.impath))[:MAX_READ_SAMPLES]\n    pool = joblib.Parallel(4)\n    mapper = joblib.delayed(load_row)\n    tasks = [mapper(row) for row in df.itertuples(False)]\n    res = pool(tqdm(tasks))\n    res = dict(res)\n    return res","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:16:49.585107Z","iopub.execute_input":"2022-02-16T11:16:49.585646Z","iopub.status.idle":"2022-02-16T11:16:49.594984Z","shell.execute_reply.started":"2022-02-16T11:16:49.585608Z","shell.execute_reply":"2022-02-16T11:16:49.594173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" pl.seed_everything(SEED)\n    \nLABEL_IDS, df = get_df()\n    \n# We cache the train set to reduce training time\naudio_image_store = load_data(df)\nlen(audio_image_store)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:16:49.597825Z","iopub.execute_input":"2022-02-16T11:16:49.59805Z","iopub.status.idle":"2022-02-16T11:17:28.656394Z","shell.execute_reply.started":"2022-02-16T11:16:49.598015Z","shell.execute_reply":"2022-02-16T11:17:28.655665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define and train model","metadata":{}},{"cell_type":"code","source":"class BirdClefDataset(Dataset):\n    def __init__(self, audio_image_store, meta, sr=SR, is_train=True, num_classes=N_CLASSES, duration=DURATION):\n        self.audio_image_store = audio_image_store\n        self.meta = meta.copy().reset_index(drop=True)\n        self.sr = sr\n        self.is_train = is_train\n        self.num_classes = num_classes\n        self.duration = duration\n        self.audio_length = self.duration*self.sr\n    \n    @staticmethod\n    def normalize(image):\n        image = image.astype(\"float32\", copy=False) / 255.0\n        image = np.stack([image, image, image])\n        return image\n\n    def __len__(self):\n        return len(self.meta)\n    \n    def __getitem__(self, idx):\n        row = self.meta.iloc[idx]\n        image = self.audio_image_store[row.filename]\n\n        image = image[np.random.choice(len(image))]\n        image = self.normalize(image)\n        \n        \n        #t = np.zeros(self.num_classes, dtype=np.float32) + 0.0025 # Label smoothing\n        #t[row.label_id] = 0.995\n        t = row.label_id\n        \n        return image, t","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:17:28.658244Z","iopub.execute_input":"2022-02-16T11:17:28.658636Z","iopub.status.idle":"2022-02-16T11:17:28.668008Z","shell.execute_reply.started":"2022-02-16T11:17:28.658596Z","shell.execute_reply":"2022-02-16T11:17:28.667296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BirdClefModel(pl.LightningModule):\n    def __init__(self, name, n_classes):\n        super().__init__()\n        self.model = get_model(name, n_classes)\n        self.f1 = F1(num_classes=n_classes, average='macro')\n\n    def forward(self, x):\n        batch_size, channels, height, width = x.size()\n        x = self.model(x)\n        return x\n\n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        logits = self(x)\n        loss = F.cross_entropy(logits, y)\n        return loss\n    \n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        logits = self.forward(x)\n        loss = F.cross_entropy(logits, y)\n        f1_score = self.f1(logits, y)\n        self.log(\"val_loss\", loss, on_epoch=True, prog_bar=True)\n        self.log(\"val_f1_score\", f1_score, on_epoch=True, prog_bar=True)\n        return {'val_loss': loss, 'val_f1_score': f1_score}\n\n    def configure_optimizers(self):\n        optimizer = optim.Adam(self.parameters(), lr=1e-3)\n        scheduler = optim.lr_scheduler.CosineAnnealingLR(\n            optimizer, eta_min=1e-5, T_max=EPOCHS)\n        return {\n            \"optimizer\":optimizer,\n            \"lr_scheduler\" : {\n                \"scheduler\" : scheduler,\n                \"monitor\" : \"val_loss\",\n                \n            }\n        }","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:17:28.670909Z","iopub.execute_input":"2022-02-16T11:17:28.671225Z","iopub.status.idle":"2022-02-16T11:17:28.682685Z","shell.execute_reply.started":"2022-02-16T11:17:28.671137Z","shell.execute_reply":"2022-02-16T11:17:28.682044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold_bar = tqdm(df.reset_index().groupby(\"fold\").index.apply(list).items(), total=df.fold.max()+1)\n\nfor fold, val_set in fold_bar:\n    if fold != USE_FOLD:\n        continue\n\n    print(f\"\\n############################### [FOLD {fold}]\")\n    fold_bar.set_description(f\"[FOLD {fold}]\")\n    train_set = np.setdiff1d(df.index, val_set)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:17:28.686638Z","iopub.execute_input":"2022-02-16T11:17:28.686822Z","iopub.status.idle":"2022-02-16T11:17:28.748084Z","shell.execute_reply.started":"2022-02-16T11:17:28.686798Z","shell.execute_reply":"2022-02-16T11:17:28.747372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = BirdClefModel(\"resnest50\", N_CLASSES)\n\ntrain_data = BirdClefDataset(audio_image_store, \n                             meta=df.iloc[train_set].reset_index(drop=True),\n                             sr=SR, duration=DURATION, is_train=True)\ntrain_dataloader = DataLoader(train_data, batch_size=TRAIN_BATCH_SIZE, \n                          num_workers=TRAIN_NUM_WORKERS, \n                          shuffle=True, pin_memory=True)\n\nval_data = BirdClefDataset(audio_image_store, \n                           meta=df.iloc[val_set].reset_index(drop=True), \n                           sr=SR, duration=DURATION, is_train=False)\nval_dataloader = DataLoader(val_data, batch_size=VAL_BATCH_SIZE,\n                        num_workers=VAL_NUM_WORKERS, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:17:28.749396Z","iopub.execute_input":"2022-02-16T11:17:28.750145Z","iopub.status.idle":"2022-02-16T11:17:30.040767Z","shell.execute_reply.started":"2022-02-16T11:17:28.750105Z","shell.execute_reply":"2022-02-16T11:17:30.040065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chk_callback = ModelCheckpoint(filename='best', monitor='val_f1_score', \n                               save_last=False, save_top_k=1, mode='max')\ntrainer = pl.Trainer(gpus=-1,\n                     max_epochs=EPOCHS,\n                     callbacks=[chk_callback]\n                    )\ntrainer.fit(model, train_dataloader, val_dataloader)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T11:18:12.539574Z","iopub.execute_input":"2022-02-16T11:18:12.53983Z","iopub.status.idle":"2022-02-16T11:26:15.687116Z","shell.execute_reply.started":"2022-02-16T11:18:12.539801Z","shell.execute_reply":"2022-02-16T11:26:15.686119Z"},"trusted":true},"execution_count":null,"outputs":[]}]}