{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm evaluations","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport os\nimport librosa\nimport psutil\nimport random\n\nimport numpy as np\nimport pandas as pd\nimport soundfile as sf\n\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm as tqdm_notebook\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import precision_recall_fscore_support\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import average_precision_score\n\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torchvision import transforms\n\nimport timm\nfrom evaluations.kaggle_2020 import row_wise_micro_averaged_f1_score\n\ndevice = torch.device(\"cuda\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class config:\n    INPUT_ROOT = \"/kaggle/input/birdclef-2021\"\n    WORK_ROOT = \"/kaggle/working\"\n    # FMIN = 20\n    # FMAX = 16000\n    # N_FFT = 2048\n    SPEC_HEIGHT = 128\n    SPEC_WIDTH = 313\n    SEED = 416\n    BATCH_SIZE = 64\n    MODEL_NAME = \"resnet18\"\n    LEAENING_RATE = 1e-3\n    T_MAX = 10\n    NUM_EPOCHS = 10\n    N_ACCUMULATE = 1\n    DATA_N_LIMIT = 200","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mono_to_color(\n    X: np.ndarray, mean=None, std=None,\n    norm_max=None, norm_min=None, eps=1e-6\n):\n    # Stack X as [X,X,X]\n    X = np.stack([X, X, X], axis=-1)\n\n    # Standardize\n    mean = mean or X.mean()\n    X = X - mean\n    std = std or X.std()\n    Xstd = X / (std + eps)\n    _min, _max = Xstd.min(), Xstd.max()\n    norm_max = norm_max or _max\n    norm_min = norm_min or _min\n    if (_max - _min) > eps:\n        # Normalize to [0, 255]\n        V = Xstd\n        V[V < norm_min] = norm_min\n        V[V > norm_max] = norm_max\n        V = 255 * (V - norm_min) / (norm_max - norm_min)\n        V = V.astype(np.uint8)\n    else:\n        # Just zero\n        V = np.zeros_like(Xstd, dtype=np.uint8)\n    return V\n\nSPEC_DATA_ROOT = [\n    \"/kaggle/input/birdclef-spectrogram-01-a\",\n    \"/kaggle/input/birdclef-spectrogram-02-b\",\n    \"/kaggle/input/birdclef-spectrogram-03-c\",\n    \"/kaggle/input/birdclef-spectrogram-04-dg\",\n    \"/kaggle/input/birdclef-spectrogram-05-hm\",\n    \"/kaggle/input/birdclef-spectrogram-06-np\",\n    \"/kaggle/input/birdclef-spectrogram-07-qs\",\n    \"/kaggle/input/birdclef-spectrogram-08-tz\",\n]\nSPEC_DATA_INITIAL = [\"a\", \"b\", \"c\", \"defg\", \"hijklm\", \"nop\", \"qrs\", \"tuvwxyz\"]\nclass BirdCLEFTrainDataset(torch.utils.data.Dataset):\n    def __init__(self, X, y, mode):\n        self.X = X\n        self.y = y\n        self.mode = mode\n        self.to_tensor = transforms.ToTensor()\n        self.norm = transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))\n        #self.crop = transforms.RandomCrop((config.SPEC_HEIGHT, config.SPEC_WIDTH), pad_if_needed=True, padding_mode=\"constant\")\n        \n    def __len__(self):\n        return len(self.y)\n\n    def __getitem__(self, idx):\n        fname = self.X[idx]\n        label = self.y[idx]\n\n\n        initial_w = label[0]\n        root_idx = [i for i, v in enumerate(SPEC_DATA_INITIAL) if initial_w in v][0]\n        spec_root = SPEC_DATA_ROOT[root_idx]\n        mel_spec = np.load(f\"{spec_root}/{label}/{fname}.npy\")\n        \n        if self.mode == \"train\":\n            mel_spec = mel_spec[:, :config.SPEC_WIDTH]  # use head 5s only\n        \n        img = mono_to_color(mel_spec)\n        img = self.to_tensor(img)\n        img = self.norm(img)\n        #if self.mode == \"train\":\n        #    img = self.crop(img)\n        #elif self.mode == \"valid\":\n        #    pass\n        \n        label_ohe = torch.eye(n_labels)[label_dic[label]]\n        \n        return img, label_ohe","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mixup(input, gamma, perm):\n    perm_input = input[perm]\n    return input.mul_(gamma).add_(1 - gamma, perm_input)\n\n#def birdclef_criterion(outputs, targets, gamma, perm):\ndef birdclef_criterion(outputs, targets):\n    clipwise_output = outputs[\"clipwise_output\"]\n    #clipwise_output = mixup(clipwise_output, gamma, perm)\n    loss = nn.BCEWithLogitsLoss(reduction=\"mean\")(clipwise_output, targets)\n    return loss\n\ndef interpolate(x: torch.Tensor, ratio: int):\n    x = x.transpose(1, 2)\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    upsampled = upsampled.transpose(1, 2)\n    return upsampled\n\nMODEL_HEADER_INFO = {\n    \"resnet18\": (-2, 512)\n}\n\nclass BirdCLEFNet(nn.Module):\n    def __init__(self, model_name):\n        super(BirdCLEFNet, self).__init__()\n        self.model_name = model_name\n        self.n_label = n_labels\n\n        base_model = timm.create_model(model_name, pretrained=True)\n        h_idx, n_dense = MODEL_HEADER_INFO[model_name]        \n        self.model_head = nn.Sequential(*list(base_model.children())[:h_idx])\n                \n        self.fc_a = nn.Conv1d(n_dense, self.n_label, 1)\n        self.fc_b = nn.Conv1d(n_dense, self.n_label, 1)\n\n    def forward(self, x):  # input x: (batch, channel, Hz, time)\n        frames_num = x.shape[3]\n        x = x.transpose(3, 2)  # (batch, channel, time, Hz)\n        h = self.model_head(x)  # (batch, unit, time, Hz)\n        \n        h = F.relu(h)\n        ti_pool = torch.mean(h, dim=3)  # (batch, unit, time)\n\n        # channel smoothing\n        #x1 = F.max_pool1d(ti_pool, kernel_size=3, stride=1, padding=1)\n        #x2 = F.avg_pool1d(ti_pool, kernel_size=3, stride=1, padding=1)\n        #ti_pool = x1 + x2\n        \n        xa = self.fc_a(ti_pool)  # (batch, n_class, time)\n        xb = self.fc_b(ti_pool)  # (batch, n_class, time)\n        xb = torch.softmax(xb, dim=2)\n\n        # time pool\n        clipwise_output = torch.sum(xa * xb, dim=2)\n        segmentwise_output= interpolate(xa, 32)\n\n        return {\n            \"clipwise_output\": clipwise_output,\n            \"segmentwise_output\": segmentwise_output,\n        }","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata_df = pd.read_csv(f\"{config.INPUT_ROOT}/train_metadata.csv\")\n\nprint(\"before filter train data:\", len(train_metadata_df))\ndfs = []\nfor primary_label, df in train_metadata_df.groupby(\"primary_label\"):\n    if len(df) > config.DATA_N_LIMIT:\n        df = df.sort_values(\"rating\", ascending=False).iloc[:config.DATA_N_LIMIT]\n    dfs.append(df)\ntrain_metadata_df = pd.concat(dfs).reset_index(drop=True)\nprint(\"after filter train data:\", len(train_metadata_df))\n\nfilenames = train_metadata_df[\"filename\"]\nprimary_labels = train_metadata_df[\"primary_label\"]\nlabel_dic = {v:i for i, v in enumerate(primary_labels.unique())}\nlabel_dic_inv = {i:v for i, v in enumerate(primary_labels.unique())}\nn_labels = len(label_dic)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_loop(train_data_loader, model, optimizer, scheduler):\n    losses = []\n    model.train()\n    optimizer.zero_grad()\n    for n_iter, (X, y) in tqdm_notebook(enumerate(train_data_loader), total=len(train_data_loader)):\n        X, y = X.to(device), y.to(device)\n        \n        # mixup\n        #gamma = np.random.beta(0.1, 0.1)\n        #perm = torch.randperm(X.size(0))\n        #X = mixup(X, gamma, perm)\n        \n        outputs = model(X)\n        #loss = birdclef_criterion(outputs, y, gamma, perm)\n        loss = birdclef_criterion(outputs, y)\n        loss.backward()\n        \n        if n_iter % config.N_ACCUMULATE == 0:\n            optimizer.step()\n            optimizer.zero_grad()\n        \n        if scheduler is not None:\n            scheduler.step()\n\n        losses.append(loss.item())\n    return losses","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_cut_image_batch(img):\n    cut_images = []\n    h_idx, t_idx = 0, config.SPEC_WIDTH\n    for idx in range(img.shape[2]):\n        if t_idx > img.shape[2]:\n            break\n        h_idx += config.SPEC_WIDTH//2\n        t_idx += config.SPEC_WIDTH//2\n        _img = img[:, :, h_idx:t_idx]\n        if _img.shape[2] != config.SPEC_WIDTH:\n            pad = torch.zeros((3, config.SPEC_HEIGHT, config.SPEC_WIDTH-_img.shape[2]))\n            _img = torch.cat([_img, pad], dim=2)\n        cut_images.append(_img)\n    cut_img_batch = torch.stack(cut_images)\n    return cut_img_batch\n\ndef valid_loop(valid_dset, model):\n    losses = []\n    predicts = []\n    model.eval()\n    for img, y in tqdm_notebook(valid_dset):\n        X = make_cut_image_batch(img)\n        X, y = X.to(device), y.to(device)\n        with torch.no_grad():\n            outputs = model(X)\n\n        clipwise_output_max, _ = outputs[\"clipwise_output\"].max(0)\n        loss = nn.BCEWithLogitsLoss(reduction=\"mean\")(clipwise_output_max, y)\n        losses.append(loss.item())\n        predicts.append(clipwise_output_max)\n    return losses, torch.vstack(predicts).cpu()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def output_to_label(clipwise_output, thr):\n    lst = []\n    for pred in tqdm_notebook(clipwise_output):\n        pred_labs = [label_dic_inv[i] for i, v in enumerate(pred) if v > thr]\n        if len(pred_labs) == 0:\n            pred_labs = \"nocall\"\n        else:\n            pred_labs = \" \".join(pred_labs)\n        lst.append(pred_labs)\n    return lst","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/input/birdclef-resnet18/","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5,  shuffle=True, random_state=config.SEED)\nfor fold, (train_index, valid_index) in enumerate(skf.split(filenames, primary_labels)):\n    print(f\"### FOLD-{fold} ###\")\n    set_seed(config.SEED)\n    \n    train_primary_labels = primary_labels.loc[train_index].values\n    valid_primary_labels = primary_labels.loc[valid_index].values\n    train_filenames = filenames.loc[train_index].values \n    valid_filenames = filenames.loc[valid_index].values\n    train_dset = BirdCLEFTrainDataset(train_filenames, train_primary_labels, \"train\")\n    train_data_loader = torch.utils.data.DataLoader(train_dset, batch_size=config.BATCH_SIZE, shuffle=True, num_workers=2)\n    valid_dset = BirdCLEFTrainDataset(valid_filenames, valid_primary_labels, \"valid\")\n    \n    model = BirdCLEFNet(config.MODEL_NAME)\n    model.to(device)\n    \n    #path = \"/kaggle/input/birdclef-resnet18/birdclefnet_f0_model_mixup_epoch20.bin\"\n    #ckpt = torch.load(path, map_location=\"cpu\")\n    #model.load_state_dict(ckpt)\n    \n    optimizer = Adam(model.parameters(), lr=config.LEAENING_RATE)\n    scheduler = CosineAnnealingLR(optimizer, T_max=len(train_data_loader)*config.T_MAX, eta_min=0.0)\n    \n    results = []\n    for epoch in range(config.NUM_EPOCHS):\n        print(f\"epoch={epoch}\")\n        train_losses = train_loop(train_data_loader, model, optimizer, scheduler)\n        valid_losses, valid_predicts = valid_loop(valid_dset, model)\n        \n        y_true = [label_dic[i] for i in valid_primary_labels]\n        y_pred = valid_predicts.argmax(1).numpy()\n        f1_micro = f1_score(y_true, y_pred, average=\"micro\")\n        \n        predict_labels = output_to_label(valid_predicts.sigmoid(), 0.5)\n        f1_micro_avg = row_wise_micro_averaged_f1_score(valid_primary_labels, predict_labels)\n\n        t_loss, v_loss = np.array(train_losses).mean(), np.array(valid_losses).mean()\n        \n        res = {\"t_loss\": t_loss, \"v_loss\": v_loss, \"f1_micro\": f1_micro, \"f1_micro_avg\": f1_micro_avg}\n        \n        print(res)\n        results.append(res)\n        torch.save(model.state_dict(), f\"{config.WORK_ROOT}/birdclefnet_f{fold}_last_model.bin\")\n        \n        gc.collect()\n        torch.cuda.empty_cache()\n        \n    break  # FOLD-0 only","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_df = pd.DataFrame(results)\nresult_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_df[\"t_loss\"].plot()\nresult_df[\"v_loss\"].plot()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_df[\"f1_micro\"].plot()\nresult_df[\"f1_micro_avg\"].plot()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}