{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook is training Transformer model used CNN.  \nI don't got good score now, but I will try more idea in this architecture.\n\n\nTrained Dataset is [Spectrogram Image](https://www.kaggle.com/takamichitoda/birdcall-spectrogram-images-cut/notebooks?sortBy=dateRun&group=profile&pageSize=20&datasetId=760815) which cut on  312x128.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport tqdm\nimport random\nimport time\n\nimport math\n\n\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\nfrom torch.optim import Adam, AdamW\nfrom torchvision.models import resnet18, resnet34, resnet50\nfrom torchvision import datasets, transforms\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, ReduceLROnPlateau\n\nimport matplotlib.pyplot as plt\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom contextlib import contextmanager\nfrom typing import Optional\nimport logging\nfrom numpy.random import beta\n\ndevice = torch.device('cuda')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n    torch.backends.cudnn.deterministic = True  # type: ignore\n    torch.backends.cudnn.benchmark = True  # type: ignore\n\nclass config:\n    SEED = 416\n    N_FOLDS = 5\n    FOLD = 0\n    PRETRAINED = True\n    INPUT = \"/kaggle/input/birdcall-spectrogram-images-cut/cut_image_from_resnet18_08\"\n    OUTPUT = \"/kaggle/working\"\n    N_LABEL = 264\n    \n    TRAIN_BS = 128\n    VALID_BS = 128\n    TRAIN_WORKS = 0\n    VALID_WORKS = 0\n    \n    DROPOUT_RATE = 0.2\n    EPOCHS = 100\n    TF_LR = 3e-5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataloder():\n    train_transform = transforms.Compose([\n        transforms.RandomCrop((128, 313), pad_if_needed=True, padding_mode=\"constant\"),\n        transforms.ToTensor(),\n    ])\n    valid_transform = transforms.Compose([\n        transforms.CenterCrop((128, 313)),\n        transforms.ToTensor()\n    ])\n    \n\n    train_datasets = datasets.ImageFolder(root=config.INPUT, transform=train_transform)\n    valid_datasets = datasets.ImageFolder(root=config.INPUT, transform=valid_transform)\n\n    skf = StratifiedKFold(n_splits=config.N_FOLDS, shuffle=True, random_state=config.SEED)\n\n    _t = train_datasets.targets\n    trn_idx, val_idx = [(trn_idx, val_idx) for trn_idx, val_idx in skf.split(_t, _t)][config.FOLD]\n\n    train_datasets = torch.utils.data.Subset(train_datasets, trn_idx)\n    valid_datasets = torch.utils.data.Subset(valid_datasets, val_idx)\n\n    train_data_loader = torch.utils.data.DataLoader(train_datasets, batch_size=config.TRAIN_BS, shuffle=True, num_workers=config.TRAIN_WORKS)\n    valid_data_loader = torch.utils.data.DataLoader(valid_datasets, batch_size=config.VALID_BS, shuffle=False, num_workers=config.VALID_WORKS)\n    \n    return train_data_loader, valid_data_loader\n\ntrain_data_loader, _ = get_dataloder()\nfor d in train_data_loader:\n    break\nimg = d[0][0]\nplt.imshow(np.rollaxis(img.numpy(), 0, 3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_fn(train_data_loader, model, optimizer):\n\n    losses, lrs = [], []\n    model.train()\n    t = tqdm.notebook.tqdm(train_data_loader, total=len(train_data_loader))\n    for (X, y) in t:\n        y_pred = model(X.to(device))\n        loss = loss_fn(y_pred,  y.to(device))\n\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        losses.append(loss.item())\n        lrs.append(np.array([param_group[\"lr\"] for param_group in optimizer.param_groups]).mean())\n    \n    return sum(losses)/len(losses), lrs\n\n        \ndef valid_fn(valid_data_loader, model, threshould=0.5):\n    losses, f1_lst = [], []\n    model.eval()\n    t = tqdm.notebook.tqdm(valid_data_loader, total=len(valid_data_loader))\n    for (X, y) in t:\n        with torch.no_grad():\n            y_pred = model(X.to(device))\n\n        loss = loss_fn(y_pred,  y.to(device))\n        losses.append(loss.item())\n\n        y_pred = y_pred.argmax(1).cpu()\n        f1 = f1_score(y, y_pred, average=\"micro\")\n        f1_lst.append(f1)\n\n    return sum(f1_lst)/len(f1_lst), sum(losses)/len(losses)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def loss_fn(output, target):\n    loss = nn.CrossEntropyLoss()(output, target)\n    return loss\n\n    \nclass BirdcallTransformer(nn.Module):\n    def __init__(self):\n        super(BirdcallTransformer, self).__init__()         \n        \n        embed = 512\n        self.conv = nn.Conv2d(in_channels=3, out_channels=embed, kernel_size=(128, 16))\n        #self.pe = PositionalEncoding(embed)  # not good work for me\n        encoder_layer = nn.TransformerEncoderLayer(d_model=embed, nhead=4)\n        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=4)\n        self.decoder = nn.Linear(embed, config.N_LABEL)\n        \n        self.init_weights()\n\n    def init_weights(self):\n        initrange = 0.1\n        self.decoder.bias.data.zero_()\n        self.decoder.weight.data.uniform_(-initrange, initrange)\n        \n    def forward(self, x):\n        h = self.conv(x).squeeze(2).permute((0, 2, 1))\n        #h = self.pe(h)  # not good work for me\n        h = self.transformer_encoder(h)\n        logits, _ = self.decoder(h).max(1)\n        return logits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"### Fold-{config.FOLD} ###\")\n\nset_seed(config.SEED+config.FOLD)\n\ntrain_data_loader, valid_data_loader = get_dataloder()\n\nmodel = BirdcallTransformer()\nmodel.to(device)\n\n\noptimizer = Adam(model.parameters(), lr=config.TF_LR)\n\nbest_loss, best_score = 9999, 0\ntrn_losses, trn_lrs, val_losses, val_scores = [], [], [], []\n\nendure = 0\nfor epoch in range(config.EPOCHS):\n    print(f\"{epoch} epoch\")\n    tloss, lrs = train_fn(train_data_loader, model, optimizer)\n    val_f1, vloss = valid_fn(valid_data_loader, model)\n\n    # save best score model\n    if best_score <= val_f1:\n        best_score = val_f1\n        torch.save(model.state_dict(), f\"{config.OUTPUT}/birdcallnet_f{config.FOLD}_best_score.bin\")\n        print(f\"Best Score Update!!! -> {best_score}\")\n\n    # save best loss model\n    if best_loss >= vloss:\n        best_loss = vloss\n        torch.save(model.state_dict(), f\"{config.OUTPUT}/birdcallnet_f{config.FOLD}_best_loss.bin\")\n        print(f\"Best Loss Update!!! -> {best_loss}\")\n        endure = 0\n    else:\n        endure += 1\n\n    # save training logs\n    trn_losses.append(tloss)\n    val_losses.append(vloss)\n    val_scores.append(val_f1)\n    trn_lrs.extend(lrs)\n    log_df = pd.DataFrame(zip(trn_losses, val_losses, val_scores), columns=[\"train loss\", \"valid loss\", \"score\"])\n    log_df.to_csv(f\"{config.OUTPUT}/valid_f1_fold{config.FOLD}_score.csv\", index=True)\n    \n    if endure > 8:\n        print(\"*** early stop ***\")\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Best Score: {best_score} / Best Loss: {best_loss}\")\nplt.plot(trn_lrs); plt.show()\nplt.plot(val_scores); plt.show()\nplt.plot(trn_losses)\nplt.plot(val_losses)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"log_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}