{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install openvino-dev[onnx]","metadata":{"execution":{"iopub.status.busy":"2023-05-14T10:58:12.173696Z","iopub.execute_input":"2023-05-14T10:58:12.174608Z","iopub.status.idle":"2023-05-14T10:58:23.583398Z","shell.execute_reply.started":"2023-05-14T10:58:12.174559Z","shell.execute_reply":"2023-05-14T10:58:23.582121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torch\nimport os\nimport pytorch_lightning as pl\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn import model_selection\nimport torchvision.transforms as transforms\nimport torchvision.io\nimport librosa\nfrom PIL import Image\nimport albumentations as alb\nimport torch.multiprocessing as mp\nimport warnings\n\n\nwarnings.filterwarnings('ignore')\nimport torch.nn as nn\nfrom torch.nn.functional import cross_entropy\nimport timm\nfrom pathlib import Path\nimport librosa as lb\nimport soundfile as sf\nfrom  soundfile import SoundFile \nimport torchaudio\n\nimport torch.nn as nn\nfrom torch.nn.functional import cross_entropy,binary_cross_entropy_with_logits\nimport torchvision\nimport time\nfrom torch.nn import functional as F\nfrom torch.distributions import Beta\nfrom torch.nn.parameter import Parameter","metadata":{"execution":{"iopub.status.busy":"2023-05-14T10:58:23.588720Z","iopub.execute_input":"2023-05-14T10:58:23.589049Z","iopub.status.idle":"2023-05-14T10:58:30.751618Z","shell.execute_reply.started":"2023-05-14T10:58:23.589012Z","shell.execute_reply":"2023-05-14T10:58:30.750038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef init_weights(model):\n    classname = model.__class__.__name__\n    if classname.find(\"Conv2d\") != -1:\n        nn.init.xavier_uniform_(model.weight, gain=np.sqrt(2))\n        model.bias.data.fill_(0)\n    elif classname.find(\"BatchNorm\") != -1:\n        model.weight.data.normal_(1.0, 0.02)\n        model.bias.data.fill_(0)\n    elif classname.find(\"GRU\") != -1:\n        for weight in model.parameters():\n            if len(weight.size()) > 1:\n                nn.init.orghogonal_(weight.data)\n    elif classname.find(\"Linear\") != -1:\n        model.weight.data.normal_(0, 0.01)\n        model.bias.data.zero_()\n\n\ndef interpolate(x: torch.Tensor, ratio: int):\n    \"\"\"Interpolate data in time domain. This is used to compensate the\n    resolution reduction in downsampling of a CNN.\n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value\n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    output = F.interpolate(\n        framewise_output.unsqueeze(1),\n        size=(frames_num, framewise_output.size(2)),\n        align_corners=True,\n        mode=\"bilinear\").squeeze(1)\n\n    return output\n\n\ndef gem(x: torch.Tensor, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1. / p)\n\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super().__init__()\n        self.p = nn.Parameter(torch.ones(1) * p)\n        self.eps = eps\n\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)\n\n    def __repr__(self):\n        return self.__class__.__name__ + f\"(p={self.p.data.tolist()[0]:.4f}, eps={self.eps})\"\n\n\nclass AttBlockV2(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)\n\nclass Mixup(nn.Module):\n    def __init__(self, mix_beta):\n\n        super(Mixup, self).__init__()\n        self.beta_distribution = Beta(mix_beta, mix_beta)\n\n    def forward(self, X, Y, weight=None):\n\n        bs = X.shape[0]\n        n_dims = len(X.shape)\n        perm = torch.randperm(bs)\n        coeffs = self.beta_distribution.rsample(torch.Size((bs,))).to(X.device)\n\n        if n_dims == 2:\n            X = coeffs.view(-1, 1) * X + (1 - coeffs.view(-1, 1)) * X[perm]\n        elif n_dims == 3:\n            X = coeffs.view(-1, 1, 1) * X + (1 - coeffs.view(-1, 1, 1)) * X[perm]\n        else:\n            X = coeffs.view(-1, 1, 1, 1) * X + (1 - coeffs.view(-1, 1, 1, 1)) * X[perm]\n\n        Y = coeffs.view(-1, 1) * Y + (1 - coeffs.view(-1, 1)) * Y[perm]\n\n        if weight is None:\n            return X, Y\n        else:\n            weight = coeffs.view(-1) * weight + (1 - coeffs.view(-1)) * weight[perm]\n            return X, Y, weight\n\nclass BirdClefModelSED(torch.nn.Module):\n    def __init__(self, model_name, num_classes = 264, pretrained = False):\n        super().__init__()\n        self.num_classes = num_classes\n        self.bn0 = nn.BatchNorm2d(128)\n\n        base_model = timm.create_model(\n           model_name, pretrained=pretrained, in_chans=3)\n        #base_model.conv_stem.stride = (1,1)\n        layers = list(base_model.children())[:-2]\n        self.encoder = nn.Sequential(*layers)\n\n        if 'efficientnet' in model_name:\n            in_features = base_model.classifier.in_features\n        elif 'eca' in model_name:\n            in_features = base_model.head.fc.in_features \n        elif 'res' in model_name:\n            in_features = base_model.fc.in_features\n        #if hasattr(base_model, \"fc\"):\n        #    in_features = base_model.fc.in_features\n        #elif hasattr(base_model.head,'fc'):\n        #    in_features = base_model.head.fc.in_features\n        #else:\n        #    in_features = base_model.head.fc.in_features\n            #in_features = base_model.classifier.in_features\n\n\n\n        self.fc1 = nn.Linear(in_features, in_features, bias=True)\n        self.att_block = AttBlockV2(\n            in_features, num_classes, activation=\"sigmoid\")\n\n        self.init_weight()\n\n        #self.loss_function = nn.BCEWithLogitsLoss(reduction='none') \n        self.loss_function = nn.CrossEntropyLoss(label_smoothing=0.1,reduction='none')\n        self.mixup = Mixup(mix_beta=5)\n\n    def init_weight(self):\n        init_layer(self.fc1)\n        init_bn(self.bn0)\n\n    def forward(self,x,tta_delta=2):\n        center=True\n        train_duration=10\n        infer_duration = 5\n        #y = batch[1]\n        #weight = batch[2]\n        #x = transform_to_spec(x,train=self.training)\n\n        x = x.permute((0,1,3,2))\n        frames_num = x.shape[2]\n\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n\n        #if self.training:\n        #    x = self.spec_augmenter(x)\n\n        x = x.transpose(2, 3)\n        # (batch_size, channels, freq, frames)\n        x = self.encoder(x)\n\n        # (batch_size, channels, frames)\n        x = torch.mean(x, dim=2)\n\n        # channel smoothing\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n \n        time_att = torch.tanh(self.att_block.att(x))\n        #norm_att = torch.softmax(time_att, dim=-1)\n        #cla = self.att_block.nonlinear_transform(self.att_block.cla(x))\n        #clipwise_output_long = torch.sum(norm_att * cla, dim=2)\n        feat_time = x.size(-1)\n        if center:\n            start = (\n                feat_time / 2 - feat_time * (infer_duration / train_duration) / 2\n            )\n            end = start + feat_time * (infer_duration / train_duration)\n            start = int(start)\n            end = int(end)\n            clipwise_output = self.attention_infer(start,end,x,time_att)\n\n            #tta_delta = 2\n\n            start_minus = start-tta_delta\n            end_minus=end-tta_delta\n            clipwise_output_minus = self.attention_infer(start_minus,end_minus,x,time_att)\n\n            start_plus = start+tta_delta\n            end_plus=end+tta_delta\n            clipwise_output_plus = self.attention_infer(start_plus,end_plus,x,time_att)\n\n            clipwise_output = 0.5*clipwise_output + 0.25*clipwise_output_minus + 0.25*clipwise_output_plus\n        else:\n            start = 0\n            end = feat_time\n            feat = x[:, :, start:end]\n            att = torch.softmax(time_att[:, :, start:end], dim=-1)\n            #             print(feat_time, start, end)\n            #             print(att_a.sum(), att.sum(), time_att.shape)\n            framewise_pred = torch.sigmoid(self.att_block.cla(feat))\n            framewise_pred_max = framewise_pred.max(dim=2)[0]\n            clipwise_output = torch.sum(framewise_pred * att, dim=-1) \n        \n        return clipwise_output\n\n    def attention_infer(self,start,end,x,time_att):\n        feat = x[:, :, start:end]\n        att = torch.softmax(time_att[:, :, start:end], dim=-1)\n        #             print(feat_time, start, end)\n        #             print(att_a.sum(), att.sum(), time_att.shape)\n        framewise_pred = torch.sigmoid(self.att_block.cla(feat))\n        framewise_pred_max = framewise_pred.max(dim=2)[0]\n        #clipwise_output = torch.sum(framewise_pred * att, dim=-1) \n        #logits = torch.sum(\n        #    self.att_block.cla(feat) * att,\n        #    dim=-1,\n        #)\n\n        #return clipwise_output\n        return framewise_pred_max","metadata":{"execution":{"iopub.status.busy":"2023-05-14T10:58:30.754112Z","iopub.execute_input":"2023-05-14T10:58:30.754498Z","iopub.status.idle":"2023-05-14T10:58:30.804059Z","shell.execute_reply.started":"2023-05-14T10:58:30.754459Z","shell.execute_reply":"2023-05-14T10:58:30.803032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = BirdClefModelSED('tf_efficientnetv2_s_in21k')\nmodel.eval()\nprint('eval')","metadata":{"execution":{"iopub.status.busy":"2023-05-14T10:58:30.807099Z","iopub.execute_input":"2023-05-14T10:58:30.807599Z","iopub.status.idle":"2023-05-14T10:58:31.840341Z","shell.execute_reply.started":"2023-05-14T10:58:30.807564Z","shell.execute_reply":"2023-05-14T10:58:31.839092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_sed = torch.randn(80, 3, 128, 384)","metadata":{"execution":{"iopub.status.busy":"2023-05-14T10:58:31.841903Z","iopub.execute_input":"2023-05-14T10:58:31.842570Z","iopub.status.idle":"2023-05-14T10:58:31.960637Z","shell.execute_reply.started":"2023-05-14T10:58:31.842520Z","shell.execute_reply":"2023-05-14T10:58:31.959614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loop = 10","metadata":{"execution":{"iopub.status.busy":"2023-05-14T10:59:40.067491Z","iopub.execute_input":"2023-05-14T10:59:40.068700Z","iopub.status.idle":"2023-05-14T10:59:40.075274Z","shell.execute_reply.started":"2023-05-14T10:59:40.068642Z","shell.execute_reply":"2023-05-14T10:59:40.073462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# pytorch infer","metadata":{}},{"cell_type":"code","source":"# pytorch\ntimes = []\nfor i in range(loop):\n    t1=time.time()\n    with torch.no_grad():\n        model(input_sed,2)\n    t2=time.time()\n    times.append(t2-t1)\nnp.mean(times)","metadata":{"execution":{"iopub.status.busy":"2023-05-14T11:01:13.378959Z","iopub.execute_input":"2023-05-14T11:01:13.380021Z","iopub.status.idle":"2023-05-14T11:02:00.659498Z","shell.execute_reply.started":"2023-05-14T11:01:13.379939Z","shell.execute_reply":"2023-05-14T11:02:00.658481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# openvino(fp32)","metadata":{}},{"cell_type":"code","source":"input_names = [ \"x\",'tta_delta' ]\noutput_names = [ \"clipwise_output\" ]\n\ntorch.onnx.export(model, input_sed, \"sed64.onnx\", verbose=False, input_names=input_names, output_names=output_names)","metadata":{"execution":{"iopub.status.busy":"2023-05-14T11:02:00.661487Z","iopub.execute_input":"2023-05-14T11:02:00.662110Z","iopub.status.idle":"2023-05-14T11:02:17.732753Z","shell.execute_reply.started":"2023-05-14T11:02:00.662071Z","shell.execute_reply":"2023-05-14T11:02:17.731530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mo --input_model sed64.onnx","metadata":{"execution":{"iopub.status.busy":"2023-05-14T11:02:17.734924Z","iopub.execute_input":"2023-05-14T11:02:17.735302Z","iopub.status.idle":"2023-05-14T11:02:22.217169Z","shell.execute_reply.started":"2023-05-14T11:02:17.735265Z","shell.execute_reply":"2023-05-14T11:02:22.216053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import openvino.runtime as ov\ncore = ov.Core()\nopenvino_model = core.read_model(model='sed64.xml')\ncompiled_model = core.compile_model(openvino_model, device_name=\"CPU\")\ninfer_request = compiled_model.create_infer_request()","metadata":{"execution":{"iopub.status.busy":"2023-05-14T11:02:22.222591Z","iopub.execute_input":"2023-05-14T11:02:22.222980Z","iopub.status.idle":"2023-05-14T11:02:22.845072Z","shell.execute_reply.started":"2023-05-14T11:02:22.222940Z","shell.execute_reply":"2023-05-14T11:02:22.844047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"times = []\nfor i in range(loop):\n    t1=time.time()\n    infer_request.infer(inputs=[input_sed.numpy(),2])\n    t2=time.time()\n    times.append(t2-t1)\nnp.mean(times)","metadata":{"execution":{"iopub.status.busy":"2023-05-14T11:02:22.847541Z","iopub.execute_input":"2023-05-14T11:02:22.847879Z","iopub.status.idle":"2023-05-14T11:03:02.857184Z","shell.execute_reply.started":"2023-05-14T11:02:22.847846Z","shell.execute_reply":"2023-05-14T11:03:02.856275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# openvino(fp16)\n## no significant improvement here but significantly inprove the inference in submission","metadata":{}},{"cell_type":"code","source":"!mo --input_model sed64.onnx --compress_to_fp16","metadata":{"execution":{"iopub.status.busy":"2023-05-14T11:03:02.858643Z","iopub.execute_input":"2023-05-14T11:03:02.859677Z","iopub.status.idle":"2023-05-14T11:03:07.585280Z","shell.execute_reply.started":"2023-05-14T11:03:02.859638Z","shell.execute_reply":"2023-05-14T11:03:07.583915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import openvino.runtime as ov\ncore = ov.Core()\nopenvino_model = core.read_model(model='sed64.xml')\ncompiled_model = core.compile_model(openvino_model, device_name=\"CPU\")\ninfer_request = compiled_model.create_infer_request()","metadata":{"execution":{"iopub.status.busy":"2023-05-14T11:03:07.589178Z","iopub.execute_input":"2023-05-14T11:03:07.590054Z","iopub.status.idle":"2023-05-14T11:03:08.273138Z","shell.execute_reply.started":"2023-05-14T11:03:07.590009Z","shell.execute_reply":"2023-05-14T11:03:08.272053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"times = []\nfor i in range(loop):\n    t1=time.time()\n    infer_request.infer(inputs=[input_sed.numpy(),2])\n    t2=time.time()\n    times.append(t2-t1)\nnp.mean(times)","metadata":{"execution":{"iopub.status.busy":"2023-05-14T11:03:08.274929Z","iopub.execute_input":"2023-05-14T11:03:08.275292Z","iopub.status.idle":"2023-05-14T11:03:48.050697Z","shell.execute_reply.started":"2023-05-14T11:03:08.275245Z","shell.execute_reply":"2023-05-14T11:03:48.049546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}