{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":70203,"databundleVersionId":8068726},{"sourceType":"datasetVersion","sourceId":8642188,"datasetId":4836532,"databundleVersionId":8791836},{"sourceType":"datasetVersion","sourceId":8478149,"datasetId":4943761,"databundleVersionId":8618114},{"sourceType":"datasetVersion","sourceId":8627238,"datasetId":4955173,"databundleVersionId":8775835},{"sourceType":"datasetVersion","sourceId":8643539,"datasetId":4786201,"databundleVersionId":8793314},{"sourceType":"datasetVersion","sourceId":8108072,"datasetId":4789213,"databundleVersionId":8226301},{"sourceType":"datasetVersion","sourceId":8363000,"datasetId":4970504,"databundleVersionId":8496293}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !export OMP_NUM_THREADS=N\n\n# !export OMP_SCHEDULE=STATIC\n# !export OMP_PROC_BIND=CLOSE\n# !export GOMP_CPU_AFFINITY=\"N-M\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-25T12:45:13.935079Z","iopub.execute_input":"2024-05-25T12:45:13.935412Z","iopub.status.idle":"2024-05-25T12:45:13.940648Z","shell.execute_reply.started":"2024-05-25T12:45:13.935385Z","shell.execute_reply":"2024-05-25T12:45:13.939406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install \"/kaggle/input/openvino-202410-py310/openvino-2024.1.0-15008-cp310-cp310-manylinux2014_x86_64.whl\" --no-index --find-links \"/kaggle/input/openvino-202410-py310\"\n# !pip install onnxruntime-gpu==1.17.1 --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:32:34.148836Z","iopub.execute_input":"2024-06-09T15:32:34.149952Z","iopub.status.idle":"2024-06-09T15:32:52.522350Z","shell.execute_reply.started":"2024-06-09T15:32:34.149895Z","shell.execute_reply":"2024-06-09T15:32:52.520790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom glob import glob\nimport torchaudio as ta\nimport torch\nimport torch.nn as nn\nimport functools\nimport librosa as lb\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pathlib import Path\n\n# import onnxruntime as ort\nimport openvino as ov\ncore = ov.Core()\n","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:32:52.525085Z","iopub.execute_input":"2024-06-09T15:32:52.525483Z","iopub.status.idle":"2024-06-09T15:32:58.228099Z","shell.execute_reply.started":"2024-06-09T15:32:52.525446Z","shell.execute_reply":"2024-06-09T15:32:58.226759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    SR = 32000\n    DURATION = 5\n    infer_duration=5\n    train_duration=10\n    \n    label_volume_ratio = 0.5\n\nnp.random.seed(0)\ntorch.manual_seed(0)","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:32:58.230301Z","iopub.execute_input":"2024-06-09T15:32:58.231259Z","iopub.status.idle":"2024-06-09T15:32:58.245184Z","shell.execute_reply.started":"2024-06-09T15:32:58.231219Z","shell.execute_reply":"2024-06-09T15:32:58.243828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 2023-2nd-CNN v9\nmodel_cnn_v9 = core.compile_model(\"/kaggle/input/birdcled2024-models/v009-quant/quant.xml\", \"CPU\")\nmodel_cnn_v9 = model_cnn_v9.create_infer_request()\n\n# 2023-2nd-CNN 187class\nmodel_cnn_187 = core.compile_model(\"/kaggle/input/birdcled2024-models/187class_quant/quant.xml\", \"CPU\")\nmodel_cnn_187 = model_cnn_187.create_infer_request()\n\n# reshapeモデルは量子化の効果が見られなかったためスキップ\nmodel_reshape_tmt = core.compile_model(\"/kaggle/input/birdcled2024-models/EXP034_fold0_quant4/quant.xml\", \"CPU\")\nmodel_reshape_tmt = model_reshape_tmt.create_infer_request()\n\n# simplecnn-rexnet\n# model_reshape_rexnet = ort.InferenceSession(\"/kaggle/input/birdcled2024-models/v028_quant3/quant.xml\", providers=['CUDAExecutionProvider'])\n\n# rexnet\nmodel_rexnet = core.compile_model(\"/kaggle/input/birdcled2024-models/v028_quant3/quant.xml\", \"CPU\")\nmodel_rexnet = model_rexnet.create_infer_request()\n\n# inception-next-nano\nmodel_inception = core.compile_model(\"/kaggle/input/birdcled2024-models/v045_quant4/quant.xml\", \"CPU\")\nmodel_inception = model_inception.create_infer_request()\n","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:32:58.251320Z","iopub.execute_input":"2024-06-09T15:32:58.251802Z","iopub.status.idle":"2024-06-09T15:33:02.786102Z","shell.execute_reply.started":"2024-06-09T15:32:58.251731Z","shell.execute_reply":"2024-06-09T15:33:02.784848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hop_length256 = Config.infer_duration*Config.SR // (256-1)\ntransforms_256 = nn.Sequential(\n    ta.transforms.MelSpectrogram(sample_rate=Config.SR, hop_length=hop_length256, n_mels=128, f_min=0, f_max=Config.SR//2, n_fft=2048, center=True, pad_mode='constant',norm='slaney',onesided=True,mel_scale='slaney'),\n    ta.transforms.AmplitudeToDB(stype='power',top_db=80)\n)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:33:02.787591Z","iopub.execute_input":"2024-06-09T15:33:02.787973Z","iopub.status.idle":"2024-06-09T15:33:02.914833Z","shell.execute_reply.started":"2024-06-09T15:33:02.787938Z","shell.execute_reply":"2024-06-09T15:33:02.912990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"birds = \"\"\"asbfly ashdro1 ashpri1 ashwoo2 asikoe2 asiope1 aspfly1\naspswi1 barfly1 barswa bcnher bkcbul1 bkrfla1 bkskit1\nbkwsti bladro1 blaeag1 blakit1 blhori1 blnmon1 blrwar1\nbncwoo3 brakit1 brasta1 brcful1 brfowl1 brnhao1 brnshr\nbrodro1 brwjac1 brwowl1 btbeat1 bwfshr1 categr chbeat1\ncohcuc1 comfla1 comgre comior1 comkin1 commoo3 commyn\ncompea comros comsan comtai1 copbar1 crbsun2 cregos1\ncrfbar1 crseag1 dafbab1 darter2 eaywag1 emedov2 eucdov\neurbla2 eurcoo forwag1 gargan gloibi goflea1 graher1\ngrbeat1 grecou1 greegr grefla1 grehor1 grejun2 grenig1\ngrewar3 grnsan grnwar1 grtdro1 gryfra grynig2 grywag\ngybpri1 gyhcaf1 heswoo1 hoopoe houcro1 houspa inbrob1\nindpit1 indrob1 indrol2 indtit1 ingori1 inpher1 insbab1\ninsowl1 integr isbduc1 jerbus2 junbab2 junmyn1 junowl1\nkenplo1 kerlau2 labcro1 laudov1 lblwar1 lesyel1 lewduc1\nlirplo litegr litgre1 litspi1 litswi1 lobsun2 maghor2\nmalpar1 maltro1 malwoo1 marsan mawthr1 moipig1 nilfly2\nniwpig1 nutman orihob2 oripip1 pabflo1 paisto1 piebus1\npiekin1 placuc3 plaflo1 plapri1 plhpar1 pomgrp2 purher1\npursun3 pursun4 purswa3 putbab1 redspu1 rerswa1 revbul\nrewbul rewlap1 rocpig rorpar rossta2 rufbab3 ruftre2\nrufwoo2 rutfly6 sbeowl1 scamin3 shikra1 smamin1 sohmyn1\nspepic1 spodov spoowl1 sqtbul1 stbkin1 sttwoo1 thbwar1\ntibfly3 tilwar1 vefnut1 vehpar1 wbbfly1 wemhar1 whbbul2\nwhbsho3 whbtre1 whbwag1 whbwat1 whbwoo2 whcbar1 whiter2\nwhrmun whtkin2 woosan wynlau1 yebbab1 yebbul3 zitcis1\"\"\".split()","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:33:02.916655Z","iopub.execute_input":"2024-06-09T15:33:02.917151Z","iopub.status.idle":"2024-06-09T15:33:02.925239Z","shell.execute_reply.started":"2024-06-09T15:33:02.917108Z","shell.execute_reply":"2024-06-09T15:33:02.923953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare Dataframe","metadata":{}},{"cell_type":"code","source":"# wave2img\ndef transform_to_spec(audio,train=True):\n    amin=1e-10\n    ref_value=1.0\n\n    spec256 = transforms_256(audio)\n    spec256_80 = (spec256 + 80) / 80\n    spec256_255 = spec256/255\n\n    return spec256_255, spec256_80\n\ndef reshp(images):\n    bs,clip_len,channel_num,mel_num,time_len = images.size()\n    images=images.reshape((bs*clip_len,channel_num,mel_num,time_len))\n    return images\n\ndef reshp_wave(waves):\n    bs,clip_len,dims = waves.size()\n    waves = waves.reshape((bs*clip_len, dims))\n    return waves, dims\n\n@functools.cache\ndef make_meta(path):\n    audio, sr = ta.load(path)\n    return audio.shape[-1], sr","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:33:02.926612Z","iopub.execute_input":"2024-06-09T15:33:02.927006Z","iopub.status.idle":"2024-06-09T15:33:02.939111Z","shell.execute_reply.started":"2024-06-09T15:33:02.926974Z","shell.execute_reply":"2024-06-09T15:33:02.937713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import itertools\n\n# データ入力用Dataframeを作成\nbg_path = [\n    \"/kaggle/input/birdclef-2024/unlabeled_soundscapes/1225526.ogg\",\n    \"/kaggle/input/birdclef-2024/unlabeled_soundscapes/4029775.ogg\",\n]\nlabel_files = sorted(glob(\"/kaggle/input/birdclef2024-additional-cleaned/*/*.ogg\"))\nfile2label = {p:Path(p).parent.name for p in label_files}\n\ndf_val = pd.DataFrame(\n    list(itertools.product(bg_path, label_files)),\n    columns=[\"bg_path\", \"label_path\"]\n)\ndf_val[\"label\"] = df_val[\"label_path\"].map(file2label)\n\ndf_val = pd.concat([\n    df_val,\n    pd.DataFrame(df_val[\"bg_path\"].apply(make_meta).tolist(), columns=[\"bg_duration\", \"bg_sr\"]),\n    pd.DataFrame(df_val[\"label_path\"].apply(make_meta).tolist(), columns=[\"label_duration\", \"label_sr\"]),\n], axis=1)\n\n\ndf_val","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:33:02.940949Z","iopub.execute_input":"2024-06-09T15:33:02.941575Z","iopub.status.idle":"2024-06-09T15:33:05.636362Z","shell.execute_reply.started":"2024-06-09T15:33:02.941529Z","shell.execute_reply":"2024-06-09T15:33:05.634748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# inject label\ndef make_random_pos(row, max_num_inject=3):\n    max_pos = row[\"bg_duration\"] - row[\"label_duration\"]\n    num_pos = np.random.randint(0, max_num_inject) + 1\n    istart = [np.random.randint(0, max_pos) for _ in range(num_pos)]\n    return istart\n\ndf_val[\"label_start\"] = df_val.apply(make_random_pos, axis=1)\ndf_val\n","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:33:05.638353Z","iopub.execute_input":"2024-06-09T15:33:05.638861Z","iopub.status.idle":"2024-06-09T15:33:05.674950Z","shell.execute_reply.started":"2024-06-09T15:33:05.638818Z","shell.execute_reply":"2024-06-09T15:33:05.673675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ans = []\n\nfor _, row in df_val.iterrows():\n    num_segments = row[\"bg_duration\"] // (Config.SR * 5)\n    ans = pd.DataFrame(np.zeros((num_segments, len(birds)), dtype=np.uint8), columns=birds)\n\n    for lstart in row[\"label_start\"] :\n        segment_start = lstart // (Config.SR * 5)\n        segment_end = (lstart + row[\"label_duration\"]) / (Config.SR * 5)\n        segment_end = int(np.ceil(segment_end))\n        ans.loc[segment_start:segment_end, row[\"label\"]] = 1\n    df_ans.append(ans)\n\ndf_ans = pd.concat(df_ans).reset_index(drop=True)\ndf_ans","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:33:05.680204Z","iopub.execute_input":"2024-06-09T15:33:05.680635Z","iopub.status.idle":"2024-06-09T15:33:05.839162Z","shell.execute_reply.started":"2024-06-09T15:33:05.680598Z","shell.execute_reply":"2024-06-09T15:33:05.837831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Datasets","metadata":{}},{"cell_type":"code","source":"class Compose:\n    def __init__(self, transforms: list):\n        self.transforms = transforms\n\n    def __call__(self, y: np.ndarray, sr):\n        for trns in self.transforms:\n            y = trns(y, sr)\n        return y\n\n\nclass AudioTransform:\n    def __init__(self, always_apply=False, p=0.5):\n        self.always_apply = always_apply\n        self.p = p\n\n    def __call__(self, y: np.ndarray, sr):\n        if self.always_apply:\n            return self.apply(y, sr=sr)\n        else:\n            if np.random.rand() < self.p:\n                return self.apply(y, sr=sr)\n            else:\n                return y\n\n    def apply(self, y: np.ndarray, **params):\n        raise NotImplementedError\n\n\nclass OneOf(Compose):\n    # https://github.com/albumentations-team/albumentations/blob/master/albumentations/core/composition.py\n    def __init__(self, transforms, p=0.5):\n        super().__init__(transforms)\n        self.p = p\n        transforms_ps = [t.p for t in transforms]\n        s = sum(transforms_ps)\n        self.transforms_ps = [t / s for t in transforms_ps]\n\n    def __call__(self, y: np.ndarray, sr):\n        data = y\n        if self.transforms_ps and (random.random() < self.p):\n            random_state = np.random.RandomState(random.randint(0, 2 ** 32 - 1))\n            t = random_state.choice(self.transforms, p=self.transforms_ps)\n            data = t(y, sr)\n        return data\n\n\nclass Normalize(AudioTransform):\n    def __init__(self, always_apply=False, p=1):\n        super().__init__(always_apply, p)\n\n    def apply(self, y: np.ndarray, **params):\n        max_vol = np.abs(y).max()\n        y_vol = y * 1 / max_vol\n        return np.asfortranarray(y_vol)","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:33:05.840651Z","iopub.execute_input":"2024-06-09T15:33:05.841063Z","iopub.status.idle":"2024-06-09T15:33:05.858242Z","shell.execute_reply.started":"2024-06-09T15:33:05.841031Z","shell.execute_reply":"2024-06-09T15:33:05.856563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BirdValDataset(torch.utils.data.Dataset):\n    def __init__(self, df, sr = Config.SR,n_mels=128, fmin=0, fmax=None, step=None, res_type=\"kaiser_fast\",resample=True, duration = Config.DURATION, downsample = 2, train = True):\n\n        self.df = df\n        self.sr = sr \n        self.n_mels = n_mels\n        self.fmin = fmin\n        self.fmax = fmax or self.sr//2\n\n        self.train = train\n        self.duration = duration\n\n        self.audio_length = self.duration*self.sr\n        self.step = step or self.audio_length\n\n        self.res_type = res_type\n        self.resample = resample\n        self.vol_transform = ta.transforms.Vol(gain=0.5)\n        \n        self.downsample = downsample\n        self.wave_transform = Compose([Normalize(p=1),])\n\n    def __len__(self):\n        return len(self.df)\n\n    def read_file(self, row):\n        # read files\n        bg_audio, orig_sr = ta.load(row[\"bg_path\"])\n        bg_audio = bg_audio.mean(0).numpy()\n\n        if self.resample and orig_sr != self.sr:\n            bg_audio = lb.resample(bg_audio, orig_sr, self.sr, res_type=self.res_type)\n\n        label_audio, orig_sr = ta.load(row[\"label_path\"])\n        label_audio = self.vol_transform(label_audio)\n        label_audio = label_audio.mean(0).numpy()\n\n        if self.resample and orig_sr != self.sr:\n            label_audio = lb.resample(label_audio, orig_sr, self.sr, res_type=self.res_type)\n\n        # mixig them\n        for lstart in row[\"label_start\"]:\n            istart = lstart\n            iend = istart + row[\"label_duration\"]\n            bg_audio[istart:iend] += label_audio\n        audio = bg_audio\n\n        # make batch\n        seconds = []\n        for i in range(self.audio_length, len(audio) + self.step, self.step):\n            start = max(0, i - self.audio_length)\n            end = start + self.audio_length\n            if end > len(audio):\n                pass\n            else:\n                seconds.append(int(end/self.sr))\n\n        audio = np.concatenate([audio,audio,audio])\n        audios = []\n        audios_clip = []\n        for i,second in enumerate(seconds):\n            end_seconds = int(second)\n            start_seconds = int(end_seconds - Config.DURATION)\n\n            # audio_clip作成\n            #             最後を入れるとややこしいので、start_index_clipにだけ2.5秒マイナスにして、全体の長さを7.5秒にする\n            end_index_clip = int(self.sr * (end_seconds)) + len(audio) // 3\n            start_index_clip = int(self.sr * (start_seconds - (Config.train_duration - Config.DURATION) / 2)) + len(audio) // 3\n#             end_pad_clip = int(self.sr * (Config.train_duration - Config.DURATION) / 2)\n            start_pad_clip = int(self.sr * (Config.train_duration - Config.DURATION) / 2)\n            \n#             3回リピートしており、最初の-2.5秒の部分はパッドとして0にする\n            y_clip = audio[start_index_clip:end_index_clip].astype(np.float32)\n            if i == 0:\n                y_clip[:start_pad_clip] = 0\n#             elif i == (len(seconds) - 1):\n#                 y_clip[-end_pad_clip:] = 0\n\n            audio_clip = self.wave_transform(y_clip, self.sr)\n            audio_clip = audio_clip[0:len(audio_clip):self.downsample]\n            audios_clip.append(audio_clip)\n        \n            # audios作成\n            end_index = int(self.sr * (end_seconds + (Config.train_duration - Config.DURATION) / 2) ) + len(audio) // 3\n            start_index = int(self.sr * (start_seconds - (Config.train_duration - Config.DURATION) / 2) ) + len(audio) // 3\n            end_pad = int(self.sr * (Config.train_duration - Config.DURATION) / 2) \n            start_pad = int(self.sr * (Config.train_duration - Config.DURATION) / 2) \n            y = audio[start_index:end_index].astype(np.float32)\n            if i==0:\n                y[:start_pad] = 0\n            elif i==(len(seconds)-1):\n                y[-end_pad:] = 0\n            audios.append(y)\n        audios = np.stack(audios)\n        audios = torch.from_numpy(audios).float().unsqueeze(1)\n        audios_clip = np.stack(audios_clip)\n        audios_clip = torch.from_numpy(audios_clip).float()\n\n        spec256_255, spec256_80 = transform_to_spec(audios,train=False)\n        return spec256_255, spec256_80, audios_clip, audios\n\n    def __getitem__(self, idx):\n        return self.read_file(self.df.iloc[idx])","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:33:05.860022Z","iopub.execute_input":"2024-06-09T15:33:05.860544Z","iopub.status.idle":"2024-06-09T15:33:05.891321Z","shell.execute_reply.started":"2024-06-09T15:33:05.860499Z","shell.execute_reply":"2024-06-09T15:33:05.889817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.utils\nimport torch.utils.data\n\n\nds = BirdValDataset(df_val)\ndl = torch.utils.data.DataLoader(ds, batch_size=1, num_workers=2, shuffle=False, drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:33:05.892633Z","iopub.execute_input":"2024-06-09T15:33:05.893097Z","iopub.status.idle":"2024-06-09T15:33:05.907650Z","shell.execute_reply.started":"2024-06-09T15:33:05.893052Z","shell.execute_reply":"2024-06-09T15:33:05.906121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spec256_255, spec256_80, audios_clip, audios = ds[0]\n","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:33:05.908919Z","iopub.execute_input":"2024-06-09T15:33:05.909374Z","iopub.status.idle":"2024-06-09T15:33:07.189256Z","shell.execute_reply.started":"2024-06-09T15:33:05.909331Z","shell.execute_reply":"2024-06-09T15:33:07.187693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"5 * Config.SR","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:33:07.190899Z","iopub.execute_input":"2024-06-09T15:33:07.191262Z","iopub.status.idle":"2024-06-09T15:33:07.201443Z","shell.execute_reply.started":"2024-06-09T15:33:07.191232Z","shell.execute_reply":"2024-06-09T15:33:07.200088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"def sigmoid(a):\n    return 1 / (1 + np.exp(-a))\n\ndef get_openvino_pred(model, x):\n    y_pred = model.infer(x)\n    return y_pred[list(y_pred.keys())[0]]\n\ndef get_onnx_pred(model, x, input_name=\"x\", output_name=\"y\"):\n    y_pred = model.run([output_name], {input_name: x.numpy()})[0]\n    return y_pred\n\n\ndf_preds = []\nfor spec256_255, spec256_80, audio_clip, audios in dl:\n    spec256_255 = reshp(spec256_255)\n    spec256_80 = reshp(spec256_80)\n    audio_clip, dims = reshp_wave(audio_clip)\n    \n    # -2.5sec ~ 7.5sec => 0sec ~ 5sec\n    spec256_255 = spec256_255[:,:,:,128:384]\n    spec256_80 = spec256_80[:,:,:,128:384]\n    audio_clip_center = audio_clip[:, int(dims*(1/3)):]\n    audios_center = audios[0,:,0,80000:240000]\n    \n    # cnn v9\n    pred_cnn_v9 = get_openvino_pred(model_cnn_v9, spec256_255)\n    pred_cnn_v9 = sigmoid(pred_cnn_v9)\n    \n    pred_cnn_v9 = pd.DataFrame(pred_cnn_v9, columns=birds)\n    pred_cnn_v9[\"model\"] = \"cnn_v9\"\n    df_preds.append(pred_cnn_v9)\n    \n    # cnn 187\n    pred_cnn_187 = get_openvino_pred(model_cnn_187, spec256_255)\n    pred_cnn_187 = pred_cnn_187[:, :-5]\n    pred_cnn_187 = sigmoid(pred_cnn_187)\n    \n    pred_cnn_187 = pd.DataFrame(pred_cnn_187, columns=birds)\n    pred_cnn_187[\"model\"] = \"cnn_187\"\n    df_preds.append(pred_cnn_187)    \n\n    # reshapeモデルは量子化の効果が見られなかったためスキップ\n    pred_reshape_tmt = get_openvino_pred(model_reshape_tmt, audio_clip_center)\n    pred_reshape_tmt = sigmoid(pred_reshape_tmt)\n    \n    pred_reshape_tmt = pd.DataFrame(pred_reshape_tmt, columns=birds)\n    pred_reshape_tmt[\"model\"] = \"reshape_tmt\"\n    df_preds.append(pred_reshape_tmt)   \n\n    # v28 rexnet\n    pred_reshape_rexnet = get_openvino_pred(model_rexnet, spec256_80)\n    pred_reshape_rexnet = sigmoid(pred_reshape_rexnet)\n    \n    pred_reshape_rexnet = pd.DataFrame(pred_reshape_rexnet, columns=birds)\n    pred_reshape_rexnet[\"model\"] = \"v28 cnn_rexnet\"\n    df_preds.append(pred_reshape_rexnet)\n    \n    # v45 inception\n    pred_inception = get_openvino_pred(model_inception, spec256_255)\n    pred_inception = sigmoid(pred_inception)\n    \n    pred_inception = pd.DataFrame(pred_inception, columns=birds)\n    pred_inception[\"model\"] = \"v45 simple_inception\"\n    df_preds.append(pred_inception)   \n    \n\ndf_preds = pd.concat(df_preds).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:33:43.836209Z","iopub.execute_input":"2024-06-09T15:33:43.836725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metrics","metadata":{}},{"cell_type":"code","source":"# 各モデルのROC_AUC\nfrom sklearn.metrics import roc_auc_score, average_precision_score\n\nlabel_col = df_ans.columns[df_ans.max(axis=0) > 0]\n\nscores = df_preds.groupby(\"model\").apply(\n    lambda df: roc_auc_score(df_ans[label_col], df[label_col]),\ninclude_groups=False)\n\nprint(\"=== ROC-AUC ===\")\ndisplay(scores)\n\n# PR_AUC\nscores = df_preds.groupby(\"model\").apply(\n    lambda df: average_precision_score(df_ans[label_col], df[label_col]),\ninclude_groups=False)\nprint(\"=== PR-AUC ===\")\ndisplay(scores)","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:33:14.202634Z","iopub.status.idle":"2024-06-09T15:33:14.203148Z","shell.execute_reply.started":"2024-06-09T15:33:14.202921Z","shell.execute_reply":"2024-06-09T15:33:14.202942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 合成valid で callが存在する鳥\nbirds_call = sorted(label_col)\n\n# 合成valid で callが存在しない鳥\nbirds_nocall = sorted(set(birds) - set(birds_call))\nprint(f\"{len(birds_call)=}  {len(birds_nocall)=}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:33:14.205042Z","iopub.status.idle":"2024-06-09T15:33:14.205490Z","shell.execute_reply.started":"2024-06-09T15:33:14.205276Z","shell.execute_reply":"2024-06-09T15:33:14.205294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nrandom.seed(0)\n\nbirds_call_plot = random.sample(birds_call, 5)\nbirds_nocall_plot = random.sample(birds_call, 5)\nprint(f\"{birds_call_plot=}\")\nprint(f\"{birds_nocall_plot=}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:33:14.207795Z","iopub.status.idle":"2024-06-09T15:33:14.208258Z","shell.execute_reply.started":"2024-06-09T15:33:14.208048Z","shell.execute_reply":"2024-06-09T15:33:14.208067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport warnings\nwarnings.simplefilter(\"ignore\", FutureWarning)\nplt.style.use('fivethirtyeight')\n\ndf_group = df_preds.groupby(\"model\")\n\ndef make_confidence_plot(bird_cols, title):\n    fig, ax = plt.subplots(nrows=5, figsize=(12,20))\n    fig.suptitle(title, y=0.9)\n    for i, col in enumerate(bird_cols):\n        for model_name, df in df_group:\n            df = df.drop(\"model\", axis=1)\n            sns.kdeplot(x=df[col], fill=True, alpha=.5, label=model_name, ax=ax[i], common_norm=False)\n        ax[i].legend()\n\n        # limit xlim\n        xmax = df_preds[col].quantile(0.995)\n        ax[i].set_xlim([-0.02, xmax])\n\nmake_confidence_plot(birds_call_plot, \"Cofidence: Call Birds\")","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:33:14.210846Z","iopub.status.idle":"2024-06-09T15:33:14.211311Z","shell.execute_reply.started":"2024-06-09T15:33:14.211100Z","shell.execute_reply":"2024-06-09T15:33:14.211120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"make_confidence_plot(birds_nocall_plot, \"Cofidence: Nocall Birds\")","metadata":{"execution":{"iopub.status.busy":"2024-06-09T15:33:14.214288Z","iopub.status.idle":"2024-06-09T15:33:14.214820Z","shell.execute_reply.started":"2024-06-09T15:33:14.214557Z","shell.execute_reply":"2024-06-09T15:33:14.214579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}