{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":44224,"databundleVersionId":5188730,"sourceType":"competition"},{"sourceId":5212841,"sourceType":"datasetVersion","datasetId":3006078,"isSourceIdPinned":true},{"sourceId":5668986,"sourceType":"datasetVersion","datasetId":3012613},{"sourceId":5763578,"sourceType":"datasetVersion","datasetId":3006070},{"sourceId":8556431,"sourceType":"datasetVersion","datasetId":5070689},{"sourceId":8771305,"sourceType":"datasetVersion","datasetId":5271088},{"sourceId":8775121,"sourceType":"datasetVersion","datasetId":5273973},{"sourceId":69148,"sourceType":"modelInstanceVersion","modelInstanceId":57681}],"dockerImageVersionId":30407,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2024-06-25T15:50:20.098372Z","iopub.execute_input":"2024-06-25T15:50:20.098846Z","iopub.status.idle":"2024-06-25T15:50:21.182118Z","shell.execute_reply.started":"2024-06-25T15:50:20.098800Z","shell.execute_reply":"2024-06-25T15:50:21.180562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/bird-clef-2023-addones/onnxruntime-1.14.0-cp37-cp37m-manylinux_2_27_x86_64.whl --no-deps","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:46:58.122346Z","iopub.execute_input":"2024-06-26T01:46:58.123473Z","iopub.status.idle":"2024-06-26T01:47:01.644995Z","shell.execute_reply.started":"2024-06-26T01:46:58.123427Z","shell.execute_reply":"2024-06-26T01:47:01.643557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip list | grep onnx","metadata":{"execution":{"iopub.status.busy":"2024-06-25T15:50:53.426813Z","iopub.execute_input":"2024-06-25T15:50:53.427374Z","iopub.status.idle":"2024-06-25T15:50:53.433766Z","shell.execute_reply.started":"2024-06-25T15:50:53.427314Z","shell.execute_reply":"2024-06-25T15:50:53.432275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install onnx2pytorch","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:47:01.647629Z","iopub.execute_input":"2024-06-26T01:47:01.648102Z","iopub.status.idle":"2024-06-26T01:47:13.761501Z","shell.execute_reply.started":"2024-06-26T01:47:01.648043Z","shell.execute_reply":"2024-06-26T01:47:13.760250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install onnxoptimizer","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:47:13.763450Z","iopub.execute_input":"2024-06-26T01:47:13.763779Z","iopub.status.idle":"2024-06-26T01:47:25.151651Z","shell.execute_reply.started":"2024-06-26T01:47:13.763747Z","shell.execute_reply":"2024-06-26T01:47:25.150485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install onnx","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:47:25.153228Z","iopub.execute_input":"2024-06-26T01:47:25.153544Z","iopub.status.idle":"2024-06-26T01:47:36.349838Z","shell.execute_reply.started":"2024-06-26T01:47:25.153513Z","shell.execute_reply":"2024-06-26T01:47:36.348557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/bird-clef-2023-code/main_folder/main_folder/')","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:47:36.352884Z","iopub.execute_input":"2024-06-26T01:47:36.353285Z","iopub.status.idle":"2024-06-26T01:47:36.359596Z","shell.execute_reply.started":"2024-06-26T01:47:36.353245Z","shell.execute_reply":"2024-06-26T01:47:36.358378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport librosa\nimport seaborn as sns\nimport os\nimport json\nimport IPython.display as ipd\nimport soundfile as sf\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport h5py\nimport onnxruntime as ort\nimport onnx\nimport onnxoptimizer\n\nfrom glob import glob\nfrom tqdm import tqdm\nfrom matplotlib import pyplot as plt\nfrom itertools import chain\nfrom os.path import join as pjoin\nfrom copy import deepcopy\nfrom scipy.spatial.distance import cosine\nfrom torchvision import transforms, models\nfrom torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\nfrom matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas\nfrom onnx2pytorch import ConvertModel\n\nfrom code_base.models import WaveCNNClasifier, WaveCNNAttenClasifier\nfrom code_base.datasets import WaveDataset, WaveAllFileDataset\nfrom code_base.utils.inference_utils import apply_avarage_weights_on_swa_path\nfrom code_base.inefernce import BirdsInference\nfrom code_base.utils import load_json, compose_submission_dataframe, groupby_np_array, stack_and_max_by_samples\nfrom code_base.utils.metrics import padded_cmap_numpy\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:47:36.360737Z","iopub.execute_input":"2024-06-26T01:47:36.361024Z","iopub.status.idle":"2024-06-26T01:47:41.423978Z","shell.execute_reply.started":"2024-06-26T01:47:36.360995Z","shell.execute_reply":"2024-06-26T01:47:41.422872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"EXP_NAME = \"convnext_small_fb_in22k_ft_in1k_384__convnextv2_tiny_fcmae_ft_in22k_in1k_384__eca_nfnet_l0_noval_v32_075Clipwise025TimeMax_GausMean\"\nTRAIN_PERIOD = 5\nprint(\"Possible checkpoints:\\n\\n{}\".format(\"\\n\".join(set([\n    os.path.basename(el) for el in glob(f\"/kaggle/input/bird-clef-2023-models/{EXP_NAME}/{EXP_NAME}/*/checkpoints/*.pt*\") if \"train\" not in os.path.basename(el)\n]))))","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:47:41.425446Z","iopub.execute_input":"2024-06-26T01:47:41.425772Z","iopub.status.idle":"2024-06-26T01:47:41.447585Z","shell.execute_reply.started":"2024-06-26T01:47:41.425740Z","shell.execute_reply":"2024-06-26T01:47:41.446252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = {\n    # Main\n    \"run_validation\": False,\n    \"run_test\": True,\n    # Inference Class\n    \"use_sigmoid\": False,\n    # Data config\n#     \"train_df_path\":\"/home/vova/data/exps/BirdCLEF_2023/birdclef_2023/train_metadata_extended.csv\",\n#     \"split_path\":\"/home/vova/data/exps/BirdCLEF_2023/cv_split_2023_v1.npy\",\n    \"folds\":[0],\n#     \"train_data_root\":\"/home/vova/data/exps/BirdCLEF_2023/birdclef_2023/train_audio/\",\n    \"test_data_root\": \"/kaggle/input/birdclef-2023/test_soundscapes/*.ogg\",\n    \"label_map_data_path\":'/kaggle/input/bird-clef-2023-models/bird2int_2023.json',\n\n    \"lookback\":None,\n    \"lookahead\":None,\n    \"segment_len\":5,\n    \"step\": None,\n    \"late_normalize\": True,\n    # Model config\n    \"exp_name\":EXP_NAME,\n#     \"model_class\": WaveCNNAttenClasifier,\n#     \"model_config\": dict(\n#         backbone=\"convnext_tiny_in22ft1k\",\n#         mel_spec_paramms={\n#             \"sample_rate\": 32000,\n#             \"n_mels\": 128,\n#             \"f_min\": 20,\n#             \"n_fft\": 2048,\n#             \"hop_length\": 512,\n#             \"normalized\": True,\n#         },\n#         head_config={\n#             \"p\": 0.5,\n#             \"num_class\": 264,\n#             \"train_period\": TRAIN_PERIOD,\n#             \"infer_period\": TRAIN_PERIOD,\n#         },\n#         pretrained=False\n#     ),\n#     \"chkp_name\":\"model.last.pth\",\n#     \"swa_checkpoint\": None,\n#     \"distributed_chkp\": False,\n}\n\nif CONFIG.get(\"use_sed_mode\", False):\n    assert CONFIG[\"step\"] is not None\nelse:\n    assert CONFIG[\"step\"] is None\n    \nif \"folds\" not in CONFIG:\n    CONFIG[\"folds\"] = list(range(CONFIG[\"n_folds\"]))","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:47:41.448998Z","iopub.execute_input":"2024-06-26T01:47:41.449357Z","iopub.status.idle":"2024-06-26T01:47:41.457752Z","shell.execute_reply.started":"2024-06-26T01:47:41.449322Z","shell.execute_reply":"2024-06-26T01:47:41.456514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data","metadata":{}},{"cell_type":"code","source":"bird2id = load_json(CONFIG[\"label_map_data_path\"])","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:47:41.459231Z","iopub.execute_input":"2024-06-26T01:47:41.459549Z","iopub.status.idle":"2024-06-26T01:47:41.479006Z","shell.execute_reply.started":"2024-06-26T01:47:41.459518Z","shell.execute_reply":"2024-06-26T01:47:41.478030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CONFIG[\"run_validation\"]:\n    df = pd.read_csv(CONFIG[\"train_df_path\"])\n    split = np.load(CONFIG[\"split_path\"], allow_pickle=True)\n    val_df = [df.iloc[split[fold_id][1]].reset_index(drop=True) for fold_id in CONFIG[\"folds\"]]","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:47:41.480206Z","iopub.execute_input":"2024-06-26T01:47:41.480515Z","iopub.status.idle":"2024-06-26T01:47:41.487164Z","shell.execute_reply.started":"2024-06-26T01:47:41.480484Z","shell.execute_reply":"2024-06-26T01:47:41.485537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CONFIG[\"run_test\"]:\n    test_au_pathes = glob(CONFIG[\"test_data_root\"])\n\n    test_df = pd.DataFrame({\n        \"filename\": test_au_pathes,\n        \"duration_s\": [librosa.get_duration(filename=el) for el in test_au_pathes]\n    })","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:47:41.490735Z","iopub.execute_input":"2024-06-26T01:47:41.491642Z","iopub.status.idle":"2024-06-26T01:47:46.288770Z","shell.execute_reply.started":"2024-06-26T01:47:41.491600Z","shell.execute_reply":"2024-06-26T01:47:46.287545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CONFIG[\"run_validation\"]:\n    val_ds_conig = {\n       \"root\": CONFIG[\"train_data_root\"],\n       \"label_str2int_mapping_path\": CONFIG[\"label_map_data_path\"],\n       \"use_audio_cache\": True,\n       \"n_cores\": 64,\n       \"verbose\": False,\n       \"segment_len\": CONFIG[\"segment_len\"],\n       \"lookback\":CONFIG[\"lookback\"],\n       \"lookahead\":CONFIG[\"lookahead\"],\n       \"sample_id\": None,\n       \"late_normalize\": CONFIG[\"late_normalize\"],\n       \"step\": CONFIG[\"step\"],\n       \"validate_sr\": 32_000\n    }\nif CONFIG[\"run_test\"]:\n    ds_config_test = {\n       \"root\": \"\",\n       \"label_str2int_mapping_path\": CONFIG[\"label_map_data_path\"],\n       \"n_cores\": 64,\n       \"use_audio_cache\": True,\n       \"test_mode\": True,\n       \"segment_len\": CONFIG[\"segment_len\"],\n       \"lookback\":CONFIG[\"lookback\"],\n       \"lookahead\":CONFIG[\"lookahead\"],\n        \"sample_id\": None,\n        \"late_normalize\": CONFIG[\"late_normalize\"],\n        \"step\": CONFIG[\"step\"],\n        \"validate_sr\": 32_000\n    }\nloader_config = {\n    \"batch_size\": 4,\n    \"drop_last\": False,\n    \"shuffle\": False,\n    \"num_workers\": 0,\n}","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:47:46.290756Z","iopub.execute_input":"2024-06-26T01:47:46.292023Z","iopub.status.idle":"2024-06-26T01:47:46.302712Z","shell.execute_reply.started":"2024-06-26T01:47:46.291970Z","shell.execute_reply":"2024-06-26T01:47:46.300867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CONFIG[\"run_test\"]:\n    ds_test = WaveAllFileDataset(df=test_df, **ds_config_test)\nif CONFIG[\"run_validation\"]:\n    ds_val = [WaveAllFileDataset(df=df, **val_ds_conig) for df in val_df]","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:47:46.304246Z","iopub.execute_input":"2024-06-26T01:47:46.304696Z","iopub.status.idle":"2024-06-26T01:47:46.323611Z","shell.execute_reply.started":"2024-06-26T01:47:46.304662Z","shell.execute_reply":"2024-06-26T01:47:46.322392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CONFIG[\"run_validation\"]:\n    loader_val = [torch.utils.data.DataLoader(\n        ds,\n        **loader_config,\n    )for ds in ds_val]\nif CONFIG[\"run_test\"]:\n    loader_test = torch.utils.data.DataLoader(\n        ds_test,\n        **loader_config,\n    )","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:47:46.325059Z","iopub.execute_input":"2024-06-26T01:47:46.325512Z","iopub.status.idle":"2024-06-26T01:47:46.332965Z","shell.execute_reply.started":"2024-06-26T01:47:46.325466Z","shell.execute_reply":"2024-06-26T01:47:46.331704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def create_model_and_upload_chkp(\n    model_class,\n    model_config,\n    model_device,\n    model_chkp,\n    use_distributed=False,\n    swa_checkpoint=None\n):\n    print(model_chkp)\n    if \"swa\" in model_chkp:\n        print(\"swa by {}\".format(os.path.splitext(os.path.basename(model_chkp))[0]))\n        t_chkp = apply_avarage_weights_on_swa_path(model_chkp, use_distributed=use_distributed, take_best=swa_checkpoint)\n    else:\n        print(\"vanilla model\")\n        t_chkp = torch.load(model_chkp, map_location=\"cpu\")\n        \n    t_model = model_class(**model_config, device=model_device)\n    t_model.load_state_dict(t_chkp)\n    t_model.eval()\n    return t_model","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:47:46.334733Z","iopub.execute_input":"2024-06-26T01:47:46.335957Z","iopub.status.idle":"2024-06-26T01:47:46.344064Z","shell.execute_reply.started":"2024-06-26T01:47:46.335901Z","shell.execute_reply":"2024-06-26T01:47:46.342924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = [create_model_and_upload_chkp(\n#         model_class=CONFIG[\"model_class\"],\n#         model_config=CONFIG['model_config'],\n#         model_device=\"cpu\",\n#         model_chkp=f\"/kaggle/input/bird-clef-2023-models/{CONFIG['exp_name']}/{CONFIG['exp_name']}/fold_{m_i}/checkpoints/{CONFIG['chkp_name']}\",\n#         swa_checkpoint=CONFIG['swa_checkpoint'],\n#         use_distributed=CONFIG['distributed_chkp']\n# ) for m_i in CONFIG[\"folds\"]]","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:47:46.345464Z","iopub.execute_input":"2024-06-26T01:47:46.345794Z","iopub.status.idle":"2024-06-26T01:47:46.358288Z","shell.execute_reply.started":"2024-06-26T01:47:46.345762Z","shell.execute_reply":"2024-06-26T01:47:46.357192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ort.InferenceSession(f\"/kaggle/input/bird-clef-2023-models/{CONFIG['exp_name']}/{CONFIG['exp_name']}/checkpoints/model_simpl.onnx\")","metadata":{"execution":{"iopub.status.busy":"2024-06-25T15:10:37.641227Z","iopub.execute_input":"2024-06-25T15:10:37.641576Z","iopub.status.idle":"2024-06-25T15:10:44.266278Z","shell.execute_reply.started":"2024-06-25T15:10:37.641520Z","shell.execute_reply":"2024-06-25T15:10:44.265224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **重訓練模型：**","metadata":{}},{"cell_type":"code","source":"# Source directory\nimport shutil\n\nsource_dir = \"/kaggle/input/forbird/onnx/bird/1/\"\n\n# Destination directory\ndestination_dir = \"/kaggle/working/model_simpl.onnx\"\n\nshutil.copytree(source_dir, destination_dir)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:47:46.359500Z","iopub.execute_input":"2024-06-26T01:47:46.359837Z","iopub.status.idle":"2024-06-26T01:47:50.380192Z","shell.execute_reply.started":"2024-06-26T01:47:46.359786Z","shell.execute_reply":"2024-06-26T01:47:50.379008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!sudo mount -uw /","metadata":{"execution":{"iopub.status.busy":"2024-06-25T15:10:48.357249Z","iopub.execute_input":"2024-06-25T15:10:48.357642Z","iopub.status.idle":"2024-06-25T15:10:48.363701Z","shell.execute_reply.started":"2024-06-25T15:10:48.357607Z","shell.execute_reply":"2024-06-25T15:10:48.362596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pprint import pprint\nimport onnxruntime\n\nonnx_path = \"/kaggle/working/model_simpl.onnx/MODEL_1.ONN\"\n# onnx_path = \"custompool/output.onnx\"\n\nprovider = \"CPUExecutionProvider\"\nonnx_session = onnxruntime.InferenceSession(onnx_path, providers=[provider])\n\nprint(\"----------------- 输入部分 -----------------\")\ninput_tensors = onnx_session.get_inputs()  # 该 API 会返回列表\nfor input_tensor in input_tensors:         # 因为可能有多个输入，所以为列表\n    \n    input_info = {\n        \"name\" : input_tensor.name,\n        \"type\" : input_tensor.type,\n        \"shape\": input_tensor.shape,\n    }\n    pprint(input_info)\n\nprint(\"----------------- 输出部分 -----------------\")\noutput_tensors = onnx_session.get_outputs()  # 该 API 会返回列表\nfor output_tensor in output_tensors:         # 因为可能有多个输出，所以为列表\n    \n    output_info = {\n        \"name\" : output_tensor.name,\n        \"type\" : output_tensor.type,\n        \"shape\": output_tensor.shape,\n    }\n    pprint(output_info)","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:47:50.381455Z","iopub.execute_input":"2024-06-26T01:47:50.381790Z","iopub.status.idle":"2024-06-26T01:47:51.764697Z","shell.execute_reply.started":"2024-06-26T01:47:50.381758Z","shell.execute_reply":"2024-06-26T01:47:51.763455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python /kaggle/input/onnx2pytroch/onnx2X-master/onnx2pytorch.py --onnx_path /kaggle/working/model_simpl.onnx/MODEL_1.ONN --simplify_path /kaggle/working/model_simpl.onnx/MODEL_1.ONN --pytorch_path /kaggle/working/ --input_shape input:30000,160000","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:47:51.766296Z","iopub.execute_input":"2024-06-26T01:47:51.766639Z","iopub.status.idle":"2024-06-26T01:49:39.761689Z","shell.execute_reply.started":"2024-06-26T01:47:51.766607Z","shell.execute_reply":"2024-06-26T01:49:39.754634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# *數據準備*","metadata":{}},{"cell_type":"code","source":"class BirdAudioDataset(Dataset):\n    def __init__(self, annotations_file, audio_base_dir, transform=None):\n        self.audio_labels = pd.read_csv(annotations_file, encoding='latin1')\n        self.audio_base_dir = audio_base_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.audio_labels)\n\n    def __getitem__(self, idx):\n        label = self.audio_labels.iloc[idx, 0]\n        audio_file = self.audio_labels.iloc[idx, -1]\n        audio_path = os.path.join(self.audio_base_dir, audio_file)\n        y, sr = librosa.load(audio_path, sr=None)\n        melspec = librosa.feature.melspectrogram(y=y, sr=sr, n_mels=128)\n        melspec_db = librosa.power_to_db(melspec, ref=np.max)\n\n        fig = plt.figure(figsize=(2.24, 2.24))\n        canvas = FigureCanvas(fig)\n        plt.imshow(melspec_db, aspect='auto', origin='lower')\n        plt.axis('off')\n        canvas.draw()\n        image = np.frombuffer(canvas.tostring_rgb(), dtype='uint8')\n        image = image.reshape(fig.canvas.get_width_height()[::-1] + (3,))\n        plt.close(fig)\n\n        image = Image.fromarray(image)\n        if self.transform:\n            image = self.transform(image)\n\n        return image, label\n\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\ntrain_dataset = BirdAudioDataset(annotations_file='/kaggle/input/newbirdset/bird3/train_metadata.csv', audio_base_dir='/kaggle/input/newbirdset/bird3/train_data', transform=transform)\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:50:39.805698Z","iopub.execute_input":"2024-06-26T01:50:39.806911Z","iopub.status.idle":"2024-06-26T01:50:39.992128Z","shell.execute_reply.started":"2024-06-26T01:50:39.806857Z","shell.execute_reply":"2024-06-26T01:50:39.991024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 加载并转换 ONNX 模型：","metadata":{}},{"cell_type":"code","source":"python .\\onnx2pytorch.py --onnx_path \\kaggle\\input\\bird-clef-2023-models\\convnext_small_fb_in22k_ft_in1k_384__convnextv2_tiny_fcmae_ft_in22k_in1k_384__eca_nfnet_l0_noval_v32_075Clipwise025TimeMax_GausMean\\convnext_small_fb_in22k_ft_in1k_384__convnextv2_tiny_fcmae_ft_in22k_in1k_384__eca_nfnet_l0_noval_v32_075Clipwise025TimeMax_GausMean\\checkpoints\\model_simpl.onnx --pytorch_path \\kaggle\\working\\mobilenetv2-7.pth --input_shape input:1,3,224,224","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:09:53.672706Z","iopub.execute_input":"2024-06-27T05:09:53.673926Z","iopub.status.idle":"2024-06-27T05:09:53.716856Z","shell.execute_reply.started":"2024-06-27T05:09:53.673867Z","shell.execute_reply":"2024-06-27T05:09:53.714970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_onnx = onnx.load(f\"/kaggle/input/bird-clef-2023-models/{CONFIG['exp_name']}/{CONFIG['exp_name']}/checkpoints/model_simpl.onnx\")  # 加载onnx模型\nonnx.checker.check_model(model_onnx)  # 验证onnx模型是否成功导出","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:50:50.302377Z","iopub.execute_input":"2024-06-26T01:50:50.303417Z","iopub.status.idle":"2024-06-26T01:50:57.741013Z","shell.execute_reply.started":"2024-06-26T01:50:50.303372Z","shell.execute_reply":"2024-06-26T01:50:57.739785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"onnx_model_path = f\"/kaggle/input/bird-clef-2023-models/{CONFIG['exp_name']}/{CONFIG['exp_name']}/checkpoints/model_simpl.onnx\"\nonnx_model = onnx.load(onnx_model_path)\n# 尝试将 ONNX 模型转换为 PyTorch 模型\ntry:\n    pytorch_model = ConvertModel(onnx_model)\n    print(\"ONNX 模型成功转换为 PyTorch 模型\")\nexcept NotImplementedError as e:\n    print(f\"ONNX 模型转换失败: {e}\")\n    # 如果遇到未实现的操作类型，则手动实现或跳过\n    def custom_op_handler(node, tensor_dict, handlers, opset_version, batch_dim):\n        # 处理特定的操作类型，例如 ReduceL2\n        if node.op_type == \"ReduceL2\":\n            input_tensor = tensor_dict[node.input[0]]\n            axis = node.attribute[0].ints\n            keepdims = node.attribute[1].i\n            reduced = torch.norm(input_tensor, p=2, dim=axis, keepdim=keepdims)\n            tensor_dict[node.output[0]] = reduced\n        else:\n            raise NotImplementedError(f\"自定义操作未实现: {node.op_type}\")\n\n    # 转换 ONNX 模型，使用自定义操作处理器\n    pytorch_model = ConvertModel(onnx_model, custom_op_handler=custom_op_handler)\n    print(\"使用自定义操作处理器成功转换 ONNX 模型为 PyTorch 模型\")","metadata":{"execution":{"iopub.status.busy":"2024-06-26T01:51:01.294007Z","iopub.execute_input":"2024-06-26T01:51:01.295155Z","iopub.status.idle":"2024-06-26T01:51:03.167461Z","shell.execute_reply.started":"2024-06-26T01:51:01.295096Z","shell.execute_reply":"2024-06-26T01:51:03.165662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 模型微調","metadata":{}},{"cell_type":"code","source":"class CustomResNet(nn.Module):\n    def __init__(self, pretrained_model):\n        super(CustomResNet, self).__init__()\n        self.features = pretrained_model.features\n        self.fc = nn.Linear(2048, 264)  # 假设有264个类别\n\n    def forward(self, x):\n        x = self.features(x)\n        x = torch.flatten(x, 1)\n        x = self.fc(x)\n        return x\n\nmodel = CustomResNet(pytorch_model)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = model.to(device)\n\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 訓練循環","metadata":{}},{"cell_type":"code","source":"num_epochs = 10\n\nfor epoch in range(num_epochs):\n    model.train()\n    running_loss = 0.0\n    for inputs, labels in train_loader:\n        inputs, labels = inputs.to(device), labels.to(device)\n\n        optimizer.zero_grad()\n\n        outputs = model(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n\n        running_loss += loss.item()\n    print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {running_loss/len(train_loader):.4f}')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 評估和保存","metadata":{}},{"cell_type":"code","source":"torch.save(model.state_dict(), 'fine_tuned_model.pth')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference Class","metadata":{}},{"cell_type":"code","source":" #音频预处理函数\ndef preprocess_audio(audio_path, segment_len=CONFIG[\"segment_len\"], sr=32000):\n    y, _ = librosa.load(audio_path, sr=sr)\n    segments = []\n    for i in range(0, len(y), segment_len * sr):\n        segment = y[i:i + segment_len * sr]\n        if len(segment) == segment_len * sr:\n            segments.append(segment)\n    return np.array(segments)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inference_class = BirdsInference(\n    device=\"gpu\",\n    verbose_tqdm=True,\n    use_sigmoid=CONFIG[\"use_sigmoid\"],\n#     model_output_key=CONFIG[\"model_output_key\"],\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Val Predict","metadata":{}},{"cell_type":"code","source":"if CONFIG[\"run_validation\"]:\n    val_tgts, val_preds, val_preds_long = inference_class.predict_val_loaders(\n        nn_models=model,\n        data_loaders=loader_val\n    )\n    print(\n        f\"Min prob {val_preds.min()}. Max prob {val_preds.max()}.\\n\"\n        f\"Padded CMAP: {padded_cmap_numpy(val_tgts, val_preds)}\"\n    )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test Pred","metadata":{}},{"cell_type":"code","source":"#if CONFIG[\"run_test\"]:\n#    test_preds, test_preds_long, test_dfidx, test_end = inference_class.predict_test_loader(\n#        nn_models=model,\n#        data_loader=loader_test,\n#        is_onnx_model=True\n#    )\n#    test_pred_df = compose_submission_dataframe(\n#        probs=test_preds,\n#        dfidxs=test_dfidx,\n#        end_seconds=test_end,\n #       filenames=loader_test.dataset.df[loader_test.dataset.name_col].copy(),\n #       bird2id=bird2id,\n#    )\n#    sample_submission = pd.read_csv(\"/kaggle/input/birdclef-2023/sample_submission.csv\")\n#    if test_pred_df.shape[1] > sample_submission.shape[1]:\n#        print(\"Shrinking columns\")\n#        test_pred_df = test_pred_df[sample_submission.columns]","metadata":{"_kg_hide-input":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_pred_df.iloc[:,1:].values.max(), test_pred_df.iloc[:,1:].values.min()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plt.hist(test_pred_df.iloc[:,1:].values.max(axis=1), bins=30);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#使用再訓練模型預測\nmodel.load_state_dict(torch.load('fine_tuned_model.pth'))\nmodel.eval()\n\ndef predict(audio_path):\n    y, sr = librosa.load(audio_path, sr=None)\n    melspec = librosa.feature.melspectrogram(y=y, sr=sr, n_mels=128)\n    melspec_db = librosa.power_to_db(melspec, ref=np.max)\n    \n    fig = plt.figure(figsize=(2.24, 2.24))\n    canvas = FigureCanvas(fig)\n    plt.imshow(melspec_db, aspect='auto', origin='lower')\n    plt.axis('off')\n    canvas.draw()\n    image = np.frombuffer(canvas.tostring_rgb(), dtype='uint8')\n    image = image.reshape(fig.canvas.get_width_height()[::-1] + (3,))\n    plt.close(fig)\n    \n    image = Image.fromarray(image)\n    image = transform(image)\n    image = image.unsqueeze(0).to(device)\n    \n    with torch.no_grad():\n        outputs = model(image)\n        _, predicted = torch.max(outputs, 1)\n    return predicted.item()\n\n# 使用测试音频文件进行预测\ntest_audio_path = 'test_audio/test_file.wav'\nprediction = predict(test_audio_path)\nprint(f'Predicted class: {prediction}')\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 推理函數\ndef predict_audio_class(audio_path):\n    segments = preprocess_audio(audio_path)\n    predictions = []\n    \n    for segment in segments:\n        input_data = segment[np.newaxis, :]  # 形狀應為 (1, num_samples)\n        ort_inputs = {model.get_inputs()[0].name: input_data}\n        ort_outs = model.run(None, ort_inputs)\n        predictions.append(ort_outs[0])\n    \n    # 平均所有片段的預測結果\n    avg_predictions = np.mean(predictions, axis=0)\n    predicted_class_idx = np.argmax(avg_predictions)\n    \n    # 反轉 bird2id 字典以獲取類別名稱\n    id2bird = {v: k for k, v in bird2id.items()}\n    predicted_class = id2bird[predicted_class_idx]\n    \n    # 計算預測的品種與每個品種的相似度\n    similarity_scores = {}\n    for bird_id, bird_name in id2bird.items():\n        similarity_scores[bird_name] = 1 - cosine(avg_predictions, np.eye(len(id2bird))[bird_id])\n    \n    # 找到最高相似度的品種\n    max_similarity_bird = max(similarity_scores, key=similarity_scores.get)\n    max_similarity_score = similarity_scores[max_similarity_bird]\n    \n    return predicted_class, avg_predictions, max_similarity_bird, max_similarity_score\n\n# 測試推理函數\naudio_path = \"/kaggle/input/birdtest/XC907019 h.ogg\"\npredicted_class, avg_predictions, max_similarity_bird, max_similarity_score = predict_audio_class(audio_path)\nprint(f\"The predicted class is: {predicted_class}\")\nprint(f\"The most similar class is: {max_similarity_bird} with a similarity score of: {max_similarity_score}\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Map Predictions","metadata":{}},{"cell_type":"code","source":"if CONFIG.get(\"check_inf_class\", False):\n    val_ds_check = WaveAllFileDataset(df=val_df[0], **{\n           \"root\": CONFIG[\"train_data_root\"],\n           \"label_str2int_mapping_path\": CONFIG[\"label_map_data_path\"],\n           \"n_cores\": 64,\n           \"use_audio_cache\": True,\n           \"test_mode\": True,\n           \"segment_len\": CONFIG[\"segment_len\"],\n           \"lookback\":CONFIG[\"lookback\"],\n           \"lookahead\":CONFIG[\"lookahead\"],\n            \"sample_id\": None,\n            \"late_normalize\": CONFIG[\"late_normalize\"],\n            \"step\": CONFIG[\"step\"],\n        }\n    )\n    val_loader_check = torch.utils.data.DataLoader(\n        val_ds_check,\n        **loader_config\n    )\n    \n    test_preds_check, test_preds_long_check, test_dfidx_check, test_end_check = inference_class.predict_test_loader(\n        nn_models=model,\n        data_loader=val_loader_check\n    )\n    test_preds_check_grouped = groupby_np_array(\n        groupby_f=test_dfidx_check,\n        array_to_group=test_preds_check,\n        apply_f=stack_and_max_by_samples,\n    )\n    print(np.allclose(\n        test_preds_check_grouped,\n        val_preds\n    ))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Boost Probs","metadata":{}},{"cell_type":"code","source":"# CLASSES = list(set(test_pred_df.columns[1:]))\n# print(len(CLASSES))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def connected_region_indices(bool_array):\n#     connected_regions = []\n#     region = []\n\n#     for i, value in enumerate(bool_array):\n#         if value:\n#             region.append(i)\n#         elif region:\n#             connected_regions.append(region)\n#             region = []\n\n#     if region:\n#         connected_regions.append(region)\n\n#     return connected_regions\n\n# def modify_probabilities(group, lower_prob=0.5, max_prob=0.75, min_seq_len=2):\n#     class_cols = ['shesta1', 'colsun2', 'amesun2', 'bcbeat1', 'marsun2']\n    \n#     mask_low = (group[CLASSES] > lower_prob).values\n#     mask_high = (group[CLASSES] > max_prob).values\n    \n#     # At least 2 chunks AND exceeds max_prob\n#     classes_to_boost = np.where((mask_low.sum(axis=0) >= min_seq_len) & mask_high.any(axis=0))[0]\n#     if len(classes_to_boost) > 0:\n#         for cls in classes_to_boost:\n#             new_probs = group[CLASSES[cls]].values.copy()\n#             connected_regions = connected_region_indices(mask_low[:,cls])\n#             for region in connected_regions:\n#                 if len(region) >= min_seq_len:\n#                     max_region_prob = group[CLASSES[cls]].iloc[region].max()\n#                     if max_region_prob > max_prob:\n#                         # print(f\"Boosting: {CLASSES[cls]}\")\n#                         new_probs[region] = max_region_prob\n#             group[CLASSES[cls]] = new_probs\n                \n#     return group","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_pred_df[\"id\"] = test_pred_df[\"row_id\"].apply(lambda x: \"_\".join(x.split(\"_\")[:-1]))\n# test_pred_df[\"sec\"] = test_pred_df[\"row_id\"].apply(lambda x: int(x.split(\"_\")[-1]))\n\n# test_pred_df = test_pred_df.sort_values([\"id\", \"sec\"]).reset_index(drop=True)\n\n# test_pred_df = test_pred_df.groupby(\"id\").apply(modify_probabilities).reset_index(drop=True)\n\n# test_pred_df = test_pred_df.drop(columns=[\"id\", \"sec\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# np.where((test_pred_df_new[CLASSES].values != test_pred_df[CLASSES].values).sum(axis=0))\n# test_pred_df_new.loc[(test_pred_df_new[CLASSES[185]] - test_pred_df[CLASSES[185]]) > 0, CLASSES[185]]\n# test_pred_df.loc[(test_pred_df_new[CLASSES[185]] - test_pred_df[CLASSES[185]]) > 0, CLASSES[185]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def boost_pred_prob(\n#     input_df,\n#     class_columns,\n#     select_prob=0.8,\n#     boost_prob=0.2\n# ):\n#     result_df = input_df.copy()\n#     for sample_id in tqdm(set(input_df[\"id\"])):\n#         sample_id_pred_df = result_df.loc[result_df[\"id\"] == sample_id, class_columns]\n#         boost_classes = class_columns[sample_id_pred_df.values.max(axis=0) > select_prob]\n#         boost_mask = sample_id_pred_df[boost_classes].values < select_prob - boost_prob\n#         result_df.loc[result_df[\"id\"] == sample_id, boost_classes] = (\n#             (boost_mask.astype(np.float32) * (sample_id_pred_df[boost_classes].values + boost_prob)) +\n#             ((~boost_mask).astype(np.float32) * sample_id_pred_df[boost_classes].values)\n#         )\n#     return result_df\n\n# test_pred_df[\"id\"] = test_pred_df[\"row_id\"].apply(lambda x: \"_\".join(x.split(\"_\")[:-1]))\n# test_pred_df[\"sec\"] = test_pred_df[\"row_id\"].apply(lambda x: int(x.split(\"_\")[-1]))\n\n# test_pred_df = boost_pred_prob(\n#     input_df=test_pred_df,\n#     class_columns=test_pred_df.columns[1:-2]\n# )\n# test_pred_df = test_pred_df.drop(columns=[\"id\", \"sec\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_pred_df.iloc[:,1:].values.max(), test_pred_df.iloc[:,1:].values.min()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.hist(test_pred_df.iloc[:,1:].values.max(axis=1), bins=30);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save Prediction","metadata":{}},{"cell_type":"code","source":"#sample_submission = pd.read_csv(\"/kaggle/input/birdclef-2023/sample_submission.csv\")\n#assert set(sample_submission.columns) == set(test_pred_df.columns)\n#test_pred_df = test_pred_df[sample_submission.columns]\n#test_pred_df.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}