{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%time\n!pip --disable-pip-version-check install --no-compile /kaggle/input/onnxruntime119/onnxruntime-1.9.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-18T04:40:33.780151Z","iopub.execute_input":"2023-05-18T04:40:33.781515Z","iopub.status.idle":"2023-05-18T04:40:50.118003Z","shell.execute_reply.started":"2023-05-18T04:40:33.781444Z","shell.execute_reply":"2023-05-18T04:40:50.116587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.utils.data as torchdata\n\nimport onnxruntime\nfrom pathlib import Path\nimport albumentations as A\n\nimport time\nimport librosa\nimport numpy as np\nimport pandas as pd\n\nimport concurrent.futures\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nonnxruntime.__version__","metadata":{"execution":{"iopub.status.busy":"2023-05-18T04:40:50.120559Z","iopub.execute_input":"2023-05-18T04:40:50.120943Z","iopub.status.idle":"2023-05-18T04:40:54.723415Z","shell.execute_reply.started":"2023-05-18T04:40:50.120904Z","shell.execute_reply":"2023-05-18T04:40:54.721973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sigmoid(x):\n \"\"\"Calculate sigmoid function.\"\"\"\n return 1 / (1 + np.exp(-x))\n\ndef compute_melspec(y, params):\n \"\"\"\n Computes a mel-spectrogram and puts it at decibel scale\n Arguments:\n y {np array} -- signal\n params {AudioParams} -- Parameters to use for the spectrogram. Expected to have the attributes sr, n_mels, f_min, f_max\n Returns:\n np array -- Mel-spectrogram\n \"\"\"\n melspec = librosa.feature.melspectrogram(\n y=y, sr=params.sr, n_mels=params.n_mels, n_fft=params.n_fft, hop_length=params.hop_length, fmin=params.fmin, fmax=params.fmax,\n )\n\n return melspec\n\n\ndef mono_to_color(X, eps=1e-6, mean=None, std=None):\n \"\"\"\n Converts a one channel array in [0, 255]\n Arguments:\n X {numpy array [H x W]} -- 2D array to convert\n Keyword Arguments:\n eps {float} -- To avoid dividing by 0 (default: {1e-6})\n mean {None or np array} -- Mean for normalization (default: {None})\n std {None or np array} -- Std for normalization (default: {None})\n Returns:\n numpy array [1 x H x W] -- RGB numpy array\n \"\"\"\n # X = np.stack([X, X, X], axis=-1)\n X = np.expand_dims(X, axis=-1)\n\n # Standardize\n mean = mean or X.mean()\n std = std or X.std()\n X = (X - mean) / (std + eps)\n\n # Normalize to [0, 255]\n _min, _max = X.min(), X.max()\n\n if (_max - _min) > eps:\n V = np.clip(X, _min, _max)\n V = 255 * (V - _min) / (_max - _min)\n V = V.astype(np.uint8)\n else:\n V = np.zeros_like(X, dtype=np.uint8)\n\n return V\n\n\nclass TestDataset(torchdata.Dataset):\n def __init__(self, \n df: pd.DataFrame, \n clip: np.ndarray,\n config=None,\n ):\n \n self.df = df\n self.clip = clip\n self.config = config\n\n def __len__(self):\n return len(self.df)\n\n def __getitem__(self, idx: int):\n\n sample = self.df.loc[idx, :]\n row_id = sample.row_id\n\n end_seconds = int(sample.seconds)\n start_seconds = int(end_seconds - 5)\n \n y = self.clip[self.config.sr * start_seconds : self.config.sr * end_seconds].astype(np.float32)\n \n image = compute_melspec(y, self.config)\n image = librosa.power_to_db(image.astype(np.float32), ref=np.max)\n \n image = mono_to_color(image)\n image = image.astype(np.uint8)\n image = albu_transforms['valid'](image=image)['image'].T\n \n return {\n \"image\": image,\n \"row_id\": row_id,\n }","metadata":{"execution":{"iopub.status.busy":"2023-05-18T04:40:54.725423Z","iopub.execute_input":"2023-05-18T04:40:54.72622Z","iopub.status.idle":"2023-05-18T04:40:54.751153Z","shell.execute_reply.started":"2023-05-18T04:40:54.726168Z","shell.execute_reply":"2023-05-18T04:40:54.749653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG_unified:\n \n batch_size=4\n num_workers=4\n \n n_mels = 128\n fmin = 16\n fmax = 16386\n n_fft = 1024\n hop_length = 320\n sr = 32000\n \n num_classes = 264\n in_channels = 1","metadata":{"execution":{"iopub.status.busy":"2023-05-18T04:40:54.754899Z","iopub.execute_input":"2023-05-18T04:40:54.75589Z","iopub.status.idle":"2023-05-18T04:40:54.767512Z","shell.execute_reply.started":"2023-05-18T04:40:54.755846Z","shell.execute_reply":"2023-05-18T04:40:54.766632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_audios = list(Path(\"../input/birdclef-2023/test_soundscapes/\").glob(\"*.ogg\"))# * 200\n\nmean = (0.485) # R only for RGB\nstd = (0.229) # R only for RGB\n\nalbu_transforms = {\n 'train' : A.Compose([\n A.Normalize(mean, std),\n ]),\n 'valid' : A.Compose([\n A.Normalize(mean, std),\n ]),\n}\n\nseconds = [i for i in range(5, 605, 5)]\ntarget_columns = \"abethr1 abhori1 abythr1 afbfly1 afdfly1 afecuc1 affeag1 afgfly1 afghor1 afmdov1 afpfly1 afpkin1 afpwag1 afrgos1 afrgrp1 afrjac1 afrthr1 amesun2 augbuz1 bagwea1 barswa bawhor2 bawman1 bcbeat1 beasun2 bkctch1 bkfruw1 blacra1 blacuc1 blakit1 blaplo1 blbpuf2 blcapa2 blfbus1 blhgon1 blhher1 blksaw1 blnmou1 blnwea1 bltapa1 bltbar1 bltori1 blwlap1 brcale1 brcsta1 brctch1 brcwea1 brican1 brobab1 broman1 brosun1 brrwhe3 brtcha1 brubru1 brwwar1 bswdov1 btweye2 bubwar2 butapa1 cabgre1 carcha1 carwoo1 categr ccbeat1 chespa1 chewea1 chibat1 chtapa3 chucis1 cibwar1 cohmar1 colsun2 combul2 combuz1 comsan crefra2 crheag1 crohor1 darbar1 darter3 didcuc1 dotbar1 dutdov1 easmog1 eaywag1 edcsun3 egygoo equaka1 eswdov1 eubeat1 fatrav1 fatwid1 fislov1 fotdro5 gabgos2 gargan gbesta1 gnbcam2 gnhsun1 gobbun1 gobsta5 gobwea1 golher1 grbcam1 grccra1 grecor greegr grewoo2 grwpyt1 gryapa1 grywrw1 gybfis1 gycwar3 gyhbus1 gyhkin1 gyhneg1 gyhspa1 gytbar1 hadibi1 hamerk1 hartur1 helgui hipbab1 hoopoe huncis1 hunsun2 joygre1 kerspa2 klacuc1 kvbsun1 laudov1 lawgol lesmaw1 lessts1 libeat1 litegr litswi1 litwea1 loceag1 lotcor1 lotlap1 luebus1 mabeat1 macshr1 malkin1 marsto1 marsun2 mcptit1 meypar1 moccha1 mouwag1 ndcsun2 nobfly1 norbro1 norcro1 norfis1 norpuf1 nubwoo1 pabspa1 palfly2 palpri1 piecro1 piekin1 pitwhy purgre2 pygbat1 quailf1 ratcis1 raybar1 rbsrob1 rebfir2 rebhor1 reboxp1 reccor reccuc1 reedov1 refbar2 refcro1 reftin1 refwar2 rehblu1 rehwea1 reisee2 rerswa1 rewsta1 rindov rocmar2 rostur1 ruegls1 rufcha2 sacibi2 sccsun2 scrcha1 scthon1 shesta1 sichor1 sincis1 slbgre1 slcbou1 sltnig1 sobfly1 somgre1 somtit4 soucit1 soufis1 spemou2 spepig1 spewea1 spfbar1 spfwea1 spmthr1 spwlap1 squher1 strher strsee1 stusta1 subbus1 supsta1 tacsun1 tafpri1 tamdov1 thrnig1 trobou1 varsun2 vibsta2 vilwea1 vimwea1 walsta1 wbgbir1 wbrcha2 wbswea1 wfbeat1 whbcan1 whbcou1 whbcro2 whbtit5 whbwea1 whbwhe3 whcpri2 whctur2 wheslf1 whhsaw1 whihel1 whrshr1 witswa1 wlwwar wookin1 woosan wtbeat1 yebapa1 yebbar1 yebduc1 yebere1 yebgre1 yebsto1 yeccan1 yefcan yelbis1 yenspu1 yertin1 yesbar1 yespet1 yetgre1 yewgre1\".split()","metadata":{"execution":{"iopub.status.busy":"2023-05-18T04:40:54.768681Z","iopub.execute_input":"2023-05-18T04:40:54.769059Z","iopub.status.idle":"2023-05-18T04:40:54.789055Z","shell.execute_reply.started":"2023-05-18T04:40:54.769025Z","shell.execute_reply":"2023-05-18T04:40:54.787187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"options = onnxruntime.SessionOptions()\n\n# Enable graph optimization\noptions.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL\n\n# Enable quantization (8-bit)\n# options.enable_quantization = True\n# options.quantization_mode = onnxruntime.QuantizationMode.QLinearOps\n\nmnasnet_models = [\n# onnxruntime.InferenceSession(\"/kaggle/input/final-onnx-bs4/mnasnet_f3.onnx\", options),\n onnxruntime.InferenceSession(\"/kaggle/input/onnx-supp/onnx_supp_bs4/mnasnet_f1.onnx\", options),\n# onnxruntime.InferenceSession(\"/kaggle/input/final-onnx-bs4/mnasnet_f2.onnx\", options),\n# onnxruntime.InferenceSession(\"/kaggle/input/onnx-finalv2/onnx_final_bs4/mnasnet_f0.onnx\", options),\n# onnxruntime.InferenceSession(\"/kaggle/input/onnx-finalv2/onnx_final_bs4/mnasnet_f1.onnx\", options),\n ]\nmobilenet_models = [onnxruntime.InferenceSession(\"/kaggle/input/final-onnx-bs4/mobilenet_f1.onnx\", options)] #2\n\nnfnet_models = [onnxruntime.InferenceSession(\"/kaggle/input/final-onnx-bs4/nfnet_f1.onnx\", options)]\n\nspnasnet_models = [\n onnxruntime.InferenceSession(\"/kaggle/input/onnx-supp/onnx_supp_bs4/spnasnet_f1.onnx\", options),\n# onnxruntime.InferenceSession(\"/kaggle/input/final-onnx-bs4/spnasnet_f1.onnx\", options),\n# onnxruntime.InferenceSession(\"/kaggle/input/onnx-finalv2/onnx_final_bs4/spnasnet_f0.onnx\", options),\n# onnxruntime.InferenceSession(\"/kaggle/input/onnx-finalv2/onnx_final_bs4/spnasnet_f1.onnx\", options), \n ]\ntfeb0_models = [\n onnxruntime.InferenceSession(\"/kaggle/input/final-onnx-bs4/tfeb0_f1.onnx\", options),\n# onnxruntime.InferenceSession(\"/kaggle/input/final-onnx-bs4/tfeb0_f3.onnx\", options) \n ]\ntfebv2m_models = [onnxruntime.InferenceSession(\"/kaggle/input/final-onnx-bs4/tfebv2m_f4.onnx\", options)]\n\n# model_ensemble = tfeb0_models+nfnet_models+tfebv2m_models\n# model_weights = [0.15, 0.15, 0.35, 0.35]\n\nmodel_ensemble = mnasnet_models+mobilenet_models+nfnet_models+spnasnet_models+tfeb0_models+tfebv2m_models\n# model_weights = [0.1, 0.12, 0.2, 0.1, 0.15, 0.2]\nmodel_weights = [0.15, 0.15, 0.20, 0.15, 0.15, 0.20]\n# model_weights = [1]*6\n\nconfig_unified = CFG_unified()\nprint(len(model_ensemble))","metadata":{"execution":{"iopub.status.busy":"2023-05-18T04:40:54.79191Z","iopub.execute_input":"2023-05-18T04:40:54.79323Z","iopub.status.idle":"2023-05-18T04:41:01.457246Z","shell.execute_reply.started":"2023-05-18T04:40:54.793186Z","shell.execute_reply":"2023-05-18T04:41:01.455806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prediction_for_clip(audio_path):\n \n clip, _ = librosa.load(audio_path, sr=32000)\n name_ = \"_\".join(audio_path.name.split(\".\")[:-1])\n row_ids = [name_+f\"_{second}\" for second in seconds]\n\n test_df = pd.DataFrame({\n \"row_id\": row_ids,\n \"seconds\": seconds\n })\n \n # inference\n prediction_dict = {}\n \n models = model_ensemble\n model_config = config_unified\n\n dataset = TestDataset(\n df=test_df, \n clip=clip,\n config=model_config,\n )\n \n loader = torchdata.DataLoader(\n dataset, \n batch_size=model_config.batch_size, \n num_workers=model_config.num_workers,\n drop_last=False,\n shuffle=False,\n pin_memory=False\n )\n \n for data in loader:\n row_ids = data['row_id']\n for row_id in row_ids:\n if row_id not in prediction_dict:\n prediction_dict[str(row_id)] = []\n \n image = data['image'].numpy()\n \n for idx, model in enumerate(models):\n inputs = {model.get_inputs()[0].name: image}\n segmentwise_output, clipwise_output, logit = model.run(None, inputs)\n \n for row_id_idx, row_id in enumerate(row_ids):\n prediction_dict[str(row_id)].append(model_weights[idx]*(sigmoid(logit[[row_id_idx]]).reshape(-1)+\\\n clipwise_output[[row_id_idx]].reshape(-1)+\\\n np.max(segmentwise_output[[row_id_idx]], 1).reshape(-1))/3)\n# prediction_dict[str(row_id)].append(model_weights[idx]*np.max(segmentwise_output[[row_id_idx]], 1).reshape(-1))\n \n for row_id in list(prediction_dict.keys()):\n \n logits = np.array(prediction_dict[row_id]).sum(0)\n prediction_dict[row_id] = {}\n for label in range(len(target_columns)):\n prediction_dict[row_id][target_columns[label]] = logits[label]\n \n return prediction_dict\n ","metadata":{"execution":{"iopub.status.busy":"2023-05-18T04:41:01.45911Z","iopub.execute_input":"2023-05-18T04:41:01.459497Z","iopub.status.idle":"2023-05-18T04:41:01.476927Z","shell.execute_reply.started":"2023-05-18T04:41:01.459459Z","shell.execute_reply":"2023-05-18T04:41:01.475513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nstart = time.time()\nwith concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:\n dicts = list(executor.map(prediction_for_clip, all_audios))\nprint(time.time()-start)","metadata":{"execution":{"iopub.status.busy":"2023-05-18T04:41:01.478136Z","iopub.execute_input":"2023-05-18T04:41:01.47855Z","iopub.status.idle":"2023-05-18T04:42:12.906016Z","shell.execute_reply.started":"2023-05-18T04:41:01.478512Z","shell.execute_reply":"2023-05-18T04:42:12.904448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction_dicts = {}\nfor d in dicts:\n prediction_dicts.update(d)","metadata":{"execution":{"iopub.status.busy":"2023-05-18T04:42:12.908518Z","iopub.execute_input":"2023-05-18T04:42:12.909631Z","iopub.status.idle":"2023-05-18T04:42:12.915467Z","shell.execute_reply.started":"2023-05-18T04:42:12.909586Z","shell.execute_reply":"2023-05-18T04:42:12.914494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import copy\n\nTHRESHOLD = 0.015\nCUTOFF = 245\n\ndef post_processing(prediction_dicts):\n \n processed_prediction_dicts = copy.deepcopy(prediction_dicts)\n# for row_id in prediction_dicts.keys():\n\n# second = int(row_id.split(\"_\")[-1])\n# prev_second = second - 5\n# next_second = second + 5\n\n# prev_row_id = \"_\".join(row_id.split(\"_\")[:-1] + [str(prev_second)])\n# next_row_id = \"_\".join(row_id.split(\"_\")[:-1] + [str(next_second)])\n \n# if prev_row_id in prediction_dicts and next_row_id in prediction_dicts:\n# for bird in prediction_dicts[row_id].keys():\n# if prediction_dicts[prev_row_id][bird] > THRESHOLD and prediction_dicts[next_row_id][bird] > THRESHOLD:\n# if prediction_dicts[row_id][bird] < THRESHOLD:\n# processed_prediction_dicts[row_id][bird] = (prediction_dicts[prev_row_id][bird]+prediction_dicts[next_row_id][bird])/2\n \n reformat_prediction_dicts = copy.deepcopy(prediction_dicts)\n \n for row_id in processed_prediction_dicts.keys():\n\n bird_logit_pairs = []\n\n for bird in processed_prediction_dicts[row_id].keys():\n bird_logit_pairs.append((bird, processed_prediction_dicts[row_id][bird]))\n\n bird_logit_pairs = sorted(bird_logit_pairs, key=lambda x : x[1], reverse=True)\n \n for i, (bird, logit) in enumerate(bird_logit_pairs):\n if i