{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Inference for 🦜BirdCLEF with Lightning⚡Flash\n\n**This is just inference version fo the original work: https://www.kaggle.com/jirkaborovec/birdclef-eda-baseline-flash-efficientnet**\n\nSee our story: [Best Practices to Rank on Kaggle Competition with PyTorch Lightning and Grid.ai Spot Instances](https://devblog.pytorchlightning.ai/best-practices-to-rank-on-kaggle-competition-with-pytorch-lightning-and-grid-ai-spot-instances-54aa5248aa8e)\n\n**Clarification about the submission format: https://www.kaggle.com/c/birdclef-2022/discussion/308009**","metadata":{}},{"cell_type":"code","source":"!ls -l /kaggle/input/birdclef-2022\n\nPATH_DATASET = \"/kaggle/input/birdclef-2022\"\nPATH_CONVERTED = \"/kaggle/input/birdclef-convert-spectrograms-noise-reduce\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-11T23:53:11.850271Z","iopub.execute_input":"2022-05-11T23:53:11.85107Z","iopub.status.idle":"2022-05-11T23:53:12.592831Z","shell.execute_reply.started":"2022-05-11T23:53:11.850979Z","shell.execute_reply":"2022-05-11T23:53:12.591947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\n# df_test = pd.read_csv(os.path.join(PATH_DATASET, \"test.csv\")).set_index(\"row_id\")\n# display(df_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T23:53:12.595189Z","iopub.execute_input":"2022-05-11T23:53:12.595419Z","iopub.status.idle":"2022-05-11T23:53:12.602696Z","shell.execute_reply.started":"2022-05-11T23:53:12.595392Z","shell.execute_reply":"2022-05-11T23:53:12.601762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Converting audio to spectogram\n\nIt is done with this notebook and the output will be attached here\n\n**https://www.kaggle.com/jirkaborovec/birdclef-convert-spectrograms-noise-reduce**","metadata":{}},{"cell_type":"code","source":"!pip install -q noisereduce --find-links /kaggle/input/birdclef-eda-baseline-flash-efficientnet/frozen_packages/ --no-index","metadata":{"_kg_hide-output":false,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-11T23:53:12.604206Z","iopub.execute_input":"2022-05-11T23:53:12.604555Z","iopub.status.idle":"2022-05-11T23:53:21.951653Z","shell.execute_reply.started":"2022-05-11T23:53:12.604518Z","shell.execute_reply":"2022-05-11T23:53:21.950793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport glob\nimport torch\nimport librosa\nimport noisereduce\nimport numpy as np\nfrom math import ceil\n\nSPECTROGRAM_PARAMS = dict(\n    sample_rate=32_000,\n    hop_length=640,\n    n_fft=800,\n    n_mels=128,\n    fmin=20,\n    fmax=16_000,\n    win_length=512\n)\n\ndef create_spectrogram(\n    fname, reduce_noise: bool = False, frame_size: int = 5, spec_params: dict = SPECTROGRAM_PARAMS,\n) -> list:\n    waveform, sample_rate = librosa.core.load(fname, sr=spec_params[\"sample_rate\"], mono=True)\n    if reduce_noise:\n        waveform = noisereduce.reduce_noise(\n            y=waveform,\n            sr=sample_rate,\n            time_constant_s=float(frame_size),\n            time_mask_smooth_ms=250,\n            n_fft=spec_params[\"n_fft\"],\n            use_tqdm=False,\n            n_jobs=2,\n        )\n    nb = int(frame_size * sample_rate)\n    count = ceil(len(waveform) / float(nb))\n    spectrograms = []\n    for i in range(count):\n        frame = waveform[i * nb:(i + 1) * nb]\n        if len(frame) < nb:\n            if i == 0:\n                rep = round(float(nb) / len(frame))\n                frame = frame.repeat(int(rep))\n            else:\n                frame = waveform[-nb:]\n        sg = librosa.feature.melspectrogram(\n            y=frame,\n            sr=sample_rate,\n            n_fft=spec_params[\"n_fft\"],\n            win_length=spec_params[\"win_length\"],\n            hop_length=spec_params[\"hop_length\"],\n            n_mels=spec_params[\"n_mels\"],\n            fmin=spec_params[\"fmin\"],\n            fmax=spec_params[\"fmax\"],\n            power=1,\n        )\n        sg = librosa.amplitude_to_db(sg, ref=np.max)\n        spectrograms.append(np.nan_to_num(sg))\n    return spectrograms","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-11T23:53:21.955023Z","iopub.execute_input":"2022-05-11T23:53:21.955292Z","iopub.status.idle":"2022-05-11T23:53:25.361023Z","shell.execute_reply.started":"2022-05-11T23:53:21.95526Z","shell.execute_reply":"2022-05-11T23:53:25.360118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_audio = glob.glob(\"/kaggle/input/birdclef-2022/test_soundscapes/soundscape_*.ogg\")[0]\nprint(path_audio)\nsgs = create_spectrogram(path_audio, reduce_noise=False)\n\nfig, axarr = plt.subplots(nrows=len(sgs), figsize=(8, 2 * len(sgs)))\nfor i, sg in enumerate(sgs):\n    print(np.min(sg), np.max(sg))\n    im = axarr[i].imshow(sg, vmin=-80, vmax=0)\n    plt.colorbar(im, ax=axarr[i])\nfig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-05-11T23:53:25.362928Z","iopub.execute_input":"2022-05-11T23:53:25.3632Z","iopub.status.idle":"2022-05-11T23:53:28.577214Z","shell.execute_reply.started":"2022-05-11T23:53:25.363163Z","shell.execute_reply":"2022-05-11T23:53:28.574806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.auto import tqdm\nfrom functools import partial\nfrom joblib import Parallel, delayed\nfrom PIL import Image\n\nimg_extension = \".png\"\n\n\ndef convert_and_export(\n    fn, path_in, path_out, reduce_noise = False, frame_size: int = 5\n) -> list:\n    path_audio = os.path.join(path_in, fn)\n    sgs = create_spectrogram(path_audio, reduce_noise=reduce_noise, frame_size=frame_size)\n    records = []\n    for i, sg in enumerate(sgs):\n        path_img = os.path.join(path_out, fn + f\".{i:03}\" + img_extension)\n        os.makedirs(os.path.dirname(path_img), exist_ok=True)\n        sg = (sg + 80) / 80.0\n        sg = np.clip(sg, a_min=0, a_max=1) * 255\n        img = Image.fromarray(sg.astype(np.uint8))\n        img.resize((256,256)).save(path_img)\n        records.append({\"img_name\": os.path.basename(path_img), \"end_time\": (i + 1) * frame_size, \"file_id\": os.path.splitext(fn)[0]})\n    return records","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-11T23:53:28.57836Z","iopub.execute_input":"2022-05-11T23:53:28.578836Z","iopub.status.idle":"2022-05-11T23:53:28.590943Z","shell.execute_reply.started":"2022-05-11T23:53:28.578795Z","shell.execute_reply":"2022-05-11T23:53:28.590058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_convert_and_export = partial(\n    convert_and_export,\n    path_in=os.path.join(PATH_DATASET, \"test_soundscapes\"),\n    path_out=os.path.join(\"/kaggle/temp\", \"test_images\"),\n)\n\nsoundscapes = glob.glob(os.path.join(PATH_DATASET, \"test_soundscapes\", \"*.ogg\"))\nsoundscapes = list(map(os.path.basename, soundscapes))\nconverted = []\nfor batch in Parallel(n_jobs=3)(delayed(_convert_and_export)(fn) for fn in tqdm(soundscapes)):\n    converted += batch\n# _= list(map(_convert_and_export, tqdm(train_meta[\"filename\"])))","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-05-11T23:53:28.59229Z","iopub.execute_input":"2022-05-11T23:53:28.592787Z","iopub.status.idle":"2022-05-11T23:53:30.75734Z","shell.execute_reply.started":"2022-05-11T23:53:28.592732Z","shell.execute_reply":"2022-05-11T23:53:30.756353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_converted = pd.DataFrame(converted)\ndisplay(df_converted.head())","metadata":{"execution":{"iopub.status.busy":"2022-05-11T23:53:30.762033Z","iopub.execute_input":"2022-05-11T23:53:30.762514Z","iopub.status.idle":"2022-05-11T23:53:30.790368Z","shell.execute_reply.started":"2022-05-11T23:53:30.762473Z","shell.execute_reply":"2022-05-11T23:53:30.78976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference with Lightning⚡Flash\n\n**Follow the example:** https://lightning-flash.readthedocs.io/en/stable/reference/audio_classification.html","metadata":{}},{"cell_type":"code","source":"!pip --version\n!pip install -q 'lightning-flash[audio,image]' -U --find-links /kaggle/input/birdclef-eda-baseline-flash-efficientnet/frozen_packages/ --no-index\n!pip install -q timm -U --find-links /kaggle/input/birdclef-submissions/packages/ --no-index\n!pip uninstall -y wandb","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-05-11T23:53:30.794052Z","iopub.execute_input":"2022-05-11T23:53:30.795931Z","iopub.status.idle":"2022-05-11T23:54:00.339929Z","shell.execute_reply.started":"2022-05-11T23:53:30.79588Z","shell.execute_reply":"2022-05-11T23:54:00.338843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\n\nimport flash\nfrom flash.audio import AudioClassificationData\nfrom flash.image import ImageClassifier","metadata":{"execution":{"iopub.status.busy":"2022-05-11T23:54:00.343289Z","iopub.execute_input":"2022-05-11T23:54:00.343519Z","iopub.status.idle":"2022-05-11T23:54:09.445538Z","shell.execute_reply.started":"2022-05-11T23:54:00.34349Z","shell.execute_reply":"2022-05-11T23:54:09.44473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1. Load the task ⚙️","metadata":{}},{"cell_type":"code","source":"model = ImageClassifier.load_from_checkpoint(\n    \"/kaggle/input/birdclef-eda-baseline-flash-efficientnet/audio_classification_model.pt\"\n#     \"/kaggle/input/birdclef-submissions/birdclef_classification_model_384px.pt\"\n)\n\nprint(model.labels)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T23:54:09.44691Z","iopub.execute_input":"2022-05-11T23:54:09.447198Z","iopub.status.idle":"2022-05-11T23:54:30.620647Z","shell.execute_reply.started":"2022-05-11T23:54:09.447163Z","shell.execute_reply":"2022-05-11T23:54:30.619915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Trainer Args\nGPUS = int(torch.cuda.is_available())  # Set to 1 if GPU is enabled for notebook\n\ntrainer = flash.Trainer(gpus=GPUS)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T23:54:30.621951Z","iopub.execute_input":"2022-05-11T23:54:30.622896Z","iopub.status.idle":"2022-05-11T23:54:30.688221Z","shell.execute_reply.started":"2022-05-11T23:54:30.622856Z","shell.execute_reply":"2022-05-11T23:54:30.68745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2. Run predictions 🎉","metadata":{}},{"cell_type":"code","source":"from dataclasses import dataclass\nfrom torchvision import transforms as T\nfrom typing import Tuple, Callable, Optional\nfrom flash.core.data.io.input_transform import InputTransform\n\n@dataclass\nclass AudioClassificationInputTransform(InputTransform):\n\n    spectrogram_size: Tuple[int, int] = (128, 128)\n    color_mean: float = 0.4173\n    color_std: float = 0.15079\n\n    def input_per_sample_transform(self) -> Callable:\n        return T.Compose([\n            T.ToTensor(),\n            T.Resize(self.spectrogram_size),\n            T.Normalize([self.color_mean] * 3, [self.color_std] * 3),\n        ])\n\n    def target_per_sample_transform(self) -> Callable:\n        return torch.as_tensor","metadata":{"execution":{"iopub.status.busy":"2022-05-11T23:54:30.69145Z","iopub.execute_input":"2022-05-11T23:54:30.692042Z","iopub.status.idle":"2022-05-11T23:54:30.703672Z","shell.execute_reply.started":"2022-05-11T23:54:30.692001Z","shell.execute_reply":"2022-05-11T23:54:30.702725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datamodule = AudioClassificationData.from_data_frame(\n    input_field=\"img_name\",\n    predict_data_frame=df_converted,\n    predict_images_root=os.path.join(\"/kaggle/temp\", \"test_images\"),\n    predict_transform=AudioClassificationInputTransform,\n    transform_kwargs=dict(spectrogram_size=(224, 224)),\n    batch_size=24,\n    num_workers=3,\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T23:54:30.707653Z","iopub.execute_input":"2022-05-11T23:54:30.707946Z","iopub.status.idle":"2022-05-11T23:54:30.724369Z","shell.execute_reply.started":"2022-05-11T23:54:30.707896Z","shell.execute_reply":"2022-05-11T23:54:30.723423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = []\nfor probs in trainer.predict(model, datamodule=datamodule, output=\"probabilities\"):\n    # lbs = [torch.argmax(p[\"preds\"].float()).item() for p in preds]\n    predictions += probs","metadata":{"execution":{"iopub.status.busy":"2022-05-11T23:54:30.725978Z","iopub.execute_input":"2022-05-11T23:54:30.726307Z","iopub.status.idle":"2022-05-11T23:54:42.094502Z","shell.execute_reply.started":"2022-05-11T23:54:30.726265Z","shell.execute_reply":"2022-05-11T23:54:42.093616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Format submission\n\nUntill this is resolved https://www.kaggle.com/c/birdclef-2022/discussion/309001","metadata":{}},{"cell_type":"code","source":"import json\n\nwith open(os.path.join(PATH_DATASET, \"scored_birds.json\")) as fp:\n    scored_birds = json.load(fp)\n\nprint(scored_birds)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T23:54:42.096287Z","iopub.execute_input":"2022-05-11T23:54:42.096892Z","iopub.status.idle":"2022-05-11T23:54:42.10735Z","shell.execute_reply.started":"2022-05-11T23:54:42.096848Z","shell.execute_reply":"2022-05-11T23:54:42.106397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = []\nfor i, row in tqdm(df_converted.iterrows(), total=len(df_converted)):\n    preds = dict(zip(model.labels, predictions[i]))\n    for bird in scored_birds:\n        submission.append({\n            \"row_id\": f\"{row['file_id']}_{bird}_{row['end_time']}\",\n            \"target\": preds[bird] > 0.5,\n        })","metadata":{"execution":{"iopub.status.busy":"2022-05-11T23:54:42.109161Z","iopub.execute_input":"2022-05-11T23:54:42.111886Z","iopub.status.idle":"2022-05-11T23:54:42.162477Z","shell.execute_reply.started":"2022-05-11T23:54:42.111846Z","shell.execute_reply":"2022-05-11T23:54:42.16171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission = pd.DataFrame(submission).set_index(\"row_id\")\ndf_submission.to_csv(\"submission.csv\")\n\n! head submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-05-11T23:54:42.163926Z","iopub.execute_input":"2022-05-11T23:54:42.164416Z","iopub.status.idle":"2022-05-11T23:54:42.992783Z","shell.execute_reply.started":"2022-05-11T23:54:42.164372Z","shell.execute_reply":"2022-05-11T23:54:42.991725Z"},"trusted":true},"execution_count":null,"outputs":[]}]}