{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"![http://www.articlesweb.org/blog/wp-content/gallery/determining-the-different-bird-species-2/Determining-the-Different-Bird-Species-2.png](http://www.articlesweb.org/blog/wp-content/gallery/determining-the-different-bird-species-2/Determining-the-Different-Bird-Species-2.png)\n\n\n# BirdCLEF 2021 - Birdcall Identification\n\nThe LifeCLEF Bird Recognition Challenge (BirdCLEF) focuses on developing machine learning algorithms to identify avian vocalizations in continuous soundscape data to aid conservation efforts worldwide. Launched in 2014, it has become one of the largest bird sound recognition competitions in terms of dataset size and species diversity.\n\nSubmissions will be evaluated based on their row-wise micro averaged F1 score.\n\nThis is inspired by @andradaolteanu wonderful EDA notebook https://www.kaggle.com/andradaolteanu/birdcall-recognition-eda-and-audio-fe\n\nI loved how she had bird images in the bar charts and am trying to replicate the same here. \n\nFor the model, I leverage @leolu1998 Leo Lu's work: https://www.kaggle.com/leolu1998/birds-call-your-name and make some changes to his\n\n# Import Libraries","metadata":{}},{"cell_type":"code","source":"!pip install ../input/timm-pytorch-image-models/pytorch-image-models-master/\n!pip install ../input/torchlibrosa/torchlibrosa-0.0.5-py3-none-any.whl","metadata":{"_kg_hide-output":true,"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport matplotlib.image as mpimg\nfrom matplotlib.offsetbox import AnnotationBbox, OffsetImage\n\nimport plotly.graph_objects as go\nimport plotly.express as px\nimport descartes\nimport geopandas as gpd\nfrom shapely.geometry import Point, Polygon\nimport librosa\nimport librosa.display\nimport IPython.display as ipd\n\n\nimport sklearn\nimport warnings\nwarnings.filterwarnings('ignore')\n\n\nimport cv2\nimport audioread\nimport logging\n\nimport random\nimport time\n\n\nimport soundfile as sf\nimport timm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as torchdata\n\nfrom contextlib import contextmanager\nfrom pathlib import Path\nfrom typing import Optional\n\nfrom albumentations.core.transforms_interface import ImageOnlyTransform\nfrom torchlibrosa.stft import LogmelFilterBank, Spectrogram\nfrom torchlibrosa.augmentation import SpecAugmentation\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import the data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/birdclef-2021/train_metadata.csv',)\ntrain_csv = pd.read_csv(\"../input/birdclef-2021/train_soundscape_labels.csv\")\ntest_csv = pd.read_csv(\"../input/birdclef-2021/test.csv\")\nsample_sub= pd.read_csv(\"../input/birdclef-2021/sample_submission.csv\")\n\ntrain","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\" The data has \",train.shape[0],\" rows\")\n\n\nprint(\"\\n The data has \",train.shape[1],\" columns. \\n The columns are: \",train.columns.values)\n\n# Unique eBird codes\nspecies = train['primary_label'].value_counts()\n\n\nprint(\"\\n There are {:,} unique bird species in the dataset.\".format(len(species)))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Make bar chart\nfig = go.Figure(data=[go.Bar(y=species.values, x=species.index)],\n                layout=go.Layout(margin=go.layout.Margin(l=0, r=0, b=10, t=50)))\n\n# Show chart\nfig.update_layout(title='Number of traning samples per species')\nfig.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is an imbalanced dataset. Some species have 500 samples and many les than 100","metadata":{}},{"cell_type":"markdown","source":"## Time of the Recording ","metadata":{}},{"cell_type":"code","source":"# Create some time features\ntrain['year'] = train['date'].apply(lambda x: x.split('-')[0])\ntrain['month'] = train['date'].apply(lambda x: x.split('-')[1])\ntrain['day_of_month'] = train['date'].apply(lambda x: x.split('-')[2])\n","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bird = mpimg.imread('../input/birdcall-recognition-data/pink bird.jpg')\nimagebox = OffsetImage(bird, zoom=0.5)\nxy = (0.5, 0.5)\nab = AnnotationBbox(imagebox, xy, frameon=False, pad=1, xybox=(20, 4000))\n\nplt.figure(figsize=(16, 6))\ntrain = train.sort_values(['year']).reset_index(drop=True)\nax = sns.countplot(train['year'], palette=\"hls\")\nax.add_artist(ab)\n\nplt.title(\"Audio Files Registration per Year Made\", fontsize=16)\nplt.xticks(rotation=90, fontsize=13)\nplt.yticks(fontsize=13)\nplt.ylabel(\"Frequency\", fontsize=14)\nplt.xlabel(\"\");","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Majority of the data was registered between 2013 and 2020, and from March to July\n\n> 0000, 0199, 0201, 0202, 2104 are likley wrong years","metadata":{}},{"cell_type":"code","source":"bird = mpimg.imread('../input/birdcall-recognition-data/green bird.jpg')\nimagebox = OffsetImage(bird, zoom=0.2)\nxy = (0.5, 0.5)\nab = AnnotationBbox(imagebox, xy, frameon=False, pad=1, xybox=(1, 7000))\n\nbird2 = mpimg.imread('../input/birdcall-recognition-data/violet bird.jpg')\nimagebox2 = OffsetImage(bird2, zoom=0.2)\n\nmn = AnnotationBbox(imagebox2, xy, frameon=False, pad=1, xybox=(10, 7000))\n\nplt.figure(figsize=(16, 6))\n\ntrain = train.sort_values(['month']).reset_index(drop=True)\nax = sns.countplot(train['month'], palette=\"hls\")\nax.add_artist(ab)\nax.add_artist(mn)\n\nplt.title(\"Audio Files Registration per Month Made\", fontsize=16)\nplt.xticks(fontsize=13)\nplt.yticks(fontsize=13)\nplt.ylabel(\"Frequency\", fontsize=14)\nplt.xlabel(\"\");","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Majority of the data was registered between March and July\n\n\n> 00 is months which are not known\n\n\n","metadata":{}},{"cell_type":"code","source":"bird = mpimg.imread('../input/birdcall-recognition-data/fluff ball.jpg')\nimagebox = OffsetImage(bird, zoom=0.45)\nxy = (0.5, 0.5)\nab = AnnotationBbox(imagebox, xy, frameon=False, pad=1, xybox=(-150, -50))\n\n# SHP file\nworld_map = gpd.read_file(\"../input/world-shapefile/world_shapefile.shp\")\n\n# Coordinate reference system\ncrs = {\"init\" : \"epsg:4326\"}\n\n# Lat and Long need to be of type float, not object\nspecies_list = ['norcar', 'houspa', 'wesblu', 'banana']\ndata = train[train['primary_label'].isin(species_list)]\ndata[\"latitude\"] = data[\"latitude\"].astype(float)\ndata[\"longitude\"] = data[\"longitude\"].astype(float)\n\n# Create geometry\ngeometry = [Point(xy) for xy in zip(data[\"longitude\"], data[\"latitude\"])]\n\n# Geo Dataframe\ngeo_df = gpd.GeoDataFrame(data, crs=crs, geometry=geometry)\n\n# Create ID for species\nspecies_id = geo_df[\"primary_label\"].value_counts().reset_index()\nspecies_id.insert(0, 'ID', range(0, 0 + len(species_id)))\n\nspecies_id.columns = [\"ID\", \"primary_label\", \"count\"]\n\n# Add ID to geo_df\ngeo_df = pd.merge(geo_df, species_id, how=\"left\", on=\"primary_label\")\n\n# === PLOT ===\nfig, ax = plt.subplots(figsize = (16, 10))\nworld_map.plot(ax=ax, alpha=0.4, color=\"grey\")\nax.add_artist(ab)\npalette = iter(sns.hls_palette(len(species_id)))\nfor i in range(len(species_list)):\n    geo_df[geo_df[\"ID\"] == i].plot(ax=ax, \n                                   markersize=20, \n                                   color=next(palette), \n                                   marker=\"o\", \n                                   label = species_id['primary_label'].values[i]);\n    \nax.legend()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most of the recordings are from the Americas and Europe. We have a few frm Africa, Australia and Asia as well... \n\n# Audio Analysis\n\nWe are taking a sample of the bird Keel-billed Toucan \n![https://cdn.shopify.com/s/files/1/0020/1926/2510/products/875-1_1024x1024@2x.jpg?v=1551504120](https://cdn.shopify.com/s/files/1/0020/1926/2510/products/875-1_1024x1024@2x.jpg?v=1551504120)\n\nhttps://ebird.org/species/kebtou1","metadata":{}},{"cell_type":"code","source":"audio_path = \"../input/birdclef-2021/train_short_audio/kebtou1/XC11517.ogg\"\nipd.Audio(audio_path)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the first 15 seconds this file using librosa\nsig, rate = librosa.load(audio_path, sr=32000, offset=None, duration=15)\n\n# The result is a 1D numpy array that conatains audio samples. \n# Take a look at the shape (seconds * sample rate == 15 * 32000 == 480000)\nprint('SIGNAL SHAPE:', sig.shape)\n\nplt.figure(figsize=(15, 5))\nlibrosa.display.waveplot(sig, sr=32000)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Try a few window lengths (should be a power of 2)","metadata":{}},{"cell_type":"code","source":"# Try a few window lengths (should be a power of 2)\nfor win_length in [128, 256, 512, 1024]:\n    \n    # We want 50% overlap between samples\n    hop_length = win_length // 2\n    \n    # Compute spec (win_length implicity also sets n_fft and vice versa)\n    spec = librosa.stft(sig, \n                        n_fft=win_length, \n                        hop_length=hop_length)\n    \n    # Scale to decibel scale\n    spec_db = librosa.amplitude_to_db(spec, ref=np.max)\n    \n    # Show plot\n    plt.figure(figsize=(15, 5))\n    plt.title('Window length: ' + str(win_length) + ', Shape: ' + str(spec_db.shape))\n    librosa.display.specshow(spec_db, \n                             sr=32000, \n                             hop_length=hop_length, \n                             x_axis='time', \n                             y_axis='hz', \n                             cmap=plt.get_cmap('viridis'))\n","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Try spectrograms of Different durations","metadata":{}},{"cell_type":"code","source":"# Desired shape of the input spectrogram\nSPEC_HEIGHT = 64\nSPEC_WIDTH = 256\n\n# Derive num_mels and hop_length from desired spec shape\n# num_mels is easy, that's just spec_height\n# hop_length is a bit more complicated\nNUM_MELS = SPEC_HEIGHT\nHOP_LENGTH = int(32000 * 5 / (SPEC_WIDTH - 1)) # sample rate * duration / spec width - 1 == 627\n\n# High- and low-pass frequencies\n# For many birds, these are a good choice\nFMIN = 500\nFMAX = 12500\n\n# Let's get all three spectrograms\nfor second in [5, 10, 15]:  \n    \n    # Get start and stop sample\n    s_start = (second - 5) * 32000\n    s_end = second * 32000\n\n    # Compute the spectrogram and apply the mel scale\n    mel_spec = librosa.feature.melspectrogram(y=sig[s_start:s_end], \n                                              sr=32000, \n                                              n_fft=1024, \n                                              hop_length=HOP_LENGTH, \n                                              n_mels=NUM_MELS, \n                                              fmin=FMIN, \n                                              fmax=FMAX)\n    \n    mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max)\n\n    # Show the spec\n    plt.figure(figsize=(15, 5))\n    plt.title('Second: ' + str(second) + ', Shape: ' + str(mel_spec_db.shape))\n    librosa.display.specshow(mel_spec_db, \n                             sr=32000, \n                             hop_length=HOP_LENGTH, \n                             x_axis='time', \n                             y_axis='mel',\n                             fmin=FMIN, \n                             fmax=FMAX, \n                             cmap=plt.get_cmap('viridis'))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# helper functions ","metadata":{}},{"cell_type":"code","source":"def set_seed(seed: int = 108):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n    torch.backends.cudnn.deterministic = True  # type: ignore\n    torch.backends.cudnn.benchmark = True  # type: ignore\n    \n    \ndef get_logger(out_file=None):\n    logger = logging.getLogger()\n    formatter = logging.Formatter(\"%(asctime)s - %(levelname)s - %(message)s\")\n    logger.handlers = []\n    logger.setLevel(logging.INFO)\n\n    handler = logging.StreamHandler()\n    handler.setFormatter(formatter)\n    handler.setLevel(logging.INFO)\n    logger.addHandler(handler)\n\n    if out_file is not None:\n        fh = logging.FileHandler(out_file)\n        fh.setFormatter(formatter)\n        fh.setLevel(logging.INFO)\n        logger.addHandler(fh)\n    logger.info(\"logger set up\")\n    return logger\n    \n    \n@contextmanager\ndef timer(name: str, logger: Optional[logging.Logger] = None):\n    t0 = time.time()\n    msg = f\"[{name}] start\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)\n    yield\n\n    msg = f\"[{name}] done in {time.time() - t0:.2f} s\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logger = get_logger(\"main.log\")\nset_seed(108)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n\n    seed = 108\n    epochs = 60\n    train = True\n    folds = [0]\n    img_size = 224\n    main_metric = \"epoch_f1_at_05\"\n    minimize_metric = False\n\n\n    train_datadir = Path(\"../input/birdclef-2021/train_short_audio\")\n    train_csv = \"../input/birdclef-2021/train_metadata.csv\"\n    train_soundscape = \"../input/birdclef-2021/train_soundscape_labels.csv\"\n\n    \n    transforms = {\n        \"train\": [{\"name\": \"Normalize\"}],\n        \"valid\": [{\"name\": \"Normalize\"}],\n        \"test\": [{\"name\": \"Normalize\"}]\n    }\n    period = 20\n    n_mels = 128\n    fmin = 20\n    fmax = 16000\n    n_fft = 2048\n    hop_length = 512\n    sample_rate = 32000\n    melspectrogram_parameters = {\n        \"n_mels\": 224,\n        \"fmin\": 20,\n        \"fmax\": 16000\n    }\n\n    target_columns = [\n        'acafly', 'acowoo', 'aldfly', 'ameavo', 'amecro',\n        'amegfi', 'amekes', 'amepip', 'amered', 'amerob',\n        'amewig', 'amtspa', 'andsol1', 'annhum', 'astfly',\n        'azaspi1', 'babwar', 'baleag', 'balori', 'banana',\n        'banswa', 'banwre1', 'barant1', 'barswa', 'batpig1',\n        'bawswa1', 'bawwar', 'baywre1', 'bbwduc', 'bcnher',\n        'belkin1', 'belvir', 'bewwre', 'bkbmag1', 'bkbplo',\n        'bkbwar', 'bkcchi', 'bkhgro', 'bkmtou1', 'bknsti', 'blbgra1',\n        'blbthr1', 'blcjay1', 'blctan1', 'blhpar1', 'blkpho',\n        'blsspa1', 'blugrb1', 'blujay', 'bncfly', 'bnhcow', 'bobfly1',\n        'bongul', 'botgra', 'brbmot1', 'brbsol1', 'brcvir1', 'brebla',\n        'brncre', 'brnjay', 'brnthr', 'brratt1', 'brwhaw', 'brwpar1',\n        'btbwar', 'btnwar', 'btywar', 'bucmot2', 'buggna', 'bugtan',\n        'buhvir', 'bulori', 'burwar1', 'bushti', 'butsal1', 'buwtea',\n        'cacgoo1', 'cacwre', 'calqua', 'caltow', 'cangoo', 'canwar',\n        'carchi', 'carwre', 'casfin', 'caskin', 'caster1', 'casvir',\n        'categr', 'ccbfin', 'cedwax', 'chbant1', 'chbchi', 'chbwre1',\n        'chcant2', 'chispa', 'chswar', 'cinfly2', 'clanut', 'clcrob',\n        'cliswa', 'cobtan1', 'cocwoo1', 'cogdov', 'colcha1', 'coltro1',\n        'comgol', 'comgra', 'comloo', 'commer', 'compau', 'compot1',\n        'comrav', 'comyel', 'coohaw', 'cotfly1', 'cowscj1', 'cregua1',\n        'creoro1', 'crfpar', 'cubthr', 'daejun', 'dowwoo', 'ducfly', 'dusfly',\n        'easblu', 'easkin', 'easmea', 'easpho', 'eastow', 'eawpew', 'eletro',\n        'eucdov', 'eursta', 'fepowl', 'fiespa', 'flrtan1', 'foxspa', 'gadwal',\n        'gamqua', 'gartro1', 'gbbgul', 'gbwwre1', 'gcrwar', 'gilwoo',\n        'gnttow', 'gnwtea', 'gocfly1', 'gockin', 'gocspa', 'goftyr1',\n        'gohque1', 'goowoo1', 'grasal1', 'grbani', 'grbher3', 'grcfly',\n        'greegr', 'grekis', 'grepew', 'grethr1', 'gretin1', 'greyel',\n        'grhcha1', 'grhowl', 'grnher', 'grnjay', 'grtgra', 'grycat',\n        'gryhaw2', 'gwfgoo', 'haiwoo', 'heptan', 'hergul', 'herthr',\n        'herwar', 'higmot1', 'hofwoo1', 'houfin', 'houspa', 'houwre',\n        'hutvir', 'incdov', 'indbun', 'kebtou1', 'killde', 'labwoo', 'larspa',\n        'laufal1', 'laugul', 'lazbun', 'leafly', 'leasan', 'lesgol', 'lesgre1',\n        'lesvio1', 'linspa', 'linwoo1', 'littin1', 'lobdow', 'lobgna5', 'logshr',\n        'lotduc', 'lotman1', 'lucwar', 'macwar', 'magwar', 'mallar3', 'marwre',\n        'mastro1', 'meapar', 'melbla1', 'monoro1', 'mouchi', 'moudov', 'mouela1',\n        'mouqua', 'mouwar', 'mutswa', 'naswar', 'norcar', 'norfli', 'normoc', 'norpar',\n        'norsho', 'norwat', 'nrwswa', 'nutwoo', 'oaktit', 'obnthr1', 'ocbfly1',\n        'oliwoo1', 'olsfly', 'orbeup1', 'orbspa1', 'orcpar', 'orcwar', 'orfpar',\n        'osprey', 'ovenbi1', 'pabspi1', 'paltan1', 'palwar', 'pasfly', 'pavpig2',\n        'phivir', 'pibgre', 'pilwoo', 'pinsis', 'pirfly1', 'plawre1', 'plaxen1',\n        'plsvir', 'plupig2', 'prowar', 'purfin', 'purgal2', 'putfru1', 'pygnut',\n        'rawwre1', 'rcatan1', 'rebnut', 'rebsap', 'rebwoo', 'redcro', 'reevir1',\n        'rehbar1', 'relpar', 'reshaw', 'rethaw', 'rewbla', 'ribgul', 'rinkin1',\n        'roahaw', 'robgro', 'rocpig', 'rotbec', 'royter1', 'rthhum', 'rtlhum',\n        'ruboro1', 'rubpep1', 'rubrob', 'rubwre1', 'ruckin', 'rucspa1', 'rucwar',\n        'rucwar1', 'rudpig', 'rudtur', 'rufhum', 'rugdov', 'rumfly1', 'runwre1',\n        'rutjac1', 'saffin', 'sancra', 'sander', 'savspa', 'saypho', 'scamac1',\n        'scatan', 'scbwre1', 'scptyr1', 'scrtan1', 'semplo', 'shicow', 'sibtan2',\n        'sinwre1', 'sltred', 'smbani', 'snogoo', 'sobtyr1', 'socfly1', 'solsan',\n        'sonspa', 'soulap1', 'sposan', 'spotow', 'spvear1', 'squcuc1', 'stbori',\n        'stejay', 'sthant1', 'sthwoo1', 'strcuc1', 'strfly1', 'strsal1', 'stvhum2',\n        'subfly', 'sumtan', 'swaspa', 'swathr', 'tenwar', 'thbeup1', 'thbkin',\n        'thswar1', 'towsol', 'treswa', 'trogna1', 'trokin', 'tromoc', 'tropar',\n        'tropew1', 'tuftit', 'tunswa', 'veery', 'verdin', 'vigswa', 'warvir',\n        'wbwwre1', 'webwoo1', 'wegspa1', 'wesant1', 'wesblu', 'weskin', 'wesmea',\n        'westan', 'wewpew', 'whbman1', 'whbnut', 'whcpar', 'whcsee1', 'whcspa',\n        'whevir', 'whfpar1', 'whimbr', 'whiwre1', 'whtdov', 'whtspa', 'whwbec1',\n        'whwdov', 'wilfly', 'willet1', 'wilsni1', 'wiltur', 'wlswar', 'wooduc',\n        'woothr', 'wrenti', 'y00475', 'yebcha', 'yebela1', 'yebfly', 'yebori1',\n        'yebsap', 'yebsee1', 'yefgra1', 'yegvir', 'yehbla', 'yehcar1', 'yelgro',\n        'yelwar', 'yeofly1', 'yerwar', 'yeteup1', 'yetvir']\n\n\n    loader_params = {\n        \"train\": {\n            \"batch_size\": 64,\n            \"num_workers\": 20,\n            \"shuffle\": True\n        },\n        \"valid\": {\n            \"batch_size\": 64,\n            \"num_workers\": 20,\n            \"shuffle\": False\n        },\n        \"test\": {\n            \"batch_size\": 64,\n            \"num_workers\": 20,\n            \"shuffle\": False\n        }\n    }\n\n    \n    split = \"StratifiedKFold\"\n    split_params = {\n        \"n_splits\": 5,\n        \"shuffle\": True,\n        \"random_state\": 108\n    }\n\n\n    \n    base_model_name = \"tf_efficientnet_b0_ns\"\n    pooling = \"max\"\n    pretrained = True\n    num_classes = 397\n    in_channels = 1\n\n\n    loss_name = \"BCEFocal2WayLoss\"\n    loss_params: dict = {}\n\n        \n    optimizer_name = \"Adam\"\n    base_optimizer = \"Adam\"\n    optimizer_params = {\n        \"lr\": 0.0015\n    }\n\n    base_optimizer = \"Adam\"\n\n    \n    scheduler_name = \"CosineAnnealingLR\"\n    scheduler_params = {\n        \"T_max\": 10\n    }","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TARGET_SR = 32000\nTEST = (len(list(Path(\"../input/birdclef-2021/test_soundscapes/\").glob(\"*.ogg\"))) != 0)\nif TEST:\n    DATADIR = Path(\"../input/birdclef-2021/test_soundscapes/\")\nelse:\n    DATADIR = Path(\"../input/birdclef-2021/train_soundscapes/\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_audios = list(DATADIR.glob(\"*.ogg\"))\nall_audio_ids = [\"_\".join(audio_id.name.split(\"_\")[:2]) for audio_id in all_audios]\nsubmission_df = pd.DataFrame({\n    \"row_id\": all_audio_ids\n})\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef init_weights(model):\n    classname = model.__class__.__name__\n    if classname.find(\"Conv2d\") != -1:\n        nn.init.xavier_uniform_(model.weight, gain=np.sqrt(2))\n        model.bias.data.fill_(0)\n    elif classname.find(\"BatchNorm\") != -1:\n        model.weight.data.normal_(1.0, 0.02)\n        model.bias.data.fill_(0)\n    elif classname.find(\"GRU\") != -1:\n        for weight in model.parameters():\n            if len(weight.size()) > 1:\n                nn.init.orghogonal_(weight.data)\n    elif classname.find(\"Linear\") != -1:\n        model.weight.data.normal_(0, 0.01)\n        model.bias.data.zero_()\n\n\ndef do_mixup(x: torch.Tensor, mixup_lambda: torch.Tensor):\n\n    out = (x[0::2].transpose(0, -1) * mixup_lambda[0::2] +\n           x[1::2].transpose(0, -1) * mixup_lambda[1::2]).transpose(0, -1)\n    return out\n\n\nclass Mixup(object):\n    def __init__(self, mixup_alpha, random_seed=108):\n\n        self.mixup_alpha = mixup_alpha\n        self.random_state = np.random.RandomState(random_seed)\n\n    def get_lambda(self, batch_size):\n        mixup_lambdas = []\n        for n in range(0, batch_size, 2):\n            lam = self.random_state.beta(\n                self.mixup_alpha, self.mixup_alpha, 1)[0]\n            mixup_lambdas.append(lam)\n            mixup_lambdas.append(1. - lam)\n\n        return torch.from_numpy(np.array(mixup_lambdas, dtype=np.float32))\n\n\ndef interpolate(x: torch.Tensor, ratio: int):\n\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n\n    output = F.interpolate(\n        framewise_output.unsqueeze(1),\n        size=(frames_num, framewise_output.size(2)),\n        align_corners=True,\n        mode=\"bilinear\").squeeze(1)\n\n    return output\n\n\ndef gem(x: torch.Tensor, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1. / p)\n\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super().__init__()\n        self.p = nn.Parameter(torch.ones(1) * p)\n        self.eps = eps\n\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)\n\n    def __repr__(self):\n        return self.__class__.__name__ + f\"(p={self.p.data.tolist()[0]:.4f}, eps={self.eps})\"\n\n\nclass AttBlockV2(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)\n\ndroprate=0.2 #0.5\n\nclass TimmSED(nn.Module):\n    def __init__(self, base_model_name: str, pretrained=False, num_classes=24, in_channels=1):\n        super().__init__()\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=CFG.n_fft, hop_length=CFG.hop_length,\n                                                 win_length=CFG.n_fft, window=\"hann\", center=True, pad_mode=\"reflect\",\n                                                 freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=CFG.sample_rate, n_fft=CFG.n_fft,\n                                                 n_mels=CFG.n_mels, fmin=CFG.fmin, fmax=CFG.fmax, ref=1.0, amin=1e-10, top_db=None,\n                                                 freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2,\n                                               freq_drop_width=8, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(CFG.n_mels)\n\n        base_model = timm.create_model(\n            base_model_name, pretrained=pretrained, in_chans=in_channels)\n        layers = list(base_model.children())[:-2]\n        self.encoder = nn.Sequential(*layers)\n\n        if hasattr(base_model, \"fc\"):\n            in_features = base_model.fc.in_features\n        else:\n            in_features = base_model.classifier.in_features\n        self.fc1 = nn.Linear(in_features, in_features, bias=True)\n        self.att_block = AttBlockV2(\n            in_features, num_classes, activation=\"sigmoid\")\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.fc1)\n        init_bn(self.bn0)\n\n    def forward(self, input):\n        # (batch_size, 1, time_steps, freq_bins)\n        x = self.spectrogram_extractor(input)\n        x = self.logmel_extractor(x)    # (batch_size, 1, time_steps, mel_bins)\n\n        frames_num = x.shape[2]\n\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n\n        if self.training:\n            x = self.spec_augmenter(x)\n\n        x = x.transpose(2, 3)\n        # (batch_size, channels, freq, frames)\n        x = self.encoder(x)\n\n        # (batch_size, channels, frames)\n        x = torch.mean(x, dim=2)\n\n        # channel smoothing\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=droprate, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=droprate, training=self.training)\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        logit = torch.sum(norm_att * self.att_block.cla(x), dim=2)\n        segmentwise_logit = self.att_block.cla(x).transpose(1, 2)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        interpolate_ratio = frames_num // segmentwise_output.size(1)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        framewise_logit = interpolate(segmentwise_logit, interpolate_ratio)\n        framewise_logit = pad_framewise_output(framewise_logit, frames_num)\n\n        output_dict = {\n            \"framewise_output\": framewise_output,\n            \"segmentwise_output\": segmentwise_output,\n            \"logit\": logit,\n            \"framewise_logit\": framewise_logit,\n            \"clipwise_output\": clipwise_output\n        }\n\n        return output_dict","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestDataset(torchdata.Dataset):\n    def __init__(self, df: pd.DataFrame, clip: np.ndarray,\n                 waveform_transforms=None):\n        self.df = df\n        self.clip = clip\n        self.waveform_transforms=waveform_transforms\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx: int):\n        SR = 32000\n        sample = self.df.loc[idx, :]\n        row_id = sample.row_id\n\n        end_seconds = int(sample.seconds)\n        start_seconds = int(end_seconds - 5)\n\n        start_index = SR * start_seconds\n        end_index = SR * end_seconds\n\n        y = self.clip[start_index:end_index].astype(np.float32)\n\n        y = np.nan_to_num(y)\n\n        if self.waveform_transforms:\n            y = self.waveform_transforms(y)\n\n        y = np.nan_to_num(y)\n\n        return y, row_id","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transforms(phase: str):\n    transforms = CFG.transforms\n    if transforms is None:\n        return None\n    else:\n        if transforms[phase] is None:\n            return None\n        trns_list = []\n        for trns_conf in transforms[phase]:\n            trns_name = trns_conf[\"name\"]\n            trns_params = {} if trns_conf.get(\"params\") is None else \\\n                trns_conf[\"params\"]\n            if globals().get(trns_name) is not None:\n                trns_cls = globals()[trns_name]\n                trns_list.append(trns_cls(**trns_params))\n\n        if len(trns_list) > 0:\n            return Compose(trns_list)\n        else:\n            return None\n\n\ndef get_waveform_transforms(config: dict, phase: str):\n    return get_transforms(config, phase)\n\n\ndef get_spectrogram_transforms(config: dict, phase: str):\n    transforms = config.get('spectrogram_transforms')\n    if transforms is None:\n        return None\n    else:\n        if transforms[phase] is None:\n            return None\n        trns_list = []\n        for trns_conf in transforms[phase]:\n            trns_name = trns_conf[\"name\"]\n            trns_params = {} if trns_conf.get(\"params\") is None else \\\n                trns_conf[\"params\"]\n            if hasattr(A, trns_name):\n                trns_cls = A.__getattribute__(trns_name)\n                trns_list.append(trns_cls(**trns_params))\n            else:\n                trns_cls = globals().get(trns_name)\n                if trns_cls is not None:\n                    trns_list.append(trns_cls(**trns_params))\n\n        if len(trns_list) > 0:\n            return A.Compose(trns_list, p=1.0)\n        else:\n            return None\n\n\nclass Normalize:\n    def __call__(self, y: np.ndarray):\n        max_vol = np.abs(y).max()\n        y_vol = y * 1 / max_vol\n        return np.asfortranarray(y_vol)\n\n\nclass NewNormalize:\n    def __call__(self, y: np.ndarray):\n        y_mm = y - y.mean()\n        return y_mm / y_mm.abs().max()\n\n\nclass Compose:\n    def __init__(self, transforms: list):\n        self.transforms = transforms\n\n    def __call__(self, y: np.ndarray):\n        for trns in self.transforms:\n            y = trns(y)\n        return y\n\n\nclass AudioTransform:\n    def __init__(self, always_apply=False, p=0.5):\n        self.always_apply = always_apply\n        self.p = p\n\n    def __call__(self, y: np.ndarray):\n        if self.always_apply:\n            return self.apply(y)\n        else:\n            if np.random.rand() < self.p:\n                return self.apply(y)\n            else:\n                return y\n\n    def apply(self, y: np.ndarray):\n        raise NotImplementedError\n\n\nclass NoiseInjection(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, max_noise_level=0.5, sr=32000):\n        super().__init__(always_apply, p)\n\n        self.noise_level = (0.0, max_noise_level)\n        self.sr = sr\n\n    def apply(self, y: np.ndarray, **params):\n        noise_level = np.random.uniform(*self.noise_level)\n        noise = np.random.randn(len(y))\n        augmented = (y + noise * noise_level).astype(y.dtype)\n        return augmented\n\n\nclass GaussianNoise(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, min_snr=5, max_snr=20, sr=32000):\n        super().__init__(always_apply, p)\n\n        self.min_snr = min_snr\n        self.max_snr = max_snr\n        self.sr = sr\n\n    def apply(self, y: np.ndarray, **params):\n        snr = np.random.uniform(self.min_snr, self.max_snr)\n        a_signal = np.sqrt(y ** 2).max()\n        a_noise = a_signal / (10 ** (snr / 20))\n\n        white_noise = np.random.randn(len(y))\n        a_white = np.sqrt(white_noise ** 2).max()\n        augmented = (y + white_noise * 1 / a_white * a_noise).astype(y.dtype)\n        return augmented\n\n\nclass PinkNoise(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, min_snr=5, max_snr=20, sr=32000):\n        super().__init__(always_apply, p)\n\n        self.min_snr = min_snr\n        self.max_snr = max_snr\n        self.sr = sr\n\n    def apply(self, y: np.ndarray, **params):\n        snr = np.random.uniform(self.min_snr, self.max_snr)\n        a_signal = np.sqrt(y ** 2).max()\n        a_noise = a_signal / (10 ** (snr / 20))\n\n        pink_noise = cn.powerlaw_psd_gaussian(1, len(y))\n        a_pink = np.sqrt(pink_noise ** 2).max()\n        augmented = (y + pink_noise * 1 / a_pink * a_noise).astype(y.dtype)\n        return augmented\n\n\nclass PitchShift(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, max_range=5, sr=32000):\n        super().__init__(always_apply, p)\n        self.max_range = max_range\n        self.sr = sr\n\n    def apply(self, y: np.ndarray, **params):\n        n_steps = np.random.randint(-self.max_range, self.max_range)\n        augmented = librosa.effects.pitch_shift(y, self.sr, n_steps)\n        return augmented\n\n\nclass TimeStretch(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, max_rate=1, sr=32000):\n        super().__init__(always_apply, p)\n        self.max_rate = max_rate\n        self.sr = sr\n\n    def apply(self, y: np.ndarray, **params):\n        rate = np.random.uniform(0, self.max_rate)\n        augmented = librosa.effects.time_stretch(y, rate)\n        return augmented\n\n\ndef _db2float(db: float, amplitude=True):\n    if amplitude:\n        return 10**(db / 20)\n    else:\n        return 10 ** (db / 10)\n\n\ndef volume_down(y: np.ndarray, db: float):\n\n    applied = y * _db2float(-db)\n    return applied\n\n\ndef volume_up(y: np.ndarray, db: float):\n\n    applied = y * _db2float(db)\n    return applied\n\n\nclass RandomVolume(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, limit=10):\n        super().__init__(always_apply, p)\n        self.limit = limit\n\n    def apply(self, y: np.ndarray, **params):\n        db = np.random.uniform(-self.limit, self.limit)\n        if db >= 0:\n            return volume_up(y, db)\n        else:\n            return volume_down(y, db)\n\n\nclass OneOf:\n    def __init__(self, transforms: list):\n        self.transforms = transforms\n\n    def __call__(self, y: np.ndarray):\n        n_trns = len(self.transforms)\n        trns_idx = np.random.choice(n_trns)\n        trns = self.transforms[trns_idx]\n        y = trns(y)\n        return y\n\n\nclass CosineVolume(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, limit=10):\n        super().__init__(always_apply, p)\n        self.limit = limit\n\n    def apply(self, y: np.ndarray, **params):\n        db = np.random.uniform(-self.limit, self.limit)\n        cosine = np.cos(np.arange(len(y)) / len(y) * np.pi * 2)\n        dbs = _db2float(cosine * db)\n        return y * dbs\n\n\ndef drop_stripes(image: np.ndarray, dim: int, drop_width: int, stripes_num: int):\n    total_width = image.shape[dim]\n    lowest_value = image.min()\n    for _ in range(stripes_num):\n        distance = np.random.randint(low=0, high=drop_width, size=(1,))[0]\n        begin = np.random.randint(\n            low=0, high=total_width - distance, size=(1,))[0]\n\n        if dim == 0:\n            image[begin:begin + distance] = lowest_value\n        elif dim == 1:\n            image[:, begin + distance] = lowest_value\n        elif dim == 2:\n            image[:, :, begin + distance] = lowest_value\n    return image\n\n\nclass TimeFreqMasking(ImageOnlyTransform):\n    def __init__(self,\n                 time_drop_width: int,\n                 time_stripes_num: int,\n                 freq_drop_width: int,\n                 freq_stripes_num: int,\n                 always_apply=False,\n                 p=0.5):\n        super().__init__(always_apply, p)\n        self.time_drop_width = time_drop_width\n        self.time_stripes_num = time_stripes_num\n        self.freq_drop_width = freq_drop_width\n        self.freq_stripes_num = freq_stripes_num\n\n    def apply(self, img, **params):\n        img_ = img.copy()\n        if img.ndim == 2:\n            img_ = drop_stripes(\n                img_, dim=0, drop_width=self.freq_drop_width, stripes_num=self.freq_stripes_num)\n            img_ = drop_stripes(\n                img_, dim=1, drop_width=self.time_drop_width, stripes_num=self.time_stripes_num)\n        return img_","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_model_for_inference(model, path: Path):\n    if not torch.cuda.is_available():\n        ckpt = torch.load(path, map_location=\"cpu\")\n    else:\n        ckpt = torch.load(path)\n    model.load_state_dict(ckpt[\"model_state_dict\"])\n    model.eval()\n    return model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prediction_for_clip(test_df: pd.DataFrame, \n                        clip: np.ndarray, \n                        model, \n                        threshold=0.5):\n\n    dataset = TestDataset(df=test_df, \n                          clip=clip,\n                          waveform_transforms=get_transforms(phase=\"test\"))\n    loader = torchdata.DataLoader(dataset, batch_size=1, shuffle=False)\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    model.eval()\n    prediction_dict = {}\n    for image, row_id in tqdm(loader):\n        row_id = row_id[0]\n        image = image.to(device)\n\n        with torch.no_grad():\n            prediction = model(image)\n            proba = prediction[\"clipwise_output\"].detach().cpu().numpy().reshape(-1)\n\n        events = proba >= threshold\n        labels = np.argwhere(events).reshape(-1).tolist()\n\n        if len(labels) == 0:\n            prediction_dict[row_id] = \"nocall\"\n        else:\n            labels_str_list = list(map(lambda x: CFG.target_columns[x], labels))\n            label_string = \" \".join(labels_str_list)\n            prediction_dict[row_id] = label_string\n    return prediction_dict","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prediction(test_audios,\n               weights_path: Path,\n               threshold=0.6):\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    model = TimmSED(base_model_name=CFG.base_model_name,\n                    pretrained=False,\n                    num_classes=CFG.num_classes,\n                    in_channels=CFG.in_channels)\n    model = prepare_model_for_inference(model, weights_path).to(device)\n\n    warnings.filterwarnings(\"ignore\")\n    prediction_dfs = []\n    for audio_path in test_audios:\n        with timer(f\"Loading {str(audio_path)}\", logger):\n            clip, _ = sf.read(audio_path)\n\n        seconds = []\n        row_ids = []\n        for second in range(5, 605, 5):\n            row_id = \"_\".join(audio_path.name.split(\"_\")[:2]) + f\"_{second}\"\n            seconds.append(second)\n            row_ids.append(row_id)\n            \n        test_df = pd.DataFrame({\n            \"row_id\": row_ids,\n            \"seconds\": seconds\n        })\n        with timer(f\"Prediction on {audio_path}\", logger):\n            prediction_dict = prediction_for_clip(test_df,\n                                                  clip=clip,\n                                                  model=model,\n                                                  threshold=threshold)\n        row_id = list(prediction_dict.keys())\n        birds = list(prediction_dict.values())\n        prediction_df = pd.DataFrame({\n            \"row_id\": row_id,\n            \"birds\": birds\n        })\n        prediction_dfs.append(prediction_df)\n    \n    prediction_df = pd.concat(prediction_dfs, axis=0, sort=False).reset_index(drop=True)\n    return prediction_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"th=0.38\nweights_path = Path(\"../input/birdclef2021-effnetb0-starter-weight/best.pth\")\nsubmission = prediction(test_audios=all_audios,\n                        weights_path=weights_path,\n                        threshold=th)\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# to be continued...","metadata":{}}]}