{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nfrom tqdm import tqdm\n#from tensorflow.python.compiler.mlcompute import mlcompute\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport librosa\n\nimport time\nimport warnings\nwarnings.filterwarnings(action='ignore')\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2021-06-08T19:57:04.149518Z","iopub.execute_input":"2021-06-08T19:57:04.150000Z","iopub.status.idle":"2021-06-08T19:57:04.156449Z","shell.execute_reply.started":"2021-06-08T19:57:04.149966Z","shell.execute_reply":"2021-06-08T19:57:04.155414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Soundscapes Dataset","metadata":{}},{"cell_type":"code","source":"train_soundscape = pd.read_csv('../input/birdclef-2021/train_soundscape_labels.csv',)\ntrain_soundscape","metadata":{"execution":{"iopub.status.busy":"2021-06-08T19:57:04.158402Z","iopub.execute_input":"2021-06-08T19:57:04.159005Z","iopub.status.idle":"2021-06-08T19:57:04.199345Z","shell.execute_reply.started":"2021-06-08T19:57:04.158961Z","shell.execute_reply":"2021-06-08T19:57:04.198265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Anzahl der Datensätze: %s\" % (\n    train_soundscape['audio_id'].value_counts().count()\n))\nprint(\"Anzahl der Audio-Files: %s\" % (\n    train_soundscape['audio_id'].count()\n))\nprint(\"Anzahl der Annotierungen: %s\" % (\n    sum(map(len, train_soundscape['birds'].str.split(' ',1)))\n))\nprint(\"Durchschnittliche Anzahl der Annotierungen pro Audio-File: %s\" % (\n    train_soundscape['audio_id'].value_counts().mean()\n))\n\n#train_soundscape.groupby(['audio_id', 'birds'])['birds'].value_counts() #.apply(display)\nfig, axs = plt.subplots(figsize=(12, 4))\ntrain_soundscape['birds'].str.split(' ',1).str.len().value_counts().plot(\n    kind='bar', rot=0, ax=axs, legend=False,\n    title='Annotierungen pro Datensatz', \n    xlabel='Anzahl Annotierungen',\n    ylabel='Datensätze',\n    color='black',\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T19:57:04.202125Z","iopub.execute_input":"2021-06-08T19:57:04.202550Z","iopub.status.idle":"2021-06-08T19:57:04.349622Z","shell.execute_reply.started":"2021-06-08T19:57:04.202499Z","shell.execute_reply":"2021-06-08T19:57:04.348651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Anzahl der Datensätze für Call / Nocall\nfig, axs = plt.subplots(figsize=(12, 4))\n\ntrain_soundscape['call_nocall'] = train_soundscape['birds']\ntrain_soundscape['call_nocall'][train_soundscape['birds'] != 'nocall'] = 'call'\n\ntrain_soundscape['call_nocall'].value_counts().plot(\n    kind='bar', rot=0, ax=axs, legend=False,\n    title='Anzahl der Datensätze für Call / Nocall', \n    xlabel='Anzahl Annotierungen',\n    ylabel='Datensätze',\n    color='black',\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T19:57:04.351130Z","iopub.execute_input":"2021-06-08T19:57:04.351410Z","iopub.status.idle":"2021-06-08T19:57:04.514571Z","shell.execute_reply.started":"2021-06-08T19:57:04.351381Z","shell.execute_reply":"2021-06-08T19:57:04.513739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(figsize=(12, 4))\n\ntrain_soundscape['birds'].str.split(' ',1).explode().value_counts().drop(labels=\"nocall\", axis=0).plot(\n    kind='bar', rot=0, ax=axs, legend=False,\n    title='Verteilung Anzahl der Datensätze pro Art', \n    xlabel='Art',\n    ylabel='Anzahl Datensätze',\n    color='black',\n)\naxs.axes.xaxis.set_ticks([]) # remove Labels for X","metadata":{"execution":{"iopub.status.busy":"2021-06-08T19:57:04.515965Z","iopub.execute_input":"2021-06-08T19:57:04.516241Z","iopub.status.idle":"2021-06-08T19:57:04.847901Z","shell.execute_reply.started":"2021-06-08T19:57:04.516214Z","shell.execute_reply":"2021-06-08T19:57:04.846877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Dataset","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/birdclef-2021/train_metadata.csv',)\ntrain","metadata":{"execution":{"iopub.status.busy":"2021-06-08T19:57:04.849484Z","iopub.execute_input":"2021-06-08T19:57:04.849915Z","iopub.status.idle":"2021-06-08T19:57:05.468305Z","shell.execute_reply.started":"2021-06-08T19:57:04.849872Z","shell.execute_reply":"2021-06-08T19:57:05.467199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lade den Deutschen Namen von https://ebird.org/species/acafly\nimport requests\n\ndef get_german_name(primary_label):\n    payload = {\n        'fmt': 'json',\n        'locale': 'de',\n        'species': primary_label\n    }\n    r = requests.get('https://api.ebird.org/v2/ref/taxonomy/ebird',\n        params=payload,\n        headers={\n            'X-eBirdApiToken': 'RXpK1BNzbJNcHfBs'\n        }\n    )\n    data = r.json()\n    if data[0] and 'comName' in data[0]:\n        return data[0]['comName']\n    return 'nicht bekannt'\n\n# sollte nicht jedes mal angefragt werden\n# german_names = {}\n# for primary_label in train.primary_label.unique():\n#    german_names[primary_label] = get_german_name(primary_label)\n\ngerman_names = {'acafly': 'Buchenschnäppertyrann', 'acowoo': 'Eichelspecht', 'aldfly': 'Erlenschnäppertyrann', 'ameavo': 'Braunhals-Säbelschnäbler', 'amecro': 'Amerikakrähe', 'amegfi': 'Goldzeisig', 'amekes': 'Buntfalke', 'amepip': 'Pazifikpieper', 'amered': 'Rotschwanz-Waldsänger', 'amerob': 'Wanderdrossel', 'amewig': 'Kanadapfeifente', 'amtspa': 'Baumammer', 'andsol1': 'Andenklarino', 'annhum': 'Annakolibri', 'astfly': 'Kalifornienschopftyrann', 'azaspi1': 'Azaradickichtschlüpfer', 'babwar': 'Braunkehl-Waldsänger', 'baleag': 'Weißkopf-Seeadler', 'balori': 'Baltimoretrupial', 'banana': 'Zuckervogel', 'banswa': 'Uferschwalbe', 'banwre1': 'Akazienzaunkönig', 'barant1': 'Bindenameisenwürger', 'barswa': 'Rauchschwalbe', 'batpig1': 'Bandtaube', 'bawswa1': 'Schwarzsteißschwalbe', 'bawwar': 'Kletterwaldsänger', 'baywre1': 'Kastanienzaunkönig', 'bbwduc': 'Rotschnabel-Pfeifgans', 'bcnher': 'Nachtreiher', 'belkin1': 'Gürtelfischer', 'belvir': 'Braunaugenvireo', 'bewwre': 'Buschzaunkönig', 'bkbmag1': 'Hudsonelster', 'bkbplo': 'Kiebitzregenpfeifer', 'bkbwar': 'Fichtenwaldsänger', 'bkcchi': 'Schwarzkopfmeise', 'bkhgro': 'Schwarzkopf-Kernknacker', 'bkmtou1': 'Goldkehltukan', 'bknsti': 'Schwarznacken-Stelzenläufer', 'blbgra1': 'Jacarinitangare', 'blbthr1': 'Schwarzschnabeldrossel', 'blcjay1': 'Schwarzbrust-Blaurabe', 'blctan1': 'Schwarzscheiteltangare', 'blhpar1': 'Schwarzohrpapagei', 'blkpho': 'Schwarzkopf-Phoebetyrann', 'blsspa1': 'Dickichtammer', 'blugrb1': 'Azurfink', 'blujay': 'Blauhäher', 'bncfly': 'Braunschopftyrann', 'bnhcow': 'Braunkopf-Kuhstärling', 'bobfly1': 'Starkschnabel-Maskentyrann', 'bongul': 'Bonapartemöwe', 'botgra': 'Bootschwanzgrackel', 'brbmot1': 'Plattschnabelmotmot', 'brbsol1': 'Braunrückenklarino', 'brcvir1': 'Braunkappenvireo', 'brebla': 'Purpurstärling', 'brncre': 'Amerikabaumläufer', 'brnjay': 'Braunhäher', 'brnthr': 'Rotspottdrossel', 'brratt1': 'Gelbbürzel-Attilatyrann', 'brwhaw': 'Breitflügelbussard', 'brwpar1': 'Glanzflügelpapagei', 'btbwar': 'Blaurücken-Waldsänger', 'btnwar': 'Grünmantel-Waldsänger', 'btywar': 'Trauerwaldsänger', 'bucmot2': 'Diademmotmot', 'buggna': 'Blaumückenfänger', 'bugtan': 'Bischofstangare', 'buhvir': 'Graukopfvireo', 'bulori': 'Bullocktrupial', 'burwar1': 'Schmätzerwaldsänger', 'bushti': 'Buschschwanzmeise', 'butsal1': 'Buntkehlsaltator', 'buwtea': 'Blauflügelente', 'cacgoo1': 'Zwergkanadagans', 'cacwre': 'Kaktuszaunkönig', 'calqua': 'Schopfwachtel', 'caltow': 'Kaliforniengrundammer', 'cangoo': 'Kanadagans', 'canwar': 'Kanadawaldsänger', 'carchi': 'Carolinameise', 'carwre': 'Carolinazaunkönig', 'casfin': 'Cassingimpel', 'caskin': 'Cassinkönigstyrann', 'caster1': 'Raubseeschwalbe', 'casvir': 'Cassinvireo', 'categr': 'Kuhreiher', 'ccbfin': 'Braunkopf-Buschammer', 'cedwax': 'Zedernseidenschwanz', 'chbant1': 'Braunrücken-Ameisenvogel', 'chbchi': 'Rotrückenmeise', 'chbwre1': 'Kastanienbrust-Zaunkönig', 'chcant2': 'Rostkappen-Ameisenpitta', 'chispa': 'Schwirrammer', 'chswar': 'Gelbscheitel-Waldsänger', 'cinfly2': 'Zimttyrann', 'clanut': 'Kiefernhäher', 'clcrob': 'Gilbdrossel', 'cliswa': 'Fahlstirnschwalbe', 'cobtan1': 'Finkengrünammer', 'cocwoo1': 'Kakaobaumsteiger', 'cogdov': 'Sperlingstäubchen', 'colcha1': 'Kolumbienguan', 'coltro1': 'Jungferntrogon', 'comgol': 'Schellente', 'comgra': 'Purpurgrackel', 'comloo': 'Eistaucher', 'commer': 'Gänsesäger', 'compau': 'Pauraquenachtschwalbe', 'compot1': 'Klagetagschläfer', 'comrav': 'Kolkrabe', 'comyel': 'Weidengelbkehlchen', 'coohaw': 'Rundschwanzhabicht', 'cotfly1': 'Graugelb-Todityrann', 'cowscj1': 'Kalifornienhäher', 'cregua1': 'Haubenguan', 'creoro1': 'Krähenstirnvogel', 'crfpar': 'Veraguasittich', 'cubthr': 'Krummschnabel-Spottdrossel', 'daejun': 'Winterammer', 'dowwoo': 'Dunenspecht', 'ducfly': 'Schwarzkappen-Schopftyrann', 'dusfly': 'Buschland-Schnäppertyrann', 'easblu': 'Rotkehl-Hüttensänger', 'easkin': 'Schieferrücken-Königstyrann', 'easmea': 'Lerchenstärling', 'easpho': 'Weißbauch-Phoebetyrann', 'eastow': 'Rötelgrundammer', 'eawpew': 'Hellbauch-Schnäppertyrann', 'eletro': 'Kupfertrogon', 'eucdov': 'Türkentaube', 'eursta': 'Star', 'fepowl': 'Brasilzwergkauz', 'fiespa': 'Klapperammer', 'flrtan1': 'Feuerbürzeltangare', 'foxspa': 'Fuchsammer', 'gadwal': 'Schnatterente', 'gamqua': 'Helmwachtel', 'gartro1': 'Grünschwanztrogon', 'gbbgul': 'Mantelmöwe', 'gbwwre1': 'Einsiedlerzaunkönig', 'gcrwar': 'Goldhähnchen-Waldsänger', 'gilwoo': 'Gilaspecht', 'gnttow': 'Grünschwanz-Grundammer', 'gnwtea': 'Krickente', 'gocfly1': 'Andenmaskentyrann', 'gockin': 'Indianergoldhähnchen', 'gocspa': 'Kronenammer', 'goftyr1': 'Goldgesicht-Kleintyrann', 'gohque1': 'Goldkopftrogon', 'goowoo1': 'Olivmantelspecht', 'grasal1': 'Grausaltator', 'grbani': 'Riefenschnabelani', 'grbher3': 'Kanadareiher', 'grcfly': 'Gelbbauch-Schopftyrann', 'greegr': 'Silberreiher', 'grekis': 'Schwefelmaskentyrann', 'grepew': 'Mexikoschnäppertyrann', 'grethr1': 'Riesendrossel', 'gretin1': 'Großtinamu', 'greyel': 'Tüpfelgelbschenkel', 'grhcha1': 'Graukopfguan', 'grhowl': 'Virginiauhu', 'grnher': 'Grünreiher', 'grnjay': 'Inkablaurabe', 'grtgra': 'Großschwanzgrackel', 'grycat': 'Katzenspottdrossel', 'gryhaw2': 'Graubussard', 'gwfgoo': 'Blässgans', 'haiwoo': 'Haarspecht', 'heptan': 'Zinnoberkardinal', 'hergul': 'Silbermöwe', 'herthr': 'Einsiedler-Musendrossel', 'herwar': 'Einsiedelwaldsänger', 'higmot1': 'Hochlandmotmot', 'hofwoo1': 'Hoffmannspecht', 'houfin': 'Hausgimpel', 'houspa': 'Haussperling', 'houwre': 'Hauszaunkönig', 'hutvir': 'Huttonvireo', 'incdov': 'Inkatäubchen', 'indbun': 'Indigofink', 'kebtou1': 'Fischertukan', 'killde': 'Keilschwanz-Regenpfeifer', 'labwoo': 'Texasspecht', 'larspa': 'Rainammer', 'laufal1': 'Lachfalke', 'laugul': 'Aztekenmöwe', 'lazbun': 'Lazulifink', 'leafly': 'Zwergschnäppertyrann', 'leasan': 'Wiesenstrandläufer', 'lesgol': 'Mexikozeisig', 'lesgre1': 'Graukappenvireo', 'lesvio1': 'Berg-Veilchenohrkolibri', 'linspa': 'Lincolnammer', 'linwoo1': 'Linienspecht', 'littin1': 'Brauntinamu', 'lobdow': 'Tundraschlammläufer', 'lobgna5': 'Schwarzschwanz-Degenschnäbler', 'logshr': 'Louisianawürger', 'lotduc': 'Eisente', 'lotman1': 'Langschwanzpipra', 'lucwar': 'Rotbürzel-Waldsänger', 'macwar': 'Dickichtwaldsänger', 'magwar': 'Magnolienwaldsänger', 'mallar3': 'Stockente', 'marwre': 'Sumpfzaunkönig', 'mastro1': 'Maskentrogon', 'meapar': 'Mülleramazone', 'melbla1': 'Trauerstärling', 'monoro1': 'Montezumastirnvogel', 'mouchi': 'Gebirgsmeise', 'moudov': 'Carolinataube', 'mouela1': 'Nordanden-Olivtyrann', 'mouqua': 'Bergwachtel', 'mouwar': 'Graukopf-Waldsänger', 'mutswa': 'Höckerschwan', 'naswar': 'Rubinfleck-Waldsänger', 'norcar': 'Rotkardinal', 'norfli': 'Goldspecht', 'normoc': 'Gartenspottdrossel', 'norpar': 'Meisenwaldsänger', 'norsho': 'Löffelente', 'norwat': 'Drosselwaldsänger', 'nrwswa': 'Graukehlschwalbe', 'nutwoo': 'Nuttallspecht', 'oaktit': 'Eichenmeise', 'obnthr1': 'Goldschnabel-Musendrossel', 'ocbfly1': 'Ockerbauch-Pipratyrann', 'oliwoo1': 'Dünnschnabel-Baumsteiger', 'olsfly': 'Olivflanken-Schnäppertyrann', 'orbeup1': 'Gelbbauchorganist', 'orbspa1': 'Goldschnabel-Buschammer', 'orcpar': 'Tovisittich', 'orcwar': 'Orangefleck-Waldsänger', 'orfpar': 'Elfenbeinsittich', 'osprey': 'Fischadler', 'ovenbi1': 'Pieperwaldsänger', 'pabspi1': 'Weißbauch-Dickichtschlüpfer', 'paltan1': 'Palmentangare', 'palwar': 'Palmenwaldsänger', 'pasfly': 'Feuchtwald-Schnäppertyrann', 'pavpig2': 'Rotrückentaube', 'phivir': 'Philadelphiavireo', 'pibgre': 'Bindentaucher', 'pilwoo': 'Helmspecht', 'pinsis': 'Fichtenzeisig', 'pirfly1': 'Kurzschnabel-Maskentyrann', 'plawre1': 'Cabaniszaunkönig', 'plaxen1': 'Braunbauch-Baumspäher', 'plsvir': 'Weißstirnvireo', 'plupig2': 'Weintaube', 'prowar': 'Zitronenwaldsänger', 'purfin': 'Purpurgimpel', 'purgal2': 'Zwergsultanshuhn', 'putfru1': 'Purpurbrustkotinga', 'pygnut': 'Zwergkleiber', 'rawwre1': 'Rotrückenzaunkönig', 'rcatan1': 'Karmesinkardinal', 'rebnut': 'Rotbrustkleiber', 'rebsap': 'Feuerkopf-Saftlecker', 'rebwoo': 'Carolinaspecht', 'redcro': 'Fichtenkreuzschnabel', 'reevir1': 'Rotaugenvireo', 'rehbar1': 'Andenbartvogel', 'relpar': 'Rotstirnamazone', 'reshaw': 'Rotschulterbussard', 'rethaw': 'Rotschwanzbussard', 'rewbla': 'Rotflügelstärling', 'ribgul': 'Ringschnabelmöwe', 'rinkin1': 'Rotbrustfischer', 'roahaw': 'Wegebussard', 'robgro': 'Rosenbrust-Kernknacker', 'rocpig': 'Felsentaube', 'rotbec': 'Rosenkehlbekarde', 'royter1': 'Königsseeschwalbe', 'rthhum': 'Rubinkehlkolibri', 'rtlhum': 'Braunschwanzamazilie', 'ruboro1': 'Breithauben-Stirnvogel', 'rubpep1': 'Rostbrauenvireo', 'rubrob': 'Rotmanteldrossel', 'rubwre1': 'Rotbrust-Zaunkönig', 'ruckin': 'Rubingoldhähnchen', 'rucspa1': 'Morgenammer', 'rucwar': 'Rotkappen-Waldsänger', 'rucwar1': 'Goldscheitel-Waldsänger', 'rudpig': 'Purpurtaube', 'rudtur': 'Steinwälzer', 'rufhum': 'Rotrücken-Zimtelfe', 'rugdov': 'Rosttäubchen', 'rumfly1': 'Rostschwingen-Maskentyrann', 'runwre1': 'Rotnacken-Zaunkönig', 'rutjac1': 'Rotschwanz-Glanzvogel', 'saffin': 'Safrangilbtangare', 'sancra': 'Kanadakranich', 'sander': 'Sanderling', 'savspa': 'Grasammer', 'saypho': 'Zimtbauch-Phoebetyrann', 'scamac1': 'Scharlachara', 'scatan': 'Scharlachkardinal', 'scbwre1': 'Schuppenbrust-Zaunkönig', 'scptyr1': 'Rot-Schuppenkopftyrann', 'scrtan1': 'Rotscheiteltangare', 'semplo': 'Eskimoregenpfeifer', 'shicow': 'Seidenkuhstärling', 'sibtan2': 'Purpurtangare', 'sinwre1': 'Sinaloazaunkönig', 'sltred': 'Larvenwaldsänger', 'smbani': 'Glattschnabelani', 'snogoo': 'Schneegans', 'sobtyr1': 'Haubenkleintyrann', 'socfly1': 'Rotscheitel-Maskentyrann', 'solsan': 'Einsiedelwasserläufer', 'sonspa': 'Singammer', 'soulap1': 'Bronzekiebitz', 'sposan': 'Drosseluferläufer', 'spotow': 'Fleckengrundammer', 'spvear1': 'Glanz-Veilchenohrkolibri', 'squcuc1': 'Eichhornkuckuck', 'stbori': 'Piroltrupial', 'stejay': 'Diademhäher', 'sthant1': 'Strichelkopf-Ameisenfänger', 'sthwoo1': 'Lanzettstrichel-Baumsteiger', 'strcuc1': 'Streifenkuckuck', 'strfly1': 'Süd-Fleckenmaskentyrann', 'strsal1': 'Strichelsaltator', 'stvhum2': 'Grünamazilie', 'subfly': 'Nord-Fleckenmaskentyrann', 'sumtan': 'Sommerkardinal', 'swaspa': 'Sumpfammer', 'swathr': 'Zwergmusendrossel', 'tenwar': 'Brauenwaldsänger', 'thbeup1': 'Dickschnabelorganist', 'thbkin': 'Dickschnabel-Königstyrann', 'thswar1': 'Dreistreifen-Waldsänger', 'towsol': 'Townsendklarino', 'treswa': 'Sumpfschwalbe', 'trogna1': 'Amazonasmückenfänger', 'trokin': 'Trauerkönigstyrann', 'tromoc': 'Tropenspottdrossel', 'tropar': 'Elfenwaldsänger', 'tropew1': 'Dunkel-Waldschnäppertyrann', 'tuftit': 'Grauhäubchenmeise', 'tunswa': 'Zwergschwan', 'veery': 'Weidenmusendrossel', 'verdin': 'Goldkopf-Beutelmeise', 'vigswa': 'Veilchenschwalbe', 'warvir': 'Sängervireo', 'wbwwre1': 'Waldzaunkönig', 'webwoo1': 'Keilschnabel-Baumsteiger', 'wegspa1': 'Weißohr-Grundammer', 'wesant1': 'Kolumbienameisenwürger', 'wesblu': 'Blaukehl-Hüttensänger', 'weskin': 'Schmalschnabel-Königstyrann', 'wesmea': 'Wiesenstärling', 'westan': 'Kiefernkardinal', 'wewpew': 'Blasskehl-Schnäppertyrann', 'whbman1': 'Weißbrustpipra', 'whbnut': 'Weißbrustkleiber', 'whcpar': 'Weißstirnpapagei', 'whcsee1': 'Schwarzbrustspelzer', 'whcspa': 'Dachsammer', 'whevir': 'Weißaugenvireo', 'whfpar1': 'Weißstirnamazone', 'whimbr': 'Regenbrachvogel', 'whiwre1': 'Bartstreif-Zaunkönig', 'whtdov': 'Blauringtaube', 'whtspa': 'Weißkehlammer', 'whwbec1': 'Weißbindenbekarde', 'whwdov': 'Weißflügeltaube', 'wilfly': 'Weidenschnäppertyrann', 'willet1': 'Schlammtreter', 'wilsni1': 'Wilsonbekassine', 'wiltur': 'Truthuhn', 'wlswar': 'Mönchswaldsänger', 'wooduc': 'Brautente', 'woothr': 'Walddrossel', 'wrenti': 'Chaparralgrasmücke', 'y00475': 'Indianerblässhuhn', 'yebcha': 'Flötenstärling', 'yebela1': 'Gelbbauch-Olivtyrann', 'yebfly': 'Birkenschnäppertyrann', 'yebori1': 'Schwarzflügeltrupial', 'yebsap': 'Gelbbauch-Saftlecker', 'yebsee1': 'Gelbbauchspelzer', 'yefgra1': 'Goldbrauen-Gimpeltangare', 'yegvir': 'Gelbflankenvireo', 'yehbla': 'Brillenstärling', 'yehcar1': 'Gelbkopfkarakara', 'yelgro': 'Gelbkopf-Kernknacker', 'yelwar': 'Goldwaldsänger', 'yeofly1': 'Olivscheitel-Breitschnabeltyrann', 'yerwar': 'Kronenwaldsänger', 'yeteup1': 'Schwalbenorganist', 'yetvir': 'Gelbkehlvireo'}","metadata":{"execution":{"iopub.status.busy":"2021-06-08T19:57:05.469613Z","iopub.execute_input":"2021-06-08T19:57:05.469926Z","iopub.status.idle":"2021-06-08T19:57:05.509320Z","shell.execute_reply.started":"2021-06-08T19:57:05.469896Z","shell.execute_reply":"2021-06-08T19:57:05.508351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize_list_string(string):\n    return string.replace('[', '').replace(']', '').replace(\"'\", '')\n\ndef count_normalized_list_string(string):\n    string = normalize_list_string(string).strip()\n    if(string == ''):\n        return 0 \n    return len(string.split(','))\n\nassert(count_normalized_list_string(\"[]\"), 0)\nassert(count_normalized_list_string(\"['abc']\"), 1)\nassert(count_normalized_list_string(\"['abc def']\"), 1)\nassert(count_normalized_list_string(\"['abc,def']\"), 2)\n\nprint(\"Anzahl der Arten: %s\" % (\n    len(train['primary_label'].value_counts())\n))\nprint(\"Anzahl der Datensätze: %s\" % (\n    len(train)\n))\n\ntrain['labels_count'] = train['secondary_labels'].apply(lambda x: count_normalized_list_string(x) + 1)\nprint(\"Durchnittliche Anzahl der Arten pro Datensatz: %s\" % (\n    train['labels_count'].mean()\n))\n\n\ntypes = []\ndef unique_type_list_string(string):\n    for typ in normalize_list_string(string).split(','):\n        typ = typ.strip()\n        if typ not in types:\n            types.append(typ)\n\ntrain['type'].apply(lambda x: unique_type_list_string(x))\nprint(\"Anzahl an Ruf-Typen: %s\" % (len(types)))\nprint(\"Ruf-Typen: %s\" % (types))\n\ntrain['type_count'] = train['type'].apply(lambda x: count_normalized_list_string(x))\nprint(\"Durchschnittliche Anzahl der Ruf-Typen: %s\" % (train['type_count'].mean()))","metadata":{"execution":{"iopub.status.busy":"2021-06-08T19:57:05.511690Z","iopub.execute_input":"2021-06-08T19:57:05.512015Z","iopub.status.idle":"2021-06-08T19:57:05.902099Z","shell.execute_reply.started":"2021-06-08T19:57:05.511983Z","shell.execute_reply":"2021-06-08T19:57:05.900898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_info_data = train\\\n    .groupby(['primary_label', 'scientific_name', 'common_name'])\\\n    .agg({\n        'rating': ['mean', 'min', 'max'],\n        'primary_label': 'size'\n    })\\\n    .reset_index()\n\nbase_info_data.columns = [\"\".join(x) for x in base_info_data.columns.ravel()]\nbase_info_data = base_info_data.set_index('primary_label')\n\n# add german translation\nbase_info_data['german_name'] = pd.Series(german_names)\n\n# sort columns\nbase_info_data = base_info_data.reindex([\n    'scientific_name',\n    'common_name',\n    'german_name',\n    'primary_labelsize',\n    'ratingmean',\n    'ratingmin',\n    'ratingmax'\n], axis=1)\n\nbase_info_data","metadata":{"execution":{"iopub.status.busy":"2021-06-08T19:57:05.904450Z","iopub.execute_input":"2021-06-08T19:57:05.904862Z","iopub.status.idle":"2021-06-08T19:57:05.972838Z","shell.execute_reply.started":"2021-06-08T19:57:05.904819Z","shell.execute_reply":"2021-06-08T19:57:05.971775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bins = [1, 100, 200, 300, 400, 500]\n\nfig, axs = plt.subplots(figsize=(12, 4))\ntrain.groupby('primary_label').size().value_counts(bins=bins, sort=False).plot(\n    kind='bar', rot=0, ax=axs, legend=False,\n    title='Verteilung Anzahl der Datensätze pro Art', \n    xlabel='Menge Datensätze',\n    ylabel='Menge Arten',\n    color='black',\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T19:57:05.973856Z","iopub.execute_input":"2021-06-08T19:57:05.974113Z","iopub.status.idle":"2021-06-08T19:57:06.182965Z","shell.execute_reply.started":"2021-06-08T19:57:05.974087Z","shell.execute_reply":"2021-06-08T19:57:06.181930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Länge der einzelnen Samples\ndef get_length(bird, file):\n    path=\"../input/birdclef-2021/train_short_audio/\"\n    audio_path=os.path.join(path,bird,file)\n    return librosa.get_duration(filename=audio_path)\n\nlengths = []\nwith tqdm(total=len(train)) as pbar:\n    for idx, row in train.iterrows():\n        pbar.update(1)\n        \n        lengths.append([\n            row['primary_label'],\n            row['filename'],\n            get_length(row['primary_label'], row['filename'])\n        ])\n\nlength_df = pd.DataFrame(lengths, columns=['primary_label', 'filename', 'length'])\nlength_df","metadata":{"execution":{"iopub.status.busy":"2021-06-08T19:57:06.184165Z","iopub.execute_input":"2021-06-08T19:57:06.184432Z","iopub.status.idle":"2021-06-08T20:30:48.008080Z","shell.execute_reply.started":"2021-06-08T19:57:06.184405Z","shell.execute_reply":"2021-06-08T20:30:48.006695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bins = [0, 5, 10, 20, 30, 40, 50, 60, 90, 120, 150, 180, 210, 240, 270, 300, 600, 1200]\n\nfig, axs = plt.subplots(figsize=(12, 4))\nlength_df['length'].value_counts(bins=bins, sort=False).plot(\n    kind='bar', rot=0, ax=axs, legend=False,\n    title='Länge der Datensätze', \n    xlabel='Länge in Sekunden',\n    ylabel='Anzahl',\n    color='black',\n)\nlength_df['length'].value_counts(bins=bins, sort=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T20:30:48.010831Z","iopub.execute_input":"2021-06-08T20:30:48.011141Z","iopub.status.idle":"2021-06-08T20:30:48.306007Z","shell.execute_reply.started":"2021-06-08T20:30:48.011091Z","shell.execute_reply":"2021-06-08T20:30:48.305312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Gesamtlänge der Aufnahmen: %s Stunden\" % (\n    length_df['length'].sum() / 60 / 60\n))\n\nprint(\"Durchschnittliche Länge der Audio-Dateien: %s Sekunden\" % (\n    length_df['length'].mean()\n))","metadata":{"execution":{"iopub.status.busy":"2021-06-08T20:30:48.307330Z","iopub.execute_input":"2021-06-08T20:30:48.307634Z","iopub.status.idle":"2021-06-08T20:30:48.314427Z","shell.execute_reply.started":"2021-06-08T20:30:48.307598Z","shell.execute_reply":"2021-06-08T20:30:48.313365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Durchschnittliche Länge der Aufnahmen pro Art hinzufügen\nmean_length = length_df.groupby('primary_label').mean('length').sort_values(by='length')\nbase_info_data['audio_length'] = mean_length\nbase_info_data","metadata":{"execution":{"iopub.status.busy":"2021-06-08T20:30:48.315722Z","iopub.execute_input":"2021-06-08T20:30:48.316011Z","iopub.status.idle":"2021-06-08T20:30:48.356825Z","shell.execute_reply.started":"2021-06-08T20:30:48.315984Z","shell.execute_reply":"2021-06-08T20:30:48.355849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"invalid_values = [ '', '0', '?', 'XX:XX', 'XX.XX', 'XX;XX', 'X', '??:??', '?:?', '.', 'PM', 'AM', 'NIGHT', 'DAWN' ]\n\n# Wann wurden die Samples aufgenommen?\ndef convert_time(str_time):\n    # None for unwanted values\n    str_time = str(str_time).upper().strip()\n    if(str_time in invalid_values):\n        return None\n    \n    # make sure AM / PM is uppercase and prefixed by whitespace\n    if('AM' in str_time or 'PM' in str_time):\n        str_time = str_time.replace('AM', ' AM')\n        str_time = str_time.replace('PM', ' PM')\n        str_time = str_time.replace('  ', ' ')\n        \n        if(str_time.count(':') == 2):\n            return time.strptime(str_time, '%H:%M:%S %p')\n        return time.strptime(str_time, '%H:%M %p')\n\n    elif(str_time.count(':') >= 1):\n        first = int(str_time.split(':')[0])\n        # kill leading zeros like 000:000\n        if(first == 24):\n            first = 0\n        if first > 23:\n            return None\n        \n        second = int(str_time.split(':')[1])\n        if second > 59:\n            return None\n        \n        return time.strptime(str(first) + ':' + str(second), '%H:%M')\n    return time.strptime(str_time, '%H:%M')\n\ndef convert_time_pd(time_input):\n    time_input = convert_time(time_input)\n    if time_input == None:\n        return None\n    return time.strftime('%H:%M:%S', time_input)\n\n\ntrain['times_cleaned'] = train['time'].apply(lambda x: convert_time_pd(x))\nprint('Anzahl aller Datensätze: %s' % (len(train['times_cleaned'])))\n\ntimes = train['times_cleaned'].dropna()\nprint('Anzahl der Datensätze mit validen Zeitinformationen: %s' % (len(times)))\n\ntimes = pd.DataFrame(times)\ntimes['times_cleaned'] = pd.to_datetime(times['times_cleaned'], format=\"%H:%M:%S\")\n#print(times)\n\nfig, axs = plt.subplots(figsize=(12, 4))\n\ntimes.groupby([times['times_cleaned'].dt.hour]).agg(len).plot(\n    kind='bar', rot=0, ax=axs, legend=False,\n    title='Zeitverteilung der Aufnahmen', \n    xlabel='Uhrzeit',\n    ylabel='Anzahl Datensätze',\n    color='black',\n)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-08T20:30:48.358149Z","iopub.execute_input":"2021-06-08T20:30:48.358450Z","iopub.status.idle":"2021-06-08T20:30:49.680265Z","shell.execute_reply.started":"2021-06-08T20:30:48.358420Z","shell.execute_reply":"2021-06-08T20:30:49.679279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(figsize=(12, 4))\n\ntrain['primary_label'].value_counts().plot(\n    kind='bar', rot=0, ax=axs, legend=False,\n    title='Verteilung Anzahl Datensätze', \n    xlabel='Vogelart',\n    ylabel='Anzahl Datensätze',\n    color='black',\n)\naxs.axes.xaxis.set_ticks([]) # remove Labels for X","metadata":{"execution":{"iopub.status.busy":"2021-06-08T20:30:49.681833Z","iopub.execute_input":"2021-06-08T20:30:49.682245Z","iopub.status.idle":"2021-06-08T20:30:51.204597Z","shell.execute_reply.started":"2021-06-08T20:30:49.682193Z","shell.execute_reply":"2021-06-08T20:30:51.203599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(figsize=(12, 4))\n\ntrain['rating'].value_counts()[[5,4.5,4,3.5,3,2.5,2,1.5,1,0.5]].plot(\n    kind='bar', rot=0, ax=axs, legend=False,\n    title='Anzahl Datensätze nach Qualität', \n    xlabel='Bewertung',\n    ylabel='Anzahl Datensätze',\n    color='black',\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T20:30:51.205932Z","iopub.execute_input":"2021-06-08T20:30:51.206223Z","iopub.status.idle":"2021-06-08T20:30:51.402599Z","shell.execute_reply.started":"2021-06-08T20:30:51.206193Z","shell.execute_reply":"2021-06-08T20:30:51.401861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\n\n# Test Audio Quality\npath=\"./train_short_audio/\"\nbirds=train.primary_label.unique()[:6]\nfile=train[train.primary_label==birds[0]]['filename'][0]\n\n\nfor i in range(0,2):\n    file=train[train.primary_label==birds[i]]['filename'].values[0]\n    audio_path=os.path.join(path,birds[i],file)\n    print(birds[i])\n    IPython.display.display(ipd.Audio(audio_path))\n    ipd.Audio(audio_path)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T20:30:51.403771Z","iopub.execute_input":"2021-06-08T20:30:51.404215Z","iopub.status.idle":"2021-06-08T20:30:51.951848Z","shell.execute_reply.started":"2021-06-08T20:30:51.404167Z","shell.execute_reply":"2021-06-08T20:30:51.950375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n# https://plotly.com/python/plotly-express/\ndf=train.groupby(['latitude','longitude'],as_index=False)['primary_label'].agg('count')\n\ndf=df[df.latitude!='Not specified']\nfig = go.Figure()\nfig.add_trace(go.Scattergeo(\n        lon = df['longitude'],\n        lat = df['latitude'],\n        text = df['primary_label'],\n        marker = dict(\n            size = df['primary_label'],\n            line_color='rgb(40,40,40)',\n            line_width=0.5,\n            sizemode = 'area'\n        )))\n\n\nfig.update_layout(\n        title_text = 'Bird Samples collected From Parts of World',\n        showlegend = False,\n        geo = dict(\n            landcolor = 'rgb(217, 217, 217)',\n        )\n    )\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-08T20:33:14.533192Z","iopub.execute_input":"2021-06-08T20:33:14.533587Z","iopub.status.idle":"2021-06-08T20:33:14.573450Z","shell.execute_reply.started":"2021-06-08T20:33:14.533554Z","shell.execute_reply":"2021-06-08T20:33:14.571734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# translate columns\ncolumns={\n    'scientific_name': 'Wissenschaftlicher Name',\n    'common_name': 'Gebäuchlicher Name',\n    \n    'ratingmean': 'Bewertungen Durchschnitt',\n    'ratingmin': 'Bewertungen Min',\n    'ratingmax': 'Bewertungen Max',\n    'primary_labelsize': 'Trainingsdaten Anzahl',\n    'german_name': 'Deutscher Name',\n    'audio_length': 'Audio-Länge Durchschnitt'\n}\nbase_info_data = base_info_data.rename(columns=columns)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-08T20:33:17.722982Z","iopub.execute_input":"2021-06-08T20:33:17.723403Z","iopub.status.idle":"2021-06-08T20:33:17.730783Z","shell.execute_reply.started":"2021-06-08T20:33:17.723359Z","shell.execute_reply":"2021-06-08T20:33:17.729862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path=\"../input/birdclef-2021/train_short_audio/\"\nbirds=train.primary_label.unique()[:6]\nfile=train[train.primary_label==birds[0]]['filename'][0]","metadata":{"execution":{"iopub.status.busy":"2021-06-08T20:33:20.610385Z","iopub.execute_input":"2021-06-08T20:33:20.610980Z","iopub.status.idle":"2021-06-08T20:33:20.633705Z","shell.execute_reply.started":"2021-06-08T20:33:20.610923Z","shell.execute_reply":"2021-06-08T20:33:20.632879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(17,20 ))\nimport librosa.display\n\nfor i in range(0,6):\n    file=train[train.primary_label==birds[i]]['filename'].values[0]\n    audio_path=os.path.join(path,birds[i],file)\n    plt.subplot(6,2,i+1)\n    x, sr = librosa.load(audio_path)\n    librosa.display.waveplot(x, sr=sr,color='black')\n    plt.gca().set_title(birds[i])\n    plt.gca().get_xaxis().set_visible(False)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T20:33:22.299037Z","iopub.execute_input":"2021-06-08T20:33:22.299678Z","iopub.status.idle":"2021-06-08T20:33:34.276117Z","shell.execute_reply.started":"2021-06-08T20:33:22.299567Z","shell.execute_reply":"2021-06-08T20:33:34.274872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}