{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nsound_filenames=[]\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        #print(os.path.join(dirname, filename))\n        if os.path.join(dirname, filename).split('/')[-1].split('.')[-1]=='ogg':\n            sound_filenames.append(os.path.join(dirname, filename))\n        \nprint(\"Total Sound Files : \",len(sound_filenames))\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-16T07:50:15.134816Z","iopub.execute_input":"2022-03-16T07:50:15.135686Z","iopub.status.idle":"2022-03-16T07:50:19.106751Z","shell.execute_reply.started":"2022-03-16T07:50:15.135562Z","shell.execute_reply":"2022-03-16T07:50:19.106130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Data Preprocessing**","metadata":{}},{"cell_type":"markdown","source":"**Plotting Sound Wave of 5 audio file.**","metadata":{}},{"cell_type":"code","source":"import librosa\nimport IPython.display as ipd\nimport matplotlib.pyplot as plt\nimport numpy as np\n%matplotlib inline\n\ndef plot_wave(SOUND_DIR):\n    # listen to the recording\n    ipd.display(ipd.Audio(SOUND_DIR))\n    # load the mp3 file\n    signal, sr = librosa.load(SOUND_DIR, duration=10)  # sr = sampling rate\n    print(signal,sr)\n    # plot recording signal\n    plt.figure(figsize=(10, 4))\n    plt.plot(signal)\n    plt.title(\"Sound Wave\")\n    plt.show()\n    print('\\n\\n')\n    \n#Plotting Sound Wave of 5 audio file.\nfor path in sound_filenames[:5]:\n    plot_wave(path)","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:50:19.108208Z","iopub.execute_input":"2022-03-16T07:50:19.108777Z","iopub.status.idle":"2022-03-16T07:50:25.399784Z","shell.execute_reply.started":"2022-03-16T07:50:19.108742Z","shell.execute_reply":"2022-03-16T07:50:25.398615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Plotting Spectogram of 5 audio files**","metadata":{}},{"cell_type":"code","source":"import librosa.display\n# Plot spectogram\ndef plot_spectogram(SOUND_DIR):\n    signal, sr = librosa.load(SOUND_DIR, duration=10)\n    print(signal,sr)\n    plt.figure(figsize=(10, 4))\n    D = librosa.amplitude_to_db(np.abs(librosa.stft(signal)), ref=np.max)\n    librosa.display.specshow(D, y_axis=\"linear\")\n    plt.colorbar(format=\"%+2.0f dB\")\n    plt.title(\"Linear-frequency power spectrogram\")\n    plt.show()\n    print('\\n\\n')\n\n#Plotting Spectogram of 5 audio files\nfor path in sound_filenames[:5]:\n    plot_spectogram(path)","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:50:25.404331Z","iopub.execute_input":"2022-03-16T07:50:25.404534Z","iopub.status.idle":"2022-03-16T07:50:29.748378Z","shell.execute_reply.started":"2022-03-16T07:50:25.404508Z","shell.execute_reply":"2022-03-16T07:50:29.747492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Classes Of Birds-**","metadata":{}},{"cell_type":"code","source":"IM_SIZE = (224, 224, 3)\ndirList=os.listdir('/kaggle/input/birdclef-2022/train_audio/')\nBIRDS=dirList\nprint('Classes of Birds : \\n',BIRDS)\nprint('\\nTotal Classes of Birds : ',len(BIRDS))","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:50:29.752129Z","iopub.execute_input":"2022-03-16T07:50:29.752343Z","iopub.status.idle":"2022-03-16T07:50:29.758436Z","shell.execute_reply.started":"2022-03-16T07:50:29.752317Z","shell.execute_reply":"2022-03-16T07:50:29.757717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Stratified K-Fold Cross Validation of train_metadata\n\n**KFold is a cross-validator that divides the dataset into k folds. Stratified is to ensure that each fold of dataset has the same proportion of observations with a given label.**","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\nSEED = 42\nDATA_PATH = \"../input/birdclef-2022/\"\nAUDIO_PATH = '../input/birdclef-2022/train_audio'\nMEAN = np.array([0.485, 0.456, 0.406])\nSTD = np.array([0.229, 0.224, 0.225])\nNUM_WORKERS = 4\nCLASSES = sorted(os.listdir(AUDIO_PATH))\nNUM_CLASSES = len(CLASSES)\nclass AudioParams:\n    \"\"\"\n    Parameters used for the audio data\n    \"\"\"\n    sr = 32000\n    duration = 5\n    # Melspectrogram\n    n_mels = 224\n    fmin = 20\n    fmax = 16000\n\n\ntrain = pd.read_csv('../input/birdclef-2022/train_metadata.csv')\ntrain[\"file_path\"] = AUDIO_PATH + '/' + train['filename']\npaths = train[\"file_path\"].values\n\nFold = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)\nfor n, (trn_index, val_index) in enumerate(Fold.split(train, train['primary_label'])):\n    train.loc[val_index, 'kfold'] = int(n)\ntrain['kfold'] = train['kfold'].astype(int)\n\ntrain.to_csv('train_folds.csv', index=False)\n\nprint(train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:50:29.759389Z","iopub.execute_input":"2022-03-16T07:50:29.759602Z","iopub.status.idle":"2022-03-16T07:50:30.046058Z","shell.execute_reply.started":"2022-03-16T07:50:29.759577Z","shell.execute_reply":"2022-03-16T07:50:30.045240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test Data","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/birdclef-2022/test.csv')\ntest\n","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:50:30.047179Z","iopub.execute_input":"2022-03-16T07:50:30.047469Z","iopub.status.idle":"2022-03-16T07:50:30.063055Z","shell.execute_reply.started":"2022-03-16T07:50:30.047437Z","shell.execute_reply":"2022-03-16T07:50:30.062400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:50:30.064042Z","iopub.execute_input":"2022-03-16T07:50:30.064565Z","iopub.status.idle":"2022-03-16T07:50:30.084235Z","shell.execute_reply.started":"2022-03-16T07:50:30.064530Z","shell.execute_reply":"2022-03-16T07:50:30.083301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# scored_birds.json data","metadata":{}},{"cell_type":"code","source":"import json\nwith open('/kaggle/input/birdclef-2022/scored_birds.json') as f:\n    scored_birds = json.load(f)\n    \nprint('scored_birds is ', len(scored_birds), ' species')\nprint(scored_birds[0:7])\nprint(scored_birds[7:14])\nprint(scored_birds[14:22])","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:50:30.087035Z","iopub.execute_input":"2022-03-16T07:50:30.087443Z","iopub.status.idle":"2022-03-16T07:50:30.101452Z","shell.execute_reply.started":"2022-03-16T07:50:30.087406Z","shell.execute_reply":"2022-03-16T07:50:30.100582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# eBird_Taxonomy_v2021.csv data","metadata":{}},{"cell_type":"code","source":"ebird = pd.read_csv('/kaggle/input/birdclef-2022/eBird_Taxonomy_v2021.csv')\nebird","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:50:30.103048Z","iopub.execute_input":"2022-03-16T07:50:30.103407Z","iopub.status.idle":"2022-03-16T07:50:30.191879Z","shell.execute_reply.started":"2022-03-16T07:50:30.103364Z","shell.execute_reply":"2022-03-16T07:50:30.190980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ebird.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:50:30.193181Z","iopub.execute_input":"2022-03-16T07:50:30.193624Z","iopub.status.idle":"2022-03-16T07:50:30.215564Z","shell.execute_reply.started":"2022-03-16T07:50:30.193550Z","shell.execute_reply":"2022-03-16T07:50:30.214502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Birds Singing Time","metadata":{}},{"cell_type":"code","source":"from datetime import timedelta\nfrom datetime import datetime\nimport seaborn as sns\n\ndef round_date(date, delta = 30, th = 10):\n    date = date.to_pydatetime()\n    x = date.minute\n    if ((x >= (delta - th)) & (x < delta)) or (x > (delta + th)):\n#         print('Up')\n        date = date + (datetime.min - date) % timedelta(minutes = delta)\n    elif ((x <= (delta+ th )) & (x > delta)) or (x < (delta - th)):\n#         print('down')\n        date = date - (date - datetime.min) % timedelta(minutes = delta)\n\n    \n    return date.time().strftime(\"%H:%M\")\n\ntrain['time_tf']  = pd.to_datetime(train['time'], errors = 'coerce').dropna().apply(lambda x:round_date(x))\ntrain.dropna(subset=['time_tf'], inplace = True)\nprint('success')\n\nplt.figure(figsize = (15,8))\nsns.countplot(x = 'time_tf', data = train.sort_values(by = 'time_tf'))\nplt.xticks(rotation=45)\nplt.xlabel('Time', fontdict = {'fontsize':18})\nplt.ylabel('Frequency', fontdict = {'fontsize':18})\nplt.title('Birds Singing Time',fontdict = {'fontsize':18})\nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:50:30.216798Z","iopub.execute_input":"2022-03-16T07:50:30.217714Z","iopub.status.idle":"2022-03-16T07:50:31.192027Z","shell.execute_reply.started":"2022-03-16T07:50:30.217677Z","shell.execute_reply":"2022-03-16T07:50:31.191272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train['secondary_labels']","metadata":{}},{"cell_type":"code","source":"train['secondary_labels']","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:50:31.193213Z","iopub.execute_input":"2022-03-16T07:50:31.193512Z","iopub.status.idle":"2022-03-16T07:50:31.200009Z","shell.execute_reply.started":"2022-03-16T07:50:31.193482Z","shell.execute_reply":"2022-03-16T07:50:31.199352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import ast\nlabels = []\nfor row in train.index:\n    labels.extend(ast.literal_eval(train.loc[row, 'secondary_labels']))\nlabels = list(set(labels))\n\nprint('Number of unique bird labels:', len(labels))","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:50:31.201214Z","iopub.execute_input":"2022-03-16T07:50:31.201485Z","iopub.status.idle":"2022-03-16T07:50:31.416312Z","shell.execute_reply.started":"2022-03-16T07:50:31.201457Z","shell.execute_reply":"2022-03-16T07:50:31.415254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nnewTrain_SecondaryLabels = train['secondary_labels'].apply(lambda x: re.findall(r\"'(\\w+)'\", x))","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:50:31.417838Z","iopub.execute_input":"2022-03-16T07:50:31.418357Z","iopub.status.idle":"2022-03-16T07:50:31.439000Z","shell.execute_reply.started":"2022-03-16T07:50:31.418311Z","shell.execute_reply":"2022-03-16T07:50:31.438336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Top 50 Birds Found on Background as Noise**","metadata":{}},{"cell_type":"code","source":"values = 50\nfig, ax = plt.subplots( figsize = (10,8))\nsns.barplot(y = 'index', x = 'secondary_labels',\n            data = newTrain_SecondaryLabels.explode().value_counts().head(values).reset_index(),\n            ax = ax)\nax.set_title(f'Top {values} Birds Found on Background as Noise', fontdict = {'fontsize':20})\nax.set_xlabel('Frequency', fontdict = {'fontsize':16})\nax.set_ylabel('Birds Common Name', fontdict = {'fontsize':16})\n\nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:50:31.440132Z","iopub.execute_input":"2022-03-16T07:50:31.440471Z","iopub.status.idle":"2022-03-16T07:50:32.437852Z","shell.execute_reply.started":"2022-03-16T07:50:31.440442Z","shell.execute_reply":"2022-03-16T07:50:32.436978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Birds Distribution Map","metadata":{}},{"cell_type":"code","source":"import geopandas as gpd\nfig, ax = plt.subplots(figsize=(26,20))\n# plot map on axis\ncountries = gpd.read_file(gpd.datasets.get_path(\"naturalearth_lowres\"))\ncountries.plot(color=\"lightgrey\", ax=ax)\n\n# plot points\ncmap = plt.cm.get_cmap('jet')\nbirds = len(train[\"primary_label\"].unique())\nprint(\"Total Classes of Birds : \",birds)\n\nfor i, (bird, dfg) in enumerate(train.groupby(\"primary_label\")):\n    dfg.longitude = np.around(dfg.longitude, 1)\n    dfg.latitude = np.around(dfg.latitude, 1)\n    dfgg = dfg.groupby([\"longitude\", \"latitude\"]).size().reset_index(name=\"counts\")\n    dfgg.plot(x=\"longitude\", y=\"latitude\", kind=\"scatter\", \n              c=cmap(float(i) / birds), s=dfgg[\"counts\"] * 5,\n              ax=ax, label=bird, alpha=0.5)\n\nax.legend(loc='upper center', bbox_to_anchor=(0.5, 1.25), ncol=15, fancybox=True, shadow=True)\n\n# get axes limits\nx_lo, x_up = ax.get_xlim()\ny_lo, y_up = ax.get_ylim()\n# add minor ticks with a specified sapcing (deg)\ndeg = 5\n# add grid\nax.set_xticks(np.arange(np.ceil(x_lo), np.ceil(x_up), deg), minor=True)\nax.set_yticks(np.arange(np.ceil(y_lo), np.ceil(y_up), deg), minor=True)\nax.grid(b=True, which=\"minor\", alpha=0.25)","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:50:32.439054Z","iopub.execute_input":"2022-03-16T07:50:32.439321Z","iopub.status.idle":"2022-03-16T07:50:47.949343Z","shell.execute_reply.started":"2022-03-16T07:50:32.439291Z","shell.execute_reply":"2022-03-16T07:50:47.948439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/birdclef-2022/sample_submission.csv')\nsubmission['target'] = True\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:52:24.791003Z","iopub.execute_input":"2022-03-16T07:52:24.791919Z","iopub.status.idle":"2022-03-16T07:52:24.811531Z","shell.execute_reply.started":"2022-03-16T07:52:24.791867Z","shell.execute_reply":"2022-03-16T07:52:24.810656Z"},"trusted":true},"execution_count":null,"outputs":[]}]}