{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Birds Book - EDA\n\nA collection of my observations regarding the data we have for this competition. The EDA may grow in the future","metadata":{}},{"cell_type":"code","source":"from __future__ import annotations\nfrom pathlib import Path\nimport json\n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport IPython.display as ipd\nfrom IPython.display import HTML\nimport torchaudio\nimport joblib\nimport torch\nfrom joblib import Parallel, delayed\nfrom torchaudio.transforms import Resample\n\nfrom tqdm.notebook import tqdm\nimport multiprocessing\n\n\ndef inline(text, tag):\n    ipd.display(HTML(f\"<{tag}>{text}</{tag}>\"))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-10T17:22:22.139344Z","iopub.execute_input":"2022-05-10T17:22:22.139601Z","iopub.status.idle":"2022-05-10T17:22:23.885553Z","shell.execute_reply.started":"2022-05-10T17:22:22.139536Z","shell.execute_reply":"2022-05-10T17:22:23.884718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('../input/birdclef2022-code')\n\nfrom demo import foobar\n\nprint(foobar(0))\nprint(foobar(1))","metadata":{"execution":{"iopub.status.busy":"2022-05-10T17:22:23.887290Z","iopub.execute_input":"2022-05-10T17:22:23.887479Z","iopub.status.idle":"2022-05-10T17:22:23.899567Z","shell.execute_reply.started":"2022-05-10T17:22:23.887450Z","shell.execute_reply":"2022-05-10T17:22:23.898701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from demo import answer\nprint(answer())","metadata":{"execution":{"iopub.status.busy":"2022-05-10T17:22:23.900632Z","iopub.execute_input":"2022-05-10T17:22:23.900890Z","iopub.status.idle":"2022-05-10T17:22:23.905210Z","shell.execute_reply.started":"2022-05-10T17:22:23.900854Z","shell.execute_reply":"2022-05-10T17:22:23.904557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"info_df = None","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:42:45.967273Z","iopub.execute_input":"2022-05-05T18:42:45.967686Z","iopub.status.idle":"2022-05-05T18:42:45.977887Z","shell.execute_reply.started":"2022-05-05T18:42:45.967654Z","shell.execute_reply":"2022-05-05T18:42:45.977272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_dir = Path('/kaggle/input/birdclef-2022/')\ntraining_dir = root_dir / 'train_audio'\ntest_dir = root_dir / 'test_soundscapes'\n\ntraining_path = root_dir / 'train_metadata.csv'\ntest_path = root_dir / 'test.csv'\nsubmission_path = root_dir / 'submission.csv'\nscored_birds_path = root_dir / 'scored_birds.json'","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:42:45.979211Z","iopub.execute_input":"2022-05-05T18:42:45.979606Z","iopub.status.idle":"2022-05-05T18:42:45.991357Z","shell.execute_reply.started":"2022-05-05T18:42:45.979572Z","shell.execute_reply":"2022-05-05T18:42:45.990312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training data EDA","metadata":{}},{"cell_type":"code","source":"too_long_columns = ['license', 'secondary_labels', 'author', 'type', 'url']\n\n@delayed\ndef load_info(name):\n    info = vars(torchaudio.info(training_dir / name))\n    info['seconds'] = info['num_frames'] / info['sample_rate']\n    info['minutes'] = info['seconds'] / 60\n    info['filename'] = name\n    return info\n\n\nwith open(root_dir / 'scored_birds.json') as fh:\n    scored_birds = json.load(fh)\n\n\ntraining_df = pd.read_csv(training_path)\ninline(\"Training dataset\", \"h3\")\nipd.display(training_df.head(3))\ninline(\"Training information\", \"h3\")\nprint(f\"Training shape: {training_df.shape}\")\nprint(f\"Unique primary labels count: {training_df.primary_label.nunique()}\")\n\n\nif info_df is None:\n    file_infos = Parallel(n_jobs=-1)(load_info(path) for path in tqdm(training_df.filename))\n    info_df = pd.DataFrame.from_records(info for info in file_infos)\n    \ninline(\"Audio files informations\", \"h3\")\nipd.display(info_df.head(3))\ntraining_df = pd.merge(training_df, info_df, on='filename')\n\ninline(\"Full training data\", \"h3\")\nipd.display(training_df.head(3).drop(['license', 'url', 'author'], axis=1))\n\nscored_df = training_df[training_df.primary_label.isin(scored_birds)].copy()\n\ntraining_df['dataset'] = 'training'\nscored_df['dataset'] = 'scored'\ncombined_df = pd.concat([training_df, scored_df]).reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:42:45.99305Z","iopub.execute_input":"2022-05-05T18:42:45.993603Z","iopub.status.idle":"2022-05-05T18:43:31.259087Z","shell.execute_reply.started":"2022-05-05T18:42:45.993559Z","shell.execute_reply":"2022-05-05T18:43:31.258117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Distribution of targets\n\nThere are some (huge) inbalances in the classes. It maybe a problem to predict if a given bird is on the recording as some birds are just not representative enough. The question is, how much will we lose if we don't learn them correctly? Open question.","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(10, 4))\norder = scored_df['primary_label'].value_counts().index\nsns.countplot(data=scored_df, y='primary_label', ax=ax, order=order)\n\nleast_represented_df = scored_df[scored_df.primary_label.isin(order[-5:])]\ninline(\"All rows of the least representative birds\", \"h4\")\nipd.display(least_represented_df.drop(too_long_columns, axis=1))","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:43:31.261159Z","iopub.execute_input":"2022-05-05T18:43:31.261482Z","iopub.status.idle":"2022-05-05T18:43:31.601129Z","shell.execute_reply.started":"2022-05-05T18:43:31.261446Z","shell.execute_reply":"2022-05-05T18:43:31.600043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Distribution of rating","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(12, 4))\n\nax[0].set_title(\"KDE plot\")\nsns.kdeplot(data=combined_df, x='rating', hue='dataset', common_norm=False, ax=ax[0])\n\nax[1].set_title(\"Boxen plot\")\nsns.boxenplot(data=combined_df, y='rating', x='dataset', ax=ax[1])","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:43:31.602369Z","iopub.execute_input":"2022-05-05T18:43:31.603036Z","iopub.status.idle":"2022-05-05T18:43:32.08926Z","shell.execute_reply.started":"2022-05-05T18:43:31.603001Z","shell.execute_reply":"2022-05-05T18:43:32.088614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are many recordings with score of 0.0. I would say that it is reasonable to remove these recording from the training set as:\n\n> Garbage in, garabe out\n\nOur models shouldn't learn on this noise, as it may hinder the performance. We can even go further, and I would say that it is safe to try to teach a model without the recordings with labels under rating of 1. That is because all of the most rare birds all have rating above 1.0 (smallest is 1.5, but I would keep it as this bird has only 2 sampels).\n\n**Cleaning Rule**\n\n> Remove all samples with rating below 1.5 (smallest rating for a rare bird we care about)","metadata":{}},{"cell_type":"markdown","source":"## The distributions of the sound file informations","metadata":{}},{"cell_type":"code","source":"inline(\"Sound file information statistics\", \"h4\")\nipd.display_html(\n    info_df.describe(percentiles=[.25, .5, .75, .95, .99, .999])\n        .style.set_table_attributes(\"style='display:inline'\")\n        .set_caption('For full data')._repr_html_() \n    + \n    scored_df[info_df.columns].describe(percentiles=[.25, .5, .75, .95, .99, .999])\n        .style.set_table_attributes(\"style='display:inline'\")\n        .set_caption('For scored only')._repr_html_(), \n    raw=True)\n\ninfo_columns = ['num_frames', 'seconds']\nfig, axs = plt.subplots(1, len(info_columns) * 2, figsize=(16, 4))\nsub_5min_df = combined_df.query(f\"seconds < {60*5}\")\n\nfor ax, column in zip(axs[:len(info_columns)], info_columns):\n    ax.set_title(f\"All data\")\n    sns.violinplot(data=combined_df, x='dataset', y=column, ax=ax)\n    ax.set_ylabel(f\"log({column})\")\n    ax.set_yscale('log')\n        \nfor ax, column in zip(axs[len(info_columns):], info_columns):\n    ax.set_title(f\"Recordings under 5 min\")\n    sns.violinplot(data=sub_5min_df, x='dataset', y=column, ax=ax)\n        \nfig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:43:32.090314Z","iopub.execute_input":"2022-05-05T18:43:32.090653Z","iopub.status.idle":"2022-05-05T18:43:34.05233Z","shell.execute_reply.started":"2022-05-05T18:43:32.090624Z","shell.execute_reply":"2022-05-05T18:43:34.051769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inline(\"Top 5 longest recordings\", \"h4\")\nipd.display(training_df.query(\"minutes > 16\").sort_values('minutes', ascending=False).drop(too_long_columns, axis=1))","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:43:34.053265Z","iopub.execute_input":"2022-05-05T18:43:34.053967Z","iopub.status.idle":"2022-05-05T18:43:34.088202Z","shell.execute_reply.started":"2022-05-05T18:43:34.053926Z","shell.execute_reply":"2022-05-05T18:43:34.087246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I would say that for the training we can drop the really long files. Why? Because:\n\n* bigger file takes longer to load, and we have plenty of other data so we don't really need these files\n* above ~16.5 minutes there are no recordings of birds that we care about\n* the problem with longer files is that we don't know at what point the bird is heard, as we must predict the presence of a bird on 5 second windows and we only have information that the bird is heard in this recording (not in what time frame). This acutally can be another topic, beacuse if we somehow obtain a \"silence\" label or \"no bird\" label then we can have better model (need more thinking).\n\n**Cleaning Rule**\n\n> Remove all sampels longer then 17 minutes","metadata":{}},{"cell_type":"markdown","source":"# Training set cleanup","metadata":{}},{"cell_type":"code","source":"def drop_poor_quality_data(df):\n    return df[df.rating >= 1.5].copy()\n\ndef drop_long_recordings(df):\n    return df[df.minutes < 17].copy()\n\ndef clean(df):\n    rules = [drop_poor_quality_data, drop_long_recordings]\n    initial_rows = df.shape[0]\n    for rule in rules:\n        df = rule(df)\n     \n    final_rows = df.shape[0]\n    print(f\"Rows before: {initial_rows}, rows after: {final_rows}, difference: {initial_rows - final_rows}\")\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:43:34.089541Z","iopub.execute_input":"2022-05-05T18:43:34.089845Z","iopub.status.idle":"2022-05-05T18:43:34.096918Z","shell.execute_reply.started":"2022-05-05T18:43:34.089803Z","shell.execute_reply":"2022-05-05T18:43:34.09595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_training_df = clean(training_df)\nclean_scored_df = clean(scored_df)\n\ninline(\"Clean training statistics\", \"h4\")\nipd.display(clean_training_df.describe())\n\ninline(\"Clean scored statistics\", \"h4\")\nipd.display(clean_scored_df.describe())","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:43:34.099855Z","iopub.execute_input":"2022-05-05T18:43:34.100151Z","iopub.status.idle":"2022-05-05T18:43:34.191201Z","shell.execute_reply.started":"2022-05-05T18:43:34.100119Z","shell.execute_reply":"2022-05-05T18:43:34.190585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example test set\n\nThe full test file will be provided durring subbmision. One recording can require multiple predictions to be made using 5 second windows. As far as I know, there *should* be a single bird per file (or at least few birds per file). If that is true we can average the scores (or for example take max of the bird). That is hard to say, because on provided recording the bird is heard only at the end, so basically we may want to apply model to each section (as sound scape can have bird in split 5-10, but no bird noises in split 10-15 etc).\n\nBut as we are required to predict on 5 second windows, then we can only create netowrk that does exactly that. Note that we need to have padding in the case the sound is shorter then 5 seconds, like in description:\n\n>  These are each within a few milliseconds of 1 minute long and in the ogg audio format.\n\nThen we need to pad as few miliseconds is not equal to 5 seconds. Also we don't know if final window will contain exactly 5 seconds (perhabs not).","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(test_path)\n# test_df.loc[1, 'bird'] = 'other'\nipd.display(test_df.head())\n\ntest_ogg_path = test_dir / 'soundscape_453028782.ogg'\nprint(torchaudio.info(test_ogg_path))\nipd.Audio(test_ogg_path)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:43:34.192112Z","iopub.execute_input":"2022-05-05T18:43:34.192737Z","iopub.status.idle":"2022-05-05T18:43:34.242311Z","shell.execute_reply.started":"2022-05-05T18:43:34.1927Z","shell.execute_reply":"2022-05-05T18:43:34.241652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test set after sumbission","metadata":{}},{"cell_type":"code","source":"# t2 = test_df.copy()\ndef unique_birds(x):\n    return x.unique()\n\ndef unique_birds_count(x):\n    return x.nunique()\n\nfile_df = test_df.groupby('file_id')['bird'].agg([unique_birds, unique_birds_count])\nfile_infos = [] \nfor file in file_df.index:\n    path =  test_dir / f'{file}.ogg'\n    if path.exists():\n        info = torchaudio.info(path)\n        info = vars(info)\n        info['index'] = file\n        info['duration_s'] = info['num_frames'] / info['sample_rate']\n        file_infos.append(info)\n\nif file_infos:\n    file_info_df = pd.DataFrame.from_records(file_infos).set_index('index')\n    file_df = file_df.join(file_info_df)\n\nanomaly_df = file_df.query('unique_birds_count > 1')","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:43:34.243363Z","iopub.execute_input":"2022-05-05T18:43:34.243614Z","iopub.status.idle":"2022-05-05T18:43:34.262269Z","shell.execute_reply.started":"2022-05-05T18:43:34.243582Z","shell.execute_reply":"2022-05-05T18:43:34.261306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test dataframe","metadata":{}},{"cell_type":"code","source":"ipd.display(file_df)\nprint(\"Statistics\")\nipd.display(file_df.describe())","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:43:34.26381Z","iopub.execute_input":"2022-05-05T18:43:34.264045Z","iopub.status.idle":"2022-05-05T18:43:34.289492Z","shell.execute_reply.started":"2022-05-05T18:43:34.264016Z","shell.execute_reply":"2022-05-05T18:43:34.288519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test dataframe where there are more then one bird in recording","metadata":{}},{"cell_type":"code","source":"ipd.display(anomaly_df)\nprint(\"Statistics\")\nipd.display(anomaly_df.describe())","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:43:34.290747Z","iopub.execute_input":"2022-05-05T18:43:34.290971Z","iopub.status.idle":"2022-05-05T18:43:34.31075Z","shell.execute_reply.started":"2022-05-05T18:43:34.290945Z","shell.execute_reply":"2022-05-05T18:43:34.30991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dummy submission\n\nWith test for data loading","metadata":{}},{"cell_type":"code","source":"scored_df = training_df[training_df.primary_label.isin(scored_birds)].copy()\n\nlookup = scored_df.groupby('primary_label')['type'].count().sort_values()\nalways_negative = set(lookup[:len(lookup)//2].index)\nalways_positive = set(lookup[len(lookup)//2:].index)\n\nassert set(lookup.index) == (always_negative | always_positive)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:43:34.312007Z","iopub.execute_input":"2022-05-05T18:43:34.312213Z","iopub.status.idle":"2022-05-05T18:43:34.322082Z","shell.execute_reply.started":"2022-05-05T18:43:34.312187Z","shell.execute_reply":"2022-05-05T18:43:34.321347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rechannel(recording, channels):\n    current_channels = recording.shape[0]\n    if current_channels == channels:\n        return recording\n    \n    if channels == 1:\n        return recording[:1, :]\n    \n    if channels == 2 and current_channels == 1:\n        return torch.cat([recording, recording])\n    \n    if channels == 2 and current_channels == 3:\n        return recording[:2, :]\n    \n    if channels == 3 and current_channels == 1:\n        return torch.cat([recording, recording, recording])\n    \n    if channels == 3 and current_channels == 2:\n        return torch.cat([\n            recording[0:1, :],\n            recording[1:2, :],\n            recording.mean(axis=0)[None, :]\n        ])\n    \n    raise ValueError(f\"Unupported target channels: {channels} and audio channels {current_channels}\")\n    \n\n# for in_c, out_c in product(range(1, 4), repeat=2):\n#     out = rechannel(torch.randn(in_c, 50), out_c)\n#     assert out.shape[0] == out_c","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:44:58.38967Z","iopub.execute_input":"2022-05-05T18:44:58.390672Z","iopub.status.idle":"2022-05-05T18:44:58.397386Z","shell.execute_reply.started":"2022-05-05T18:44:58.390626Z","shell.execute_reply":"2022-05-05T18:44:58.396477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['target'] = [b in always_positive for b in test_df['bird']]\nsubmission_df = test_df[['row_id', 'target']]\nipd.display(submission_df)\nsubmission_df.to_csv('submission.csv', index=False)\n\n# Testing time required to load test set:\n\n\n# for file_id in tqdm(test_df.file_id):\n# files = [f+'.ogg' for f in test_df.file_id]\n\n# files = list(test_dir.glob('*.ogg'))\n# for file_id in tqdm(files):\n#     audio, rate = torchaudio.load(test_dir / file_id)\n#     resampler = Resample(rate, 1000)\n\n#     audo = rechannel(audio, 2)\n#     audio = resampler(audio)\n    \n#     del audio\n\n# submission_df = pd.merge(test_df, file_df.reset_index(), on='file_id', how='outer')\n# ipd.display(submission_df)\n# submission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:44:58.71639Z","iopub.execute_input":"2022-05-05T18:44:58.716705Z","iopub.status.idle":"2022-05-05T18:44:58.940744Z","shell.execute_reply.started":"2022-05-05T18:44:58.71667Z","shell.execute_reply":"2022-05-05T18:44:58.939873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}