{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pkg_resources\n\ndef placeholder(x):\n    raise pkg_resources.DistributionNotFound\npkg_resources.get_distribution = placeholder\n\n!pip uninstall fastai fastcore torchaudio -y\n#!pip install torch==1.8.1 torchaudio==0.8.1 fastcore==1.3.20\n!pip install torch==1.9.0 torchaudio==0.9.0\n!pip install fastaudio","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-05-13T04:27:47.392062Z","iopub.execute_input":"2022-05-13T04:27:47.392857Z","iopub.status.idle":"2022-05-13T04:29:35.251406Z","shell.execute_reply.started":"2022-05-13T04:27:47.392812Z","shell.execute_reply":"2022-05-13T04:29:35.2502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    import pycaret\nexcept:\n    !pip install pycaret-nightly","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-05-13T04:29:47.305057Z","iopub.execute_input":"2022-05-13T04:29:47.305371Z","iopub.status.idle":"2022-05-13T04:29:47.311274Z","shell.execute_reply.started":"2022-05-13T04:29:47.305338Z","shell.execute_reply":"2022-05-13T04:29:47.310117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**I would like to organize this notebook in the following order.**\n> 1. Understand each dataset. Check which features are important features using a simple model.\n> 2. Understand audio data.\n> 3. Transform audio data and design dataloader to process them.\n> 4. Design a model and train it by introducing various methods to improve performance.","metadata":{}},{"cell_type":"code","source":"import os\nimport librosa\nfrom tqdm import tqdm\n\nimport pandas as pd\nfrom fastaudio.all import *\nfrom fastai.vision.all import *\n\nimport torch\nimport torchaudio\nimport fastcore\nimport fastai\nimport fastaudio\nimport torchaudio\ntorchaudio.set_audio_backend(\"sox_io\")\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom matplotlib import rcParams\nimport plotly.express as px\n\nfrom sklearn.preprocessing import LabelEncoder\n\nimport warnings\nwarnings.filterwarnings(action='ignore')","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-05-13T04:29:49.93925Z","iopub.execute_input":"2022-05-13T04:29:49.939549Z","iopub.status.idle":"2022-05-13T04:29:49.955326Z","shell.execute_reply.started":"2022-05-13T04:29:49.939517Z","shell.execute_reply":"2022-05-13T04:29:49.954271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set(style=\"ticks\", context=\"talk\",font_scale = 1.2)\nplt.style.use(\"seaborn-paper\")\nplt.subplots_adjust(wspace=1)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:29:54.107827Z","iopub.execute_input":"2022-05-13T04:29:54.108487Z","iopub.status.idle":"2022-05-13T04:29:54.16198Z","shell.execute_reply.started":"2022-05-13T04:29:54.108433Z","shell.execute_reply":"2022-05-13T04:29:54.161036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"------------------------------------------------------\n# EDA","metadata":{}},{"cell_type":"code","source":"# CONFIGURATIONS\nDATA_DIR = Path('../input/birdclef-2022/train_audio')","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:29:56.729098Z","iopub.execute_input":"2022-05-13T04:29:56.72942Z","iopub.status.idle":"2022-05-13T04:29:56.735258Z","shell.execute_reply.started":"2022-05-13T04:29:56.729385Z","shell.execute_reply":"2022-05-13T04:29:56.734037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"audio_fns = get_audio_files(DATA_DIR)\nprint(f'No. of audio files: {len(audio_fns)}')\n\n# to save time, I subset training data\nmeta_df = pd.read_csv('../input/birdclef-2022/train_metadata.csv')\ntest_df = pd.read_csv('../input/birdclef-2022/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:30:00.859769Z","iopub.execute_input":"2022-05-13T04:30:00.860141Z","iopub.status.idle":"2022-05-13T04:30:03.445244Z","shell.execute_reply.started":"2022-05-13T04:30:00.860104Z","shell.execute_reply":"2022-05-13T04:30:03.44402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--------------------------------------------\n# Analyzing Meta Data","metadata":{}},{"cell_type":"code","source":"meta_df.head().T.style.set_properties(**{'background-color': 'black',\n                           'color': 'white',\n                           'border-color': 'white'})","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:30:06.128733Z","iopub.execute_input":"2022-05-13T04:30:06.129138Z","iopub.status.idle":"2022-05-13T04:30:06.204383Z","shell.execute_reply.started":"2022-05-13T04:30:06.129098Z","shell.execute_reply":"2022-05-13T04:30:06.203512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_df.nunique()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:30:08.439751Z","iopub.execute_input":"2022-05-13T04:30:08.440221Z","iopub.status.idle":"2022-05-13T04:30:08.493914Z","shell.execute_reply.started":"2022-05-13T04:30:08.440173Z","shell.execute_reply":"2022-05-13T04:30:08.493115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head().T.style.set_properties(**{'background-color': 'black',\n                           'color': 'white',\n                           'border-color': 'white'})","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:30:10.32989Z","iopub.execute_input":"2022-05-13T04:30:10.330638Z","iopub.status.idle":"2022-05-13T04:30:10.350174Z","shell.execute_reply.started":"2022-05-13T04:30:10.330588Z","shell.execute_reply":"2022-05-13T04:30:10.348992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = meta_df.drop(['url','filename','scientific_name','license','time','common_name','secondary_labels']\n                        ,axis=1\n                        ,errors='ignore')\ntrain_df = train_df.sample(1000)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:30:12.301701Z","iopub.execute_input":"2022-05-13T04:30:12.302407Z","iopub.status.idle":"2022-05-13T04:30:12.325484Z","shell.execute_reply.started":"2022-05-13T04:30:12.302345Z","shell.execute_reply":"2022-05-13T04:30:12.324381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enc_list = ['primary_label','author','type']\nfor feature in enc_list:\n    le = LabelEncoder()\n    le = le.fit(train_df[feature])\n    train_df[feature] = le.transform(train_df[feature])","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:30:13.767823Z","iopub.execute_input":"2022-05-13T04:30:13.768494Z","iopub.status.idle":"2022-05-13T04:30:13.779882Z","shell.execute_reply.started":"2022-05-13T04:30:13.768435Z","shell.execute_reply":"2022-05-13T04:30:13.779168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"------------------------------------------------------------------------\n# Understaning Metadata from the simple model","metadata":{}},{"cell_type":"code","source":"from pycaret.classification import *","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:30:16.256245Z","iopub.execute_input":"2022-05-13T04:30:16.256901Z","iopub.status.idle":"2022-05-13T04:30:16.261682Z","shell.execute_reply.started":"2022-05-13T04:30:16.256842Z","shell.execute_reply":"2022-05-13T04:30:16.261011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"setup(data = train_df, \n      target = 'primary_label',\n      preprocess = True,\n      silent = True\n     )","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-05-13T04:30:17.781854Z","iopub.execute_input":"2022-05-13T04:30:17.782836Z","iopub.status.idle":"2022-05-13T04:30:24.836884Z","shell.execute_reply.started":"2022-05-13T04:30:17.782755Z","shell.execute_reply":"2022-05-13T04:30:24.833647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt = create_model('dt',verbose = False)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-05-13T04:30:24.843861Z","iopub.execute_input":"2022-05-13T04:30:24.848022Z","iopub.status.idle":"2022-05-13T04:30:41.104814Z","shell.execute_reply.started":"2022-05-13T04:30:24.847834Z","shell.execute_reply":"2022-05-13T04:30:41.103232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model(dt, plot='feature')","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:30:43.707266Z","iopub.execute_input":"2022-05-13T04:30:43.707586Z","iopub.status.idle":"2022-05-13T04:30:44.703613Z","shell.execute_reply.started":"2022-05-13T04:30:43.707552Z","shell.execute_reply":"2022-05-13T04:30:44.702843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color:Blue\"> Observation:\n* latitue and longitude are important features.\n* rating is of relatively low importance.\n    \nI still don't know if this information is helpful for learning. If it is helpful, I still don't know how to use it for learning.","metadata":{}},{"cell_type":"code","source":"fig = px.scatter_geo(\n    meta_df,\n    lat=\"latitude\",\n    lon=\"longitude\",\n    color=\"primary_label\",\n    width=1000,\n    height=500,\n    title=\"Bird Distribution\",\n)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:30:59.170242Z","iopub.execute_input":"2022-05-13T04:30:59.170723Z","iopub.status.idle":"2022-05-13T04:30:59.865623Z","shell.execute_reply.started":"2022-05-13T04:30:59.170689Z","shell.execute_reply":"2022-05-13T04:30:59.864495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"------------------------------------\n# Understanding Train Dataset\n\nTrain data are audio files. Let's hear it for ourselves and see the waveform to see what strategy can solve this problem.","metadata":{}},{"cell_type":"code","source":"def show_bird(audios):\n    for fn in audios:\n        audio = AudioTensor.create(fn)\n        audio.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:31:05.723391Z","iopub.execute_input":"2022-05-13T04:31:05.723716Z","iopub.status.idle":"2022-05-13T04:31:05.729114Z","shell.execute_reply.started":"2022-05-13T04:31:05.723675Z","shell.execute_reply":"2022-05-13T04:31:05.728273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"-------------------------------\n## Normoc\n\n![](https://live.staticflickr.com/7070/6873951614_2dd80c1d7c_b.jpg)\n\nRef: https://cdn.download.ams.birds.cornell.edu","metadata":{}},{"cell_type":"code","source":"fig = px.scatter_geo(\n    meta_df[meta_df['primary_label'] == 'normoc'],\n    lat=\"latitude\",\n    lon=\"longitude\",\n    color=\"primary_label\",\n    width=1000,\n    height=500,\n    title=\"Bird Distribution\",\n)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:31:10.269107Z","iopub.execute_input":"2022-05-13T04:31:10.269953Z","iopub.status.idle":"2022-05-13T04:31:10.3539Z","shell.execute_reply.started":"2022-05-13T04:31:10.269896Z","shell.execute_reply":"2022-05-13T04:31:10.353057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"normoc_fns = get_audio_files( '../input/birdclef-2022/train_audio/normoc')","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:58:48.100219Z","iopub.execute_input":"2022-05-12T23:58:48.100465Z","iopub.status.idle":"2022-05-12T23:58:48.109888Z","shell.execute_reply.started":"2022-05-12T23:58:48.100434Z","shell.execute_reply":"2022-05-12T23:58:48.109037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_bird(normoc_fns[:3])","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:58:48.111336Z","iopub.execute_input":"2022-05-12T23:58:48.11186Z","iopub.status.idle":"2022-05-12T23:58:52.944663Z","shell.execute_reply.started":"2022-05-12T23:58:48.111816Z","shell.execute_reply":"2022-05-12T23:58:52.944067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color:Blue\"> Observation:\n* They are of the same bird species, but sound different.\n* It seems that there are audio files in which noise is heard.\n* Even if you look at the Waveform, there is no similarity.","metadata":{}},{"cell_type":"markdown","source":"--------------------------------\n## Norcar\n\n![](https://upload.wikimedia.org/wikipedia/commons/d/da/Cardinal.jpg)\n\nRef: https://upload.wikimedia.org","metadata":{}},{"cell_type":"code","source":"fig = px.scatter_geo(\n    meta_df[meta_df['primary_label'] == 'norcar'],\n    lat=\"latitude\",\n    lon=\"longitude\",\n    color=\"primary_label\",\n    width=1000,\n    height=500,\n    title=\"Bird Distribution\",\n)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:58:52.945884Z","iopub.execute_input":"2022-05-12T23:58:52.946148Z","iopub.status.idle":"2022-05-12T23:58:53.016961Z","shell.execute_reply.started":"2022-05-12T23:58:52.946118Z","shell.execute_reply":"2022-05-12T23:58:53.015981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"norcar_fns = get_audio_files( '../input/birdclef-2022/train_audio/norcar')","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:58:53.018137Z","iopub.execute_input":"2022-05-12T23:58:53.018693Z","iopub.status.idle":"2022-05-12T23:58:53.029636Z","shell.execute_reply.started":"2022-05-12T23:58:53.018659Z","shell.execute_reply":"2022-05-12T23:58:53.028765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_bird(norcar_fns[:3])","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:58:53.030826Z","iopub.execute_input":"2022-05-12T23:58:53.031575Z","iopub.status.idle":"2022-05-12T23:59:00.435191Z","shell.execute_reply.started":"2022-05-12T23:58:53.031536Z","shell.execute_reply":"2022-05-12T23:59:00.434021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color:Blue\"> Observation:\n* They are of the same bird species, but sound different.\n* It seems that there are audio files in which noise is heard.\n* Even if you look at the Waveform, there is no similarity.","metadata":{}},{"cell_type":"markdown","source":"----------------------------------------\n## Bcnher\n\n![](https://www.birdingintaiwan.com/Black-crowned%20Night-Heron.600b.jpg)\n\nPicture Credit: https://www.birdingintaiwan.com","metadata":{}},{"cell_type":"code","source":"fig = px.scatter_geo(\n    meta_df[meta_df['primary_label'] == 'bcnher'],\n    lat=\"latitude\",\n    lon=\"longitude\",\n    color=\"primary_label\",\n    width=1000,\n    height=500,\n    title=\"Bird Distribution\",\n)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:59:00.437336Z","iopub.execute_input":"2022-05-12T23:59:00.437707Z","iopub.status.idle":"2022-05-12T23:59:00.512753Z","shell.execute_reply.started":"2022-05-12T23:59:00.437659Z","shell.execute_reply":"2022-05-12T23:59:00.511785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bcnher_fns = get_audio_files( '../input/birdclef-2022/train_audio/bcnher')","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:59:00.514033Z","iopub.execute_input":"2022-05-12T23:59:00.514287Z","iopub.status.idle":"2022-05-12T23:59:00.525413Z","shell.execute_reply.started":"2022-05-12T23:59:00.514248Z","shell.execute_reply":"2022-05-12T23:59:00.52445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_bird(bcnher_fns[:3])","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:59:00.527178Z","iopub.execute_input":"2022-05-12T23:59:00.527787Z","iopub.status.idle":"2022-05-12T23:59:04.443055Z","shell.execute_reply.started":"2022-05-12T23:59:00.527741Z","shell.execute_reply":"2022-05-12T23:59:04.442099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color:Blue\"> Observation:\n* They are of the same bird species, but sound different.\n* It seems that there are audio files in which noise is heard.\n* Even if you look at the Waveform, there is no similarity.","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"'''\n\"afrsil1\",\n        \"akekee\",\n        \"akepa1\",\n        \"akiapo\",\n        \"akikik\",\n        \"amewig\",\n        \"aniani\",\n        \"apapan\",\n        \"arcter\",\n        \"barpet\",\n        \"belkin1\",\n        \"bkbplo\",\n        \"bknsti\",\n        \"bkwpet\",\n        \"blkfra\",\n        \"blknod\",\n        \"bongul\",\n        \"brant\",\n        \"brnboo\",\n        \"brnnod\",\n        \"brnowl\",\n        \"brtcur\",\n        \"bubsan\",\n        \"buffle\",\n        \"bulpet\",\n        \"burpar\",\n        \"buwtea\",\n        \"cacgoo1\",\n        \"calqua\",\n        \"cangoo\",\n        \"canvas\",\n        \"caster1\",\n        \"categr\",\n        \"chbsan\",\n        \"chemun\",\n        \"chukar\",\n        \"cintea\",\n        \"comgal1\",\n        \"commyn\",\n        \"compea\",\n        \"comsan\",\n        \"comwax\",\n        \"coopet\",\n        \"crehon\",\n        \"dunlin\",\n        \"elepai\",\n        \"ercfra\",\n        \"eurwig\",\n        \"fragul\",\n        \"gadwal\",\n        \"gamqua\",\n        \"glwgul\",\n        \"gnwtea\",\n        \"golphe\",\n        \"grbher3\",\n        \"grefri\",\n        \"gresca\",\n        \"gryfra\",\n        \"gwfgoo\",\n        \"hawama\",\n        \"hawcoo\",\n        \"hawcre\",\n        \"hawgoo\",\n        \"hawhaw\",\n        \"hawpet1\",\n        \"hoomer\",\n        \"houfin\",\n        \"houspa\",\n        \"hudgod\",\n        \"iiwi\",\n        \"incter1\",\n        \"jabwar\",\n        \"japqua\",\n        \"kalphe\",\n        \"kauama\",\n        \"laugul\",\n        \"layalb\",\n        \"lcspet\",\n        \"leasan\",\n        \"leater1\",\n        \"lessca\",\n        \"lesyel\",\n        \"lobdow\",\n        \"lotjae\",\n        \"madpet\",\n        \"magpet1\",\n        \"mallar3\",\n        \"masboo\",\n        \"mauala\",\n        \"maupar\",\n        \"merlin\",\n        \"mitpar\",\n        \"moudov\",\n     \n        \"norhar2\",\n        \"norpin\",\n        \"norsho\",\n        \"nutman\",\n        \"oahama\",\n        \"omao\",\n        \"osprey\",\n        \"pagplo\",\n        \"palila\",\n        \"parjae\",\n        \"pecsan\",\n        \"peflov\",\n        \"perfal\",\n        \"pibgre\",\n        \"pomjae\",\n        \"puaioh\",\n        \"reccar\",\n        \"redava\",\n        \"redjun\",\n        \"redpha1\",\n        \"refboo\",\n        \"rempar\",\n        \"rettro\",\n        \"ribgul\",\n        \"rinduc\",\n        \"rinphe\",\n        \"rocpig\",\n        \"rorpar\",\n        \"rudtur\",\n        \"ruff\",\n        \"saffin\",\n        \"sander\",\n        \"semplo\",\n        \"sheowl\",\n        \"shtsan\",\n        \"skylar\",\n        \"snogoo\",\n        \"sooshe\",\n        \"sooter1\",\n        \"sopsku1\",\n        \"sora\",\n        \"spodov\",\n        \"sposan\",\n        \"towsol\",\n        \"wantat1\",\n        \"warwhe1\",\n        \"wesmea\",\n        \"wessan\",\n        \"wetshe\",\n        \"whfibi\",\n        \"whiter\",\n        \"whttro\",\n        \"wiltur\",\n        \"yebcar\",\n        \"yefcan\",\n        \"zebdov\",\n        '''","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:31:32.499178Z","iopub.execute_input":"2022-05-13T04:31:32.49965Z","iopub.status.idle":"2022-05-13T04:31:32.510103Z","shell.execute_reply.started":"2022-05-13T04:31:32.499615Z","shell.execute_reply":"2022-05-13T04:31:32.508918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--------------------------------------------\n# Making fingerprints of audios\n\n![](https://www.researchgate.net/publication/335398843/figure/fig1/AS:796124961058818@1566822390492/MFCC-mel-frequency-cepstral-coefficients-characteristic-vectors-extraction-flow.png)\n\nPicture Credit: https://www.researchgate.net\n\nWe try to solve the problem of audio by changing it to an image problem. Regarding Image, there are proven CNN models with high performance, and many methods have been devised to improve the performance.\n\nFortunately, there are APIs related to MFCC provided by fastai, so let's use them to transform audio into image. In other words, we are going to take a method that makes audio fingerprints so that we can classify them with these fingerprints.","metadata":{}},{"cell_type":"markdown","source":"**What is Mel-frequency cepstral coefficients (MFCCs)?**\n\n> In sound processing, the mel-frequency cepstrum (MFC) is a representation of the short-term power spectrum of a sound, based on a linear cosine transform of a log power spectrum on a nonlinear mel scale of frequency.\n> \n> Mel-frequency cepstral coefficients (MFCCs) are coefficients that collectively make up an MFC. They are derived from a type of cepstral representation of the audio clip (a nonlinear \"spectrum-of-a-spectrum\"). The difference between the cepstrum and the mel-frequency cepstrum is that in the MFC, the frequency bands are equally spaced on the mel scale, which approximates the human auditory system's response more closely than the linearly-spaced frequency bands used in the normal spectrum. This frequency warping can allow for better representation of sound, for example, in audio compression.\n\nRef: https://en.wikipedia.org/wiki/Mel-frequency_cepstrum","metadata":{}},{"cell_type":"code","source":"aud2mfcc = AudioToMFCC(n_mfcc=40, melkwargs={'n_fft':2048, 'hop_length':256,'n_mels':128})","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:31:39.645334Z","iopub.execute_input":"2022-05-13T04:31:39.645663Z","iopub.status.idle":"2022-05-13T04:31:39.717654Z","shell.execute_reply.started":"2022-05-13T04:31:39.64563Z","shell.execute_reply":"2022-05-13T04:31:39.716511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Making Pipeline, Dataloaders\n\nBuild the pipeline in the following order.\n> Resample -> DownmixMono -> RemoveSilence -> ResizeSignal -> AudioToMFCC -> Delta -> Tensor","metadata":{}},{"cell_type":"code","source":"item_tfms = [RemoveSilence(),ResizeSignal(1000), aud2mfcc, Delta()]","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:59:04.477073Z","iopub.execute_input":"2022-05-12T23:59:04.477748Z","iopub.status.idle":"2022-05-12T23:59:04.484569Z","shell.execute_reply.started":"2022-05-12T23:59:04.477698Z","shell.execute_reply":"2022-05-12T23:59:04.483565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aud_digit = DataBlock(blocks=(AudioBlock, CategoryBlock),  \n                 get_items=get_audio_files, \n                 splitter=RandomSplitter(),\n                 item_tfms = item_tfms,\n                 get_y=parent_label)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:31:42.16605Z","iopub.execute_input":"2022-05-13T04:31:42.166954Z","iopub.status.idle":"2022-05-13T04:31:42.172562Z","shell.execute_reply.started":"2022-05-13T04:31:42.166875Z","shell.execute_reply":"2022-05-13T04:31:42.171606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aud_digit.summary(DATA_DIR)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:31:44.15443Z","iopub.execute_input":"2022-05-13T04:31:44.154956Z","iopub.status.idle":"2022-05-13T04:31:48.848244Z","shell.execute_reply.started":"2022-05-13T04:31:44.154902Z","shell.execute_reply":"2022-05-13T04:31:48.847274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = aud_digit.dataloaders(DATA_DIR, bs=16)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:31:57.084019Z","iopub.execute_input":"2022-05-13T04:31:57.084334Z","iopub.status.idle":"2022-05-13T04:31:57.731751Z","shell.execute_reply.started":"2022-05-13T04:31:57.084302Z","shell.execute_reply":"2022-05-13T04:31:57.730295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.c","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:32:00.470967Z","iopub.execute_input":"2022-05-13T04:32:00.471343Z","iopub.status.idle":"2022-05-13T04:32:00.480109Z","shell.execute_reply.started":"2022-05-13T04:32:00.471301Z","shell.execute_reply":"2022-05-13T04:32:00.478311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color:Blue\"> Observation:\n    \n* Target is composed of 152 classes.","metadata":{}},{"cell_type":"markdown","source":"--------------------------------------------------------------------\n## Checking batch","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 8))\ndls.show_batch(max_n=3,figsize=(20,10))","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:32:03.943586Z","iopub.execute_input":"2022-05-13T04:32:03.943962Z","iopub.status.idle":"2022-05-13T04:32:08.642284Z","shell.execute_reply.started":"2022-05-13T04:32:03.943903Z","shell.execute_reply":"2022-05-13T04:32:08.64153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color:Blue\"> Observation:\n* I converted audio to image. \nThe format of the question has been changed. Looking at the picture above, it can be seen that the problem of audio classification has been changed to the problem of image classification.","metadata":{}},{"cell_type":"markdown","source":"------------------------------------------------------\n# Modeling","metadata":{}},{"cell_type":"code","source":"def audio_learner(dls, arch, loss_func, metrics):\n  \"Prepares a `Learner` for audio processing\"\n  learn = Learner(dls, arch, loss_func, metrics=metrics, \n                  cbs = [EarlyStoppingCallback(monitor='accuracy', patience=5),ActivationStats(with_hist=True)]).to_fp16()\n  n_c = dls.one_batch()[0].shape[1]\n  if n_c == 1: alter_learner(learn)\n  return learn","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:32:11.235515Z","iopub.execute_input":"2022-05-13T04:32:11.236045Z","iopub.status.idle":"2022-05-13T04:32:11.243025Z","shell.execute_reply.started":"2022-05-13T04:32:11.235982Z","shell.execute_reply":"2022-05-13T04:32:11.241962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = audio_learner(dls, \n                      xresnet18(), \n                      LabelSmoothingCrossEntropy(), \n                      accuracy)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:32:13.771342Z","iopub.execute_input":"2022-05-13T04:32:13.771841Z","iopub.status.idle":"2022-05-13T04:32:17.788745Z","shell.execute_reply.started":"2022-05-13T04:32:13.771804Z","shell.execute_reply":"2022-05-13T04:32:17.787942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sr = learn.lr_find()\nsr","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:32:19.191199Z","iopub.execute_input":"2022-05-13T04:32:19.191546Z","iopub.status.idle":"2022-05-13T04:35:34.084256Z","shell.execute_reply.started":"2022-05-13T04:32:19.191507Z","shell.execute_reply":"2022-05-13T04:35:34.083008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"------------------------------------------------\n# Training","metadata":{}},{"cell_type":"code","source":"learn.fit_one_cycle(100, sr.lr_steep)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T05:14:41.051189Z","iopub.execute_input":"2022-05-13T05:14:41.052002Z","iopub.status.idle":"2022-05-13T05:40:01.223914Z","shell.execute_reply.started":"2022-05-13T05:14:41.051879Z","shell.execute_reply":"2022-05-13T05:40:01.21749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"-------------------------------------------\n# Interpreting Model","metadata":{}},{"cell_type":"code","source":"def plot_layer_stats(self, idx):\n    plt,axs = subplots(1, 3, figsize=(15,3))\n    plt.subplots_adjust(wspace=0.5)\n    for o,ax,title in zip(self.layer_stats(idx),axs,('mean','std','% near zero')):\n        ax.plot(o)\n        ax.set_title(title)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:27:41.62851Z","iopub.status.idle":"2022-05-13T04:27:41.629129Z","shell.execute_reply.started":"2022-05-13T04:27:41.628852Z","shell.execute_reply":"2022-05-13T04:27:41.628883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_layer_stats(learn.activation_stats,-2)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:27:41.631387Z","iopub.status.idle":"2022-05-13T04:27:41.631997Z","shell.execute_reply.started":"2022-05-13T04:27:41.631665Z","shell.execute_reply":"2022-05-13T04:27:41.631695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_layer_stats(learn.activation_stats,-1)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:27:41.63432Z","iopub.status.idle":"2022-05-13T04:27:41.634899Z","shell.execute_reply.started":"2022-05-13T04:27:41.63458Z","shell.execute_reply":"2022-05-13T04:27:41.63461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color:Blue\"> Observation:\n* The activation distribution is well distributed.","metadata":{}},{"cell_type":"markdown","source":"-------------------------------------------\n# Checking Underfitting and Overfitting\n\n![](https://vitalflux.com/wp-content/uploads/2020/12/overfitting-and-underfitting-wrt-model-error-vs-complexity.png)\n\nPicture Credit: https://vitalflux.com/wp-content","metadata":{}},{"cell_type":"code","source":"learn.recorder.plot_loss()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:27:41.63672Z","iopub.status.idle":"2022-05-13T04:27:41.637304Z","shell.execute_reply.started":"2022-05-13T04:27:41.636991Z","shell.execute_reply":"2022-05-13T04:27:41.637021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color:Blue\"> Observation:\n* It seems that learning has ended before overffing. However, I still don't know if it is the optimal point. It is burdensome to increase the epoch because the training time is too long.","metadata":{}},{"cell_type":"markdown","source":"----------------------------------------------\n# Checking Results","metadata":{}},{"cell_type":"code","source":"interp = ClassificationInterpretation.from_learner(learn)\ninterp.plot_confusion_matrix(figsize=(30,30), dpi=240)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:27:41.639787Z","iopub.status.idle":"2022-05-13T04:27:41.640368Z","shell.execute_reply.started":"2022-05-13T04:27:41.640061Z","shell.execute_reply":"2022-05-13T04:27:41.640093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color:Blue\"> Observation:\n* Because there are many classes of Target, it is not good to look at.\n* If you look at the diagonal of the matrix, it seems that it has learned to some extent. However, it is not satisfactory.","metadata":{}},{"cell_type":"markdown","source":"**Let's check the cases where the model made the most mistakes!**","metadata":{}},{"cell_type":"code","source":"interp.most_confused(min_val=10)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T04:27:41.642292Z","iopub.status.idle":"2022-05-13T04:27:41.642706Z","shell.execute_reply.started":"2022-05-13T04:27:41.642497Z","shell.execute_reply":"2022-05-13T04:27:41.642523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color:Blue\"> Observation:\n* In the combination of 'bcnher' and 'brnowl', there are many cases where the judgment is exceptionally wrong. I guess I'll have to check more on why. ","metadata":{}}]}