{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n## <font size='5' color='red'>목표</font>\n​\n​\n​\n​\n이 대회는 코넬 조류 연구소의 연구원들이 새 울음소리 오디오를 사용하여 새 종을 식별할 수 있는 AI를 만들고 분류방법들을 목표로 합니다.\n​\n여기 노트에서는 EDA와 오디오 시각화가 작성되었습니다.\n​\n이번 kaggle 제출을 통해 얻어가고자 했던 목표는 음성을 처리하는 방법과 시각화, 여러 개념들, 오디오 분류방법까지 목표로 삼았습니다.\n​\n​\n- EDA : O\n- 시각화 : O\n- 오디오 처리 방법 : O\n- 머신러닝을 통한 분류방법 : O","metadata":{}},{"cell_type":"markdown","source":"## MFCC Feature extraction for Audio Classification\n\n\nMFCC를 추출한 후 MFCC값을 통해 오디오 분류를 진행함","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nimport glob\nimport librosa\nimport librosa.display\nfrom tqdm import tqdm_notebook as tqdm\nfrom keras.models import Model\nfrom keras.utils import np_utils\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\n%matplotlib inline \nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-14T12:51:24.125010Z","iopub.execute_input":"2022-12-14T12:51:24.125343Z","iopub.status.idle":"2022-12-14T12:51:32.834424Z","shell.execute_reply.started":"2022-12-14T12:51:24.125306Z","shell.execute_reply":"2022-12-14T12:51:32.832959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LIMIT = 3","metadata":{"execution":{"iopub.status.busy":"2022-12-14T12:51:32.837112Z","iopub.execute_input":"2022-12-14T12:51:32.837478Z","iopub.status.idle":"2022-12-14T12:51:32.843168Z","shell.execute_reply.started":"2022-12-14T12:51:32.837440Z","shell.execute_reply":"2022-12-14T12:51:32.841935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../input/birdsong-recognition","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2022-12-14T12:51:32.844222Z","iopub.execute_input":"2022-12-14T12:51:32.844470Z","iopub.status.idle":"2022-12-14T12:51:33.137030Z","shell.execute_reply.started":"2022-12-14T12:51:32.844445Z","shell.execute_reply":"2022-12-14T12:51:33.135619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('../input/birdsong-recognition/train.csv')\ndf_train","metadata":{"execution":{"iopub.status.busy":"2022-12-14T12:51:33.139510Z","iopub.execute_input":"2022-12-14T12:51:33.139951Z","iopub.status.idle":"2022-12-14T12:51:33.658294Z","shell.execute_reply.started":"2022-12-14T12:51:33.139907Z","shell.execute_reply":"2022-12-14T12:51:33.656864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../input/birdsong-recognition/train_audio\n\ntrain_dir = '../input/birdsong-recognition/train_audio'\ntest_idr = '../input/birdsong-recognition/test_audio'","metadata":{"execution":{"iopub.status.busy":"2022-12-14T12:51:33.659652Z","iopub.execute_input":"2022-12-14T12:51:33.659994Z","iopub.status.idle":"2022-12-14T12:51:33.972654Z","shell.execute_reply.started":"2022-12-14T12:51:33.659960Z","shell.execute_reply":"2022-12-14T12:51:33.970915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Extract Feature using MFCC()","metadata":{}},{"cell_type":"code","source":"def mfcc_extract(filename):\n    try:\n        y, sr  = librosa.load(filename, sr = 44100)\n        mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=13, n_fft=int(0.02*sr),hop_length=int(0.01*sr))\n        return mfcc\n    except:\n        return","metadata":{"execution":{"iopub.status.busy":"2022-12-14T12:51:33.976284Z","iopub.execute_input":"2022-12-14T12:51:33.976651Z","iopub.status.idle":"2022-12-14T12:51:33.983719Z","shell.execute_reply.started":"2022-12-14T12:51:33.976617Z","shell.execute_reply":"2022-12-14T12:51:33.982075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def parse_audio_files(parent_dir, sub_dirs, limit):\n    labels = []\n    features = []\n    for label, sub_dir in enumerate(tqdm(sub_dirs)):\n        i = 0\n        for fn in glob.glob(os.path.join(parent_dir,sub_dir,\"*.mp3\")):\n            if i >= limit:\n                break\n            features.append(mfcc_extract(fn))\n            labels.append(label)\n            i+=1\n    return features, labels","metadata":{"execution":{"iopub.status.busy":"2022-12-14T12:51:33.985362Z","iopub.execute_input":"2022-12-14T12:51:33.985697Z","iopub.status.idle":"2022-12-14T12:51:33.995211Z","shell.execute_reply.started":"2022-12-14T12:51:33.985663Z","shell.execute_reply":"2022-12-14T12:51:33.994482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain_cat_dirs = glob.glob(train_dir+'/*')\ntrain_cat = []\nfor cat_dir in train_cat_dirs:\n    tmp = cat_dir.split('/')[-1]\n    train_cat.append(tmp)\nprint('the number of kinds:', len(train_cat))\n\nclass_num = len(train_cat)\nfeatures, labels = parse_audio_files(train_dir, train_cat, LIMIT)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T12:51:33.996271Z","iopub.execute_input":"2022-12-14T12:51:33.996609Z","iopub.status.idle":"2022-12-14T13:07:30.072321Z","shell.execute_reply.started":"2022-12-14T12:51:33.996576Z","shell.execute_reply":"2022-12-14T13:07:30.071418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(features))\nprint(features[0].shape)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:07:30.073957Z","iopub.execute_input":"2022-12-14T13:07:30.074225Z","iopub.status.idle":"2022-12-14T13:07:30.080259Z","shell.execute_reply.started":"2022-12-14T13:07:30.074195Z","shell.execute_reply":"2022-12-14T13:07:30.079337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot few features\n\nfig = plt.figure(figsize=(28,24))\nfor i,mfcc in enumerate(tqdm(features[:100])):\n    if i%40 < 3 : \n        sub = plt.subplot(10,3,i%40+3*(i/40)+1)\n        librosa.display.specshow(mfcc,vmin=-700,vmax=300)\n        if ((i%40+3*(i/40)+1)%3==0) : \n            plt.colorbar()\n        sub.set_title(train_cat[labels[i]])\nplt.show()  ","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:07:30.081490Z","iopub.execute_input":"2022-12-14T13:07:30.081738Z","iopub.status.idle":"2022-12-14T13:07:31.266929Z","shell.execute_reply.started":"2022-12-14T13:07:30.081712Z","shell.execute_reply":"2022-12-14T13:07:31.265431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission = pd.read_csv('../input/birdsong-recognition/sample_submission.csv')\ndf_submission.to_csv('submission.csv', index = None)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:07:31.268220Z","iopub.execute_input":"2022-12-14T13:07:31.268481Z","iopub.status.idle":"2022-12-14T13:07:31.511350Z","shell.execute_reply.started":"2022-12-14T13:07:31.268452Z","shell.execute_reply":"2022-12-14T13:07:31.510338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA\n\n외부 참조: https://www.kaggle.com/code/shahules/bird-watch-complete-eda-fe","metadata":{}},{"cell_type":"markdown","source":"## <font size='4' color='red'>Importing Libraries</font><a id='1'></a>\n\n음성처리나 음성인식에 대한 라이브러리로 librosa 라이브러리를 현재 가장 많이 사용중입니다.\n\nPython에서 음원 데이터를 분석해주며 STFT나 Mel-Spectrogram, MFCC 등 다양한 기능을 제공합니다.\n","metadata":{}},{"cell_type":"code","source":"!pip install librosa","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:07:56.443217Z","iopub.execute_input":"2022-12-14T13:07:56.443687Z","iopub.status.idle":"2022-12-14T13:08:23.509891Z","shell.execute_reply.started":"2022-12-14T13:07:56.443656Z","shell.execute_reply":"2022-12-14T13:08:23.509046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from plotly.subplots import make_subplots\nimport plotly.graph_objects as go\nimport matplotlib.pyplot as plt\nimport matplotlib.pyplot as plt\nimport IPython.display as ipd\nimport plotly.express as px\nimport librosa.display\nimport pandas as pd\nimport numpy as  np\nimport librosa\nimport warnings\nimport IPython\nimport os\n\nplt.style.use(\"ggplot\")","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:28.314067Z","iopub.execute_input":"2022-12-14T13:08:28.314416Z","iopub.status.idle":"2022-12-14T13:08:29.183300Z","shell.execute_reply.started":"2022-12-14T13:08:28.314381Z","shell.execute_reply":"2022-12-14T13:08:29.182040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.filterwarnings(action='ignore')","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:30.478335Z","iopub.execute_input":"2022-12-14T13:08:30.478634Z","iopub.status.idle":"2022-12-14T13:08:30.488097Z","shell.execute_reply.started":"2022-12-14T13:08:30.478607Z","shell.execute_reply":"2022-12-14T13:08:30.486642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/birdsong-recognition/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:34.041597Z","iopub.execute_input":"2022-12-14T13:08:34.041970Z","iopub.status.idle":"2022-12-14T13:08:34.271743Z","shell.execute_reply.started":"2022-12-14T13:08:34.041934Z","shell.execute_reply":"2022-12-14T13:08:34.270343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Train dataset has {} rows and {} columns\".format(*train.shape))","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:35.090376Z","iopub.execute_input":"2022-12-14T13:08:35.090686Z","iopub.status.idle":"2022-12-14T13:08:35.096210Z","shell.execute_reply.started":"2022-12-14T13:08:35.090660Z","shell.execute_reply":"2022-12-14T13:08:35.095335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <font size='4' color='red'>Bird Species</font>\n\n조류 종의 이름","metadata":{}},{"cell_type":"code","source":"print(\"There are {} unique species of birds in train dataset\".format(train.species.nunique()))","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:36.395735Z","iopub.execute_input":"2022-12-14T13:08:36.396049Z","iopub.status.idle":"2022-12-14T13:08:36.408399Z","shell.execute_reply.started":"2022-12-14T13:08:36.396023Z","shell.execute_reply":"2022-12-14T13:08:36.406856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"species=train.species.value_counts()\nprint(species)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:38.174658Z","iopub.execute_input":"2022-12-14T13:08:38.175030Z","iopub.status.idle":"2022-12-14T13:08:38.187038Z","shell.execute_reply.started":"2022-12-14T13:08:38.174994Z","shell.execute_reply":"2022-12-14T13:08:38.185469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"go는 graph objects로 브라우저 기반 시각화 라이브러리 입니다.","metadata":{}},{"cell_type":"code","source":"fig = go.Figure(data=[\n    go.Bar(y=species.values, x=species.index,marker_color='deeppink')\n])\n\nfig.update_layout(title='새 종의 분포')\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:39.572032Z","iopub.execute_input":"2022-12-14T13:08:39.572346Z","iopub.status.idle":"2022-12-14T13:08:39.725903Z","shell.execute_reply.started":"2022-12-14T13:08:39.572321Z","shell.execute_reply":"2022-12-14T13:08:39.724284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 절반의 종이 100개의 데이터를 가지고 있습니다.\n- '다이빙 오리'는 9개의 데이터만을 가지고 있습니다.","metadata":{}},{"cell_type":"markdown","source":"## <font size='4' color='red'>Country</font>\n\n관측이 이루어진 국가별 분포","metadata":{}},{"cell_type":"code","source":"country = train.country.value_counts()[:20]\nfig = go.Figure(data=[\n    go.Bar(x=country.index, y=country.values,marker_color='deeppink')\n])\n\nfig.update_layout(title='데이터를 얻은 국가')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:41.596576Z","iopub.execute_input":"2022-12-14T13:08:41.596961Z","iopub.status.idle":"2022-12-14T13:08:41.615579Z","shell.execute_reply.started":"2022-12-14T13:08:41.596925Z","shell.execute_reply":"2022-12-14T13:08:41.614045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 대부분의 데이터는 미국에서 수집되었습니다.\n- 또한 북미에서 대부분 수집이 이루어졌습니다.","metadata":{}},{"cell_type":"markdown","source":"## <font size='4' color='red'>Date</font>\n관찰이 이루어진 날짜","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12, 8))\ntrain['date'].value_counts().sort_index().plot(color='pink',alpha=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:42.149837Z","iopub.execute_input":"2022-12-14T13:08:42.150397Z","iopub.status.idle":"2022-12-14T13:08:42.338926Z","shell.execute_reply.started":"2022-12-14T13:08:42.150363Z","shell.execute_reply":"2022-12-14T13:08:42.337142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train['date'].value_counts().sort_index())","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:42.341135Z","iopub.execute_input":"2022-12-14T13:08:42.341539Z","iopub.status.idle":"2022-12-14T13:08:42.354625Z","shell.execute_reply.started":"2022-12-14T13:08:42.341502Z","shell.execute_reply":"2022-12-14T13:08:42.353608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 수집이 시작된 날짜는 1979년부터 2019년까지 입니다.\n- 2013년부터 2015년에 대부분의 데이터가 수집되었습니다.","metadata":{}},{"cell_type":"markdown","source":"## <font size='4' color='red'>Time</font>\n24시간 기준으로 관찰이 이루어진 시간","metadata":{}},{"cell_type":"code","source":"\n\nhist_data = pd.to_datetime(train.time,errors='coerce').dropna().dt.hour.values.tolist() #datetime 자료형으로 변환\nfig = go.Figure(data=[go.Histogram(x=hist_data, histnorm='probability',marker_color='deeppink')]) #histnorm은 정규화 유형을 지정\nfig.update_layout(title='하루 중 데이터를 수집한 시간')\n\nfig.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:42.808113Z","iopub.execute_input":"2022-12-14T13:08:42.808748Z","iopub.status.idle":"2022-12-14T13:08:43.051618Z","shell.execute_reply.started":"2022-12-14T13:08:42.808709Z","shell.execute_reply":"2022-12-14T13:08:43.049998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 오전 8시에 대부분의 데이터를 가져왔습니다.\n- 또한 대부분의 샘플은 아침 시간에 기록되었습니다.","metadata":{}},{"cell_type":"markdown","source":"## <font size='4' color='red'>Duration</font>\n관찰 기간\n","metadata":{}},{"cell_type":"code","source":"\n\nhist_data = train.duration.values.tolist()\nfig = go.Figure(data=[go.Histogram(x=hist_data,marker_color='deeppink')])\nfig.update_layout(title='관찰 기간')\n\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:43.281520Z","iopub.execute_input":"2022-12-14T13:08:43.281916Z","iopub.status.idle":"2022-12-14T13:08:43.451760Z","shell.execute_reply.started":"2022-12-14T13:08:43.281853Z","shell.execute_reply":"2022-12-14T13:08:43.450405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 5일에서 9일, 10일에서 14일이 가장 많은 관찰 기간이였습니다.\n- 또한 3년 넘게 관찰된 기간도 있었습니다.","metadata":{}},{"cell_type":"markdown","source":"## <font size='4' color='red'>Rating</font>\n관찰에 부여된 등급","metadata":{}},{"cell_type":"code","source":"\nhist_data = train.rating.values.tolist()\nfig = go.Figure(data=[go.Histogram(x=hist_data,marker_color='deeppink')])\nfig.update_layout(title='관측치 등급')\n\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:43.772769Z","iopub.execute_input":"2022-12-14T13:08:43.773156Z","iopub.status.idle":"2022-12-14T13:08:43.947077Z","shell.execute_reply.started":"2022-12-14T13:08:43.773123Z","shell.execute_reply":"2022-12-14T13:08:43.946089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 어떤 기준으로 등급을 나눴는지 참조 문서나 검색했을때 파악할 수 없었습니다.","metadata":{}},{"cell_type":"markdown","source":"## <font size='4' color='red'>Bird Seen</font>\nRecording 중 새를 본 경우","metadata":{}},{"cell_type":"code","source":"colors = ['gold', 'mediumturquoise', 'darkorange', 'lightgreen']\ndf = train.bird_seen.value_counts()\nfig = px.pie(df,df.index,df.values,labels={'index':'Bird Seen'})\nfig.update_traces(hoverinfo='label+percent', textinfo='value', textfont_size=20,\n                  marker=dict(colors=colors, line=dict(color='#000000', width=2)))\nfig.update_layout(title='Bird Seen')\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:44.221822Z","iopub.execute_input":"2022-12-14T13:08:44.222162Z","iopub.status.idle":"2022-12-14T13:08:45.466638Z","shell.execute_reply.started":"2022-12-14T13:08:44.222135Z","shell.execute_reply":"2022-12-14T13:08:45.465752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 80% 이상이 오디오를 녹음하는 동안 새를 시각적으로 확인 할 수 있었습니다.","metadata":{}},{"cell_type":"markdown","source":"## <font size='4' color='red'>Sampling rate</font>\nsampling_rate는 소리를 녹음할때 얼마나 자주 기록하는지를 의미합니다.\n\n1초당 들리는 sample의 개수를 단위로 나타내며 신호를 측정하는 간격\n\nsampling : 신호를 측정하는 과정 자체를 의미\n","metadata":{}},{"cell_type":"code","source":"rec = train.sampling_rate.value_counts()\nfig = go.Figure(data=[\n    go.Bar(x=rec.index, y=rec.values,marker_color='deeppink')\n])\n\nfig.update_layout(title='Top Recordists')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:45.468444Z","iopub.execute_input":"2022-12-14T13:08:45.468778Z","iopub.status.idle":"2022-12-14T13:08:45.487824Z","shell.execute_reply.started":"2022-12-14T13:08:45.468740Z","shell.execute_reply":"2022-12-14T13:08:45.486199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 44kHz은 일반적으로 사용되는 Sr입니다.","metadata":{}},{"cell_type":"markdown","source":"## <font size='3' color='red'>Channel</font>\n채널은 신호나 데이터가 전송되는 경로\n\n일반적인 채널을 모노라고 하며 더 많은 채널을 스테레오, 서라운드 사운드 등으로 나타냄","metadata":{}},{"cell_type":"code","source":"rec = train.channels.value_counts()\nfig = go.Figure(data=[\n    go.Bar(x=rec.index, y=rec.values,marker_color='deeppink')\n])\n\nfig.update_layout(title='Top Recordists')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:45.489341Z","iopub.execute_input":"2022-12-14T13:08:45.489611Z","iopub.status.idle":"2022-12-14T13:08:45.506631Z","shell.execute_reply.started":"2022-12-14T13:08:45.489585Z","shell.execute_reply":"2022-12-14T13:08:45.504725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <font size='4' color='red'>Length</font>\n오디오 신호의 길이","metadata":{}},{"cell_type":"code","source":"df=train.length.value_counts()\nfig = px.pie(df,df.index,df.values,labels={'index':'length of audio'})\nfig.update_layout(title='Length of audio signal')\nfig.update_traces(hoverinfo='label+percent', textinfo='value', textfont_size=20,\n                  marker=dict(colors=colors, line=dict(color='#000000', width=2)))\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:45.509291Z","iopub.execute_input":"2022-12-14T13:08:45.509628Z","iopub.status.idle":"2022-12-14T13:08:45.559423Z","shell.execute_reply.started":"2022-12-14T13:08:45.509599Z","shell.execute_reply":"2022-12-14T13:08:45.558695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <font size='4' color='red'>Geographical Analysis</font>\n\n지리적인 분석","metadata":{}},{"cell_type":"code","source":"df=train.groupby(['latitude','longitude'],as_index=False)['ebird_code'].agg('count')","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:51.373586Z","iopub.execute_input":"2022-12-14T13:08:51.374167Z","iopub.status.idle":"2022-12-14T13:08:51.400570Z","shell.execute_reply.started":"2022-12-14T13:08:51.374127Z","shell.execute_reply":"2022-12-14T13:08:51.399486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:51.511453Z","iopub.execute_input":"2022-12-14T13:08:51.511956Z","iopub.status.idle":"2022-12-14T13:08:51.525678Z","shell.execute_reply.started":"2022-12-14T13:08:51.511918Z","shell.execute_reply":"2022-12-14T13:08:51.524282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=df[df.latitude!='Not specified']\nfig = go.Figure()\nfig.add_trace(go.Scattergeo(\n        lon = df['longitude'],\n        lat = df['latitude'],\n        text = df['ebird_code'],\n        marker = dict(\n            size = df['ebird_code'],\n            line_color='rgb(40,40,40)',\n            line_width=0.5,\n            sizemode = 'area'\n        )))\n\n\nfig.update_layout(\n        title_text = '전세계에서 수집된 조류의 샘플',\n        showlegend = True,\n        geo = dict(\n            landcolor = 'rgb(217, 217, 217)',\n        )\n    )\n\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:51.654321Z","iopub.execute_input":"2022-12-14T13:08:51.654779Z","iopub.status.idle":"2022-12-14T13:08:51.758245Z","shell.execute_reply.started":"2022-12-14T13:08:51.654749Z","shell.execute_reply":"2022-12-14T13:08:51.756728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 대부분의 새들이 미국에서 수집된걸 확인할 수 있습니다.","metadata":{}},{"cell_type":"markdown","source":"### <font size='3' color='red'>Samples from USA</font>\n\n미국을 집약적으로 확인해봄","metadata":{}},{"cell_type":"code","source":"fig = go.Figure()\nfig.add_trace(go.Scattergeo( #그래프의 종류와 데이터 옵션을 지정\n        locationmode = 'USA-states', #위치의 항목을 지도의 영역과 일치시키는 데 사용되는 위치 집합을 결정\n        lon = df['longitude'],\n        lat = df['latitude'],\n        text = df['ebird_code'],\n        marker = dict(\n            size = df['ebird_code'],\n            line_color='rgb(40,40,40)',\n            line_width=0.5,\n            sizemode = 'area'\n        )))\n\n\nfig.update_layout(\n        title_text = 'Bird Samples collected From USA',\n        showlegend = True,\n        geo = dict(\n            scope = 'usa',\n            landcolor = 'rgb(217, 217, 217)',\n        )\n    )\n\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:52.060778Z","iopub.execute_input":"2022-12-14T13:08:52.061172Z","iopub.status.idle":"2022-12-14T13:08:52.147274Z","shell.execute_reply.started":"2022-12-14T13:08:52.061136Z","shell.execute_reply":"2022-12-14T13:08:52.145821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 뉴욕이나 로스앤젤레스, 샌프란시스코 등에서 주로 수집되었습니다.","metadata":{}},{"cell_type":"markdown","source":"## <font size='4' color='red'>Playing audio</font><a id='2'></a>","metadata":{}},{"cell_type":"code","source":"path=\"../input/birdsong-recognition/train_audio/\"\nbirds=train.ebird_code.unique()[:6]\nfile=train[train.ebird_code==birds[0]]['filename'][0]","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:52.430285Z","iopub.execute_input":"2022-12-14T13:08:52.431029Z","iopub.status.idle":"2022-12-14T13:08:52.442603Z","shell.execute_reply.started":"2022-12-14T13:08:52.430863Z","shell.execute_reply":"2022-12-14T13:08:52.441164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfor i in range(0,2):\n    file=train[train.ebird_code==birds[i]]['filename'].values[0]\n    audio_path=os.path.join(path,birds[i],file)\n    print(birds[i])\n    IPython.display.display(ipd.Audio(audio_path))\n","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:52.564855Z","iopub.execute_input":"2022-12-14T13:08:52.565357Z","iopub.status.idle":"2022-12-14T13:08:52.617067Z","shell.execute_reply.started":"2022-12-14T13:08:52.565319Z","shell.execute_reply":"2022-12-14T13:08:52.616147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <font size='4' color='red'>Visualizing Audio</font>\n\n\n오디오 신호를 2D plot으로 시각화합니다.","metadata":{}},{"cell_type":"markdown","source":"\n\n### Waveform\n\n- waveform은 신호를 전송하는 방식을 나타내는데 전송방식에 따라 파형이 달라집니다.\n\n- x축은 time, y축은 amplitude","metadata":{}},{"cell_type":"code","source":"\nplt.figure(figsize=(17,20 ))\n\n\nfor i in range(0,6):\n    file=train[train.ebird_code==birds[i]]['filename'].values[0]\n    audio_path=os.path.join(path,birds[i],file)\n    plt.subplot(6,2,i+1)\n    x , sr = librosa.load(audio_path) #오디오 파일을 읽어 샘플을 리스트로 반환해줌\n    librosa.display.waveplot(x, sr=sr,color='r')\n    plt.gca().set_title(birds[i])\n    plt.gca().get_xaxis().set_visible(False)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:08:55.660178Z","iopub.execute_input":"2022-12-14T13:08:55.660747Z","iopub.status.idle":"2022-12-14T13:09:05.798183Z","shell.execute_reply.started":"2022-12-14T13:08:55.660706Z","shell.execute_reply":"2022-12-14T13:09:05.796718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <font size='4' color='red'>Spectrogram</font>\n\n스펙트로그램은 소리나 파동을 시각화하여 파악히 위한 도구로, 파형과 스펙트럼의 특징이 조합됨.\n\nx축은 Time, y축은 Frequency, z축은 진폭이 적혀있다.\n\n파열, 진동, VOT, 포먼트 등을 파악할 수 있고 어떤 것을 발음하였는지 알수 있다는 의미","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(17,20 ))\n\n\nfor i in range(0,6):\n    file=train[train.ebird_code==birds[i]]['filename'].values[0]\n    audio_path=os.path.join(path,birds[i],file)\n    plt.subplot(6,2,i+1)\n    x , sr = librosa.load(audio_path)\n    x = librosa.stft(x) # 데이터의 스펙트로그램을 리턴\n    Xdb = librosa.amplitude_to_db(abs(x)) # 스펙트로그램을 dB 스케일\n    librosa.display.specshow(Xdb, sr=sr, x_axis='time', y_axis='hz')\n    plt.gca().set_title(birds[i])\n    plt.gca().get_xaxis().set_visible(False)\n    plt.colorbar()","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:09:05.800429Z","iopub.execute_input":"2022-12-14T13:09:05.800728Z","iopub.status.idle":"2022-12-14T13:09:20.220655Z","shell.execute_reply.started":"2022-12-14T13:09:05.800696Z","shell.execute_reply":"2022-12-14T13:09:20.219437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"진하게 표현될수록 주파수가 크다는것을 의미","metadata":{}},{"cell_type":"markdown","source":"## <font size='4' color='red'>Mel-Spectrogram</font>\n\n주파수 특성이 시간에 따라 달라지는 오디오를 분석하기 위한 특징 추출 기법\n\n음성을 인식할때 낮은 주파수를 높은 주파수보다 더 예민하게 받아들입니다.\n\nMel-scale은 이러한 pitch에서 발견한 사람의 음을 인지하는 기준(threshold)을 반영한 scale 변환 함수이다.","metadata":{}},{"cell_type":"code","source":"### n_fft = 2048\nwin_length = 2048\nhop_length = 1024\nn_mels = 128\n\nplt.figure(figsize=(17,20 ))\n\n\nfor i in range(0,6):\n    file=train[train.ebird_code==birds[i]]['filename'].values[0]\n    audio_path=os.path.join(path,birds[i],file)\n    plt.subplot(6,2,i+1)\n    x , sr = librosa.load(audio_path)\n    x = librosa.stft(x) # 데이터의 스펙트로그램을 리턴\n    mel_spec = librosa.feature.melspectrogram(S=x, sr=sr, n_mels=n_mels, hop_length=hop_length, win_length=win_length)\n    Xdb = librosa.amplitude_to_db(abs(mel_spec)) # 스펙트로그램을 dB 스케일\n    librosa.display.specshow(Xdb, sr=sr, x_axis='time', y_axis='hz')\n    plt.gca().set_title(birds[i])\n    plt.gca().get_xaxis().set_visible(False)\n    plt.colorbar()","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:09:23.679517Z","iopub.execute_input":"2022-12-14T13:09:23.679864Z","iopub.status.idle":"2022-12-14T13:09:34.774612Z","shell.execute_reply.started":"2022-12-14T13:09:23.679820Z","shell.execute_reply":"2022-12-14T13:09:34.773690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"외부 코드 참조: https://hyongdoc.tistory.com/402","metadata":{}},{"cell_type":"markdown","source":"## <font size='5' color='blue'>Feature extraction from Audio signal</font><a id='3'></a>\n\n주파수 성분, 스펙트럼 중심, 밀도, 롤오프 등과 같이 푸리에 변환을 통해 시간 기반 신호를 주파수 영역으로 변환하여 얻은 스펙트럼 특징들\n\n(주파수 기반 특징)\n","metadata":{}},{"cell_type":"markdown","source":"### <font size='4' color='red'>Spectral Centroid</font>\n\n스펙트럼의 에너지가 중심이 되는 주파수 또는 소리의 중심이 어디에 있는지를 나타냅니다.\n\n(주파수에서 스펙트럼의 가중 평균과 같음)\n\n일반적으로 소리신호가 높은 주파수를 많이 포함하고 있으면 중심값은 높아지고 밝은 소리를 낸다고 합니다.\n","metadata":{}},{"cell_type":"code","source":"import sklearn\n# 시각화를 위해서 스펙트럼 중심을 정규화 하는 함수\ndef normalize(x, axis=0):\n    return sklearn.preprocessing.minmax_scale(x, axis=axis)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:09:34.776453Z","iopub.execute_input":"2022-12-14T13:09:34.776700Z","iopub.status.idle":"2022-12-14T13:09:34.781488Z","shell.execute_reply.started":"2022-12-14T13:09:34.776674Z","shell.execute_reply":"2022-12-14T13:09:34.780349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nplt.figure(figsize=(17,20 ))\n\n\nfor i in range(0,6):\n    file=train[train.ebird_code==birds[i]]['filename'].values[0]\n    audio_path=os.path.join(path,birds[i],file)\n    plt.subplot(6,2,i+1)\n    x , sr = librosa.load(audio_path)\n    spectral_centroids = librosa.feature.spectral_centroid(x, sr=sr)[0]\n    frames = range(len(spectral_centroids))\n    t = librosa.frames_to_time(frames)\n    librosa.display.waveplot(x, sr=sr, alpha=0.4)\n    plt.plot(t, normalize(spectral_centroids), color='b')\n    plt.gca().set_title(birds[i])\n    plt.gca().get_xaxis().set_visible(False)\n    \n\n","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:09:34.782528Z","iopub.execute_input":"2022-12-14T13:09:34.782743Z","iopub.status.idle":"2022-12-14T13:09:45.212248Z","shell.execute_reply.started":"2022-12-14T13:09:34.782720Z","shell.execute_reply":"2022-12-14T13:09:45.210338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###  Zero-Crossing Rate\n\n\n신호가 0을 지나는 음파가 양에서 음으로 또는 음에서 양으로 바뀌는 비율, 신호의 부호가 바뀌는 비율을 말합니다.\n\n서로 다른 오디오 신호를 특징짓는데 유용하며 분류하는 알고리즘에 많이 사용됨\n\n0(Zero)을 많이 지날수록 노이즈가 많다고 볼 수 있습니다.\n","metadata":{}},{"cell_type":"code","source":"x , sr = librosa.load(audio_path)\nplt.figure(figsize=(14, 5))\nlibrosa.display.waveplot(x, sr=sr)\n# Zooming in\nn0 = 9000\nn1 = 9100\nplt.figure(figsize=(14, 5))\nplt.plot(x[n0:n1])\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:09:45.215099Z","iopub.execute_input":"2022-12-14T13:09:45.215650Z","iopub.status.idle":"2022-12-14T13:09:46.681132Z","shell.execute_reply.started":"2022-12-14T13:09:45.215613Z","shell.execute_reply":"2022-12-14T13:09:46.679327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zero_crossings = librosa.zero_crossings(x[n0:n1], pad=False)\nprint(sum(zero_crossings))","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:09:46.684056Z","iopub.execute_input":"2022-12-14T13:09:46.684348Z","iopub.status.idle":"2022-12-14T13:09:46.691793Z","shell.execute_reply.started":"2022-12-14T13:09:46.684321Z","shell.execute_reply":"2022-12-14T13:09:46.690607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"7개의 포인트에서 wave cross zero가 발생합니다.","metadata":{}},{"cell_type":"markdown","source":"### <font size='4' color='red'>Mel-Frequency Cepstral Coefficients(MFCCs)</font>\n\n음성데이터의 특징을 추출하는 방법으로 두가지가 대표적으로 사용되는데 MFCC와 Mel-Spectrogram입니다.\n\nMFCC는 음성데이터를 특징벡터화 해주는 알고리즘\n\n데이터를 벡터화 한다는것은 학습이 가능하다를 의미하기에 머신러닝에서 중요한 부분을 차지\n\nMFCC는 오디오 신호에서 추출할 수 있는 feature로, 소리의 고유한 특징을 나타내는 수치","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(17, 20))\n\nfor i in range(0,6):\n    file=train[train.ebird_code==birds[i]]['filename'].values[0]\n    audio_path=os.path.join(path,birds[i],file)\n    plt.subplot(6,2,i+1)\n    x , sr = librosa.load(audio_path)\n    mfccs = librosa.feature.mfcc(x, sr=sr)\n    librosa.display.specshow(mfccs, sr=sr, x_axis='time')\n    plt.gca().set_title(birds[i])\n    plt.gca().get_xaxis().set_visible(False)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:09:46.693827Z","iopub.execute_input":"2022-12-14T13:09:46.694183Z","iopub.status.idle":"2022-12-14T13:09:56.711778Z","shell.execute_reply.started":"2022-12-14T13:09:46.694147Z","shell.execute_reply":"2022-12-14T13:09:56.710410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"픽셀 관점이 아니라 주파수 관점에서 접근해야합니다. 즉 보이는 이미지가 특별한 의미가 있다기 보다는, 주파수가 어떻게 분포되는지가 더 중요하다고 볼 수 있는 것 같습니다. ","metadata":{}},{"cell_type":"markdown","source":"### <font size='4' color='red'>Chroma feature</font>\n\n12계의 음계를 나타냅니다.\n\n각 음계의 에너지가 신호에 얼마나 존재하는지를 나타내는 12요소 특성 벡터\n","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(17, 20))\n\nfor i in range(0,6):\n    file=train[train.ebird_code==birds[i]]['filename'].values[0]\n    audio_path=os.path.join(path,birds[i],file)\n    plt.subplot(6,3,i+1)\n    x , sr = librosa.load(audio_path)\n    chromagram = librosa.feature.chroma_stft(x, sr=sr)\n    librosa.display.specshow(chromagram, x_axis='time', y_axis='chroma')\n    plt.gca().set_title(birds[i])\n    plt.gca().get_xaxis().set_visible(False)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:09:56.713511Z","iopub.execute_input":"2022-12-14T13:09:56.713789Z","iopub.status.idle":"2022-12-14T13:10:07.651665Z","shell.execute_reply.started":"2022-12-14T13:09:56.713761Z","shell.execute_reply":"2022-12-14T13:10:07.650422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <font size='4' color='blue'>새들의 스펙트로그램과, MFCC, chroma feature 비교</font><a id='4'></a>","metadata":{}},{"cell_type":"code","source":"fig=plt.figure(figsize=(15,15))\nk=1\nfor i in range(5):\n    \n    file=train[train.ebird_code==birds[i]]['filename'].values[0]\n    audio_path=os.path.join(path,birds[i],file)\n    plt.subplot(5,3,k)\n    k+=1\n    x , sr = librosa.load(audio_path)\n    s = librosa.stft(x)\n    Xdb = librosa.amplitude_to_db(abs(s))\n    librosa.display.specshow(Xdb, sr=sr, x_axis='time', y_axis='hz')\n    plt.gca().set_title('Spectrogram')\n    plt.gca().set_ylabel(birds[i])\n    plt.gca().get_xaxis().set_visible(False)\n\n    plt.subplot(5,3,k)\n    k+=1\n    mfccs = librosa.feature.mfcc(x, sr=sr)\n    librosa.display.specshow(mfccs, sr=sr, x_axis='time')\n    plt.gca().set_title('MFFC features ')\n    plt.gca().get_xaxis().set_visible(False)\n\n    plt.subplot(5,3,k)\n    k+=1\n    chromagram = librosa.feature.chroma_stft(x, sr=sr)\n    librosa.display.specshow(chromagram, x_axis='time', y_axis='chroma', cmap='coolwarm')\n    plt.gca().set_title('Chroma feature')\n    plt.gca().get_xaxis().set_visible(False)\n  \n\n    \n#fig.suptitle('Comparing audio features for bird species')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:10:07.653971Z","iopub.execute_input":"2022-12-14T13:10:07.654332Z","iopub.status.idle":"2022-12-14T13:10:21.656157Z","shell.execute_reply.started":"2022-12-14T13:10:07.654296Z","shell.execute_reply":"2022-12-14T13:10:21.654863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Audio Classification \n\n일반적인 Flow\n\n데이터 전처리 - 특징 추출 - 모델링 - 예측\n\n\n- 전처리 : Input 데이터에 대한 길이 조정\n- 특징 추출 : Spectrogram, MFCC, Waveform, Fourier transform\n- 모델링 : Train, Test 분리, 모델에 대한 구조와 파라메터 정의\n- 예측 : confusion matrix, loss\n\n참조 : https://dacon.io/codeshare/5201?dtype=recent","metadata":{}}]}