{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\nHey, thanks for viewing my Kernel!\n\nIf you like my work, please, leave an upvote: it will be really appreciated and it will motivate me in offering more content to the Kaggle community ! 😊","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nimport plotly.express as px\n\n# For exploring audio files\nimport librosa\nimport librosa.display\nimport IPython.display as ipd\n\nsns.set()\nBASE_DIR = '../input/birdclef-2022/'\ntrain = pd.read_csv(f'{BASE_DIR}/train_metadata.csv')\ntest = pd.read_csv(f'{BASE_DIR}/test.csv')\nebird = pd.read_csv(f'{BASE_DIR}/eBird_Taxonomy_v2021.csv')\nss = pd.read_csv(f'{BASE_DIR}/sample_submission.csv')\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:19.598996Z","iopub.execute_input":"2022-03-02T06:15:19.599351Z","iopub.status.idle":"2022-03-02T06:15:23.837539Z","shell.execute_reply.started":"2022-03-02T06:15:19.599263Z","shell.execute_reply":"2022-03-02T06:15:23.836277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:23.840241Z","iopub.execute_input":"2022-03-02T06:15:23.841103Z","iopub.status.idle":"2022-03-02T06:15:23.868221Z","shell.execute_reply.started":"2022-03-02T06:15:23.841044Z","shell.execute_reply":"2022-03-02T06:15:23.867063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['time_dt'] = pd.to_datetime(train['time'], errors='coerce')\ntrain['time_dt'] = train['time_dt'].dt.round('30min')\ntrain['time_H_M'] = train['time_dt'].dt.strftime('%H:%M')\nprint('Error Times :', train['time_dt'].isna().sum())","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:23.869946Z","iopub.execute_input":"2022-03-02T06:15:23.870290Z","iopub.status.idle":"2022-03-02T06:15:24.087293Z","shell.execute_reply.started":"2022-03-02T06:15:23.870258Z","shell.execute_reply":"2022-03-02T06:15:24.086276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.loc[train['time_dt'].isna(), ['time', 'time_H_M']].head()","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:24.089858Z","iopub.execute_input":"2022-03-02T06:15:24.090191Z","iopub.status.idle":"2022-03-02T06:15:24.111188Z","shell.execute_reply.started":"2022-03-02T06:15:24.090146Z","shell.execute_reply":"2022-03-02T06:15:24.110426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:24.112292Z","iopub.execute_input":"2022-03-02T06:15:24.112511Z","iopub.status.idle":"2022-03-02T06:15:24.119401Z","shell.execute_reply.started":"2022-03-02T06:15:24.112484Z","shell.execute_reply":"2022-03-02T06:15:24.118798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.duplicated().sum()","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:24.120620Z","iopub.execute_input":"2022-03-02T06:15:24.121041Z","iopub.status.idle":"2022-03-02T06:15:24.159648Z","shell.execute_reply.started":"2022-03-02T06:15:24.121004Z","shell.execute_reply":"2022-03-02T06:15:24.158619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seperate_list_col(str_list, max_len=6):\n    if str_list != '[]':\n        seperated_list = str_list.split(',')\n        seperated_list[0] = seperated_list[0][1:]\n        seperated_list[-1] = seperated_list[-1][:-1]\n        seperated_list.extend([None] * (6 - len(seperated_list)))\n        return seperated_list\n    else:\n        return [None] * max_len","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-02T06:15:24.161312Z","iopub.execute_input":"2022-03-02T06:15:24.161624Z","iopub.status.idle":"2022-03-02T06:15:24.168788Z","shell.execute_reply.started":"2022-03-02T06:15:24.161584Z","shell.execute_reply":"2022-03-02T06:15:24.167132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['secondary_label_len'] = train['secondary_labels'].apply(lambda x: len(x.split(',')))\ntrain['type_len'] = train['type'].apply(lambda x: len(x.split(',')))","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:24.170263Z","iopub.execute_input":"2022-03-02T06:15:24.170504Z","iopub.status.idle":"2022-03-02T06:15:24.205341Z","shell.execute_reply.started":"2022-03-02T06:15:24.170471Z","shell.execute_reply":"2022-03-02T06:15:24.204511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"values_secondary_labels = train['secondary_labels'].apply(lambda x: seperate_list_col(x, max_len=6))\ndf_secondary_labels = pd.DataFrame(values_secondary_labels.to_list(), columns=['l2', 'l3', 'l4', 'l5', 'l6', 'l7'])\ndf_secondary_labels.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:24.206359Z","iopub.execute_input":"2022-03-02T06:15:24.206580Z","iopub.status.idle":"2022-03-02T06:15:24.239364Z","shell.execute_reply.started":"2022-03-02T06:15:24.206554Z","shell.execute_reply":"2022-03-02T06:15:24.238399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['secondary_label_len'] = np.where(df_secondary_labels['l2'].values != None, train['secondary_label_len'], 0)","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:20:26.193348Z","iopub.execute_input":"2022-03-02T06:20:26.194112Z","iopub.status.idle":"2022-03-02T06:20:26.200070Z","shell.execute_reply.started":"2022-03-02T06:20:26.194068Z","shell.execute_reply":"2022-03-02T06:20:26.199170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"values_type = train['type'].apply(lambda x: seperate_list_col(x, max_len=9))\ndf_type = pd.DataFrame(values_type.to_list(), columns=['t1', 't2', 't3', 't4', 't5', 't6', 't7', 't8', 't9'])\ndf_type.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:24.242567Z","iopub.execute_input":"2022-03-02T06:15:24.242818Z","iopub.status.idle":"2022-03-02T06:15:24.295305Z","shell.execute_reply.started":"2022-03-02T06:15:24.242785Z","shell.execute_reply":"2022-03-02T06:15:24.294372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['type_len'] = np.where(df_type['t1'].values != None, train['type_len'], 0)","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:22:23.900369Z","iopub.execute_input":"2022-03-02T06:22:23.900679Z","iopub.status.idle":"2022-03-02T06:22:23.907831Z","shell.execute_reply.started":"2022-03-02T06:22:23.900645Z","shell.execute_reply":"2022-03-02T06:22:23.907021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Distributions","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(24, 8))\nsns.countplot(data=train, x='common_name', ax=ax, order=train['common_name'].value_counts().index)\nplt.xticks(rotation=90);","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:24.296852Z","iopub.execute_input":"2022-03-02T06:15:24.297161Z","iopub.status.idle":"2022-03-02T06:15:30.754401Z","shell.execute_reply.started":"2022-03-02T06:15:24.297130Z","shell.execute_reply":"2022-03-02T06:15:30.753696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(16, 8))\nsns.countplot(data=train, x='rating', ax=ax);","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:30.755424Z","iopub.execute_input":"2022-03-02T06:15:30.756091Z","iopub.status.idle":"2022-03-02T06:15:31.079988Z","shell.execute_reply.started":"2022-03-02T06:15:30.756053Z","shell.execute_reply":"2022-03-02T06:15:31.079287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(16, 8))\nx_labels = pd.date_range(start='00:00', periods=48, freq='30min')\nx_labels = list(x_labels.strftime('%H:%M'))\nsns.countplot(data=train, x='time_H_M', ax=ax)\nax.set_xticklabels(x_labels, rotation=90);","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:31.081103Z","iopub.execute_input":"2022-03-02T06:15:31.081428Z","iopub.status.idle":"2022-03-02T06:15:31.932816Z","shell.execute_reply.started":"2022-03-02T06:15:31.081401Z","shell.execute_reply":"2022-03-02T06:15:31.932216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(16, 8))\nsns.countplot(data=train, x='secondary_label_len', ax=ax);","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:20:48.589924Z","iopub.execute_input":"2022-03-02T06:20:48.590225Z","iopub.status.idle":"2022-03-02T06:20:48.814355Z","shell.execute_reply.started":"2022-03-02T06:20:48.590193Z","shell.execute_reply":"2022-03-02T06:20:48.813780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"values = [df_secondary_labels[col].values.tolist() for col in df_secondary_labels.columns]\nvalues = np.array(values)\ndf_values = pd.DataFrame(values.flatten(), columns=['secondary_labels'])\n\nfig, ax = plt.subplots(figsize=(24, 8))\nsns.countplot(data = df_values, x='secondary_labels', ax=ax, order=df_values['secondary_labels'].value_counts().index)\nplt.xticks(rotation=90);","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:32.198916Z","iopub.execute_input":"2022-03-02T06:15:32.199618Z","iopub.status.idle":"2022-03-02T06:15:37.768229Z","shell.execute_reply.started":"2022-03-02T06:15:32.199584Z","shell.execute_reply":"2022-03-02T06:15:37.767266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(16, 8))\nsns.countplot(data=train, x='type_len', ax=ax);","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:37.769616Z","iopub.execute_input":"2022-03-02T06:15:37.769877Z","iopub.status.idle":"2022-03-02T06:15:38.041509Z","shell.execute_reply.started":"2022-03-02T06:15:37.769844Z","shell.execute_reply":"2022-03-02T06:15:38.039624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"values = [df_type[col].values.tolist() for col in df_type.columns]\nvalues = np.array(values)\ndf_values = pd.DataFrame(values.flatten(), columns=['df_type'])\nth = 10\ncounts = df_values['df_type'].value_counts()\nignore_values = counts[counts < th].index\ndf_values.loc[df_values['df_type'].isin(ignore_values), 'df_type'] = 'OTHERS'\n\nfig, ax = plt.subplots(figsize=(24, 8))\nsns.countplot(data = df_values, x='df_type', ax=ax, order=df_values['df_type'].value_counts().index)\nplt.xticks(rotation=90);","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:38.042590Z","iopub.execute_input":"2022-03-02T06:15:38.043025Z","iopub.status.idle":"2022-03-02T06:15:40.358901Z","shell.execute_reply.started":"2022-03-02T06:15:38.042984Z","shell.execute_reply":"2022-03-02T06:15:40.358029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(16, 8))\nsns.countplot(data=train, x='license', ax=ax)\nplt.xticks(rotation=45);","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:40.360065Z","iopub.execute_input":"2022-03-02T06:15:40.360303Z","iopub.status.idle":"2022-03-02T06:15:40.660633Z","shell.execute_reply.started":"2022-03-02T06:15:40.360274Z","shell.execute_reply":"2022-03-02T06:15:40.659812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.scatter_geo(\n    train,\n    lat=\"latitude\",\n    lon=\"longitude\",\n    color=\"common_name\",\n    width=1_000,\n    height=500,\n    title=\"BirdCLEF 2022 Training Data\",\n)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:40.661922Z","iopub.execute_input":"2022-03-02T06:15:40.662392Z","iopub.status.idle":"2022-03-02T06:15:42.493392Z","shell.execute_reply.started":"2022-03-02T06:15:40.662351Z","shell.execute_reply":"2022-03-02T06:15:42.492477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Noise Reduction\n\n\"Noise reduction is the process of removing noise from a signal. Noise reduction techniques exist for audio and images. Noise reduction algorithms may distort the signal to some degree.\" [Source](https://en.wikipedia.org/wiki/Noise_reduction)\n\nIn this work, we use a low pass filter for noise reduction. We can control noise reduction rate by changin **\"th\"** parameter.","metadata":{}},{"cell_type":"code","source":"# Listen to the audio for the second training example\nfn = train[\"filename\"].values[1]\nipd.Audio(f\"{BASE_DIR}train_audio/{fn}\")","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:42.495037Z","iopub.execute_input":"2022-03-02T06:15:42.495560Z","iopub.status.idle":"2022-03-02T06:15:42.543695Z","shell.execute_reply.started":"2022-03-02T06:15:42.495525Z","shell.execute_reply":"2022-03-02T06:15:42.542985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y, sr = librosa.load(f\"{BASE_DIR}train_audio/{fn}\")\nfig, ax = plt.subplots(figsize=(16, 8))\nsns.lineplot(x=np.arange(len(y)), y=y, ax=ax);","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:15:42.545015Z","iopub.execute_input":"2022-03-02T06:15:42.545462Z","iopub.status.idle":"2022-03-02T06:16:21.880020Z","shell.execute_reply.started":"2022-03-02T06:15:42.545417Z","shell.execute_reply":"2022-03-02T06:16:21.879259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def noise_reduction(y, sr, plot=True, th=0.3):\n    from scipy.fft import fft, fftfreq, ifft\n    \n    SAMPLE_RATE = 1\n    DURATION = len(y) / SAMPLE_RATE\n    N = int(SAMPLE_RATE * DURATION)\n\n    yf = fft(y)\n    xf = fftfreq(N, 1 / SAMPLE_RATE)\n    \n    if plot:\n        fig, axes = plt.subplots(1, 2, figsize=(24, 8))\n        axes[0].plot(np.arange(len(y)), y)\n        axes[0].set_title('Before Time-Domain')\n        axes[1].plot(xf, np.abs(yf))\n        axes[1].set_title('Before Frequency-Domain')\n        plt.show()\n    \n    # Filtering Low-Pass\n    new_yf = yf.copy()\n    middle = len(y) / 2\n    new_yf[int(middle - len(y) * th):int(middle + len(y) * th)] = 0\n    new_y = ifft(new_yf)\n    new_y = new_y.real\n    \n    if plot:\n        fig, axes = plt.subplots(1, 2, figsize=(24, 8))\n        axes[0].plot(np.arange(len(y)), new_y)\n        axes[0].set_title('After Time-Domain')\n        axes[1].plot(xf, np.abs(new_yf))\n        axes[1].set_title('After Frequency-Domain')\n        plt.show()\n    \n    return new_y, sr","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-02T06:16:21.881324Z","iopub.execute_input":"2022-03-02T06:16:21.881734Z","iopub.status.idle":"2022-03-02T06:16:21.892800Z","shell.execute_reply.started":"2022-03-02T06:16:21.881681Z","shell.execute_reply":"2022-03-02T06:16:21.892071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_y, sr = noise_reduction(y, sr, plot=True, th=0.3)","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:16:21.894119Z","iopub.execute_input":"2022-03-02T06:16:21.894357Z","iopub.status.idle":"2022-03-02T06:16:23.484527Z","shell.execute_reply.started":"2022-03-02T06:16:21.894329Z","shell.execute_reply":"2022-03-02T06:16:23.483477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(\"Old Audio\", ipd.Audio(data=y, rate=sr))\ndisplay(\"New Audio\", ipd.Audio(data=new_y, rate=sr))","metadata":{"execution":{"iopub.status.busy":"2022-03-02T06:16:23.486283Z","iopub.execute_input":"2022-03-02T06:16:23.486616Z","iopub.status.idle":"2022-03-02T06:16:23.610861Z","shell.execute_reply.started":"2022-03-02T06:16:23.486570Z","shell.execute_reply":"2022-03-02T06:16:23.610199Z"},"trusted":true},"execution_count":null,"outputs":[]}]}