{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"}],"dockerImageVersionId":30699,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# BirdCLEF 2024","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"# Import all libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set_style('darkgrid')\nsns.set()\n%config InlineBackend.figure_format = 'retina'\nimport geopandas as gpd\n\nimport librosa\nimport optuna","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:13.146864Z","iopub.execute_input":"2024-05-17T13:09:13.147314Z","iopub.status.idle":"2024-05-17T13:09:14.931136Z","shell.execute_reply.started":"2024-05-17T13:09:13.147281Z","shell.execute_reply":"2024-05-17T13:09:14.929838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Understanding and Exploratory Data Analysis (EDA)","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/birdclef-2024/train_metadata.csv')\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:14.932786Z","iopub.execute_input":"2024-05-17T13:09:14.933256Z","iopub.status.idle":"2024-05-17T13:09:15.173922Z","shell.execute_reply.started":"2024-05-17T13:09:14.933217Z","shell.execute_reply":"2024-05-17T13:09:15.172768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:15.175394Z","iopub.execute_input":"2024-05-17T13:09:15.175855Z","iopub.status.idle":"2024-05-17T13:09:15.183380Z","shell.execute_reply.started":"2024-05-17T13:09:15.175815Z","shell.execute_reply":"2024-05-17T13:09:15.181945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.columns","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:15.187222Z","iopub.execute_input":"2024-05-17T13:09:15.187573Z","iopub.status.idle":"2024-05-17T13:09:15.195905Z","shell.execute_reply.started":"2024-05-17T13:09:15.187544Z","shell.execute_reply":"2024-05-17T13:09:15.194860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.info()","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:15.197423Z","iopub.execute_input":"2024-05-17T13:09:15.197826Z","iopub.status.idle":"2024-05-17T13:09:15.253486Z","shell.execute_reply.started":"2024-05-17T13:09:15.197795Z","shell.execute_reply":"2024-05-17T13:09:15.252205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.describe(include='all')","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:15.254814Z","iopub.execute_input":"2024-05-17T13:09:15.255119Z","iopub.status.idle":"2024-05-17T13:09:15.391389Z","shell.execute_reply.started":"2024-05-17T13:09:15.255093Z","shell.execute_reply":"2024-05-17T13:09:15.390322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Number of unique names of birds: \", train_data.primary_label.nunique())\nprint(\"There are birds: \")\ntrain_data.primary_label.unique()","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:15.392438Z","iopub.execute_input":"2024-05-17T13:09:15.393145Z","iopub.status.idle":"2024-05-17T13:09:15.407666Z","shell.execute_reply.started":"2024-05-17T13:09:15.393114Z","shell.execute_reply":"2024-05-17T13:09:15.406456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"value_counts_bird = train_data.primary_label.value_counts().reset_index()\nprint(\"Value Counts of birds in train data:\")\nvalue_counts_bird","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:15.409361Z","iopub.execute_input":"2024-05-17T13:09:15.409913Z","iopub.status.idle":"2024-05-17T13:09:15.435755Z","shell.execute_reply.started":"2024-05-17T13:09:15.409860Z","shell.execute_reply":"2024-05-17T13:09:15.434280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Unique numbers of count birds:\")\nvalue_counts_bird['count'].unique()","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:15.437275Z","iopub.execute_input":"2024-05-17T13:09:15.437719Z","iopub.status.idle":"2024-05-17T13:09:15.452299Z","shell.execute_reply.started":"2024-05-17T13:09:15.437685Z","shell.execute_reply":"2024-05-17T13:09:15.451214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"How many times of birds have exactly counts in our dataset:\")\nvalue_counts_bird.groupby('count').count().sort_values(by='primary_label', ascending=False).head(5)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:15.455518Z","iopub.execute_input":"2024-05-17T13:09:15.455949Z","iopub.status.idle":"2024-05-17T13:09:15.471185Z","shell.execute_reply.started":"2024-05-17T13:09:15.455918Z","shell.execute_reply":"2024-05-17T13:09:15.469894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Numbers of unique values of secondary labels birds: \", train_data.secondary_labels.nunique())\ntrain_data.secondary_labels.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:15.552756Z","iopub.execute_input":"2024-05-17T13:09:15.553135Z","iopub.status.idle":"2024-05-17T13:09:15.571249Z","shell.execute_reply.started":"2024-05-17T13:09:15.553107Z","shell.execute_reply":"2024-05-17T13:09:15.569889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Numbers of unique values of type songs birds: \", train_data.type.nunique())\ntrain_data.type.value_counts()[train_data.type.value_counts() > 99]","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:16.001308Z","iopub.execute_input":"2024-05-17T13:09:16.001739Z","iopub.status.idle":"2024-05-17T13:09:16.024787Z","shell.execute_reply.started":"2024-05-17T13:09:16.001707Z","shell.execute_reply":"2024-05-17T13:09:16.023669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Number of unique scientific names of birds: \", train_data.scientific_name.nunique())\nprint(\"Number of unique common names of birds: \", train_data.common_name.nunique())","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:16.563168Z","iopub.execute_input":"2024-05-17T13:09:16.563554Z","iopub.status.idle":"2024-05-17T13:09:16.575071Z","shell.execute_reply.started":"2024-05-17T13:09:16.563525Z","shell.execute_reply":"2024-05-17T13:09:16.573836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Number of unique authors: \", train_data.author.nunique())\nprint(f\"Most popular author - {train_data.author.mode()[0]} ({sum(train_data.author == 'José Carlos Sires')} times)\")","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:17.074275Z","iopub.execute_input":"2024-05-17T13:09:17.074665Z","iopub.status.idle":"2024-05-17T13:09:17.096607Z","shell.execute_reply.started":"2024-05-17T13:09:17.074635Z","shell.execute_reply":"2024-05-17T13:09:17.095489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Uniqiue licenses:\")\nprint(train_data.license.unique())","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:17.477980Z","iopub.execute_input":"2024-05-17T13:09:17.478426Z","iopub.status.idle":"2024-05-17T13:09:17.486946Z","shell.execute_reply.started":"2024-05-17T13:09:17.478392Z","shell.execute_reply":"2024-05-17T13:09:17.485836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gdf = gpd.GeoDataFrame(train_data, geometry=gpd.points_from_xy(train_data.longitude, train_data.latitude))\n\nworld = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))\nax = world.plot(figsize=(10, 6), color='white', edgecolor='black')\ngdf.plot(ax=ax, color='red', markersize=0.2)\n\nplt.title('World Map with Longitude and Latitude Points')\nplt.axis(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:18.073490Z","iopub.execute_input":"2024-05-17T13:09:18.073937Z","iopub.status.idle":"2024-05-17T13:09:24.217798Z","shell.execute_reply.started":"2024-05-17T13:09:18.073902Z","shell.execute_reply":"2024-05-17T13:09:24.216871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(5, 4))\nax = fig.add_axes([0, 0, 1, 1])\n\nplt.pie(train_data.license.value_counts().values, \n        labels=train_data.license.value_counts().index,\n        explode=(0.1, 0.5, 0.5, 0.5),\n        autopct='%1.1f%%')\n\nax.set_title('Pie Plot of License', fontweight='bold')\n\nplt.axis('equal')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:24.219340Z","iopub.execute_input":"2024-05-17T13:09:24.220088Z","iopub.status.idle":"2024-05-17T13:09:24.686927Z","shell.execute_reply.started":"2024-05-17T13:09:24.220055Z","shell.execute_reply":"2024-05-17T13:09:24.685911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.rating.describe()['mean']","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:24.688412Z","iopub.execute_input":"2024-05-17T13:09:24.688762Z","iopub.status.idle":"2024-05-17T13:09:24.699055Z","shell.execute_reply.started":"2024-05-17T13:09:24.688732Z","shell.execute_reply":"2024-05-17T13:09:24.697785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(5, 4))\nax = fig.add_axes([0, 0, 1, 1])\n\nsns.histplot(x='rating', data=train_data, ax=ax, bins=train_data.rating.nunique())\nplt.axvline(train_data.rating.mean(), color='red', ls='--')\nplt.axvline(train_data.rating.median(), color='green', ls='-.')\n\n\nax.annotate('Mean', xy=(train_data.rating.mean(), 6000), xytext=(train_data.rating.mean() - 1, 5500),\n            arrowprops=dict(color='black',  arrowstyle='->'), color='red')\nax.annotate('Median', xy=(train_data.rating.median(), 6000), xytext=(train_data.rating.median() + 0.5, 5500),\n            arrowprops=dict(color='black', arrowstyle='->'), color='green')\n\nax.set_ylabel('')\nax.set_xlabel('Rating')\nax.set_title(\"Histplot of Rating\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:24.701495Z","iopub.execute_input":"2024-05-17T13:09:24.701838Z","iopub.status.idle":"2024-05-17T13:09:25.296111Z","shell.execute_reply.started":"2024-05-17T13:09:24.701801Z","shell.execute_reply":"2024-05-17T13:09:25.294887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load train data","metadata":{}},{"cell_type":"code","source":"min_samples_per_species = 50\n\nsampled_data = train_data.groupby('primary_label', group_keys=False).apply(lambda x: x.sample(min(len(x), min_samples_per_species), random_state=42))\n\nsampled_data = sampled_data.sample(frac=1, random_state=42).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:25.297656Z","iopub.execute_input":"2024-05-17T13:09:25.298825Z","iopub.status.idle":"2024-05-17T13:09:25.590802Z","shell.execute_reply.started":"2024-05-17T13:09:25.298770Z","shell.execute_reply":"2024-05-17T13:09:25.589446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sampled_data.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:09:27.576287Z","iopub.execute_input":"2024-05-17T13:09:27.576747Z","iopub.status.idle":"2024-05-17T13:09:27.583868Z","shell.execute_reply.started":"2024-05-17T13:09:27.576702Z","shell.execute_reply":"2024-05-17T13:09:27.582644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SR = 512 # 100\nDURATION = 10\nN_FFT = 128 # 100","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:46:54.825215Z","iopub.execute_input":"2024-05-17T13:46:54.825631Z","iopub.status.idle":"2024-05-17T13:46:54.829970Z","shell.execute_reply.started":"2024-05-17T13:46:54.825599Z","shell.execute_reply":"2024-05-17T13:46:54.828911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsound_data = []\nsound_rate_data = []\nfor file_name in sampled_data.filename:\n    file_path = os.path.join('/kaggle/input/birdclef-2024/train_audio/', file_name)    \n    sound, sr = librosa.load(file_path, sr=SR, duration=DURATION)\n#     rms_file = librosa.feature.rms(y=sound, hop_length=30)\n    mfcc = librosa.feature.mfcc(y=sound, sr=sr, n_fft=N_FFT)\n    mfcc_sound = mfcc.mean(axis=1)\n    sound_data.append(mfcc_sound)\n    sound_rate_data.append(sr)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:46:55.695387Z","iopub.execute_input":"2024-05-17T13:46:55.695805Z","iopub.status.idle":"2024-05-17T13:49:23.799984Z","shell.execute_reply.started":"2024-05-17T13:46:55.695775Z","shell.execute_reply":"2024-05-17T13:49:23.798645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sound_data[:5]","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:20:32.015086Z","iopub.execute_input":"2024-05-17T13:20:32.015553Z","iopub.status.idle":"2024-05-17T13:20:32.026663Z","shell.execute_reply.started":"2024-05-17T13:20:32.015521Z","shell.execute_reply":"2024-05-17T13:20:32.025333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sound_rate_data[:5]","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:20:34.283894Z","iopub.execute_input":"2024-05-17T13:20:34.284329Z","iopub.status.idle":"2024-05-17T13:20:34.293188Z","shell.execute_reply.started":"2024-05-17T13:20:34.284292Z","shell.execute_reply":"2024-05-17T13:20:34.291795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sound, sr = librosa.load('/kaggle/input/birdclef-2024/train_audio/asbfly/XC134896.ogg')\n\nfig, axes = plt.subplots(ncols=1, nrows=2, figsize=(10, 7))\n\nlibrosa.display.waveshow(sound, sr=sr, ax=axes[0])\n\nhar, per = librosa.effects.hpss(sound)\nlibrosa.display.waveshow(har, sr=sr, ax=axes[1], alpha=1.0, color='blue', label='Harmonic')\nlibrosa.display.waveshow(per, sr=sr, ax=axes[1], alpha=0.3, color='red', label='Percussive')\n\naxes[0].label_outer()\naxes[0].set_title(\"Envelope view, mono\", fontsize=15)\naxes[1].legend()\naxes[1].set_title(\"Multiple waveforms\", fontsize=15)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:14:38.126120Z","iopub.execute_input":"2024-05-17T13:14:38.127011Z","iopub.status.idle":"2024-05-17T13:14:43.493489Z","shell.execute_reply.started":"2024-05-17T13:14:38.126945Z","shell.execute_reply":"2024-05-17T13:14:43.492591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nle = LabelEncoder()\ny = le.fit_transform(sampled_data.primary_label)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:14:47.318145Z","iopub.execute_input":"2024-05-17T13:14:47.318615Z","iopub.status.idle":"2024-05-17T13:14:47.327156Z","shell.execute_reply.started":"2024-05-17T13:14:47.318553Z","shell.execute_reply":"2024-05-17T13:14:47.325884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sound_data[0].shape[0]","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:20:38.304265Z","iopub.execute_input":"2024-05-17T13:20:38.304692Z","iopub.status.idle":"2024-05-17T13:20:38.312769Z","shell.execute_reply.started":"2024-05-17T13:20:38.304662Z","shell.execute_reply":"2024-05-17T13:20:38.311549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sound_data[0]","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:20:41.105647Z","iopub.execute_input":"2024-05-17T13:20:41.106032Z","iopub.status.idle":"2024-05-17T13:20:41.113857Z","shell.execute_reply.started":"2024-05-17T13:20:41.106005Z","shell.execute_reply":"2024-05-17T13:20:41.112510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_shape = 100000000000\nfor audio in sound_data:\n    if audio.shape[0] < min_shape:\n        min_shape = audio.shape[0]\nprint(min_shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:20:43.061276Z","iopub.execute_input":"2024-05-17T13:20:43.061678Z","iopub.status.idle":"2024-05-17T13:20:43.073027Z","shell.execute_reply.started":"2024-05-17T13:20:43.061649Z","shell.execute_reply":"2024-05-17T13:20:43.071543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"equal_data = [librosa.util.fix_length(sound, size=20) for sound in sound_data]","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:20:44.195932Z","iopub.execute_input":"2024-05-17T13:20:44.197804Z","iopub.status.idle":"2024-05-17T13:20:44.365231Z","shell.execute_reply.started":"2024-05-17T13:20:44.197745Z","shell.execute_reply":"2024-05-17T13:20:44.363694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_shape = 100000000000\nfor audio in equal_data:\n    if audio.shape[0] < min_shape:\n        min_shape = audio.shape[0]\nprint(min_shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:20:45.824633Z","iopub.execute_input":"2024-05-17T13:20:45.825068Z","iopub.status.idle":"2024-05-17T13:20:45.835100Z","shell.execute_reply.started":"2024-05-17T13:20:45.825023Z","shell.execute_reply":"2024-05-17T13:20:45.833688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_sound_data = np.array(sound_data)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:20:47.140185Z","iopub.execute_input":"2024-05-17T13:20:47.140579Z","iopub.status.idle":"2024-05-17T13:20:47.152961Z","shell.execute_reply.started":"2024-05-17T13:20:47.140550Z","shell.execute_reply":"2024-05-17T13:20:47.151662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_sound_data.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:20:48.492965Z","iopub.execute_input":"2024-05-17T13:20:48.493396Z","iopub.status.idle":"2024-05-17T13:20:48.500485Z","shell.execute_reply.started":"2024-05-17T13:20:48.493362Z","shell.execute_reply":"2024-05-17T13:20:48.499327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Augmentation","metadata":{}},{"cell_type":"code","source":"# !pip install audiomentations","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:05:30.775786Z","iopub.execute_input":"2024-05-17T13:05:30.776601Z","iopub.status.idle":"2024-05-17T13:05:30.781656Z","shell.execute_reply.started":"2024-05-17T13:05:30.776550Z","shell.execute_reply":"2024-05-17T13:05:30.780460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from audiomentations import Compose, AddGaussianNoise, TimeStretch, PitchShift, HighPassFilter, Shift\n\n# augment = Compose([\n#     AddGaussianNoise(min_amplitude=0.001, max_amplitude=0.015, p=0.5),\n#     TimeStretch(min_rate=0.8, max_rate=1.25, p=0.5),\n#     PitchShift(min_semitones=-4, max_semitones=4, p=0.5),\n#     HighPassFilter(min_cutoff_freq=2000, max_cutoff_freq=4000, p=1),\n#     Shift(min_shift=-0.5, max_shift=0.5, p=0.5)\n# ])\n\n# # Augment/transform/perturb the audio data\n# augmented_samples = augment(samples=filtered_sound_data.T, sample_rate=10000).T","metadata":{"execution":{"iopub.status.busy":"2024-05-15T15:11:47.134063Z","iopub.execute_input":"2024-05-15T15:11:47.134932Z","iopub.status.idle":"2024-05-15T15:11:47.139409Z","shell.execute_reply.started":"2024-05-15T15:11:47.134895Z","shell.execute_reply":"2024-05-15T15:11:47.138114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# augmented_samples.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-14T13:19:23.973351Z","iopub.status.idle":"2024-05-14T13:19:23.973667Z","shell.execute_reply.started":"2024-05-14T13:19:23.973509Z","shell.execute_reply":"2024-05-14T13:19:23.973522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# augmented_samples[:10]","metadata":{"execution":{"iopub.status.busy":"2024-05-14T13:19:23.974741Z","iopub.status.idle":"2024-05-14T13:19:23.975168Z","shell.execute_reply.started":"2024-05-14T13:19:23.974949Z","shell.execute_reply":"2024-05-14T13:19:23.974969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Balance target label","metadata":{}},{"cell_type":"code","source":"from imblearn.over_sampling import RandomOverSampler\n\nros = RandomOverSampler(sampling_strategy={class_label: 100 for class_label in range(182)}, random_state=42)\nX_ros, y_ros = ros.fit_resample(filtered_sound_data, y)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:28:01.210188Z","iopub.execute_input":"2024-05-17T13:28:01.210610Z","iopub.status.idle":"2024-05-17T13:28:01.240051Z","shell.execute_reply.started":"2024-05-17T13:28:01.210559Z","shell.execute_reply":"2024-05-17T13:28:01.238787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_ros.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:28:01.844333Z","iopub.execute_input":"2024-05-17T13:28:01.845124Z","iopub.status.idle":"2024-05-17T13:28:01.852400Z","shell.execute_reply.started":"2024-05-17T13:28:01.845088Z","shell.execute_reply":"2024-05-17T13:28:01.851028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(y_ros, return_counts=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:28:02.081616Z","iopub.execute_input":"2024-05-17T13:28:02.082017Z","iopub.status.idle":"2024-05-17T13:28:02.093928Z","shell.execute_reply.started":"2024-05-17T13:28:02.081987Z","shell.execute_reply":"2024-05-17T13:28:02.092760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create tradional models","metadata":{}},{"cell_type":"code","source":"# from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV, cross_val_score, StratifiedKFold\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.compose import make_column_transformer, make_column_selector\n# from sklearn.pipeline import Pipeline\n# from sklearn.metrics import make_scorer, precision_score, recall_score, f1_score\n\n\n# class Model:\n#     RANDOM_STATE = 42\n\n#     def __init__(self, features, label):\n#         #         label = self.columns_transformer().fit_transform(label)\n#         self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(features,\n#                                                                                 label,\n#                                                                                 test_size=0.3,\n#                                                                                 random_state=self.RANDOM_STATE)\n\n#     def __call__(self, estimator, param_grid, random: bool = False):\n#         searcher = self.random_search(estimator, param_grid) if random else self.grid_search(estimator, param_grid)\n#         self.print_results_searching(searcher)\n\n#         estimator.set_params(**searcher.best_params_).fit(self.X_train, self.y_train)\n#         self.print_results(estimator)\n\n#         return estimator\n\n#     def grid_search(self, estimator, param_grid):\n#         searching = GridSearchCV(self.make_model(estimator),\n#                                  param_grid=param_grid,\n#                                  scoring=self.make_f1_scorer(),\n#                                  cv=StratifiedKFold(n_splits=5, shuffle=True, random_state=self.RANDOM_STATE),\n#                                  n_jobs=-1,\n#                                  verbose=1)\n#         return searching.fit(self.X_train, self.y_train)\n\n#     def random_search(self, estimator, param_distributions):\n#         searching = RandomizedSearchCV(self.make_model(estimator),\n#                                        param_distributions=param_distributions,\n#                                        n_iter=10,\n#                                        scoring=self.make_f1_scorer(),\n#                                        cv=StratifiedKFold(n_splits=5, shuffle=True, random_state=self.RANDOM_STATE),\n#                                        n_jobs=-1,\n#                                        verbose=1)\n#         return searching.fit(self.X_train, self.y_train)\n    \n#     @staticmethod\n#     def print_results_searching(searcher):\n#         best_params = searcher.best_params_\n#         best_score = searcher.best_score_\n#         print(\"-\" * 50)\n#         print(\"***RESULTS SEARCHING***\")\n#         print(f\"Best estimator's params: {best_params}\")\n#         print(f\"Best searching score: {best_score}\")\n#         print(\"-\" * 50)\n\n#     def print_results(self, model):\n#         y_pred = model.predict(self.X_test)\n\n#         print(\"***RESULTS MODEL***\")\n#         print(f\"Accuracy train score: {model.score(self.X_train, self.y_train)}\")\n#         print(f\"Accuracy test score: {model.score(self.X_test, self.y_test)}\")\n#         print(\n#             f\"Cross validation score: {cross_val_score(model, self.X_test, self.y_test, scoring=self.make_f1_scorer(), cv=StratifiedKFold(shuffle=True, random_state=self.RANDOM_STATE))}\")\n#         print(f\"Precision score: {precision_score(self.y_test, y_pred, average='macro')}\")\n#         print(f\"Recall score: {recall_score(self.y_test, y_pred, average='macro')}\")\n#         print(f\"F1 score: {f1_score(self.y_test, y_pred, average='macro')}\")\n#         print(\"-\" * 50)\n        \n#     @staticmethod\n#     def make_model(estimator):\n#         model = Pipeline([\n#             ('scaler', StandardScaler()),\n#             ('estimator', estimator)\n#         ])\n\n#         return model\n\n#         #     def columns_transformer(self):\n#         #         transformer = make_column_transformer((LabelEncoder()),\n#         #                                                remainder='passthrough')\n\n#         # return transformer\n\n#     @staticmethod\n#     def make_f1_scorer():\n#         return make_scorer(f1_score, average='macro')","metadata":{"execution":{"iopub.status.busy":"2024-05-14T13:27:22.611332Z","iopub.execute_input":"2024-05-14T13:27:22.611672Z","iopub.status.idle":"2024-05-14T13:27:22.620016Z","shell.execute_reply.started":"2024-05-14T13:27:22.611647Z","shell.execute_reply":"2024-05-14T13:27:22.618802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV, cross_val_score, StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\nfrom sklearn.compose import make_column_transformer, make_column_selector\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import make_scorer, precision_score, recall_score, f1_score\n\n\nclass Searcher:\n    RANDOM_STATE = 42\n\n    def __init__(self, features, label):\n        #         label = self.columns_transformer().fit_transform(label)\n        self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(features,\n                                                                                label,\n                                                                                test_size=0.2,\n                                                                                random_state=self.RANDOM_STATE)\n\n    def searcher(self, estimator, param_grid, random: bool = False):\n        searcher = self.random_search(estimator, param_grid) if random else self.grid_search(estimator, param_grid)\n        self.print_results_searching(searcher)\n\n        return searcher\n\n    def grid_search(self, estimator, param_grid):\n        searching = GridSearchCV(self.make_model(estimator),\n                                 param_grid=param_grid,\n                                 scoring=self.make_f1_scorer(),\n                                 cv=StratifiedKFold(n_splits=5, shuffle=True, random_state=self.RANDOM_STATE),\n                                 n_jobs=-1,\n                                 verbose=1)\n        return searching.fit(self.X_train, self.y_train)\n\n    def random_search(self, estimator, param_distributions):\n        searching = RandomizedSearchCV(self.make_model(estimator),\n                                       param_distributions=param_distributions,\n                                       n_iter=10,\n                                       scoring=self.make_f1_scorer(),\n                                       cv=StratifiedKFold(n_splits=5, shuffle=True, random_state=self.RANDOM_STATE),\n                                       n_jobs=-1,\n                                       verbose=1)\n        return searching.fit(self.X_train, self.y_train)\n\n    @staticmethod\n    def print_results_searching(searcher):\n        best_params = searcher.best_params_\n        best_score = searcher.best_score_\n        print(\"-\" * 50)\n        print(\"***RESULTS SEARCHING***\")\n        print(f\"Best estimator's params: {best_params}\")\n        print(f\"Best searching score: {best_score}\")\n        print(\"-\" * 50)\n\n    @staticmethod\n    def make_model(estimator):\n        model = Pipeline([\n            ('scaler', StandardScaler()),\n            ('estimator', estimator)\n        ])\n\n        return model\n\n        #     def columns_transformer(self):\n        #         transformer = make_column_transformer((LabelEncoder()),\n        #                                                remainder='passthrough')\n\n        # return transformer\n\n    @staticmethod\n    def make_f1_scorer():\n        return make_scorer(f1_score, average='macro')\n\n\nclass Model(Searcher):\n    def __init__(self, features, label):\n        super().__init__(features, label)\n\n    def __call__(self, estimator, param_grid, random: bool = False):\n        searcher = self.searcher(estimator, param_grid, random)\n        best_params = self.get_searching_params(searcher.best_params_)\n\n        estimator.set_params(**best_params).fit(self.X_train, self.y_train)\n        self.print_results(estimator)\n        \n        return estimator\n\n    def print_results(self, model):\n        y_pred = model.predict(self.X_test)\n\n        print(\"***RESULTS MODEL***\")\n        print(f\"Accuracy train score: {model.score(self.X_train, self.y_train)}\")\n        print(f\"Accuracy test score: {model.score(self.X_test, self.y_test)}\")\n        print(\n            f\"Cross validation score: {cross_val_score(model, self.X_test, self.y_test, scoring=self.make_f1_scorer(), cv=StratifiedKFold(shuffle=True, random_state=self.RANDOM_STATE))}\")\n        print(f\"Precision score: {precision_score(self.y_test, y_pred, average='macro')}\")\n        print(f\"Recall score: {recall_score(self.y_test, y_pred, average='macro')}\")\n        print(f\"F1 score: {f1_score(self.y_test, y_pred, average='macro')}\")\n        print(\"-\" * 50)\n\n    @staticmethod\n    def get_searching_params(params):\n        return {param_name.split('estimator__')[1]: param for param_name, param in params.items()}","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:21:11.171009Z","iopub.execute_input":"2024-05-17T13:21:11.171400Z","iopub.status.idle":"2024-05-17T13:21:11.204419Z","shell.execute_reply.started":"2024-05-17T13:21:11.171372Z","shell.execute_reply":"2024-05-17T13:21:11.202915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Model(X_ros, y_ros)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:28:10.343437Z","iopub.execute_input":"2024-05-17T13:28:10.343914Z","iopub.status.idle":"2024-05-17T13:28:10.353136Z","shell.execute_reply.started":"2024-05-17T13:28:10.343873Z","shell.execute_reply":"2024-05-17T13:28:10.351878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, VotingClassifier\nfrom xgboost import XGBClassifier\nfrom lightgbm import LGBMClassifier","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:27:49.605715Z","iopub.execute_input":"2024-05-17T13:27:49.606124Z","iopub.status.idle":"2024-05-17T13:27:49.612116Z","shell.execute_reply.started":"2024-05-17T13:27:49.606091Z","shell.execute_reply":"2024-05-17T13:27:49.610670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nknn_params = {\n    'estimator__n_neighbors': [1, 2, 3, 4, 5],\n    'estimator__leaf_size': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],\n    'estimator__weights': ['uniform', 'distance'],\n}\n\nknn_model = model(KNeighborsClassifier(),  knn_params)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:28:11.383258Z","iopub.execute_input":"2024-05-17T13:28:11.383717Z","iopub.status.idle":"2024-05-17T13:29:20.446133Z","shell.execute_reply.started":"2024-05-17T13:28:11.383680Z","shell.execute_reply":"2024-05-17T13:29:20.444684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# tree_params = {\n#     'estimator__splitter': ['best', 'random'],\n#     'estimator__max_depth': [89, 91, 92, 93, 94, 95],\n#     'estimator__min_samples_split': [1, 2, 3, 4, 5],\n#     'estimator__class_weight': ['balanced', None]\n# }\n\n# tree_model = model(DecisionTreeClassifier(), tree_params)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:20.973846Z","iopub.execute_input":"2024-05-17T13:06:20.974919Z","iopub.status.idle":"2024-05-17T13:06:20.979181Z","shell.execute_reply.started":"2024-05-17T13:06:20.974878Z","shell.execute_reply":"2024-05-17T13:06:20.978363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# xgboosting_params = {\n#     'estimator__n_estimators': [100, 200, 300],\n#     'estimator__max_depth': np.random.randint(3, 10, 100),\n#     'estimator__learning_rate': np.random.uniform(0.01, 0.3, 100),\n#     'estimator__subsample': np.random.uniform(0.5, 1.0, 100),\n#     'estimator__colsample_bytree': np.random.uniform(0.5, 1.0, 100),\n#     'estimator__gamma': np.random.uniform(0, 0.5, 100),\n#     'estimator__reg_alpha': np.random.uniform(0, 0.5, 100),\n#     'estimator__reg_lambda': np.random.uniform(0, 0.5, 100),\n#     'estimator__min_child_weight': np.random.randint(1, 10, 100),\n# }\n\n# xbg_model = model(XGBClassifier(), xgboosting_params, random=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:21.156811Z","iopub.execute_input":"2024-05-17T13:06:21.157200Z","iopub.status.idle":"2024-05-17T13:06:21.162039Z","shell.execute_reply.started":"2024-05-17T13:06:21.157168Z","shell.execute_reply":"2024-05-17T13:06:21.160967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lgbm_params = {'estimator__boosting_type': ['gbdt', 'dart'],\n#                'estimator__n_estimators': [100, 200, 300],\n#                'estimator__num_leaves': np.arange(10, 101, 10),\n#                'estimator__learning_rate': np.random.uniform(0.01, 0.3, 100),\n#                'estimator__max_depth': np.random.randint(3, 50, 100),\n#                'estimator__min_child_samples': np.arange(10, 101, 10),\n#                'estimator__min_split_gain': np.logspace(-3, 2, num=1000, endpoint=True, base=10),\n#                'estimator__estimator__min_child_weight': np.logspace(-3, 2, num=1000, endpoint=True, base=10),\n#                'estimator__verbose': [-1]}\n\n# lgbm_model = model(LGBMClassifier(), lgbm_params, random=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:21.287334Z","iopub.execute_input":"2024-05-17T13:06:21.288479Z","iopub.status.idle":"2024-05-17T13:06:21.295255Z","shell.execute_reply.started":"2024-05-17T13:06:21.288428Z","shell.execute_reply":"2024-05-17T13:06:21.293806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import optuna\n# from sklearn.neighbors import KNeighborsClassifier\n# from sklearn.tree import DecisionTreeClassifier\n# from sklearn.metrics import f1_score\n# from sklearn.model_selection import train_test_split\n# import pandas as pd\n\n# # Assuming X_ros and y_ros are your features and labels\n# X_train, X_test, y_train, y_test = train_test_split(X_ros, y_ros, test_size=0.2, random_state=42)\n\n# def objective(trial):\n#     classifier_name = trial.suggest_categorical('classifier', ['n_neighbors', 'tree'])\n    \n#     if classifier_name == 'n_neighbors':\n#         n_neighbors = trial.suggest_int('n_neighbors', 1, 20)\n#         leaf_size = trial.suggest_int('leaf_size', 1, 10)\n#         weights = trial.suggest_categorical('weights', ['uniform', 'distance'])\n#         classifier_obj = KNeighborsClassifier(n_neighbors=n_neighbors, \n#                                               leaf_size=leaf_size, \n#                                               weights=weights)\n        \n#     elif classifier_name == 'tree':\n#         splitter = trial.suggest_categorical('splitter', ['best', 'random'])\n#         max_depth = trial.suggest_int('max_depth', 1, 100)\n#         min_samples_split = trial.suggest_int('min_samples_split', 1, 10)\n#         class_weight = trial.suggest_categorical('class_weight', ['balanced', None])\n#         classifier_obj = DecisionTreeClassifier(splitter=splitter, \n#                                                 max_depth=max_depth, \n#                                                 min_samples_split=min_samples_split,\n#                                                 class_weight=class_weight)\n    \n#     classifier_obj.fit(X_train, y_train)\n#     y_pred = classifier_obj.predict(X_test)\n#     accuracy = f1_score(y_test, y_pred)\n#     print(classifier_name)\n#     print(f\"Trial {trial.number}: F1 Score - {accuracy}\")\n#     return accuracy","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:21.423882Z","iopub.execute_input":"2024-05-17T13:06:21.424700Z","iopub.status.idle":"2024-05-17T13:06:21.431649Z","shell.execute_reply.started":"2024-05-17T13:06:21.424653Z","shell.execute_reply":"2024-05-17T13:06:21.430459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# study = optuna.create_study(direction='maximize')\n# study.optimize(objective, n_trials=50)\n# optuna.logging.set_verbosity(optuna.logging.WARNING)\n# best_params = study.best_params\n# best_accuracy = study.best_value","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(\"Best parameters:\", best_params)\n# print(\"Best accuracy:\", best_accuracy)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import optuna.visualization as optuna_vis\n\n# optuna_vis.plot_optimization_history(study)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:21.554713Z","iopub.execute_input":"2024-05-17T13:06:21.555126Z","iopub.status.idle":"2024-05-17T13:06:21.562383Z","shell.execute_reply.started":"2024-05-17T13:06:21.555098Z","shell.execute_reply":"2024-05-17T13:06:21.560861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Neural Networks","metadata":{}},{"cell_type":"code","source":"# from imblearn.over_sampling import RandomOverSampler\n\n# ros = RandomOverSampler(sampling_strategy={class_label: 1000 for class_label in range(182)}, random_state=42)\n# X_ros, y_ros = ros.fit_resample(filtered_sound_data, y)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:21.848247Z","iopub.execute_input":"2024-05-17T13:06:21.848898Z","iopub.status.idle":"2024-05-17T13:06:21.855756Z","shell.execute_reply.started":"2024-05-17T13:06:21.848860Z","shell.execute_reply":"2024-05-17T13:06:21.853989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_ros.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:22.121747Z","iopub.execute_input":"2024-05-17T13:06:22.122166Z","iopub.status.idle":"2024-05-17T13:06:22.128420Z","shell.execute_reply.started":"2024-05-17T13:06:22.122133Z","shell.execute_reply":"2024-05-17T13:06:22.126907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import tensorflow as tf\n# from tensorflow.keras.layers import Input, Dense, Conv1D, LSTM, Dropout, Reshape, CategoryEncoding\n# from tensorflow.keras.models import Model, clone_model\n# from tensorflow.keras.losses import CategoricalCrossentropy\n# from tensorflow.keras.optimizers import Adam\n# from tensorflow.keras.metrics import Accuracy\n# from tensorflow.data import Dataset, AUTOTUNE\n# from tensorflow.keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau, EarlyStopping\n# from tensorflow.keras.utils import plot_model","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:22.454575Z","iopub.execute_input":"2024-05-17T13:06:22.455011Z","iopub.status.idle":"2024-05-17T13:06:22.461065Z","shell.execute_reply.started":"2024-05-17T13:06:22.454980Z","shell.execute_reply":"2024-05-17T13:06:22.459609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# INPUT_SHAPE = (X_ros.shape[1],)\n# COUNTS_CLASSES = len(np.unique(y_ros))\n# BATCH_SIZE = 32","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:22.955120Z","iopub.execute_input":"2024-05-17T13:06:22.955528Z","iopub.status.idle":"2024-05-17T13:06:22.960662Z","shell.execute_reply.started":"2024-05-17T13:06:22.955494Z","shell.execute_reply":"2024-05-17T13:06:22.959288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.preprocessing import MinMaxScaler\n\n# X_train, X_test, y_train, y_test = train_test_split(X_ros, y_ros, test_size=0.3, random_state=42)\n\n# # minmax = MinMaxScaler()\n\n# # X_train_scaled = minmax.fit_transform(X_train)\n# # X_test_scaled = minmax.fit_transoform(X_test)\n\n# X_train_tensor = Dataset.from_tensor_slices(X_train).batch(BATCH_SIZE).prefetch(AUTOTUNE)\n# X_test_tensor = Dataset.from_tensor_slices(X_test).batch(BATCH_SIZE).prefetch(AUTOTUNE)\n\n# y_train_one_hot = tf.one_hot(y_train, depth=COUNTS_CLASSES)\n# y_test_one_hot = tf.one_hot(y_test, depth=COUNTS_CLASSES)\n\n# y_train_tensor = Dataset.from_tensor_slices(y_train_one_hot).batch(BATCH_SIZE).prefetch(AUTOTUNE)\n# y_test_tensor = Dataset.from_tensor_slices(y_test_one_hot).batch(BATCH_SIZE).prefetch(AUTOTUNE)\n\n# train_tensor = Dataset.zip((X_train_tensor, y_train_tensor))\n# test_tensor = Dataset.zip((X_test_tensor, y_test_tensor))\n\n# train_tensor, test_tensor","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:23.240050Z","iopub.execute_input":"2024-05-17T13:06:23.240477Z","iopub.status.idle":"2024-05-17T13:06:23.247211Z","shell.execute_reply.started":"2024-05-17T13:06:23.240446Z","shell.execute_reply":"2024-05-17T13:06:23.245539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# scheduler = LearningRateScheduler(lambda epoch: 1e-4 * 100 ** (epoch / 10))\n# early_stopping = EarlyStopping(monitor='val_accuracy', min_delta=0.01, patience=10, restore_best_weights=True)\n# # checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', save_best_only=True)\n# reduce = ReduceLROnPlateau(monitor='val_accuracy', factor=0.1, min_delta=0.01, patience=10)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:23.382674Z","iopub.execute_input":"2024-05-17T13:06:23.383073Z","iopub.status.idle":"2024-05-17T13:06:23.388531Z","shell.execute_reply.started":"2024-05-17T13:06:23.383043Z","shell.execute_reply":"2024-05-17T13:06:23.386995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# inputs = Input(shape=INPUT_SHAPE)\n# x = Dense(324, activation='relu')(inputs)\n# # dropout_1 = Dropout(0.1)(x)\n# x = Dense(324, activation='relu')(x)\n# # dropout_2 = Dropout(0.1)(x)\n# x = Dense(324, activation='relu')(x)\n# # dropout_3 = Dropout(0.1)(x)\n# outputs = Dense(COUNTS_CLASSES, activation='softmax')(x)\n\n# model_1 = Model(inputs, outputs, name='model_1')\n\n# model_1.compile(loss=CategoricalCrossentropy(),\n#                 optimizer=Adam(),\n#                 metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:23.542961Z","iopub.execute_input":"2024-05-17T13:06:23.543353Z","iopub.status.idle":"2024-05-17T13:06:23.549469Z","shell.execute_reply.started":"2024-05-17T13:06:23.543315Z","shell.execute_reply":"2024-05-17T13:06:23.548040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# steps_per_epoch = len(X_train) // BATCH_SIZE\n# validation_steps = len(X_test) // BATCH_SIZE\n# steps_per_epoch, validation_steps","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:23.688864Z","iopub.execute_input":"2024-05-17T13:06:23.689252Z","iopub.status.idle":"2024-05-17T13:06:23.694997Z","shell.execute_reply.started":"2024-05-17T13:06:23.689222Z","shell.execute_reply":"2024-05-17T13:06:23.693468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history_1 = model_1.fit(train_tensor,\n#                         epochs=50,\n#                         steps_per_epoch=steps_per_epoch // 3,\n#                         validation_data=test_tensor,\n#                         validation_steps=validation_steps,\n#                         callbacks=[scheduler],\n#                         verbose=0)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:23.823032Z","iopub.execute_input":"2024-05-17T13:06:23.823405Z","iopub.status.idle":"2024-05-17T13:06:23.828692Z","shell.execute_reply.started":"2024-05-17T13:06:23.823377Z","shell.execute_reply":"2024-05-17T13:06:23.827491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def get_best_learning_rate(history: dict, metric='val_accuracy'):\n#     \"\"\"\n#     Returns the best learning rate based on a given metric from a model's training history.\n\n#     Parameters:\n#     - history (dict): A dictionary containing the training history of a model, typically obtained from model.fit().\n#     - metric (str): The metric used to determine the best learning rate. Default is 'val_accuracy'.\n\n#     Returns:\n#     - best_lr (float): The learning rate that corresponds to the highest value of the specified metric.\n\n#     Example:\n#     ```python\n#     history = {\n#         'learning_rate': [0.001, 0.01, 0.1],\n#         'accuracy': [0,92, 0.95, 0.97],\n#         'val_accuracy': [0.9, 0.92, 0.94],\n#         'loss': [10, 8, 5]\n#     }\n#     best_lr = get_best_learning_rate(history, metric='val_accuracy')\n#     ```\n#     \"\"\"\n#     best_lr = history['learning_rate'][np.argmax(history[metric])]\n#     accuracy = np.max(history[metric])\n#     print(f\"Best learning rate is {best_lr} with accuracy {accuracy}\")\n    \n#     return best_lr","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:23.965717Z","iopub.execute_input":"2024-05-17T13:06:23.967123Z","iopub.status.idle":"2024-05-17T13:06:23.973510Z","shell.execute_reply.started":"2024-05-17T13:06:23.967047Z","shell.execute_reply":"2024-05-17T13:06:23.972218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# best_lr_1 = get_best_learning_rate(history_1.history, metric='val_accuracy')","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:24.286277Z","iopub.execute_input":"2024-05-17T13:06:24.286958Z","iopub.status.idle":"2024-05-17T13:06:24.293264Z","shell.execute_reply.started":"2024-05-17T13:06:24.286922Z","shell.execute_reply":"2024-05-17T13:06:24.291842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# inputs = Input(shape=INPUT_SHAPE)\n# x = Dense(324, activation='relu')(inputs)\n# # dropout_1 = Dropout(0.1)(x)\n# x = Dense(324, activation='relu')(x)\n# # dropout_2 = Dropout(0.1)(x)\n# x = Dense(324, activation='relu')(x)\n# # dropout_3 = Dropout(0.1)(x)\n# outputs = Dense(COUNTS_CLASSES, activation='softmax')(x)\n\n# model_2 = Model(inputs, outputs, name='model_2')\n\n# model_2.compile(loss=CategoricalCrossentropy(),\n#                 optimizer=Adam(best_lr_1),\n#                 metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:24.502867Z","iopub.execute_input":"2024-05-17T13:06:24.503282Z","iopub.status.idle":"2024-05-17T13:06:24.509472Z","shell.execute_reply.started":"2024-05-17T13:06:24.503252Z","shell.execute_reply":"2024-05-17T13:06:24.508151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_2.fit(train_tensor,\n#             epochs=100,\n#             validation_data=test_tensor,\n#             steps_per_epoch=steps_per_epoch // 3,\n#             validation_steps=validation_steps,\n#             callbacks=[early_stopping, reduce])","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:24.664850Z","iopub.execute_input":"2024-05-17T13:06:24.665315Z","iopub.status.idle":"2024-05-17T13:06:24.670228Z","shell.execute_reply.started":"2024-05-17T13:06:24.665280Z","shell.execute_reply":"2024-05-17T13:06:24.669076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot_model(model_2, show_shapes=True, show_layer_names=True, show_layer_activations=True, dpi=50)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:24.994390Z","iopub.execute_input":"2024-05-17T13:06:24.994832Z","iopub.status.idle":"2024-05-17T13:06:24.999363Z","shell.execute_reply.started":"2024-05-17T13:06:24.994788Z","shell.execute_reply":"2024-05-17T13:06:24.998457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_2.evaluate(train_tensor)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:25.332723Z","iopub.execute_input":"2024-05-17T13:06:25.333143Z","iopub.status.idle":"2024-05-17T13:06:25.337885Z","shell.execute_reply.started":"2024-05-17T13:06:25.333110Z","shell.execute_reply":"2024-05-17T13:06:25.336691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_2.evaluate(test_tensor)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:25.789387Z","iopub.execute_input":"2024-05-17T13:06:25.789815Z","iopub.status.idle":"2024-05-17T13:06:25.794474Z","shell.execute_reply.started":"2024-05-17T13:06:25.789772Z","shell.execute_reply":"2024-05-17T13:06:25.793254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# inputs = Input(INPUT_SHAPE)\n# x = Dense(128, activation='relu')(inputs)\n# reshaper = Reshape((1, 128))(x)\n# x = LSTM(128, activation='relu', return_sequences=True)(reshaper)\n# x = LSTM(128, activation='relu')(x)\n# x = Dense(128, activation='relu')(x)\n# outputs = Dense(COUNTS_CLASSES, activation='softmax')(x)\n\n# model_3 = Model(inputs, outputs, name='recurrent_model_preparing')\n\n# model_3.compile(loss=CategoricalCrossentropy(),\n#                 optimizer=Adam(),\n#                 metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:26.076386Z","iopub.execute_input":"2024-05-17T13:06:26.076846Z","iopub.status.idle":"2024-05-17T13:06:26.081613Z","shell.execute_reply.started":"2024-05-17T13:06:26.076809Z","shell.execute_reply":"2024-05-17T13:06:26.080674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history_3 = model_3.fit(train_tensor,\n#                         epochs=50,\n#                         validation_data=test_tensor,\n#                         steps_per_epoch=steps_per_epoch // 3,\n#                         validation_steps=validation_steps,\n#                         callbacks=[scheduler], \n#                         verbose=0)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:26.524692Z","iopub.execute_input":"2024-05-17T13:06:26.525105Z","iopub.status.idle":"2024-05-17T13:06:26.530145Z","shell.execute_reply.started":"2024-05-17T13:06:26.525070Z","shell.execute_reply":"2024-05-17T13:06:26.528782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# best_learning_rate_3 = get_best_learning_rate(history_3.history)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:27.516991Z","iopub.execute_input":"2024-05-17T13:06:27.517369Z","iopub.status.idle":"2024-05-17T13:06:27.522983Z","shell.execute_reply.started":"2024-05-17T13:06:27.517341Z","shell.execute_reply":"2024-05-17T13:06:27.521656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_4 = clone_model(model_3)\n\n# model_4.compile(loss=CategoricalCrossentropy(),\n#                 optimizer=Adam(best_learning_rate_3),\n#                 metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:28.784361Z","iopub.execute_input":"2024-05-17T13:06:28.784789Z","iopub.status.idle":"2024-05-17T13:06:28.790507Z","shell.execute_reply.started":"2024-05-17T13:06:28.784757Z","shell.execute_reply":"2024-05-17T13:06:28.789289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history_4 = model_4.fit(train_tensor,\n#                         epochs=50,\n#                         validation_data=test_tensor,\n#                         steps_per_epoch=steps_per_epoch // 3,\n#                         validation_steps=validation_steps,\n#                         callbacks=[early_stopping, reduce])","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:28.969709Z","iopub.execute_input":"2024-05-17T13:06:28.970105Z","iopub.status.idle":"2024-05-17T13:06:28.974223Z","shell.execute_reply.started":"2024-05-17T13:06:28.970076Z","shell.execute_reply":"2024-05-17T13:06:28.973320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot_model(model_4, show_shapes=True, show_layer_names=True, show_layer_activations=True, dpi=50)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:29.896554Z","iopub.execute_input":"2024-05-17T13:06:29.896989Z","iopub.status.idle":"2024-05-17T13:06:29.901976Z","shell.execute_reply.started":"2024-05-17T13:06:29.896959Z","shell.execute_reply":"2024-05-17T13:06:29.900966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_4.evaluate(train_tensor)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:30.198207Z","iopub.execute_input":"2024-05-17T13:06:30.198622Z","iopub.status.idle":"2024-05-17T13:06:30.203376Z","shell.execute_reply.started":"2024-05-17T13:06:30.198574Z","shell.execute_reply":"2024-05-17T13:06:30.202460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_4.evaluate(test_tensor)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:30.536164Z","iopub.execute_input":"2024-05-17T13:06:30.536561Z","iopub.status.idle":"2024-05-17T13:06:30.542425Z","shell.execute_reply.started":"2024-05-17T13:06:30.536533Z","shell.execute_reply":"2024-05-17T13:06:30.541103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import tensorflow_hub as hub\n\n# # Load the model.\n# bird_vocal_classifier = hub.KerasLayer('https://www.kaggle.com/models/google/bird-vocalization-classifier/TensorFlow2/bird-vocalization-classifier/8',\n#                                        input_shape=(27,),\n#                                        trainable=False,\n#                                        signature='serving_default',\n#                                        output_key='output',\n#                                        output_shape=(182,),\n#                                        name='bird_vocal_classifier')\n                                      ","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:31.859020Z","iopub.execute_input":"2024-05-17T13:06:31.859447Z","iopub.status.idle":"2024-05-17T13:06:31.866396Z","shell.execute_reply.started":"2024-05-17T13:06:31.859412Z","shell.execute_reply":"2024-05-17T13:06:31.865388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class MyWrapperLayer(tf.keras.layers.Layer):\n#     def __init__(self, model, **kwargs):\n#         super(MyWrapperLayer, self).__init__(**kwargs)\n#         self.model = model\n\n#     def call(self, inputs):\n#         return self.model(inputs)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:32.014777Z","iopub.execute_input":"2024-05-17T13:06:32.015294Z","iopub.status.idle":"2024-05-17T13:06:32.020681Z","shell.execute_reply.started":"2024-05-17T13:06:32.015260Z","shell.execute_reply":"2024-05-17T13:06:32.019031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# inputs = Input(shape=(27,))\n# x = MyWrapperLayer(bird_vocal_classifier)(inputs)\n# outputs = Dense(182, activation='softmax')(x)\n\n# model = Model(inputs, outputs)\n\n# model.compile(losses=CategoricalCrossentropy(),\n#               optimizer=Adam(),\n#               metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2024-05-17T13:06:32.162746Z","iopub.execute_input":"2024-05-17T13:06:32.163147Z","iopub.status.idle":"2024-05-17T13:06:32.169320Z","shell.execute_reply.started":"2024-05-17T13:06:32.163119Z","shell.execute_reply":"2024-05-17T13:06:32.167673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# x.summary()","metadata":{"execution":{"iopub.status.busy":"2024-05-10T15:38:06.644334Z","iopub.execute_input":"2024-05-10T15:38:06.645180Z","iopub.status.idle":"2024-05-10T15:38:06.690690Z","shell.execute_reply.started":"2024-05-10T15:38:06.645145Z","shell.execute_reply":"2024-05-10T15:38:06.689491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsound_test_data = []\nsound_test_rate_data = []\nidx = []\nfor sound_name in os.listdir('/kaggle/input/birdclef-2024/unlabeled_soundscapes'):\n    sound, sr = librosa.load(os.path.join('/kaggle/input/birdclef-2024/unlabeled_soundscapes', sound_name), sr=1000, duration=30)\n    mfcc = librosa.feature.mfcc(y=sound, sr=sr)\n    mfcc_sound = mfcc.mean(axis=1)\n    sound_test_data.append(mfcc_sound)\n    sound_test_rate_data.append(sr)\n    idx.append(sound_name)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T14:52:59.812465Z","iopub.execute_input":"2024-05-15T14:52:59.812833Z","iopub.status.idle":"2024-05-15T15:04:41.908071Z","shell.execute_reply.started":"2024-05-15T14:52:59.812805Z","shell.execute_reply":"2024-05-15T15:04:41.907076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx_without_extension = list(map(lambda x: int(x.rstrip('.ogg')), idx))","metadata":{"execution":{"iopub.status.busy":"2024-05-15T15:41:55.707772Z","iopub.execute_input":"2024-05-15T15:41:55.708038Z","iopub.status.idle":"2024-05-15T15:41:55.750297Z","shell.execute_reply.started":"2024-05-15T15:41:55.708014Z","shell.execute_reply":"2024-05-15T15:41:55.748580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred = knn_model.predict_proba(sound_test_data)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T15:41:55.751129Z","iopub.status.idle":"2024-05-15T15:41:55.751537Z","shell.execute_reply.started":"2024-05-15T15:41:55.751329Z","shell.execute_reply":"2024-05-15T15:41:55.751346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sound_test_data = tf.constant(np.squeeze(sound_test_data))","metadata":{"execution":{"iopub.status.busy":"2024-05-15T15:41:55.753634Z","iopub.status.idle":"2024-05-15T15:41:55.754512Z","shell.execute_reply.started":"2024-05-15T15:41:55.754215Z","shell.execute_reply":"2024-05-15T15:41:55.754246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sound_test_data.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-15T14:22:09.174074Z","iopub.execute_input":"2024-05-15T14:22:09.174747Z","iopub.status.idle":"2024-05-15T14:22:09.180608Z","shell.execute_reply.started":"2024-05-15T14:22:09.174714Z","shell.execute_reply":"2024-05-15T14:22:09.179638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_pred = model_2.predict(sound_test_data)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T14:22:00.357083Z","iopub.execute_input":"2024-05-15T14:22:00.357821Z","iopub.status.idle":"2024-05-15T14:22:00.860534Z","shell.execute_reply.started":"2024-05-15T14:22:00.357788Z","shell.execute_reply":"2024-05-15T14:22:00.859455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(data=test_pred, columns=train_data.primary_label.unique(), index=idx_without_extension)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T15:10:35.184204Z","iopub.execute_input":"2024-05-15T15:10:35.184981Z","iopub.status.idle":"2024-05-15T15:10:35.196020Z","shell.execute_reply.started":"2024-05-15T15:10:35.184948Z","shell.execute_reply":"2024-05-15T15:10:35.195059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-15T15:10:36.207362Z","iopub.execute_input":"2024-05-15T15:10:36.208078Z","iopub.status.idle":"2024-05-15T15:10:36.239876Z","shell.execute_reply.started":"2024-05-15T15:10:36.208035Z","shell.execute_reply":"2024-05-15T15:10:36.238919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:58:30.177481Z","iopub.execute_input":"2024-05-13T14:58:30.178276Z","iopub.status.idle":"2024-05-13T14:58:31.222228Z","shell.execute_reply.started":"2024-05-13T14:58:30.178239Z","shell.execute_reply":"2024-05-13T14:58:31.221343Z"},"trusted":true},"execution_count":null,"outputs":[]}]}