{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n\nimport warnings\nwarnings.filterwarnings(action='ignore')\n\nimport pandas as pd\nimport librosa\nimport numpy as np\n\nfrom sklearn.utils import shuffle\nfrom PIL import Image\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\n\n# Global vars\nRANDOM_SEED = 1337\nSAMPLE_RATE = 32000\nSIGNAL_LENGTH = 5 # seconds\nSPEC_SHAPE = (48, 128) # height x width\nFMIN = 500\nFMAX = 12500\nMAX_AUDIO_FILES = 3000","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-09T04:13:41.011769Z","iopub.execute_input":"2021-06-09T04:13:41.012111Z","iopub.status.idle":"2021-06-09T04:13:47.672453Z","shell.execute_reply.started":"2021-06-09T04:13:41.01203Z","shell.execute_reply":"2021-06-09T04:13:47.671576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load metadata file\ntrain = pd.read_csv('../input/birdclef-2021/train_metadata.csv',)\n\n# Limit the number of training samples and classes\n# First, only use high quality samples\ntrain = train.query('rating>=4')\n\n# A species needs at least 200 recordings with a rating above 4 to be considered common\nbirds_count = {}\nfor bird_species, count in zip(train.primary_label.unique(), \n                               train.groupby('primary_label')['primary_label'].count().values):\n    birds_count[bird_species] = count\nmost_represented_birds = [key for key,value in birds_count.items() if value >= 200] \n\nTRAIN = train.query('primary_label in @most_represented_birds')\nLABELS = sorted(TRAIN.primary_label.unique())\n\n# Let's see how many species and samples we have left\nprint('NUMBER OF SPECIES IN TRAIN DATA:', len(LABELS))\nprint('NUMBER OF SAMPLES IN TRAIN DATA:', len(TRAIN))\nprint('LABELS:', most_represented_birds)","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:13:47.673988Z","iopub.execute_input":"2021-06-09T04:13:47.674303Z","iopub.status.idle":"2021-06-09T04:13:48.124074Z","shell.execute_reply.started":"2021-06-09T04:13:47.674268Z","shell.execute_reply":"2021-06-09T04:13:48.123236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN = shuffle(TRAIN, random_state=RANDOM_SEED)[:MAX_AUDIO_FILES]\n\n# Define a function that splits an audio file, \n# extracts spectrograms and saves them in a working directory\ndef get_spectrograms(filepath, primary_label, output_dir):\n    \n    # Open the file with librosa (limited to the first 15 seconds)\n    sig, rate = librosa.load(filepath, sr=SAMPLE_RATE, offset=None, duration=15)\n    \n    # Split signal into five second chunks\n    sig_splits = []\n    for i in range(0, len(sig), int(SIGNAL_LENGTH * SAMPLE_RATE)):\n        split = sig[i:i + int(SIGNAL_LENGTH * SAMPLE_RATE)]\n\n        # End of signal?\n        if len(split) < int(SIGNAL_LENGTH * SAMPLE_RATE):\n            break\n        \n        sig_splits.append(split)\n        \n    # Extract mel spectrograms for each audio chunk\n    s_cnt = 0\n    saved_samples = []\n    for chunk in sig_splits:\n        \n        hop_length = int(SIGNAL_LENGTH * SAMPLE_RATE / (SPEC_SHAPE[1] - 1))\n        mel_spec = librosa.feature.melspectrogram(y=chunk, \n                                                  sr=SAMPLE_RATE, \n                                                  n_fft=1024, \n                                                  hop_length=hop_length, \n                                                  n_mels=SPEC_SHAPE[0], \n                                                  fmin=FMIN, \n                                                  fmax=FMAX)\n    \n        mel_spec = librosa.power_to_db(mel_spec, ref=np.max) \n        \n        # Normalize\n        mel_spec -= mel_spec.min()\n        mel_spec /= mel_spec.max()\n        \n        # Save as image file\n        save_dir = os.path.join(output_dir, primary_label)\n        if not os.path.exists(save_dir):\n            os.makedirs(save_dir)\n        save_path = os.path.join(save_dir, filepath.rsplit(os.sep, 1)[-1].rsplit('.', 1)[0] + \n                                 '_' + str(s_cnt) + '.png')\n        im = Image.fromarray(mel_spec * 255.0).convert(\"L\")\n        im.save(save_path)\n        \n        saved_samples.append(save_path)\n        s_cnt += 1\n        \n        \n    return saved_samples\n\nprint('FINAL NUMBER OF AUDIO FILES IN TRAINING DATA:', len(TRAIN))","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:13:48.125799Z","iopub.execute_input":"2021-06-09T04:13:48.126055Z","iopub.status.idle":"2021-06-09T04:13:48.143348Z","shell.execute_reply.started":"2021-06-09T04:13:48.126031Z","shell.execute_reply":"2021-06-09T04:13:48.14157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Parse audio files and extract training samples\ninput_dir = '../input/birdclef-2021/train_short_audio/'\noutput_dir = '../working/melspectrogram_dataset/'\nsamples = []\nwith tqdm(total=len(TRAIN)) as pbar:\n    for idx, row in TRAIN.iterrows():\n        pbar.update(1)\n        \n        if row.primary_label in most_represented_birds:\n            audio_file_path = os.path.join(input_dir, row.primary_label, row.filename)\n            samples += get_spectrograms(audio_file_path, row.primary_label, output_dir)\n            \nTRAIN_SPECS = shuffle(samples, random_state=RANDOM_SEED)\nprint('SUCCESSFULLY EXTRACTED {} SPECTROGRAMS'.format(len(TRAIN_SPECS)))","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:13:48.145476Z","iopub.execute_input":"2021-06-09T04:13:48.145968Z","iopub.status.idle":"2021-06-09T04:17:13.130079Z","shell.execute_reply.started":"2021-06-09T04:13:48.145886Z","shell.execute_reply":"2021-06-09T04:17:13.129204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the first 12 spectrograms of TRAIN_SPECS\nplt.figure(figsize=(15, 7))\nfor i in range(12):\n    spec = Image.open(TRAIN_SPECS[i])\n    plt.subplot(3, 4, i + 1)\n    plt.title(TRAIN_SPECS[i].split(os.sep)[-1])\n    plt.imshow(spec, origin='lower')","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:17:13.134066Z","iopub.execute_input":"2021-06-09T04:17:13.136197Z","iopub.status.idle":"2021-06-09T04:17:14.304779Z","shell.execute_reply.started":"2021-06-09T04:17:13.136152Z","shell.execute_reply":"2021-06-09T04:17:14.303953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_specs, train_labels = [], []\nwith tqdm(total=len(TRAIN_SPECS)) as pbar:\n    for path in TRAIN_SPECS:\n        pbar.update(1)\n\n        # Open image\n        spec = Image.open(path)\n\n        # Convert to numpy array\n        spec = np.array(spec, dtype='float32')\n        \n        # Normalize between 0.0 and 1.0\n        # and exclude samples with nan \n        spec -= spec.min()\n        spec /= spec.max()\n        if not spec.max() == 1.0 or not spec.min() == 0.0:\n            continue\n\n        # Add channel axis to 2D array\n        spec = np.expand_dims(spec, -1)\n\n        # Add new dimension for batch size\n        spec = np.expand_dims(spec, 0)\n\n        # Add to train data\n        if len(train_specs) == 0:\n            train_specs = spec\n        else:\n            train_specs = np.vstack((train_specs, spec))\n\n        # Add to label data\n        target = np.zeros((len(LABELS)), dtype='float32')\n        bird = path.split(os.sep)[-2]\n        target[LABELS.index(bird)] = 1.0\n        if len(train_labels) == 0:\n            train_labels = target\n        else:\n            train_labels = np.vstack((train_labels, target))","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:17:14.305784Z","iopub.execute_input":"2021-06-09T04:17:14.306083Z","iopub.status.idle":"2021-06-09T04:22:26.385891Z","shell.execute_reply.started":"2021-06-09T04:17:14.30605Z","shell.execute_reply":"2021-06-09T04:22:26.384921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import librosa\naudio_file = '../input/birdclef-2021/train_short_audio/astfly/XC118723.ogg'\nx , sr = librosa.load(audio_file)\nimport IPython.display as play\nplay.Audio(audio_file)","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:27:21.61249Z","iopub.execute_input":"2021-06-09T04:27:21.612854Z","iopub.status.idle":"2021-06-09T04:27:23.256342Z","shell.execute_reply.started":"2021-06-09T04:27:21.612823Z","shell.execute_reply":"2021-06-09T04:27:23.255403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"integers = np.argmax(train_labels, axis=1)\nunique_elements, counts_elements = np.unique(integers, return_counts=True)\nprint(\"Frequency of unique values of the said array:\")\nprint(np.asarray((unique_elements, counts_elements)))","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:22:26.387215Z","iopub.execute_input":"2021-06-09T04:22:26.387575Z","iopub.status.idle":"2021-06-09T04:22:26.394774Z","shell.execute_reply.started":"2021-06-09T04:22:26.387539Z","shell.execute_reply":"2021-06-09T04:22:26.393835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = train_specs[0:7500]\ny_train = train_labels[0:7500]\nX_test = train_specs[7500:]\ny_test = train_labels[7500:]","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:22:26.398063Z","iopub.execute_input":"2021-06-09T04:22:26.398461Z","iopub.status.idle":"2021-06-09T04:22:26.406315Z","shell.execute_reply.started":"2021-06-09T04:22:26.398406Z","shell.execute_reply":"2021-06-09T04:22:26.405384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_specs.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:22:26.408336Z","iopub.execute_input":"2021-06-09T04:22:26.408814Z","iopub.status.idle":"2021-06-09T04:22:26.416209Z","shell.execute_reply.started":"2021-06-09T04:22:26.408732Z","shell.execute_reply":"2021-06-09T04:22:26.41533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## AlexNet","metadata":{}},{"cell_type":"code","source":"#Importing library\nimport keras\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Activation, Dropout, Flatten, Conv2D, MaxPooling2D\nfrom keras.layers.normalization import BatchNormalization\nimport numpy as np\nfrom keras.utils.vis_utils import plot_model\nnp.random.seed(1000)\n\n#Instantiation\nAlexNet = Sequential()\n\n#1st Convolutional Layer\nAlexNet.add(Conv2D(filters=96, input_shape=(SPEC_SHAPE[0], SPEC_SHAPE[1], 1), kernel_size=(11,11), strides=(4,4), padding='same'))\nAlexNet.add(BatchNormalization())\nAlexNet.add(Activation('relu'))\nAlexNet.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='same'))\n\n#2nd Convolutional Layer\nAlexNet.add(Conv2D(filters=256, kernel_size=(5, 5), strides=(1,1), padding='same'))\nAlexNet.add(BatchNormalization())\nAlexNet.add(Activation('relu'))\nAlexNet.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='same'))\n\n#3rd Convolutional Layer\nAlexNet.add(Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), padding='same'))\nAlexNet.add(BatchNormalization())\nAlexNet.add(Activation('relu'))\n\n#4th Convolutional Layer\nAlexNet.add(Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), padding='same'))\nAlexNet.add(BatchNormalization())\nAlexNet.add(Activation('relu'))\n\n#5th Convolutional Layer\nAlexNet.add(Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), padding='same'))\nAlexNet.add(BatchNormalization())\nAlexNet.add(Activation('relu'))\nAlexNet.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='same'))\n\n#Passing it to a Fully Connected layer\nAlexNet.add(Flatten())\n# 1st Fully Connected Layer\nAlexNet.add(Dense(4096, input_shape=(32,32,3,)))\nAlexNet.add(BatchNormalization())\nAlexNet.add(Activation('relu'))\n# Add Dropout to prevent overfitting\nAlexNet.add(Dropout(0.4))\n\n#3rd Fully Connected Layer\nAlexNet.add(Dense(1000))\nAlexNet.add(BatchNormalization())\nAlexNet.add(Activation('relu'))\n#Add Dropout\nAlexNet.add(Dropout(0.4))\n\n#Output Layer\nAlexNet.add(Dense(len(LABELS)))\nAlexNet.add(BatchNormalization())\nAlexNet.add(Activation('softmax'))\n\n#Model Summary\nAlexNet.summary()","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:22:26.417463Z","iopub.execute_input":"2021-06-09T04:22:26.41786Z","iopub.status.idle":"2021-06-09T04:22:28.652786Z","shell.execute_reply.started":"2021-06-09T04:22:26.417827Z","shell.execute_reply":"2021-06-09T04:22:28.651882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AlexNet.compile(optimizer=tf.keras.optimizers.Adam(lr=0.001),\n              loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.01),\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:22:28.654243Z","iopub.execute_input":"2021-06-09T04:22:28.65465Z","iopub.status.idle":"2021-06-09T04:22:28.673548Z","shell.execute_reply.started":"2021-06-09T04:22:28.65461Z","shell.execute_reply":"2021-06-09T04:22:28.672363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"callbacks = [tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', \n                                                  patience=2, \n                                                  verbose=1, \n                                                  factor=0.25),\n             tf.keras.callbacks.EarlyStopping(monitor='val_loss', \n                                              verbose=1,\n                                              patience=3),\n             tf.keras.callbacks.ModelCheckpoint(filepath='best_model.h5', \n                                                monitor='val_loss',\n                                                verbose=0,\n                                                save_best_only=True)]","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:22:28.675631Z","iopub.execute_input":"2021-06-09T04:22:28.676016Z","iopub.status.idle":"2021-06-09T04:22:28.68291Z","shell.execute_reply.started":"2021-06-09T04:22:28.675976Z","shell.execute_reply":"2021-06-09T04:22:28.681619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = AlexNet.fit(X_train,y_train,validation_data=(X_test, y_test),callbacks=callbacks,epochs=100)","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:22:28.684351Z","iopub.execute_input":"2021-06-09T04:22:28.684986Z","iopub.status.idle":"2021-06-09T04:23:44.645747Z","shell.execute_reply.started":"2021-06-09T04:22:28.684944Z","shell.execute_reply":"2021-06-09T04:23:44.644425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nprint(history.history.keys())\n# summarize history for accuracy\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.savefig('AccuAlexNet.png')\nplt.show()\n# summarize history for loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.savefig('LossAlexNet.png')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:23:44.647403Z","iopub.execute_input":"2021-06-09T04:23:44.6478Z","iopub.status.idle":"2021-06-09T04:23:44.974379Z","shell.execute_reply.started":"2021-06-09T04:23:44.64775Z","shell.execute_reply":"2021-06-09T04:23:44.973389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred=AlexNet.predict_classes(X_test)\ny_true=np.argmax(y_test,axis=1)\n\nfrom sklearn.metrics import precision_recall_fscore_support, f1_score\n\n\nprint(precision_recall_fscore_support(y_true, y_pred,average='weighted'))\nf1_score(y_true, y_pred, average='weighted')\n","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:23:44.975877Z","iopub.execute_input":"2021-06-09T04:23:44.976223Z","iopub.status.idle":"2021-06-09T04:23:45.258048Z","shell.execute_reply.started":"2021-06-09T04:23:44.976186Z","shell.execute_reply":"2021-06-09T04:23:45.257307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Custom CNN","metadata":{}},{"cell_type":"code","source":"# Make sure your experiments are reproducible\ntf.random.set_seed(RANDOM_SEED)\n\n# Build a simple model as a sequence of  convolutional blocks.\n# Each block has the sequence CONV --> RELU --> BNORM --> MAXPOOL.\n# Finally, perform global average pooling and add 2 dense layers.\n# The last layer is our classification layer and is softmax activated.\n# (Well it's a multi-label task so sigmoid might actually be a better choice)\nmodel = tf.keras.Sequential([\n    \n    # First conv block\n    tf.keras.layers.Conv2D(16, (3, 3), activation='relu',input_shape=(SPEC_SHAPE[0], SPEC_SHAPE[1], 1)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D((2, 2)),\n    # Second conv block\n    tf.keras.layers.Conv2D(32, (3, 3), activation='relu'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D((2, 2)), \n    # Third conv block\n    tf.keras.layers.Conv2D(64, (3, 3), activation='relu'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D((2, 2)),  \n    # Fourth conv block\n    tf.keras.layers.Conv2D(256, (3, 3), activation='relu'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D((2, 2)),  \n    # Global pooling instead of flatten()\n    tf.keras.layers.GlobalAveragePooling2D(), \n    # Dense block\n    tf.keras.layers.Dense(512, activation='relu'),   \n    tf.keras.layers.Dropout(0.5),  \n    tf.keras.layers.Dense(512, activation='relu'),   \n    tf.keras.layers.Dropout(0.5),\n    # Classification layer\n    tf.keras.layers.Dense(len(LABELS), activation='softmax')\n])\nplot_model(model, to_file='Custom.png', show_shapes=True, show_layer_names=True)\nprint('MODEL HAS {} PARAMETERS.'.format(model.count_params()))","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:23:45.259325Z","iopub.execute_input":"2021-06-09T04:23:45.259676Z","iopub.status.idle":"2021-06-09T04:23:45.884019Z","shell.execute_reply.started":"2021-06-09T04:23:45.25964Z","shell.execute_reply":"2021-06-09T04:23:45.882986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:23:45.885877Z","iopub.execute_input":"2021-06-09T04:23:45.886238Z","iopub.status.idle":"2021-06-09T04:23:45.900915Z","shell.execute_reply.started":"2021-06-09T04:23:45.886199Z","shell.execute_reply":"2021-06-09T04:23:45.899506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer=tf.keras.optimizers.Adam(lr=0.001),\n              loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.01),\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:23:45.902561Z","iopub.execute_input":"2021-06-09T04:23:45.903025Z","iopub.status.idle":"2021-06-09T04:23:45.918279Z","shell.execute_reply.started":"2021-06-09T04:23:45.902986Z","shell.execute_reply":"2021-06-09T04:23:45.917358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"callbacks = [tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', \n                                                  patience=2, \n                                                  verbose=1, \n                                                  factor=0.25),\n             tf.keras.callbacks.EarlyStopping(monitor='val_loss', \n                                              verbose=1,\n                                              patience=3),\n             tf.keras.callbacks.ModelCheckpoint(filepath='best_model.h5', \n                                                monitor='val_loss',\n                                                verbose=0,\n                                                save_best_only=True)]","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:23:45.920605Z","iopub.execute_input":"2021-06-09T04:23:45.920959Z","iopub.status.idle":"2021-06-09T04:23:45.926536Z","shell.execute_reply.started":"2021-06-09T04:23:45.920923Z","shell.execute_reply":"2021-06-09T04:23:45.925421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(X_train,y_train,validation_data=(X_test, y_test),callbacks=callbacks,epochs=100)","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:23:45.928085Z","iopub.execute_input":"2021-06-09T04:23:45.928575Z","iopub.status.idle":"2021-06-09T04:24:21.849964Z","shell.execute_reply.started":"2021-06-09T04:23:45.928464Z","shell.execute_reply":"2021-06-09T04:24:21.849166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nprint(history.history.keys())\n# summarize history for accuracy\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.savefig('AccuCustom.png')\nplt.show()\n# summarize history for loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.savefig('LossCustom.png')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:24:21.851379Z","iopub.execute_input":"2021-06-09T04:24:21.851735Z","iopub.status.idle":"2021-06-09T04:24:22.210205Z","shell.execute_reply.started":"2021-06-09T04:24:21.851699Z","shell.execute_reply":"2021-06-09T04:24:22.209418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred=model.predict_classes(X_test)\ny_true=np.argmax(y_test,axis=1)\n\nfrom sklearn.metrics import precision_recall_fscore_support, f1_score\n\n\nprint(precision_recall_fscore_support(y_true, y_pred,average='weighted'))\nf1_score(y_true, y_pred, average='weighted')\n","metadata":{"execution":{"iopub.status.busy":"2021-06-09T04:24:22.211489Z","iopub.execute_input":"2021-06-09T04:24:22.211855Z","iopub.status.idle":"2021-06-09T04:24:22.461125Z","shell.execute_reply.started":"2021-06-09T04:24:22.211818Z","shell.execute_reply":"2021-06-09T04:24:22.460371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}