{"cells":[{"metadata":{},"cell_type":"markdown","source":"In this Kaggle kernel, we will perform an ablation study to find the connection between model size and validation accuracy with the dataset that I have created in this [Kaggle kernel](https://www.kaggle.com/ayuraj/rainforest-create-image-dataset-with-w-b). \n\nWe will set up a [Weights and Biases Sweep](https://docs.wandb.com/sweeps) to find the best model for our dataset. We will also see the effect of batch size and learning rate along with the model size. "},{"metadata":{},"cell_type":"markdown","source":"# Imports and Setups"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"SEED = 666\n\nimport tensorflow as tf\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.models import *\ntf.random.set_seed(SEED)\n\nfrom tensorflow import keras\nfrom tensorflow.keras.datasets import cifar10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport csv\nos.environ[\"TF_DETERMINISTIC_OPS\"] = \"1\"\n\nimport numpy as np\nnp.random.seed(SEED)\n\nimport pandas as pd\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\n\n%matplotlib inline\n\nfrom mpl_toolkits.axes_grid1 import ImageGrid\nfrom PIL import Image\n\nfrom sklearn.metrics import classification_report\nfrom sklearn.model_selection import train_test_split\n\nimport librosa as lb \nimport librosa.display\nimport matplotlib.pyplot as plt\nimport IPython.display as ipd\n\nfrom skimage.transform import resize\nfrom scipy import stats","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import wandb\nfrom wandb.keras import WandbCallback\n\nwandb.login()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Download Dataset from W&B Artifacts and Prepare"},{"metadata":{"trusted":true},"cell_type":"code","source":"run = wandb.init(project='rainforest', job_type='download_dataset')\n\nartifact = run.use_artifact('wandb/rainforest/spectrogram-dataset_nfft_2024_hop_512:v0', type='dataset')\nartifact_dir = artifact.download()\n\nrun.join()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_DIR = Path(artifact_dir+'/')\nIMG_PATH = list(map(str, list(IMG_DIR.glob('*.bmp'))))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train-test Split"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path, valid_path = train_test_split(IMG_PATH, test_size=0.20, shuffle=True, random_state=42)\nlen(IMG_PATH), len(train_path), len(valid_path)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataloader"},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nBATCH_SIZE = 32\nIMG_WIDTH = 400\nIMG_HEIGHT = 224\nCHANNELS = 3\nNUM_CLASSES = 24","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef parse_data(image_path):\n    # parse image\n    image = tf.io.read_file(image_path)\n    image = tf.image.decode_image(image)\n    image = tf.image.convert_image_dtype(image, tf.float32)\n    # normalize image\n    image = tf.image.per_image_standardization(image)\n    \n    # parse data\n    label = tf.strings.split(image_path, sep='_')[-2]\n    label = tf.strings.to_number(label, out_type=tf.int32)\n    label = tf.one_hot(label, NUM_CLASSES) \n    \n    return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model(base_model_name):\n    if base_model_name=='resnet':\n        base_model = tf.keras.applications.ResNet50V2(include_top=False, weights=\"imagenet\", input_shape=(224, 224, 3))\n        \n    elif base_model_name=='efficientnetb0':\n        base_model = tf.keras.applications.EfficientNetB0(include_top=False, weights=\"imagenet\", input_shape=(224, 224, 3))\n    elif base_model_name=='efficientnetb1':\n        base_model = tf.keras.applications.EfficientNetB1(include_top=False, weights=\"imagenet\", input_shape=(224, 224, 3))\n    elif base_model_name=='efficientnetb2':\n        base_model = tf.keras.applications.EfficientNetB2(include_top=False, weights=\"imagenet\", input_shape=(224, 224, 3))\n    \n    elif base_model_name=='xception':\n        base_model = tf.keras.applications.Xception(include_top=False, weights=\"imagenet\", input_shape=(224, 224, 3))\n    \n    elif base_model_name=='densenet':\n        base_model = tf.keras.applications.DenseNet169(include_top=False, weights=\"imagenet\", input_shape=(224, 224, 3))\n\n    base_model.trainabe = True  \n    \n    inputs = Input((IMG_HEIGHT, IMG_WIDTH, 3))\n    resize = experimental.preprocessing.Resizing(224,224)(inputs) \n    x = base_model(resize, training=True)\n    x = GlobalAveragePooling2D()(x)\n    x = Dropout(0.5)(x)  \n    outputs = Dense(NUM_CLASSES, activation='sigmoid')(x)\n    \n    return Model(inputs, outputs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Setup Sweep "},{"metadata":{"trusted":true},"cell_type":"code","source":"def train():\n    # Specify the hyperparameter to be tuned along with\n    # an initial value\n    config_defaults = {\n        'base_model_name': 'resnet',\n        'dataset_name': 'nfft_2024_hop_512'\n        }\n\n    # Initialize wandb with a sample project name\n    run = wandb.init(config=config_defaults)\n\n    # Specify the other hyperparameters to the configuration, if any\n    wandb.config.epochs = 30\n\n    # download and prepare data\n    data_artifact = run.use_artifact('wandb/rainforest/spectrogram-dataset_{}:latest'.format(wandb.config.dataset_name))\n    artifact_dir = data_artifact.download()\n    \n    IMG_DIR = Path(artifact_dir+'/')\n    IMG_PATH = list(map(str, list(IMG_DIR.glob('*.bmp'))))\n    \n    # train-validation split\n    train_path, valid_path = train_test_split(IMG_PATH, test_size=0.20, shuffle=True, random_state=42)\n    \n    # dataloader\n    trainloader = tf.data.Dataset.list_files((train_path))\n    validloader = tf.data.Dataset.list_files((valid_path))\n\n    trainloader = (\n        trainloader\n        .shuffle(1024)\n        .map(parse_data, num_parallel_calls=AUTO)\n        .batch(BATCH_SIZE)\n        .prefetch(AUTO)\n    )\n\n    validloader = (\n        validloader\n        .map(parse_data, num_parallel_calls=AUTO)\n        .batch(BATCH_SIZE)\n        .prefetch(AUTO)\n    )\n\n\n    # Iniialize model with hyperparameters\n    keras.backend.clear_session()\n    model = get_model(wandb.config.base_model_name)\n    \n    # Compile the model\n    opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n    model.compile(opt, 'binary_crossentropy', metrics=['acc'])\n    \n    # Train the model\n    _ = model.fit(trainloader,\n                  epochs=wandb.config.epochs, \n                  validation_data=validloader,\n                  callbacks=[WandbCallback()]) # WandbCallback to automatically track metrics\n                            \n    # Evaluate    \n    loss, accuracy = model.evaluate(validloader, callbacks=[WandbCallback()])\n    print('Test Error Rate: ', round((1-accuracy)*100, 2))\n    wandb.log({'Test Error Rate': round((1-accuracy)*100, 2)}) # wandb.log to track custom metrics\n    \n    # save model\n    model.save('model.h5')\n\n    # initialize a new artifact to save the model\n    model_artifact =  wandb.Artifact(\"trained-model\", \n                                     type=\"model\", \n                                     description=\"Simple model trained with spectrogram dataset formed with nfft 2024 and hop length of 512\",\n                                     metadata={'optimizer': 'Adam',\n                                              'Loss': 'Binary Cross Entropy',\n                                              'Learning Rate': 0.001})\n\n    model_artifact.add_file('model.h5')\n    run.log_artifact(model_artifact)\n    \n    run.join()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Sweep Config"},{"metadata":{"trusted":true},"cell_type":"code","source":"sweep_config = {\n  'method': 'bayes', \n  'metric': {\n      'name': 'val_loss',\n      'goal': 'minimize'\n  },\n  'early_terminate':{\n      'type': 'hyperband',\n      'max_iter': 27,\n      's': 2,\n      'eta': 3\n  },\n  'parameters': {\n      'base_model_name': {\n          'values': ['resnet',\n                     'efficientnetb0',\n                     'efficientnetb1',\n                     'efficientnetb2',\n                     'xception',\n                     'densenet']\n      },\n      'dataset_name': {\n          'values': ['nfft_2024_hop_512',\n                     'nfft_1024_hop_512',\n                     'nfft_1024_hop_256']\n      }\n  }\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Initialize Sweep and Run Agent"},{"metadata":{"trusted":true},"cell_type":"code","source":"sweep_id = wandb.sweep(sweep_config, project=\"rainforest\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wandb.agent(sweep_id, function=train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}