{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Objective\n\nIn this notebook i am going to use 8TPU cores for 8Folds training\n\nReference : https://www.kaggle.com/abhishek/super-duper-fast-pytorch-tpu-kernel","execution_count":null},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev\n!pip install pretrainedmodels\n!pip install pydub","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport librosa\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport random\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import KFold, StratifiedKFold\nimport math\nfrom collections import OrderedDict\n\nfrom PIL import Image\nimport albumentations\nfrom pydub import AudioSegment\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\n\nimport pretrainedmodels\nimport torchvision.models as models\n\n\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.distributed.xla_multiprocessing as xmp\n\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\n\nfrom joblib import Parallel, delayed\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom tqdm.autonotebook import tqdm\nimport time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocessing <a id=\"3\"></a>","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train= pd.read_csv(\"../input/birdsong-recognition/train.csv\")\ntest = pd.read_csv(\"../input/birdsong-recognition/test.csv\")\nsubmission = pd.read_csv(\"../input/birdsong-recognition/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### e-bird code\n\na code for the bird species. we need to predict `ebird_code` using metadata and audio data \n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape, '\\n columns', train.columns)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print(\"Number of Unique birds : \", train.ebird_code.nunique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_quality = train[['species','ebird_code', 'rating', 'filename']]\ngb_quality = df_quality.groupby(['species', 'ebird_code']).agg({'rating':'mean', 'filename':'count'}).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# normalizations\ngb_quality['normalized_quality'] = gb_quality['rating']/max(gb_quality['rating'])\ngb_quality['normalized_counts'] = gb_quality['filename']/max(gb_quality['filename'])\n# quality index (total_rating X amount_of_samples)\ngb_quality['quality_index'] = gb_quality['normalized_quality'] * gb_quality['normalized_counts']\ngb_quality = gb_quality.sort_values(by=['quality_index'],ascending=False).reset_index()\ngb_quality.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(23,8))\n\n\nplt.plot(gb_quality['quality_index'], label = 'quality')\nplt.title('data quality vs. species', color=\"red\", fontsize = 14)\nplt.legend(fontsize = 16)\nplt.ylabel('normalized amount/quality of samples', color=\"red\", fontsize = 14)\nplt.xlabel('species', color=\"red\", fontsize = 14)\nplt.xticks(rotation='vertical')\nplt.rcParams['figure.constrained_layout.use'] = True\nplt.savefig(\"null.png\", format=\"PNG\", dpi = 100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"durations = train['length'].value_counts().reset_index()\ndurations","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"duration is not very informative, since 2/3 are not specified","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### top10 Birds\nwe are taking top10 birds to build stater model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"top10_birds = gb_quality['ebird_code'][:10].tolist() #list(train.ebird_code.value_counts().index[:10])\ntop10_birds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#top10_birds = list(train.ebird_code.value_counts().index[:10])\n\ntrain = train[train.ebird_code.isin(top10_birds)]\n\n# label encoding for target values\ntrain[\"ebird_label\"] = LabelEncoder().fit_transform(train.ebird_code.values)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### K-Fold","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"FOLDS = 8\n\ntrain.loc[:, \"kfold\"] = -1\n\ntrain= train.sample(frac=1).reset_index(drop=True)\n\nX = train.filename.values\ny = train.ebird_code.values\n\nkfold = StratifiedKFold(n_splits=8)\n\nfor fold, (t_idx, v_idx) in enumerate(kfold.split(X, y)):\n    train.loc[v_idx, \"kfold\"] = fold\n\nprint(train.kfold.value_counts())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Arguments","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class args:\n    \n    ROOT_PATH = \"../input/birdsong-recognition/train_audio\"\n    \n    num_classes = 10\n    max_duration= 5 # seconds\n    \n    sample_rate = 32000\n    \n    img_height = 128\n    img_width = 313\n    \n    batch_size = 16\n    num_workers = 4\n    epochs = 2\n    \n    lr = 0.0009\n    wd = 1e-5\n    momentum = 0.9\n    eps = 1e-8\n    betas = (0.9, 0.999)\n    \n    melspectrogram_parameters = {\n        \"n_mels\": 128,\n        \"fmin\": 20,\n        \"fmax\": 16000\n    }\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Loading Audio Files","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_audio(path):\n    try:\n        sound = AudioSegment.from_mp3(path)\n        sound = sound.set_frame_rate(args.sample_rate)\n        sound_array = np.array(sound.get_array_of_samples(), dtype=np.float32)\n    except:\n        sound_array = np.zeros(args.sample_rate * args.max_duration, dtype=np.float32)\n        \n    return sound_array, args.sample_rate","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Audo Albumentations\n\n- check my other notebook [Audio Albumentations](https://www.kaggle.com/gopidurgaprasad/audio-albumentations)","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from albumentations.core.transforms_interface import DualTransform, BasicTransform\n\nclass AudioTransform(BasicTransform):\n    \"\"\"Transform for Audio task\"\"\"\n\n    @property\n    def targets(self):\n        return {\"data\": self.apply}\n    \n    def update_params(self, params, **kwargs):\n        if hasattr(self, \"interpolation\"):\n            params[\"interpolation\"] = self.interpolation\n        if hasattr(self, \"fill_value\"):\n            params[\"fill_value\"] = self.fill_value\n        return params\n\nclass NoiseInjection(AudioTransform):\n    \"\"\"It simply add some random value into data by using numpy\"\"\"\n    def __init__(self, noise_levels=(0, 0.5), always_apply=False, p=0.5):\n        super(NoiseInjection, self).__init__(always_apply, p)\n\n        self.noise_levels = noise_levels\n    \n    def apply(self, data, **params):\n        sound, sr = data\n        noise_level = np.random.uniform(*self.noise_levels)\n        noise = np.random.randn(len(sound))\n        augmented_sound = sound + noise_level * noise\n        # Cast back to same data type\n        augmented_sound = augmented_sound.astype(type(sound[0]))\n\n        return augmented_sound, sr\n\nclass ShiftingTime(AudioTransform):\n    \"\"\"Shifting time axis\"\"\"\n    def __init__(self, always_apply=False, p=0.5):\n        super(ShiftingTime, self).__init__(always_apply, p)\n    \n    def apply(self, data, **params):\n        sound, sr = data\n\n        shift_max = np.random.randint(1,len(sound))\n        shift = np.random.randint(int(sr * shift_max))\n        direction = np.random.randint(0,2)\n        if direction == 1:\n            shift = -shift\n\n        augmented_sound = np.roll(sound, shift)\n        # Set to silence for heading/ tailing\n        if shift > 0:\n            augmented_sound[:shift] = 0\n        else:\n            augmented_sound[shift:] = 0\n\n        return augmented_sound, sr\n\nclass PitchShift(AudioTransform):\n    \n    def __init__(self, always_apply=False, p=0.5):\n        super(PitchShift, self).__init__(always_apply, p)\n    \n    def apply(self, data, **params):\n        sound, sr = data\n\n        n_steps = np.random.randint(-10, 10)\n        augmented_sound = librosa.effects.pitch_shift(sound, sr, n_steps)\n\n        return augmented_sound, sr\n\nclass TimeStretch(AudioTransform):\n    \n    def __init__(self, always_apply=False, p=0.5):\n        super(TimeStretch, self).__init__(always_apply, p)\n    \n    def apply(self, data, **params):\n        sound, sr = data\n\n        rate = np.random.uniform(0, 2)\n        augmented_sound = librosa.effects.time_stretch(sound, rate)\n\n        return augmented_sound, sr\n\nclass RandomAudio(AudioTransform):\n    \n    def __init__(self,  seconds=5, always_apply=False, p=0.5):\n        super(RandomAudio, self).__init__(always_apply, p)\n\n        self.seconds = seconds\n    \n    def apply(self, data, **params):\n        sound, sr = data\n\n        shift = np.random.randint(len(sound))\n        trim_sound = np.roll(sound, shift)\n\n        min_samples = int(sr * self.seconds)\n\n        if len(trim_sound) < min_samples:\n            padding = min_samples - len(trim_sound)\n            offset = padding // 2\n            trim_sound = np.pad(trim_sound, (offset, padding - offset), \"constant\")\n        else:\n            trim_sound = trim_sound[:min_samples]\n\n        return trim_sound, sr\n\nclass MelSpectrogram(AudioTransform):\n\n    def __init__(self, parameters, always_apply=False, p=0.5):\n        super(MelSpectrogram, self).__init__(always_apply, p)\n\n        self.parameters = parameters\n    \n    def apply(self, data, **params):\n        sound, sr = data\n\n        melspec = librosa.feature.melspectrogram(sound, sr=sr, **self.parameters)\n        melspec = librosa.power_to_db(melspec)\n        melspec = melspec.astype(np.float32)\n\n        return melspec, sr\n\nclass SpecAugment(AudioTransform):\n    \n    def __init__(self, num_mask=2, freq_masking=0.15, time_masking=0.20, always_apply=False, p=0.5):\n        super(SpecAugment, self).__init__(always_apply, p)\n\n        self.num_mask = num_mask\n        self.freq_masking = freq_masking\n        self.time_masking = time_masking\n    \n    def apply(self, data, **params):\n        melspec, sr = data\n\n        spec_aug = self.spec_augment(melspec, \n                                     self.num_mask,\n                                     self.freq_masking,\n                                     self.time_masking,\n                                     melspec.min())\n        \n\n\n        return spec_aug, sr\n    \n    # Source: https://www.kaggle.com/davids1992/specaugment-quick-implementation\n    def spec_augment(self, \n                    spec: np.ndarray,\n                    num_mask=2,\n                    freq_masking=0.15,\n                    time_masking=0.20,\n                    value=0):\n        spec = spec.copy()\n        num_mask = random.randint(1, num_mask)\n        for i in range(num_mask):\n            all_freqs_num, all_frames_num  = spec.shape\n            freq_percentage = random.uniform(0.0, freq_masking)\n\n            num_freqs_to_mask = int(freq_percentage * all_freqs_num)\n            f0 = np.random.uniform(low=0.0, high=all_freqs_num - num_freqs_to_mask)\n            f0 = int(f0)\n            spec[f0:f0 + num_freqs_to_mask, :] = value\n\n            time_percentage = random.uniform(0.0, time_masking)\n\n            num_frames_to_mask = int(time_percentage * all_frames_num)\n            t0 = np.random.uniform(low=0.0, high=all_frames_num - num_frames_to_mask)\n            t0 = int(t0)\n            spec[:, t0:t0 + num_frames_to_mask] = value\n\n        return spec\n\nclass SpectToImage(AudioTransform):\n\n    def __init__(self, always_apply=False, p=0.5):\n        super(SpectToImage, self).__init__(always_apply, p)\n    \n    def apply(self, data, **params):\n        image, sr = data\n        delta = librosa.feature.delta(image)\n        accelerate = librosa.feature.delta(image, order=2)\n        image = np.stack([image, delta, accelerate], axis=0)\n        image = image.astype(np.float32) / 100.0\n\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Example\n\ntrain_audio_augmentation = albumentations.Compose([\n     RandomAudio(seconds=args.max_duration, always_apply=True),\n     NoiseInjection(p=0.33),\n     MelSpectrogram(parameters=args.melspectrogram_parameters,always_apply=True),\n     SpecAugment(p=0.33),\n     SpectToImage(always_apply=True)\n])\n\nvalid_audio_augmentation = albumentations.Compose([\n     RandomAudio(seconds=args.max_duration, always_apply=True),\n     MelSpectrogram(parameters=args.melspectrogram_parameters,always_apply=True),\n     SpectToImage(always_apply=True)\n])\n\npath = f\"{args.ROOT_PATH}/aldfly/XC134874.mp3\"\ndata = load_audio(path)\nimage = train_audio_augmentation(data=data)['data']\n\nplt.imshow(image.transpose(1,2,0))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Pytorch DataLoader","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class BirdDataset:\n    def __init__(self, df, valid=False):\n        \n        self.filename = df.filename.values\n        self.ebird_label = df.ebird_label.values\n        self.ebird_code = df.ebird_code.values\n        \n        if valid:\n            self.aug = valid_audio_augmentation\n        else:\n            self.aug = train_audio_augmentation\n        \n    \n    def __len__(self):\n        return len(self.filename)\n    \n    def __getitem__(self, item):\n        \n        filename = self.filename[item]\n        ebird_code = self.ebird_code[item]\n        ebird_label = self.ebird_label[item]\n\n        data = load_audio(f\"{args.ROOT_PATH}/{ebird_code}/{filename}\")\n        spect = self.aug(data=data)[\"data\"]\n        \n        target = ebird_label\n        \n        return {\n            \"spect\" : torch.tensor(spect, dtype=torch.float), \n            \"target\" : torch.tensor(target, dtype=torch.long)\n        }\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Example \ndataset = BirdDataset(train)\nd = dataset.__getitem__(10)\n\nprint(d[\"spect\"].shape, d[\"target\"])\n\nplt.imshow(d[\"spect\"].permute(1,2,0))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"###  Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.applications import DenseNet201\n\ndef get_model():\n    with strategy.scope():\n        rnet = DenseNet201(\n            input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),\n            weights='imagenet',\n            include_top=False\n        )\n        # trainable rnet\n        rnet.trainable = True\n        model = tf.keras.Sequential([\n            rnet,\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')\n        ])\n    model.compile(\n        optimizer='adam',\n        loss = 'sparse_categorical_crossentropy',\n        metrics=['sparse_categorical_accuracy']\n    )\n    return model\n\ndef train_cross_validate(folds = 8):\n    histories = []\n    models = []\n    early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 3)\n    kfold = KFold(folds, shuffle = True, random_state = SEED)\n    for f, (trn_ind, val_ind) in enumerate(kfold.split(TRAINING_FILENAMES)):\n        print(); print('#'*25)\n        print('### FOLD',f+1)\n        print('#'*25)\n        train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[trn_ind]['TRAINING_FILENAMES']), labeled = True)\n        val_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[val_ind]['TRAINING_FILENAMES']), labeled = True, ordered = True)\n        model = get_model()\n        history = model.fit(\n            get_training_dataset(train_dataset), \n            steps_per_epoch = STEPS_PER_EPOCH,\n            epochs = EPOCHS,\n            callbacks = [lr_callback],#, early_stopping],\n            validation_data = get_validation_dataset(val_dataset),\n            verbose=2\n        )\n        models.append(model)\n        histories.append(history)\n    return histories, models\n\ndef train_and_predict(folds = 8):\n    test_ds = get_test_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and ids, order matters.\n    test_images_ds = test_ds.map(lambda image, idnum: image)\n    print('Start training %i folds'%folds)\n    histories, models = train_cross_validate(folds = folds)\n    print('Computing predictions...')\n    # get the mean probability of the folds models\n    probabilities = np.average([models[i].predict(test_images_ds) for i in range(folds)], axis = 0)\n    predictions = np.argmax(probabilities, axis=-1)\n    print('Generating submission.csv file...')\n    test_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\n    test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\n    np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')\n    return histories, models\n    \n# run train and predict\nhistories, models = train_and_predict(folds = FOLDS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Utility functions","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### 8 Folds","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}