{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# AGENDA\n#### TO DO LIST\n1. Plotting confusion matrix, plot prediction for top losses (need typedispatch on AudioTensor)\n\n#### Completed\n1. [11/05/2021] Set up stratified KFold\n2. [11/05/2021] Crop training set data\n3. [14/05/2021] Try multiclass v.s. multilabel\n4. [17/05/2021] Prototyped W&B integration\n5. [17/05/2021] Fixed `Normalize` is missing in both train and valid\n6. [23/05/2021] Fixed sample rate != 48000 error\n7. [23/05/2021] Fixed imagenet normalization on train+val \n8. [03/06/2021] Prototype of Pseudo Labeling\n\n#### Reference\n1. code starts from: https://www.kaggle.com/scart97/fastaudio-starter-kit/notebook","metadata":{}},{"cell_type":"markdown","source":"### 0. Install Packages","metadata":{}},{"cell_type":"code","source":"!pip install --up../input/rfcx-species-audio-detection/he-torch\n# specify version to resolve version compability\n!ltt install torch==1.7.1 torchvision==0.8.2 torchaudio==0.7.2\n#!pip install --upgrade git+https://github.com/fastaudio/fastaudio.git\n!pip install fastaudio==1.0.0\n!pip install wandb --upgrade","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-13T17:17:26.729884Z","iopub.execute_input":"2021-06-13T17:17:26.730165Z","iopub.status.idle":"2021-06-13T17:17:49.007464Z","shell.execute_reply.started":"2021-06-13T17:17:26.730139Z","shell.execute_reply":"2021-06-13T17:17:49.006436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pkg_resources\n\ndef placeholder(x):\n    raise pkg_resources.DistributionNotFound\npkg_resources.get_distribution = placeholder","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:17:49.011078Z","iopub.execute_input":"2021-06-13T17:17:49.011382Z","iopub.status.idle":"2021-06-13T17:17:49.019206Z","shell.execute_reply.started":"2021-06-13T17:17:49.01135Z","shell.execute_reply":"2021-06-13T17:17:49.018344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from enum import Enum\nfrom math import ceil\n\nimport wandb\nfrom kaggle_secrets import UserSecretsClient\n\nimport pandas as pd\nfrom fastaudio.all import *\nfrom fastai.vision.all import *\nfrom fastai.callback.wandb import *\n\nimport torch.nn as nn\n# migrate to sox_io\nimport torchaudio\ntorchaudio.set_audio_backend(\"sox_io\")\n\nfrom sklearn.model_selection import StratifiedKFold\n\n# check cuda is available\nimport torch\nprint(torch.cuda.is_available())","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:17:49.023337Z","iopub.execute_input":"2021-06-13T17:17:49.023679Z","iopub.status.idle":"2021-06-13T17:17:54.55252Z","shell.execute_reply.started":"2021-06-13T17:17:49.023653Z","shell.execute_reply":"2021-06-13T17:17:54.551659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport fastai\nimport fastaudio\nimport torchaudio\nprint(f'torch version: {torch.__version__}')\nprint(f'torchaudio version: {torchaudio.__version__}')\nprint(f'fastaudio version: {fastaudio.__version__}')\nprint(f'fastai version: {fastai.__version__}')","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:17:54.554186Z","iopub.execute_input":"2021-06-13T17:17:54.554548Z","iopub.status.idle":"2021-06-13T17:17:54.563153Z","shell.execute_reply.started":"2021-06-13T17:17:54.55451Z","shell.execute_reply":"2021-06-13T17:17:54.562254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fix failure to preserve metadata in multiprocessing in DataLoader\ndef _rebuild_from_type(func, type, args, dict):\n    ret = func(*args).as_subclass(type)\n    ret.__dict__ = dict\n    return ret\n\n@patch\ndef __reduce_ex__(self: TensorBase, proto):\n    from fastai.torch_core import _fa_rebuild_qtensor, _fa_rebuild_tensor\n    torch.utils.hooks.warn_if_has_hooks(self)\n    args = (type(self), self.storage(), self.storage_offset(), tuple(self.size()), self.stride())\n    if self.is_quantized: args = args + (self.q_scale(), self.q_zero_point())\n    args = args + (self.requires_grad, OrderedDict())\n    f = _fa_rebuild_qtensor if self.is_quantized else  _fa_rebuild_tensor\n    return (_rebuild_from_type, (f, type(self), args, self.__dict__))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:17:54.56463Z","iopub.execute_input":"2021-06-13T17:17:54.56516Z","iopub.status.idle":"2021-06-13T17:17:54.775148Z","shell.execute_reply.started":"2021-06-13T17:17:54.565122Z","shell.execute_reply":"2021-06-13T17:17:54.774118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1a. Configuration","metadata":{}},{"cell_type":"code","source":"class LossFunction(str, Enum):\n    FOCAL_LOSS = 'focal_loss'\n    BCE_LOGIT_LOSS = 'bce_logit_loss'\n    CE_SOFTMAX_LOSS = 'ce_softmax_loss'\n    MASKED_BCE_WITH_TPFP = 'masked_bce_with_tpfp'\n\n\nclass RecordMetric(str, Enum):\n    LWRAP = 'LWRAP'\n    VALID_LOSS = 'valid_loss'\n    \n\nclass Normalizer(str, Enum):\n    IMAGENET_STATS = 'imagenet_stats'\n    SAMPLE_STATS = 'sample_stats'\n\n    \nclass Channel(int, Enum):\n    SINGLE = 1\n    THREE = 3\n\n    \nassert LossFunction.FOCAL_LOSS == 'focal_loss'","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:17:54.77773Z","iopub.execute_input":"2021-06-13T17:17:54.778718Z","iopub.status.idle":"2021-06-13T17:17:54.790275Z","shell.execute_reply.started":"2021-06-13T17:17:54.778675Z","shell.execute_reply":"2021-06-13T17:17:54.789255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# W&B CONFIG\nDISABLE_WB = False\nWB_CONFIG = {\n    \"project\": \"RFCX Experiment Tracker\", \n    \"name\": \"baseline with n_mels=256, masked bce with TPFP, monitor LWRAP (10+30)\",\n    \"notes\": \"rerun baseline again, without mixup and audio aug, n_mels=256, masked bce loss with TPFP, monitor LWRAP (10+30)\",\n    \"job_type\": \"train\",\n    \"tags\": ['resnet34', \"TPFP\", \"Masked BCE loss\", \"imagenet norm\"]\n}\n\n\n# DATA CONFIG\nDATA_DIR = Path('../input/rfcx-species-audio-detection')\nMODEL_DIR = Path('../input/fastiai-fastaudio-rainforest-starter')\n\n\n# MORE DATA\nIS_PSEUDO_LABEL = False\nPSEUDO_LABEL_CSV = Path('../input/rainforest-model-checkpoints/df_train_pseudo_3s.csv')\n\n\n# AUDIO TRANSFORM CONFIG\nNORMALIZER = Normalizer.IMAGENET_STATS.value\nSR = 48000\nINTERVAL = 10 # crop len in sec\nRANDOM_INTERVAL = 8 # random crop among 10 sec\nMELSPEC_CONFIG = {\n    'n_fft': 2048,\n    'sample_rate': SR,\n    'n_mels': 256\n#     'mel': False,\n#     'f_max': 14000\n}\nIS_MIN_MAX_RESCALE = False\nRESIZE_MEL = None\n\n\n# TRAINING SCHEME CONFIG\nIS_AUDIO_AUG = False\nIS_MIXUP = False\nNUM_WORKERS = 0\nCHANNEL = Channel.SINGLE.value\nLOSS_FUNCTION = LossFunction.MASKED_BCE_WITH_TPFP.value # 'celoss'/ 'focal'\nIS_MULTILABEL = True\nMONITOR_METRIC = RecordMetric.LWRAP.value\nFOLD_ID = 0\nBATCH_SIZE = 128\n# multilabel: 0.033, multiclass: 0.0132\n#LR = 0.033 if IS_MULTILABEL else 0.0132\n# Masked BCE Loss: lr_min=0.33, lr_steep=0.0052\nLR = 5e-2\nFREEZE_EPOCH = 10\nUNFREEZE_EPOCH = 30\n\n\nRANDOM_SEED = 144\n\n\n# https://www.kaggle.com/c/rfcx-species-audio-detection/discussion/220389\n# MELSPEC_CONFIG = {\n#     'sample_rate': 32000,\n#     'n_fft': 2048,\n#     'n_mels': 384,\n#     'hop_length': 512,\n#     'win_length': 2048\n# }","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:17:54.795077Z","iopub.execute_input":"2021-06-13T17:17:54.795507Z","iopub.status.idle":"2021-06-13T17:17:54.810382Z","shell.execute_reply.started":"2021-06-13T17:17:54.79547Z","shell.execute_reply":"2021-06-13T17:17:54.809437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1b. Preset Files and DataFrame","metadata":{}},{"cell_type":"code","source":"# extract dataset\ntrain_path = DATA_DIR/ 'train'\ntest_path = DATA_DIR/ 'test'\ntrain_fns = get_audio_files(train_path)\n\n# massage df\ndf_train_tp = pd.read_csv(DATA_DIR/ 'train_tp.csv')\ndf_train_tp['recording_id'] = df_train_tp['recording_id'].map(lambda x: 'train/' + x)\n\n# K FOLD STRATIFICATION\ndf_train_tp['species_id'] = df_train_tp['species_id'].astype(str)\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=RANDOM_SEED)\nfor fold_id, (train_idxs, val_idxs) in enumerate(skf.split(df_train_tp.recording_id.values, df_train_tp.species_id.values)):\n    kfold_col = f'fold_{fold_id}'\n    df_train_tp[kfold_col] = 0\n    df_train_tp.loc[val_idxs, kfold_col] = 1\ntarget_fold = f'fold_{FOLD_ID}'\n\nif LOSS_FUNCTION == LossFunction.MASKED_BCE_WITH_TPFP.value:\n    print('Concatenating TPFP when using Masked Loss')\n    df_train_fp = pd.read_csv(DATA_DIR/ 'train_fp.csv')\n    df_train_fp['recording_id'] = df_train_fp['recording_id'].map(lambda x: 'train/' + x)\n    df_train_fp['species_id'] = df_train_fp['species_id'].astype(str)\n    for fold_id in range(5):\n        df_train_fp[f'fold_{fold_id}'] = 0\n    df_train_tp['label_type'] = 'TP'\n    df_train_fp['label_type'] = 'FP'\n    df_train_tp = pd.concat([df_train_tp, df_train_fp]).reset_index(drop=True)\n    print(f'DataFrame size after concat TPFP: {df_train_tp.shape[0]}')\n    print(f'# Val after concat TPFP: {df_train_tp[df_train_tp[target_fold] == 1].shape[0]}')\n    \n\n# sanity check\nassert DATA_DIR.is_dir()\nassert MODEL_DIR.is_dir()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:17:54.81726Z","iopub.execute_input":"2021-06-13T17:17:54.819585Z","iopub.status.idle":"2021-06-13T17:17:58.151971Z","shell.execute_reply.started":"2021-06-13T17:17:54.819543Z","shell.execute_reply":"2021-06-13T17:17:58.151145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_tp[df_train_tp[target_fold]==1].head(3)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:17:58.153629Z","iopub.execute_input":"2021-06-13T17:17:58.153991Z","iopub.status.idle":"2021-06-13T17:17:58.174368Z","shell.execute_reply.started":"2021-06-13T17:17:58.153954Z","shell.execute_reply":"2021-06-13T17:17:58.173427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2a. Sample Create Cropped AudioTensor","metadata":{}},{"cell_type":"code","source":"SAMPLE_IDX = 1\ntest_row = df_train_tp.loc[SAMPLE_IDX]\ntest_row","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:17:58.17565Z","iopub.execute_input":"2021-06-13T17:17:58.176Z","iopub.status.idle":"2021-06-13T17:17:58.185247Z","shell.execute_reply.started":"2021-06-13T17:17:58.175964Z","shell.execute_reply":"2021-06-13T17:17:58.184393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_audio_tensor(row, is_truncate=True):\n    if is_truncate:\n        t_min, t_max = row.t_min, row.t_max\n        center = (t_max + t_min) / 2\n        start_t = center - (INTERVAL/2.)\n        _frame_offset = int(max(0, start_t) * SR)\n        _num_frames = int(INTERVAL * SR)\n    else:\n        _frame_offset = 0\n        _num_frames = -1\n    \n    # debug\n    #print(f'offset: {_frame_offset}')\n    #print(f'num_frames: {_num_frames}')\n    audio_fn = DATA_DIR.resolve()/f'{row.recording_id}.flac'\n    audio = AudioTensor.create(audio_fn,\n                               frame_offset=_frame_offset, \n                               num_frames=_num_frames)\n    return audio","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:17:58.186434Z","iopub.execute_input":"2021-06-13T17:17:58.186687Z","iopub.status.idle":"2021-06-13T17:17:58.195525Z","shell.execute_reply.started":"2021-06-13T17:17:58.186658Z","shell.execute_reply":"2021-06-13T17:17:58.19467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"truncated_audio = create_audio_tensor(test_row, is_truncate=True)\norig_audio = create_audio_tensor(test_row, is_truncate=False)\n\nassert truncated_audio.shape[1] <= INTERVAL * SR\nassert orig_audio.shape[1] == 60 * SR","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:17:58.197124Z","iopub.execute_input":"2021-06-13T17:17:58.197593Z","iopub.status.idle":"2021-06-13T17:17:58.48229Z","shell.execute_reply.started":"2021-06-13T17:17:58.197556Z","shell.execute_reply":"2021-06-13T17:17:58.481385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"truncated_audio.show();","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:17:58.483609Z","iopub.execute_input":"2021-06-13T17:17:58.48398Z","iopub.status.idle":"2021-06-13T17:17:58.815013Z","shell.execute_reply.started":"2021-06-13T17:17:58.483941Z","shell.execute_reply":"2021-06-13T17:17:58.814167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"orig_audio.show();","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:17:58.816254Z","iopub.execute_input":"2021-06-13T17:17:58.816746Z","iopub.status.idle":"2021-06-13T17:17:59.440534Z","shell.execute_reply.started":"2021-06-13T17:17:58.816708Z","shell.execute_reply":"2021-06-13T17:17:59.439736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2b. Set up `DataBlock`, `DataLoaders`","metadata":{}},{"cell_type":"code","source":"# define custom block for audio\nif NORMALIZER == Normalizer.IMAGENET_STATS.value:\n    batch_tfms = IntToFloatTensor\n    \nelif NORMALIZER == Normalizer.SAMPLE_STATS.value:\n    class SampleNormalize(Transform):\n        order=99\n        def encodes(self, o:AudioSpectrogram):\n            # (BS, C, W, H)\n            means = o.mean(axis=(1,2,3), keepdim=True)\n            stds = o.std(axis=(1,2,3), keepdim=True)\n            return (o-means)/stds\n    batch_tfms = SampleNormalize\n    \nelse:\n    raise NotImplementedError\n    \nclass CustomAudioBlock(TransformBlock):\n    \"A `TransformBlock` for audios\"\n    @delegates(audio_item_tfms)\n    def __init__(self, batch_tfms, cache_folder=None, **kwargs):\n        item_tfms = audio_item_tfms(**kwargs)\n        type_tfm = None\n        return super().__init__(type_tfms=type_tfm, \n                                item_tfms=item_tfms, \n                                batch_tfms=batch_tfms)\n\n    \nvocab = list(map(str, range(24)))\nif LOSS_FUNCTION == LossFunction.MASKED_BCE_WITH_TPFP.value:\n    def label_encodes_dict(self, d: dict):\n        o = d['label']\n        if not all(elem in self.vocab.o2i.keys() for elem in o):\n            diff = [elem for elem in o if elem not in self.vocab.o2i.keys()]\n            diff_str = \"', '\".join(diff)\n            raise KeyError(f\"Labels '{diff_str}' were not included in the training dataset\")\n        t = TensorMultiCategory([self.vocab.o2i[o_] for o_ in o])\n        t.label_type = d['type']\n        return t\n    \n    def one_hot_with_metadata(self, o: TensorMultiCategory):\n        t = TensorMultiCategory(one_hot(o, self.c).float())\n        t.__dict__ = o.__dict__\n        return t\n    \n    class NegateFP(Transform):\n        order = OneHotEncode.order + 1\n        def encodes(self, o: TensorMultiCategory):\n            assert getattr(o, 'label_type')\n            assert o.label_type in ('TP', 'FP')\n            if o.label_type == 'FP':\n                o *= -1\n            return o\n\n        def decodes(self, o: TensorMultiCategory):\n            assert getattr(o, 'label_type')\n            assert o.label_type in ('TP', 'FP')\n            if o.label_type == 'FP':\n                o *= -1\n            return o        \n    \n    def MultiCategoryBlockForMetadata(vocab):\n        tfm = [MultiCategorize(vocab=vocab), OneHotEncode, NegateFP]\n        return TransformBlock(type_tfms=tfm)\n    \n    MultiCategorize.encodes.add(label_encodes_dict)\n    OneHotEncode.encodes.add(one_hot_with_metadata)\n    blocks = (\n        partial(CustomAudioBlock, batch_tfms=batch_tfms, sample_rate=SR),\n        partial(MultiCategoryBlockForMetadata, vocab=vocab)\n    )\n\nelif IS_MULTILABEL:\n    blocks = (\n        partial(CustomAudioBlock, batch_tfms=batch_tfms, sample_rate=SR), \n        partial(MultiCategoryBlock, vocab=vocab)\n    )\nelse:\n    blocks = (\n        partial(CustomAudioBlock, batch_tfms=batch_tfms, sample_rate=SR),\n        partial(CategoryBlock, vocab=vocab)\n    )\n\n\n# duration in ms\nitem_tfms = [ResizeSignal(duration=RANDOM_INTERVAL*1000, pad_mode=AudioPadType.Repeat)]\nif IS_AUDIO_AUG:\n    # data augmentation on raw audio\n    data_augmentation = [\n        AddNoise(color=NoiseColor.White, noise_level=0.1), \n        SignalShifter(max_pct=0.3)\n    ]\n    item_tfms += data_augmentation\n    \n\n\n# batch/ datablock setup\ndb_batch_tfms = []\ndb_batch_tfms += [\n    AudioToSpec.from_cfg(\n        AudioConfig.BasicMelSpectrogram(**MELSPEC_CONFIG)\n    )\n]\n\nif IS_MIN_MAX_RESCALE:\n    class MinMaxRescale(Transform):\n        def encodes(self, o: AudioSpectrogram):\n            BS, C, W, H = o.shape\n            _min = o.reshape(BS, C, W*H).min(axis=2, keepdim=True)[0][:,:,:,None]\n            _max = o.reshape(BS, C, W*H).max(axis=2, keepdim=True)[0][:,:,:,None]\n            return (o - _min) / _max\n    db_batch_tfms += [MinMaxRescale()]\n    \nif RESIZE_MEL is not None:\n    assert isinstance(RESIZE_MEL, tuple)\n    resize_tfms = TfmResizeGPU(size=RESIZE_MEL)\n    db_batch_tfms += [resize_tfms]\n\nif CHANNEL == Channel.THREE.value:\n    class SpectrogramStacker(Transform):\n        def encodes(self, o: AudioSpectrogram):\n            return o.expand(-1, 3, -1, -1)\n        def decodes(self, o: AudioSpectrogram):\n            return o[:, :1, :, :]\n    db_batch_tfms += [SpectrogramStacker()]\n\n\n#  val split\nif IS_PSEUDO_LABEL:\n    df_pseudo = pd.read_csv(PSEUDO_LABEL_CSV)\n    df_pseudo.recording_id = df_pseudo.recording_id.apply(lambda _id: f'train/{_id}')\n    for _idx in range(5):\n        df_pseudo[f'fold_{_idx}'] = None\n    df_train_tp = pd.concat([df_train_tp, df_pseudo])\n    df_train_tp = df_train_tp.reset_index(drop=True)\n    df_train_tp['species_id'] = df_train_tp['species_id'].astype(str)\nval_idxs = df_train_tp[df_train_tp[target_fold] == 1].index.tolist()\nsplitter=IndexSplitter(val_idxs)\n\n\n# get label from DataFrame row\nif LOSS_FUNCTION == LossFunction.MASKED_BCE_WITH_TPFP.value:\n    def label_to_list_v2(row):\n        \"\"\" propagate metadata to pipeline \"\"\"\n        y_reader = ColReader('species_id')\n        label = y_reader(row)\n        try:\n            label_type = row.label_type\n        except:\n            label_type = 'TP'\n        return {'label': [label], 'type': label_type}\n    \n    y_getter = label_to_list_v2\nelse:\n    y_reader = ColReader('species_id')\n    \n    def label_to_list(row):\n        label = y_reader(row)\n        return [label]\n\n    y_getter = label_to_list if IS_MULTILABEL else y_reader\n\n\n# get the datablock\ndatablock = DataBlock(\n    blocks=blocks,\n    item_tfms=item_tfms,\n    batch_tfms=db_batch_tfms,\n    get_x=create_audio_tensor,\n    get_y=y_getter,\n    #splitter=RandomSplitter(valid_pct=0.2, seed=RANDOM_SEED)\n    splitter=splitter\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:17:59.442424Z","iopub.execute_input":"2021-06-13T17:17:59.442893Z","iopub.status.idle":"2021-06-13T17:17:59.529254Z","shell.execute_reply.started":"2021-06-13T17:17:59.442852Z","shell.execute_reply":"2021-06-13T17:17:59.528439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = datablock.dataloaders(source=df_train_tp, \n                            bs=BATCH_SIZE,\n                            num_workers=NUM_WORKERS)\none_batch = dls.one_batch()\n\n# sanity check\nassert one_batch[0].shape[0] == BATCH_SIZE\n# no OneHot transform in CategoryBlock\n# because torch.nn.CrossEntropyLoss not require one hot encode\nfor idx in [1, 3, 6]:\n    test_row = dls.train.items.loc[idx]\n    gt_category = int(test_row.species_id)\n    tfms_category = dls.train.tfms[1](test_row)\n    if IS_MULTILABEL:\n        assert tfms_category.sum().numpy() == 1.\n        assert tfms_category[gt_category].numpy() == 1.\n    else:\n        assert isinstance(tfms_category, TensorCategory)\n\nprint(len(dls.train.items))\nprint(len(dls.valid.items))\nprint('DataLoader transform completed and checked')","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:17:59.530578Z","iopub.execute_input":"2021-06-13T17:17:59.531119Z","iopub.status.idle":"2021-06-13T17:18:13.314195Z","shell.execute_reply.started":"2021-06-13T17:17:59.53108Z","shell.execute_reply":"2021-06-13T17:18:13.312552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if sample norm, its scale may not look right coz its not reversable\ndls.show_batch(ncols=2, nrows=3, figsize=(15, 10))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:18:13.315508Z","iopub.execute_input":"2021-06-13T17:18:13.315845Z","iopub.status.idle":"2021-06-13T17:18:22.085346Z","shell.execute_reply.started":"2021-06-13T17:18:13.315806Z","shell.execute_reply":"2021-06-13T17:18:22.084305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CHANNEL == Channel.SINGLE.value:\n    assert one_batch[0].shape[1] == 1\nelif CHANNEL == Channel.THREE.value:\n    assert one_batch[0].shape[1] == 3\nelse:\n    raise NotImplementedError\nassert one_batch[0].sr == SR\none_batch[0].shape, type(one_batch[0]), dls.num_workers, dls.after_batch","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:18:22.086602Z","iopub.execute_input":"2021-06-13T17:18:22.086927Z","iopub.status.idle":"2021-06-13T17:18:22.098027Z","shell.execute_reply.started":"2021-06-13T17:18:22.086892Z","shell.execute_reply":"2021-06-13T17:18:22.096807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3a. Set up `Learner` and `AccumMetric`\n- nn.BCEWithLogitsLoss: for each instance, sigmoid for each class, BCE for each class, average across classes","metadata":{}},{"cell_type":"code","source":"# source: https://www.kaggle.com/c/rfcx-species-audio-detection/discussion/198418\ndef LWRAP(preds, labels):\n    \n    # labels: (BS, ) for mutliclass, (BS, C) for multilabel\n    if not IS_MULTILABEL:\n        labels = torch.nn.functional.one_hot(labels, 24)\n        \n    # Ranks of the predictions\n    ranked_classes = torch.argsort(preds, dim=-1, descending=True)\n    # i, j corresponds to rank of prediction in row i\n    class_ranks = torch.zeros_like(ranked_classes)\n    for i in range(ranked_classes.size(0)):\n        for j in range(ranked_classes.size(1)):\n            class_ranks[i, ranked_classes[i][j]] = j + 1\n    # Mask out to only use the ranks of relevant GT labels\n    ground_truth_ranks = class_ranks * labels + (1e6) * (1 - labels)\n    # All the GT ranks are in front now\n    sorted_ground_truth_ranks, _ = torch.sort(ground_truth_ranks, dim=-1, descending=False)\n    # Number of GT labels per instance\n    num_labels = labels.sum(-1)\n    pos_matrix = torch.tensor(np.array([i+1 for i in range(labels.size(-1))])).unsqueeze(0)\n    score_matrix = pos_matrix / sorted_ground_truth_ranks\n    score_mask_matrix, _ = torch.sort(labels, dim=-1, descending=True)\n    scores = score_matrix * score_mask_matrix\n    score = scores.sum() / labels.sum()\n    return score.item()\n\nactivation_type = 'Softmax' if IS_MULTILABEL else 'Sigmoid'\nlwrap_metric = AccumMetric(\n    func=LWRAP, activation=activation_type,\n    to_np=False, flatten=False\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:25:10.734977Z","iopub.execute_input":"2021-06-13T17:25:10.735299Z","iopub.status.idle":"2021-06-13T17:25:10.744512Z","shell.execute_reply.started":"2021-06-13T17:25:10.735269Z","shell.execute_reply":"2021-06-13T17:25:10.743234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# source: https://www.kaggle.com/c/rfcx-species-audio-detection/discussion/213075\nclass BinaryFocalLoss(nn.Module):\n    def __init__(self, gamma, alpha=1.):\n        super().__init__()\n        self.gamma = gamma\n        self.alpha = alpha\n        self.loss_func = nn.BCEWithLogitsLoss(reduction='none')\n    \n    def forward(self, preds, targets):\n        bce_loss = self.loss_func(preds, targets)\n        probas = torch.sigmoid(preds)\n        loss = torch.where(\n            targets>=0.5, \n            self.alpha*((1.-probas)**self.gamma)*bce_loss, \n            (probas**self.gamma)*bce_loss\n        )\n        loss = loss.mean()\n        return loss\n    \n\nclass MaskedLossWithTPFP(BCEWithLogitsLossFlat):\n    def __init__(self, **kwargs):\n        super().__init__(reduction='sum', **kwargs)\n    \n    def __call__(self, inp, targ, **kwargs):\n        weight = (targ != 0.).float().to(targ.device)\n        # apply masking on non-TP/ non-FP\n        self.func.weight = weight.view(-1)\n        # set FP to 0.\n        targ[targ == -1] = 0.\n        n = weight.sum()\n        masked_sum = super().__call__(inp, targ, **kwargs)\n        return masked_sum / n","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:25:11.502454Z","iopub.execute_input":"2021-06-13T17:25:11.502808Z","iopub.status.idle":"2021-06-13T17:25:11.511451Z","shell.execute_reply.started":"2021-06-13T17:25:11.502779Z","shell.execute_reply":"2021-06-13T17:25:11.510498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get the metrics\nif LOSS_FUNCTION == LossFunction.FOCAL_LOSS.value:\n    loss_func = BinaryFocalLoss(gamma=2., alpha=1.)\nelif LOSS_FUNCTION == LossFunction.BCE_LOGIT_LOSS.value:\n    loss_func = BCEWithLogitsLossFlat()\nelif LOSS_FUNCTION == LossFunction.CE_SOFTMAX_LOSS.value:\n    loss_func = CrossEntropyLossFlat()\nelif LOSS_FUNCTION == LossFunction.MASKED_BCE_WITH_TPFP.value:\n    loss_func = MaskedLossWithTPFP()\nelse:\n    raise ValueError('Invalid loss function')\n    \n    \nif CHANNEL == Channel.SINGLE.value:\n    config_dict = {'n_in': 1}\nelif CHANNEL == Channel.THREE.value:\n    config_dict = {'n_in': 3}\nelse:\n    raise NotImplementedError\n\n\ncbs = MixUp(1.) if IS_MIXUP else None\n\n    \nif NORMALIZER == Normalizer.IMAGENET_STATS.value:\n    # type-dispatch for Normalizer to work on AudioSpec tensor\n    def encode_tensorimage(self, x:TensorImage): \n        return (x-self.mean) / self.std\n    def encode_audiospec(self, x:AudioSpectrogram): \n        return (x-self.mean) / self.std\n    Normalize.encodes = TypeDispatch([encode_tensorimage, encode_audiospec])\n    \n    learner = cnn_learner(\n        dls, resnet34, \n        pretrained=True,\n        normalize=True,\n        config=config_dict,\n        loss_func=loss_func,\n        metrics=[lwrap_metric],\n        cbs=cbs\n    )\n    \nelif NORMALIZER == Normalizer.SAMPLE_STATS.value:\n    learner = cnn_learner(\n        dls, resnet34, \n        pretrained=True,\n        normalize=False,\n        config=config_dict,\n        loss_func=loss_func,\n        metrics=[lwrap_metric],\n        cbs=cbs\n    )\n\nelse:\n    raise NotImplementedError\n\nprint('Learner is set')","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:25:12.915157Z","iopub.execute_input":"2021-06-13T17:25:12.915501Z","iopub.status.idle":"2021-06-13T17:25:14.980338Z","shell.execute_reply.started":"2021-06-13T17:25:12.915468Z","shell.execute_reply":"2021-06-13T17:25:14.979354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (NORMALIZER == Normalizer.IMAGENET_STATS.value):\n    if CHANNEL == (Channel.SINGLE.value):\n        # make sure normalizer can handle AudioSpec tensor\n        learner.dls.after_batch[-1].mean = learner.dls.after_batch[-1].mean[:, 0:1, :, :]\n        learner.dls.after_batch[-1].std = learner.dls.after_batch[-1].std[:, 0:1, :, :]    \n        # assert normalizer stats are 1 channel\n        assert learner.dls.after_batch[-1].mean.shape[1] == 1\n        assert learner.dls.train.after_batch[-1].mean.shape[1] == 1\n        \n    # add Normalize transform into valid (bug fix in latest release)\n    if CHANNEL == (Channel.SINGLE.value):\n        mean=learner.dls.after_batch[-1].mean.cpu().numpy().flatten()[0]\n        std=learner.dls.after_batch[-1].std.cpu().numpy().flatten()[0]\n    elif CHANNEL == (Channel.THREE.value):\n        mean=learner.dls.after_batch[-1].mean.cpu().numpy().flatten()\n        std=learner.dls.after_batch[-1].std.cpu().numpy().flatten()\n    else:\n        raise NotImplementedError\n        \n    learner.dls.valid.after_batch.add(\n        Normalize.from_stats(mean=mean, std=std),\n        'after_batch'\n    )\n\nprint(f'Train batch_tfms: \\n{learner.dls.train.after_batch}')\nprint(f'Valid batch_tfms: \\n{learner.dls.valid.after_batch}')","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:25:14.984097Z","iopub.execute_input":"2021-06-13T17:25:14.984446Z","iopub.status.idle":"2021-06-13T17:25:15.00847Z","shell.execute_reply.started":"2021-06-13T17:25:14.984393Z","shell.execute_reply":"2021-06-13T17:25:15.007229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner.loss_func","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:25:15.603025Z","iopub.execute_input":"2021-06-13T17:25:15.603346Z","iopub.status.idle":"2021-06-13T17:25:15.609389Z","shell.execute_reply.started":"2021-06-13T17:25:15.603315Z","shell.execute_reply":"2021-06-13T17:25:15.60851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner.show_training_loop()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T17:25:16.230955Z","iopub.execute_input":"2021-06-13T17:25:16.23127Z","iopub.status.idle":"2021-06-13T17:25:16.247035Z","shell.execute_reply.started":"2021-06-13T17:25:16.231241Z","shell.execute_reply":"2021-06-13T17:25:16.246245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3b. Finding Optimal Learning Rate","metadata":{}},{"cell_type":"code","source":"#learner.lr_find() ","metadata":{"execution":{"iopub.status.busy":"2021-06-09T17:25:13.797083Z","iopub.execute_input":"2021-06-09T17:25:13.797494Z","iopub.status.idle":"2021-06-09T17:36:07.781676Z","shell.execute_reply.started":"2021-06-09T17:25:13.797467Z","shell.execute_reply":"2021-06-09T17:36:07.780458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#learner.fine_tune(epochs=1, base_lr=0.033)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T18:20:46.620037Z","iopub.execute_input":"2021-05-22T18:20:46.620403Z","iopub.status.idle":"2021-05-22T18:22:13.789762Z","shell.execute_reply.started":"2021-05-22T18:20:46.62036Z","shell.execute_reply":"2021-05-22T18:22:13.78887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3c. Start Training\nIssue Tracker:\n- Low GPU utility (0%), and very high CPU utility","metadata":{}},{"cell_type":"code","source":"cbs = [SaveModelCallback(monitor=MONITOR_METRIC)]\nif not DISABLE_WB:\n    # start up W&B run\n    user_secrets = UserSecretsClient()\n    wandb_api = user_secrets.get_secret(\"wandb_key\")\n    wandb.login(key=wandb_api)\n    wandb.init(**WB_CONFIG)\n    config = wandb.config\n    # additionally log the mel spectrogram transform config\n    config.audio_to_spec = AudioConfig.BasicMelSpectrogram(**MELSPEC_CONFIG).__repr__()\n    cbs += [WandbCallback()]\n    print('Completed setup for W&B run')\n\n\nprint('Start training model')\nlearner.fine_tune(\n    epochs=UNFREEZE_EPOCH, base_lr=LR, \n    freeze_epochs=FREEZE_EPOCH, cbs=cbs\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner.recorder.plot_loss()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner.save('model_last_epoch')\nprint('Model checkpoint saved')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### How to de-register a callback?\n- Learner.remove_cbs: arg is the `Callback` class, NOT its instance\n- Details\n    - remove attribute `name` from the callback\n    - set its attribute `learn` = None\n    - remove from `Learner.cbs` list (calling `cbs.remove`, inherited from `list`)","metadata":{}},{"cell_type":"code","source":"#learner.show_training_loop();\ntry:\n    learner.remove_cbs(FetchPredsCallback)\nexcept:\n    print('Failed to remove FetchPredsCallback, probably its absent')\nlearner.cbs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Sanity Check Best Model Validation Loss","metadata":{}},{"cell_type":"code","source":"preds, targs, loss = learner.get_preds(with_loss=True)\nl = loss.mean().cpu().numpy()\nprint(f'Loss on validation: {l:.2f}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4. Prepare Submission","metadata":{}},{"cell_type":"code","source":"def sample_one_subclips(row, i):    \n    length = int(RANDOM_INTERVAL * SR)\n    total_length = int(60 * SR)\n    # Last segment going from the end\n    if (i + 1) * length > total_length:\n        _frame_offset = total_length - length\n    else:\n        _frame_offset = i * length\n        \n    audio_fn = test_path.resolve()/f'{row.recording_id}.flac'\n    audio_subclip = AudioTensor.create(audio_fn, \n                                        frame_offset=_frame_offset, \n                                        num_frames=length)\n    return audio_subclip","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(DATA_DIR/'sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.train_ds.tls[0].tfms","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.valid_ds.tls[0].tfms = Pipeline(partial(sample_one_subclips, i=0))\nsample_test_dl = dls.test_dl(test_df, with_labels=False)\nprint(sample_test_dl.after_batch)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subclips_preds = []\nsegment_n = ceil((60*SR)/(RANDOM_INTERVAL*SR))\ncp_pipeline = learner.dls.valid_ds.tls[0].tfms\n\nfor crt_i in range(segment_n):\n    print(f'Running prediction on subclip {crt_i} for all test samples...')\n    #dls.valid_ds.tls[0].tfms = Pipeline(partial(sample_one_subclips, i=crt_i))\n    crt_test_dl = learner.dls.test_dl(test_df, with_labels=False)\n    crt_test_dl.tls[0].tfms = Pipeline(partial(sample_one_subclips, i=crt_i))\n    # predict on a subclip for all test samples\n    _preds = learner.get_preds(dl=crt_test_dl)\n    \n    # softmax makes output statistics of each class dependent\n    # softmax could distort the order if u take max across all subclips\n    if IS_MULTILABEL:\n        _preds = _preds[0]\n    else:\n        _preds = torch.nn.Softmax(dim=-1)(_preds[0])\n        \n    subclips_preds.append(_preds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sanity check if normalizer is present in test dataloader\ncrt_test_dl.after_batch","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_preds = torch.stack(subclips_preds, dim=1)\nfinal_preds = all_preds.max(dim=1)[0]\n\ntest_df.iloc[:, 1:] = final_preds\ntest_df.to_csv('my_submission.csv', index=False)\nprint('Submission CSV written')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Revalidate the save model is working","metadata":{}},{"cell_type":"code","source":"learner.dls.valid_ds.tls[0].tfms","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner = learner.load('model')\nlearner.dls.valid_ds.tls[0].tfms = cp_pipeline\npreds, targs, loss = learner.get_preds(with_loss=True)\nl = loss.mean().cpu().numpy()\nprint(f'Loss on validation after model load: {l:.2f}')","metadata":{},"execution_count":null,"outputs":[]}]}