{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# AGENDA\n#### TO DO LIST\n1. Apply TTA on inference\n2. Plotting confusion matrix, plot prediction for top losses (need typedispatch on AudioTensor)\n\n#### Completed\n1. [11/05/2021] Set up stratified KFold\n2. [11/05/2021] Crop training set data\n3. [14/05/2021] Try multiclass v.s. multilabel\n4. [17/05/2021] Prototyped W&B integration\n5. [17/05/2021] Fixed `Normalize` is missing in both train and valid\n6. [23/05/2021] Fixed sample rate != 48000 error\n7. [23/05/2021] Fixed imagenet normalization on train+val \n\n#### Reference\n1. code starts from: https://www.kaggle.com/scart97/fastaudio-starter-kit/notebook","metadata":{}},{"cell_type":"markdown","source":"### 0. Install Packages","metadata":{}},{"cell_type":"code","source":"# !pip install --up../input/rfcx-species-audio-detection/he-torch\n# Change to just \"!pip install --upgrade fastaudio\" when 1.0.1 is ready\n!pip install --upgrade git+https://github.com/fastaudio/fastaudio.git@update_fastai_2.3\n!pip install wandb --upgrade","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After installing fastaudio, restart the env before importing the library\n\n![Restart](https://i.imgur.com/xlAOnbW.png)","metadata":{}},{"cell_type":"code","source":"from enum import Enum\nfrom math import ceil\n\nimport wandb\nfrom kaggle_secrets import UserSecretsClient\n\nimport pandas as pd\nfrom fastaudio.all import *\nfrom fastai.vision.all import *\nfrom fastai.callback.wandb import *\n\nimport torch.nn as nn\n# migrate to sox_io\nimport torchaudio\ntorchaudio.set_audio_backend(\"sox_io\")\n\nfrom sklearn.model_selection import StratifiedKFold\n\n# check cuda is available\nimport torch\nprint(torch.cuda.is_available())","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:34:11.032114Z","iopub.execute_input":"2021-05-24T21:34:11.032545Z","iopub.status.idle":"2021-05-24T21:34:16.759996Z","shell.execute_reply.started":"2021-05-24T21:34:11.032433Z","shell.execute_reply":"2021-05-24T21:34:16.757051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import fastai\nprint(fastai.__version__)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:34:49.511091Z","iopub.execute_input":"2021-05-24T21:34:49.511502Z","iopub.status.idle":"2021-05-24T21:34:49.516740Z","shell.execute_reply.started":"2021-05-24T21:34:49.511443Z","shell.execute_reply":"2021-05-24T21:34:49.515738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1a. Configuration","metadata":{}},{"cell_type":"code","source":"class LossFunction(str, Enum):\n    FOCAL_LOSS = 'focal_loss'\n    BCE_LOGIT_LOSS = 'bce_logit_loss'\n    CE_SOFTMAX_LOSS = 'ce_softmax_loss'\n\n\nclass RecordMetric(str, Enum):\n    LWRAP = 'LWRAP'\n    VALID_LOSS = 'valid_loss'\n    \n\nclass Normalizer(str, Enum):\n    IMAGENET_STATS = 'imagenet_stats'\n    SAMPLE_STATS = 'sample_stats'\n\n    \nclass Channel(int, Enum):\n    SINGLE = 1\n    THREE = 3\n\n    \nassert LossFunction.FOCAL_LOSS == 'focal_loss'","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:35:03.327233Z","iopub.execute_input":"2021-05-24T21:35:03.327627Z","iopub.status.idle":"2021-05-24T21:35:03.334116Z","shell.execute_reply.started":"2021-05-24T21:35:03.327594Z","shell.execute_reply":"2021-05-24T21:35:03.332852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# W&B CONFIG\nDISABLE_WB = False\nWB_CONFIG = {\n    \"project\": \"RFCX Experiment Tracker\", \n    \"name\": \"mixup train longer\",\n    \"notes\": \"mixup train longer (i.e. 15+25)\",\n    \"job_type\": \"train\",\n    \"tags\": ['resnet34', 'TP', \"BCE loss\", \"imagenet norm\", \"mixup\"]\n}\n\n# DATA CONFIG\nDATA_DIR = Path('../input/rfcx-species-audio-detection')\nMODEL_DIR = Path('../input/fastiai-fastaudio-rainforest-starter')\n\n\n# AUDIO TRANSFORM CONFIG\nNORMALIZER = Normalizer.IMAGENET_STATS.value\nSR = 48000\nINTERVAL = 10 # crop len in sec\nRANDOM_INTERVAL = 8 # random crop among 10 sec\nMELSPEC_CONFIG = {\n    'n_fft': 2048,\n    'sample_rate': SR\n}\nIS_MIN_MAX_RESCALE = False\n\n\n# TRAINING SCHEME CONFIG\nIS_MIXUP = True\nNUM_WORKERS = 0\nCHANNEL = Channel.SINGLE.value\nLOSS_FUNCTION = LossFunction.BCE_LOGIT_LOSS.value # 'celoss'/ 'focal'\nIS_MULTILABEL = True\nMONITOR_METRIC = RecordMetric.VALID_LOSS.value\nFOLD_ID = 0\nBATCH_SIZE = 128\n# multilabel: 0.033, multiclass: 0.0132\n#LR = 0.033 if IS_MULTILABEL else 0.0132\nLR = 0.033\nFREEZE_EPOCH = 15\nUNFREEZE_EPOCH = 25\n\n\nRANDOM_SEED = 144\n\n\n# https://www.kaggle.com/c/rfcx-species-audio-detection/discussion/220389\n# MELSPEC_CONFIG = {\n#     'sample_rate': 32000,\n#     'n_fft': 2048,\n#     'n_mels': 384,\n#     'hop_length': 512,\n#     'win_length': 2048\n# }","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:35:16.895917Z","iopub.execute_input":"2021-05-24T21:35:16.896264Z","iopub.status.idle":"2021-05-24T21:35:16.903734Z","shell.execute_reply.started":"2021-05-24T21:35:16.896227Z","shell.execute_reply":"2021-05-24T21:35:16.902876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1b. Preset Files and DataFrame","metadata":{}},{"cell_type":"code","source":"# extract dataset\ntrain_path = DATA_DIR/ 'train'\ntest_path = DATA_DIR/ 'test'\ntrain_fns = get_audio_files(train_path)\n\n# massage df\ndf_train_tp = pd.read_csv(DATA_DIR/ 'train_tp.csv')\ndf_train_tp['recording_id'] = df_train_tp['recording_id'].map(lambda x: 'train/' + x)\n\n# GROUP BY RECORDING ID FOR MULTILABEL\n# df_train_tp = df_train_tp.drop(['t_min', 't_max', 'f_min', 'f_max', 'songtype_id'], axis=1)\n# df_train_tp['species_id'] = df_train_tp['species_id'].astype(str)\n# before_n = df_train_tp.shape[0]\n# # group labels for same recording_id\n# # remove duplicated labels\n# df_train_tp['species_id'] = df_train_tp.groupby('recording_id')['species_id'].\\\n#                             transform(lambda ls: \",\".join(list(set(ls))))\n# df_train_tp.drop_duplicates(subset=['recording_id'], inplace=True)\n# df_train_tp.reset_index(drop=True, inplace=True)\n# after_n = df_train_tp.shape[0]\n# print(f'Dataframe after deduplicate: {after_n}')\n# assert before_n > after_n\n\n# K FOLD STRATIFICATION\ndf_train_tp['species_id'] = df_train_tp['species_id'].astype(str)\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=RANDOM_SEED)\nfor fold_id, (train_idxs, val_idxs) in enumerate(skf.split(df_train_tp.recording_id.values, df_train_tp.species_id.values)):\n    kfold_col = f'fold_{fold_id}'\n    df_train_tp[kfold_col] = 0\n    df_train_tp.loc[val_idxs, kfold_col] = 1\ntarget_fold = f'fold_{FOLD_ID}'\n\n\n# sanity check\nassert DATA_DIR.is_dir()\nassert MODEL_DIR.is_dir()","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:35:23.770473Z","iopub.execute_input":"2021-05-24T21:35:23.770816Z","iopub.status.idle":"2021-05-24T21:35:31.284871Z","shell.execute_reply.started":"2021-05-24T21:35:23.770786Z","shell.execute_reply":"2021-05-24T21:35:31.283086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_tp[df_train_tp[target_fold]==1]","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:35:55.415425Z","iopub.execute_input":"2021-05-24T21:35:55.415802Z","iopub.status.idle":"2021-05-24T21:35:55.449021Z","shell.execute_reply.started":"2021-05-24T21:35:55.415769Z","shell.execute_reply":"2021-05-24T21:35:55.447978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2a. Sample Create Cropped AudioTensor","metadata":{}},{"cell_type":"code","source":"SAMPLE_IDX = 1\ntest_row = df_train_tp.loc[SAMPLE_IDX]\ntest_row","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:35:58.917128Z","iopub.execute_input":"2021-05-24T21:35:58.917498Z","iopub.status.idle":"2021-05-24T21:35:58.927529Z","shell.execute_reply.started":"2021-05-24T21:35:58.917463Z","shell.execute_reply":"2021-05-24T21:35:58.926677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_audio_tensor(row, is_truncate=True):\n    if is_truncate:\n        t_min, t_max = row.t_min, row.t_max\n        center = (t_max + t_min) / 2\n        start_t = center - (INTERVAL/2.)\n        _frame_offset = int(max(0, start_t) * SR)\n        _num_frames = int(INTERVAL * SR)\n    else:\n        _frame_offset = 0\n        _num_frames = -1\n    \n    # debug\n    #print(f'offset: {_frame_offset}')\n    #print(f'num_frames: {_num_frames}')\n    audio_fn = DATA_DIR.resolve()/f'{row.recording_id}.flac'\n    audio = AudioTensor.create(audio_fn,\n                               frame_offset=_frame_offset, \n                               num_frames=_num_frames)\n    return audio","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:36:03.584154Z","iopub.execute_input":"2021-05-24T21:36:03.584507Z","iopub.status.idle":"2021-05-24T21:36:03.591870Z","shell.execute_reply.started":"2021-05-24T21:36:03.584477Z","shell.execute_reply":"2021-05-24T21:36:03.590726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"truncated_audio = create_audio_tensor(test_row, is_truncate=True)\norig_audio = create_audio_tensor(test_row, is_truncate=False)\n\nassert truncated_audio.shape[1] <= INTERVAL * SR\nassert orig_audio.shape[1] == 60 * SR","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:36:04.387226Z","iopub.execute_input":"2021-05-24T21:36:04.387614Z","iopub.status.idle":"2021-05-24T21:36:04.543237Z","shell.execute_reply.started":"2021-05-24T21:36:04.387581Z","shell.execute_reply":"2021-05-24T21:36:04.542369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"truncated_audio.show();","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:36:07.992899Z","iopub.execute_input":"2021-05-24T21:36:07.993242Z","iopub.status.idle":"2021-05-24T21:36:08.340974Z","shell.execute_reply.started":"2021-05-24T21:36:07.993210Z","shell.execute_reply":"2021-05-24T21:36:08.340073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"orig_audio.show();","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:36:49.817575Z","iopub.execute_input":"2021-05-24T21:36:49.817910Z","iopub.status.idle":"2021-05-24T21:36:50.449741Z","shell.execute_reply.started":"2021-05-24T21:36:49.817878Z","shell.execute_reply":"2021-05-24T21:36:50.448796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2b. Set up `DataBlock`, `DataLoaders`","metadata":{}},{"cell_type":"code","source":"# define custom block for audio\nif NORMALIZER == Normalizer.IMAGENET_STATS.value:\n    batch_tfms = IntToFloatTensor\n    \nelif NORMALIZER == Normalizer.SAMPLE_STATS.value:\n    class SampleNormalize(Transform):\n        order=99\n        def encodes(self, o:AudioSpectrogram):\n            # (BS, C, W, H)\n            means = o.mean(axis=(1,2,3), keepdim=True)\n            stds = o.std(axis=(1,2,3), keepdim=True)\n            return (o-means)/stds\n    batch_tfms = SampleNormalize\n    \nelse:\n    raise NotImplementedError\n    \nclass CustomAudioBlock(TransformBlock):\n    \"A `TransformBlock` for audios\"\n    @delegates(audio_item_tfms)\n    def __init__(self, batch_tfms, cache_folder=None, **kwargs):\n        item_tfms = audio_item_tfms(**kwargs)\n        type_tfm = None\n        return super().__init__(type_tfms=type_tfm, \n                                item_tfms=item_tfms, \n                                batch_tfms=batch_tfms)\n\nvocab = list(map(str, range(24)))\nif IS_MULTILABEL:\n    blocks = (\n        partial(CustomAudioBlock, batch_tfms=batch_tfms, sample_rate=SR), \n        partial(MultiCategoryBlock, vocab=vocab)\n    )\nelse:\n    blocks = (\n        partial(CustomAudioBlock, batch_tfms=batch_tfms, sample_rate=SR),\n        partial(CategoryBlock, vocab=vocab)\n    )\n\n\n# data augmentation\ndata_augmentation = [\n    AddNoise(color=NoiseColor.White, noise_level=0.1), \n    SignalShifter(max_pct=0.3)\n]\n# duration in ms\nitem_tfms = [ResizeSignal(duration=RANDOM_INTERVAL*1000, pad_mode=AudioPadType.Repeat)] + data_augmentation\n\n\n# batch/ datablock setup\ndb_batch_tfms = []\ndb_batch_tfms += [\n    AudioToSpec.from_cfg(\n        AudioConfig.BasicMelSpectrogram(**MELSPEC_CONFIG)\n    )\n]\n\nif IS_MIN_MAX_RESCALE:\n    class MinMaxRescale(Transform):\n        def encodes(self, o: AudioSpectrogram):\n            BS, C, W, H = o.shape\n            _min = o.reshape(BS, C, W*H).min(axis=2, keepdim=True)[0][:,:,:,None]\n            _max = o.reshape(BS, C, W*H).max(axis=2, keepdim=True)[0][:,:,:,None]\n            return (o - _min) / _max\n    db_batch_tfms += [MinMaxRescale()]\n\nif CHANNEL == Channel.THREE.value:\n    class SpectrogramStacker(Transform):\n        def encodes(self, o: AudioSpectrogram):\n            return o.expand(-1, 3, -1, -1)\n        def decodes(self, o: AudioSpectrogram):\n            return o[:, :1, :, :]\n    db_batch_tfms += [SpectrogramStacker()]\n\n\n#  val split\nval_idxs = df_train_tp[df_train_tp[target_fold] == 1].index.tolist()\nsplitter=IndexSplitter(val_idxs)\n\n\n# get label from DataFrame row\ny_reader = ColReader('species_id')\ndef label_to_list(row):\n    label = y_reader(row)\n    return [label]\n\ny_getter = label_to_list if IS_MULTILABEL else y_reader\n\n\n# get the datablock\ndatablock = DataBlock(\n    blocks=blocks,\n    item_tfms=item_tfms,\n    batch_tfms=db_batch_tfms,\n    get_x=create_audio_tensor,\n    get_y=y_getter,\n    #splitter=RandomSplitter(valid_pct=0.2, seed=RANDOM_SEED)\n    splitter=splitter\n)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:37:02.139341Z","iopub.execute_input":"2021-05-24T21:37:02.139730Z","iopub.status.idle":"2021-05-24T21:37:02.163516Z","shell.execute_reply.started":"2021-05-24T21:37:02.139697Z","shell.execute_reply":"2021-05-24T21:37:02.162695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = datablock.dataloaders(source=df_train_tp, \n                            bs=BATCH_SIZE,\n                            num_workers=NUM_WORKERS)\none_batch = dls.one_batch()\n\n# sanity check\nassert one_batch[0].shape[0] == BATCH_SIZE\n# no OneHot transform in CategoryBlock\n# because torch.nn.CrossEntropyLoss not require one hot encode\nfor idx in [1, 3, 6]:\n    test_row = dls.train.items.loc[idx]\n    gt_category = int(test_row.species_id)\n    tfms_category = dls.train.tfms[1](test_row)\n    if IS_MULTILABEL:\n        assert tfms_category.sum().numpy() == 1.\n        assert tfms_category[gt_category].numpy() == 1.\n    else:\n        assert isinstance(tfms_category, TensorCategory)\n    \nprint('DataLoader transform completed and checked')","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:37:05.808692Z","iopub.execute_input":"2021-05-24T21:37:05.809037Z","iopub.status.idle":"2021-05-24T21:37:16.330372Z","shell.execute_reply.started":"2021-05-24T21:37:05.809006Z","shell.execute_reply":"2021-05-24T21:37:16.329516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if sample norm, its scale may not look right coz its not reversable\ndls.show_batch(ncols=2, nrows=3, figsize=(15, 10))","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:37:18.409118Z","iopub.execute_input":"2021-05-24T21:37:18.409463Z","iopub.status.idle":"2021-05-24T21:37:27.736374Z","shell.execute_reply.started":"2021-05-24T21:37:18.409429Z","shell.execute_reply":"2021-05-24T21:37:27.735334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CHANNEL == Channel.SINGLE.value:\n    assert one_batch[0].shape[1] == 1\nelif CHANNEL == Channel.THREE.value:\n    assert one_batch[0].shape[1] == 3\nelse:\n    raise NotImplementedError\nassert one_batch[0].sr == SR\none_batch[0].shape, type(one_batch[0]), dls.num_workers, dls.after_batch","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:37:59.460759Z","iopub.execute_input":"2021-05-24T21:37:59.461106Z","iopub.status.idle":"2021-05-24T21:37:59.468906Z","shell.execute_reply.started":"2021-05-24T21:37:59.461075Z","shell.execute_reply":"2021-05-24T21:37:59.467914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3a. Set up `Learner` and `AccumMetric`\n- nn.BCEWithLogitsLoss: for each instance, sigmoid for each class, BCE for each class, average across classes","metadata":{}},{"cell_type":"code","source":"# source: https://www.kaggle.com/c/rfcx-species-audio-detection/discussion/198418\ndef LWRAP(preds, labels):\n    \n    # labels: (BS, ) for mutliclass, (BS, C) for multilabel\n    if not IS_MULTILABEL:\n        labels = torch.nn.functional.one_hot(labels, 24)\n        \n    # Ranks of the predictions\n    ranked_classes = torch.argsort(preds, dim=-1, descending=True)\n    # i, j corresponds to rank of prediction in row i\n    class_ranks = torch.zeros_like(ranked_classes)\n    for i in range(ranked_classes.size(0)):\n        for j in range(ranked_classes.size(1)):\n            class_ranks[i, ranked_classes[i][j]] = j + 1\n    # Mask out to only use the ranks of relevant GT labels\n    ground_truth_ranks = class_ranks * labels + (1e6) * (1 - labels)\n    # All the GT ranks are in front now\n    sorted_ground_truth_ranks, _ = torch.sort(ground_truth_ranks, dim=-1, descending=False)\n    # Number of GT labels per instance\n    num_labels = labels.sum(-1)\n    pos_matrix = torch.tensor(np.array([i+1 for i in range(labels.size(-1))])).unsqueeze(0)\n    score_matrix = pos_matrix / sorted_ground_truth_ranks\n    score_mask_matrix, _ = torch.sort(labels, dim=-1, descending=True)\n    scores = score_matrix * score_mask_matrix\n    score = scores.sum() / labels.sum()\n    return score.item()\n\nactivation_type = 'Softmax' if IS_MULTILABEL else 'Sigmoid'\nlwrap_metric = AccumMetric(\n    func=LWRAP, activation=activation_type,\n    to_np=False, flatten=False\n)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:38:05.378870Z","iopub.execute_input":"2021-05-24T21:38:05.379218Z","iopub.status.idle":"2021-05-24T21:38:05.387905Z","shell.execute_reply.started":"2021-05-24T21:38:05.379187Z","shell.execute_reply":"2021-05-24T21:38:05.387015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# source: https://www.kaggle.com/c/rfcx-species-audio-detection/discussion/213075\nclass BinaryFocalLoss(nn.Module):\n    def __init__(self, gamma, alpha=1.):\n        super().__init__()\n        self.gamma = gamma\n        self.alpha = alpha\n        self.loss_func = nn.BCEWithLogitsLoss(reduction='none')\n    \n    def forward(self, preds, targets):\n        bce_loss = self.loss_func(preds, targets)\n        probas = torch.sigmoid(preds)\n        loss = torch.where(\n            targets>=0.5, \n            self.alpha*((1.-probas)**self.gamma)*bce_loss, \n            (probas**self.gamma)*bce_loss\n        )\n        loss = loss.mean()\n        return loss","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:38:09.509889Z","iopub.execute_input":"2021-05-24T21:38:09.510220Z","iopub.status.idle":"2021-05-24T21:38:09.516183Z","shell.execute_reply.started":"2021-05-24T21:38:09.510191Z","shell.execute_reply":"2021-05-24T21:38:09.515383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get the metrics\nif LOSS_FUNCTION == LossFunction.FOCAL_LOSS.value:\n    loss_func = BinaryFocalLoss(gamma=2., alpha=1.)\nelif LOSS_FUNCTION == LossFunction.BCE_LOGIT_LOSS.value:\n    loss_func = BCEWithLogitsLossFlat()\nelif LOSS_FUNCTION == LossFunction.CE_SOFTMAX_LOSS.value:\n    loss_func = CrossEntropyLossFlat()\n\n    \nif CHANNEL == Channel.SINGLE.value:\n    config_dict = {'n_in': 1}\nelif CHANNEL == Channel.THREE.value:\n    config_dict = {'n_in': 3}\nelse:\n    raise NotImplementedError\n\n\ncbs = MixUp(1.) if IS_MIXUP else None\n\n    \nif NORMALIZER == Normalizer.IMAGENET_STATS.value:\n    # type-dispatch for Normalizer to work on AudioSpec tensor\n    def encode_tensorimage(self, x:TensorImage): \n        return (x-self.mean) / self.std\n    def encode_audiospec(self, x:AudioSpectrogram): \n        return (x-self.mean) / self.std\n    Normalize.encodes = TypeDispatch([encode_tensorimage, encode_audiospec])\n    \n    learner = cnn_learner(\n        dls, resnet34, \n        pretrained=True,\n        normalize=True,\n        config=config_dict,\n        loss_func=loss_func,\n        metrics=[lwrap_metric],\n        cbs=cbs\n    )\n    \nelif NORMALIZER == Normalizer.SAMPLE_STATS.value:\n    learner = cnn_learner(\n        dls, resnet34, \n        pretrained=True,\n        normalize=False,\n        config=config_dict,\n        loss_func=loss_func,\n        metrics=[lwrap_metric],\n        cbs=cbs\n    )\n\nelse:\n    raise NotImplementedError\n\nprint('Learner is set')","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:38:10.342849Z","iopub.execute_input":"2021-05-24T21:38:10.343183Z","iopub.status.idle":"2021-05-24T21:38:14.790562Z","shell.execute_reply.started":"2021-05-24T21:38:10.343152Z","shell.execute_reply":"2021-05-24T21:38:14.789547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (NORMALIZER == Normalizer.IMAGENET_STATS.value):\n    if CHANNEL == (Channel.SINGLE.value):\n        # make sure normalizer can handle AudioSpec tensor\n        learner.dls.after_batch[-1].mean = learner.dls.after_batch[-1].mean[:, 0:1, :, :]\n        learner.dls.after_batch[-1].std = learner.dls.after_batch[-1].std[:, 0:1, :, :]    \n        # assert normalizer stats are 1 channel\n        assert learner.dls.after_batch[-1].mean.shape[1] == 1\n        assert learner.dls.train.after_batch[-1].mean.shape[1] == 1\n        \n    # add Normalize transform into valid (bug fix in latest release)\n    if CHANNEL == (Channel.SINGLE.value):\n        mean=learner.dls.after_batch[-1].mean.cpu().numpy().flatten()[0]\n        std=learner.dls.after_batch[-1].std.cpu().numpy().flatten()[0]\n    elif CHANNEL == (Channel.THREE.value):\n        mean=learner.dls.after_batch[-1].mean.cpu().numpy().flatten()\n        std=learner.dls.after_batch[-1].std.cpu().numpy().flatten()\n    else:\n        raise NotImplementedError\n        \n    learner.dls.valid.after_batch.add(\n        Normalize.from_stats(mean=mean, std=std),\n        'after_batch'\n    )\n\nprint(f'Train batch_tfms: \\n{learner.dls.train.after_batch}')\nprint(f'Valid batch_tfms: \\n{learner.dls.valid.after_batch}')","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:38:55.320281Z","iopub.execute_input":"2021-05-24T21:38:55.320730Z","iopub.status.idle":"2021-05-24T21:38:55.346861Z","shell.execute_reply.started":"2021-05-24T21:38:55.320688Z","shell.execute_reply":"2021-05-24T21:38:55.345914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner.loss_func","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:38:57.071780Z","iopub.execute_input":"2021-05-24T21:38:57.072115Z","iopub.status.idle":"2021-05-24T21:38:57.077199Z","shell.execute_reply.started":"2021-05-24T21:38:57.072082Z","shell.execute_reply":"2021-05-24T21:38:57.076396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner.show_training_loop()","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:38:57.078708Z","iopub.execute_input":"2021-05-24T21:38:57.079243Z","iopub.status.idle":"2021-05-24T21:38:57.096479Z","shell.execute_reply.started":"2021-05-24T21:38:57.079196Z","shell.execute_reply":"2021-05-24T21:38:57.095636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"b = dls.one_batch()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3b. Finding Optimal Learning Rate","metadata":{}},{"cell_type":"code","source":"#learner.lr_find()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#learner.fine_tune(epochs=1, base_lr=0.033)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3c. Start Training\nIssue Tracker:\n- Low GPU utility (0%), and very high CPU utility","metadata":{}},{"cell_type":"code","source":"cbs = [SaveModelCallback(monitor=MONITOR_METRIC)]\nif not DISABLE_WB:\n    # start up W&B run\n    user_secrets = UserSecretsClient()\n    wandb_api = user_secrets.get_secret(\"wandb_key\")\n    wandb.login(key=wandb_api)\n    wandb.init(**WB_CONFIG)\n    config = wandb.config\n    # additionally log the mel spectrogram transform config\n    config.audio_to_spec = AudioConfig.BasicMelSpectrogram(**MELSPEC_CONFIG).__repr__()\n    cbs += [WandbCallback()]\n    print('Completed setup for W&B run')\n\n\nprint('Start training model')\nlearner.fine_tune(\n    epochs=UNFREEZE_EPOCH, base_lr=LR, \n    freeze_epochs=FREEZE_EPOCH, cbs=cbs\n)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T21:39:20.889871Z","iopub.execute_input":"2021-05-24T21:39:20.890225Z","iopub.status.idle":"2021-05-24T21:40:39.168032Z","shell.execute_reply.started":"2021-05-24T21:39:20.890196Z","shell.execute_reply":"2021-05-24T21:40:39.164449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner.recorder.plot_loss()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner.save('model_last_epoch')\nprint('Model checkpoint saved')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### How to de-register a callback?\n- Learner.remove_cbs: arg is the `Callback` class, NOT its instance\n- Details\n    - remove attribute `name` from the callback\n    - set its attribute `learn` = None\n    - remove from `Learner.cbs` list (calling `cbs.remove`, inherited from `list`)","metadata":{}},{"cell_type":"code","source":"#learner.show_training_loop();\ntry:\n    learner.remove_cbs(FetchPredsCallback)\nexcept:\n    print('Failed to remove FetchPredsCallback, probably its absent')\nlearner.cbs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4. Prepare Submission","metadata":{}},{"cell_type":"code","source":"def sample_one_subclips(row, i):    \n    length = int(RANDOM_INTERVAL * SR)\n    total_length = int(60 * SR)\n    # Last segment going from the end\n    if (i + 1) * length > total_length:\n        _frame_offset = total_length - length\n    else:\n        _frame_offset = i * length\n        \n    audio_fn = test_path.resolve()/f'{row.recording_id}.flac'\n    audio_subclip = AudioTensor.create(audio_fn, \n                                        frame_offset=_frame_offset, \n                                        num_frames=length)\n    return audio_subclip","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(DATA_DIR/'sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.train_ds.tls[0].tfms","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.valid_ds.tls[0].tfms = Pipeline(partial(sample_one_subclips, i=0))\nsample_test_dl = dls.test_dl(test_df, with_labels=False)\nprint(sample_test_dl.after_batch)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subclips_preds = []\nsegment_n = ceil((60*SR)/(RANDOM_INTERVAL*SR))\n\nfor crt_i in range(segment_n):\n    print(f'Running prediction on subclip {crt_i} for all test samples...')\n    dls.valid_ds.tls[0].tfms = Pipeline(partial(sample_one_subclips, i=crt_i))\n    crt_test_dl = dls.test_dl(test_df, with_labels=False)\n    # predict on a subclip for all test samples\n    _preds = learner.get_preds(dl=crt_test_dl)\n    \n    # softmax makes output statistics of each class dependent\n    # softmax could distort the order if u take max across all subclips\n    if IS_MULTILABEL:\n        _preds = _preds[0]\n    else:\n        _preds = torch.nn.Softmax(dim=-1)(_preds[0])\n        \n    subclips_preds.append(_preds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sanity check if normalizer is present in test dataloader\ncrt_test_dl.after_batch","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_preds = torch.stack(subclips_preds, dim=1)\nfinal_preds = all_preds.max(dim=1)[0]\n\ntest_df.iloc[:, 1:] = final_preds\ntest_df.to_csv('my_submission.csv', index=False)\nprint('Submission CSV written')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission_df = pd.read_csv(DATA_DIR/ 'sample_submission.csv')\n# submission_df[\"recording_id\"] = submission_df[\"recording_id\"].map(lambda x: \"test/\"+x)\n# submission_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Easily create test dataloader and get the predictions\n# test_dl = dls.test_dl(submission_df)\n# preds = learner.get_preds(dl = test_dl)\n# preds[0].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Copy the predictions into the submission dataframe\n# submission_df.iloc[:, 1:] = preds[0]\n\n# # It's ready to submit\n# submission_df[\"recording_id\"] = submission_df[\"recording_id\"].map(lambda x: x.split(\"/\")[1])\n\n# submission_df.to_csv('my_submission.csv', index=False)\n# print('Submission file written')\n# submission_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}