{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook\nimport IPython\nimport IPython.display\nimport PIL\nimport pickle\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def _one_sample_positive_class_precisions(scores, truth):\n    \"\"\"Calculate precisions for each true class for a single sample.\n\n    Args:\n      scores: np.array of (num_classes,) giving the individual classifier scores.\n      truth: np.array of (num_classes,) bools indicating which classes are true.\n\n    Returns:\n      pos_class_indices: np.array of indices of the true classes for this sample.\n      pos_class_precisions: np.array of precisions corresponding to each of those\n        classes.\n    \"\"\"\n    num_classes = scores.shape[0]\n    pos_class_indices = np.flatnonzero(truth > 0)\n    # Only calculate precisions if there are some true classes.\n    if not len(pos_class_indices):\n        return pos_class_indices, np.zeros(0)\n    # Retrieval list of classes for this sample.\n    retrieved_classes = np.argsort(scores)[::-1]\n    # class_rankings[top_scoring_class_index] == 0 etc.\n    class_rankings = np.zeros(num_classes, dtype=np.int)\n    class_rankings[retrieved_classes] = range(num_classes)\n    # Which of these is a true label?\n    retrieved_class_true = np.zeros(num_classes, dtype=np.bool)\n    retrieved_class_true[class_rankings[pos_class_indices]] = True\n    # Num hits for every truncated retrieval list.\n    retrieved_cumulative_hits = np.cumsum(retrieved_class_true)\n    # Precision of retrieval list truncated at each hit, in order of pos_labels.\n    precision_at_hits = (\n            retrieved_cumulative_hits[class_rankings[pos_class_indices]] /\n            (1 + class_rankings[pos_class_indices].astype(np.float)))\n    return pos_class_indices, precision_at_hits\n\n\ndef calculate_per_class_lwlrap(truth, scores):\n    \"\"\"Calculate label-weighted label-ranking average precision.\n\n    Arguments:\n      truth: np.array of (num_samples, num_classes) giving boolean ground-truth\n        of presence of that class in that sample.\n      scores: np.array of (num_samples, num_classes) giving the classifier-under-\n        test's real-valued score for each class for each sample.\n\n    Returns:\n      per_class_lwlrap: np.array of (num_classes,) giving the lwlrap for each\n        class.\n      weight_per_class: np.array of (num_classes,) giving the prior of each\n        class within the truth labels.  Then the overall unbalanced lwlrap is\n        simply np.sum(per_class_lwlrap * weight_per_class)\n    \"\"\"\n    assert truth.shape == scores.shape\n    num_samples, num_classes = scores.shape\n    # Space to store a distinct precision value for each class on each sample.\n    # Only the classes that are true for each sample will be filled in.\n    precisions_for_samples_by_classes = np.zeros((num_samples, num_classes))\n    for sample_num in range(num_samples):\n        pos_class_indices, precision_at_hits = (\n            _one_sample_positive_class_precisions(scores[sample_num, :],\n                                                  truth[sample_num, :]))\n        precisions_for_samples_by_classes[sample_num, pos_class_indices] = (\n            precision_at_hits)\n    labels_per_class = np.sum(truth > 0, axis=0)\n    weight_per_class = labels_per_class / float(np.sum(labels_per_class))\n    # Form average of each column, i.e. all the precisions assigned to labels in\n    # a particular class.\n    per_class_lwlrap = (np.sum(precisions_for_samples_by_classes, axis=0) /\n                        np.maximum(1, labels_per_class))\n    # overall_lwlrap = simple average of all the actual per-class, per-sample precisions\n    #                = np.sum(precisions_for_samples_by_classes) / np.sum(precisions_for_samples_by_classes > 0)\n    #           also = weighted mean of per-class lwlraps, weighted by class label prior across samples\n    #                = np.sum(per_class_lwlrap * weight_per_class)\n    return per_class_lwlrap, weight_per_class","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## File/folder definitions\n\n- `df` will handle training data.\n- `test_df` will handle test data."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"DATA = Path('../input/freesound-audio-tagging-2019')\nPREPROCESSED = Path('../input/fat19-dataprepare')\nWORK = Path('work')\nPath(WORK).mkdir(exist_ok=True, parents=True)\n\nPKL_TRN_CURATED = PREPROCESSED/'mels_train_curated.pkl'\nPKL_TRN_NOISY_BEST50S = PREPROCESSED/'mels_trn_noisy_best50s.pkl'\n\ntrn_curated = pickle.load(open(PKL_TRN_CURATED, 'rb'))\ntrn_noisy_best50s = pickle.load(open(PKL_TRN_NOISY_BEST50S, 'rb'))\n\nX_curated_data = trn_curated['data']\nX_noisy_best50s_data = trn_noisy_best50s['data']\n\ntrn_curated_df = trn_curated.drop(['data'],axis=1)\ntrn_noisy50s_df = trn_noisy_best50s.drop(['data'],axis=1)\n\ndf = pd.concat([trn_curated_df, trn_noisy50s_df], ignore_index=True, sort=True)\nX_train = list(X_curated_data) + list(X_noisy_best50s_data)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Custom `open_image` for fast.ai library to load data from memory\n\n- Important note: Random cropping 1 sec, this is working like augmentation."},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai import *\nfrom fastai.vision import *\nfrom fastai.vision.data import *\nfrom fastai.callbacks import *\nimport random\n\nCUR_X_FILES, CUR_X = list(df.fname.values), X_train\n\ndef open_fat2019_image(fn, convert_mode, after_open)->Image:\n    # open\n    idx = CUR_X_FILES.index(fn.split('/')[-1])\n    x = PIL.Image.fromarray(CUR_X[idx])\n    # crop 1sec\n    time_dim, base_dim = x.size\n    crop_x = random.randint(0, time_dim - base_dim)\n    x = x.crop([crop_x, 0, crop_x+base_dim, base_dim])    \n    # standardize\n    return Image(pil2tensor(x, np.float32).div_(255))\n\nvision.data.open_image = open_fat2019_image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Follow multi-label classification\n\n- Almost following fast.ai course: https://nbviewer.jupyter.org/github/fastai/course-v3/blob/master/nbs/dl1/lesson3-planet.ipynb\n- But `pretrained=False`"},{"metadata":{"trusted":true},"cell_type":"code","source":"def lwlrap(y_pred,y_true):\n    score, weight = calculate_per_class_lwlrap(y_true.cpu().numpy(), y_pred.cpu().numpy())\n    lwlrap = (score * weight).sum()\n    return torch.from_numpy(np.array(lwlrap))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MixUpCallback(LearnerCallback):\n    \"Callback that creates the mixed-up input and target.\"\n    def __init__(self, learn:Learner, alpha:float=0.4, stack_x:bool=False, stack_y:bool=True):\n        super().__init__(learn)\n        self.alpha,self.stack_x,self.stack_y = alpha,stack_x,stack_y\n    \n    def on_train_begin(self, **kwargs):\n        if self.stack_y: self.learn.loss_func = MixUpLoss(self.learn.loss_func)\n        \n    def on_batch_begin(self, last_input, last_target, train, **kwargs):\n        \"Applies mixup to `last_input` and `last_target` if `train`.\"\n        if not train: return\n        lambd = np.random.beta(self.alpha, self.alpha, last_target.size(0))\n        lambd = np.concatenate([lambd[:,None], 1-lambd[:,None]], 1).max(1)\n        lambd = last_input.new(lambd)\n        shuffle = torch.randperm(last_target.size(0)).to(last_input.device)\n        x1, y1 = last_input[shuffle], last_target[shuffle]\n        if self.stack_x:\n            new_input = [last_input, last_input[shuffle], lambd]\n        else: \n            new_input = (last_input * lambd.view(lambd.size(0),1,1,1) + x1 * (1-lambd).view(lambd.size(0),1,1,1))\n        if self.stack_y:\n            new_target = torch.cat([last_target[:,None].float(), y1[:,None].float(), lambd[:,None].float()], 1)\n        else:\n            if len(last_target.shape) == 2:\n                lambd = lambd.unsqueeze(1).float()\n            new_target = last_target.float() * lambd + y1.float() * (1-lambd)\n        return {'last_input': new_input, 'last_target': new_target}  \n    \n    def on_train_end(self, **kwargs):\n        if self.stack_y: self.learn.loss_func = self.learn.loss_func.get_old()\n        \n\nclass MixUpLoss(nn.Module):\n    \"Adapt the loss function `crit` to go with mixup.\"\n    \n    def __init__(self, crit, reduction='mean'):\n        super().__init__()\n        if hasattr(crit, 'reduction'): \n            self.crit = crit\n            self.old_red = crit.reduction\n            setattr(self.crit, 'reduction', 'none')\n        else: \n            self.crit = partial(crit, reduction='none')\n            self.old_crit = crit\n        self.reduction = reduction\n        \n    def forward(self, output, target):\n        if len(target.size()) == 2:\n            loss1, loss2 = self.crit(output,target[:,0].long()), self.crit(output,target[:,1].long())\n            d = (loss1 * target[:,2] + loss2 * (1-target[:,2])).mean()\n        else:  d = self.crit(output, target)\n        if self.reduction == 'mean': return d.mean()\n        elif self.reduction == 'sum':            return d.sum()\n        return d\n    \n    def get_old(self):\n        if hasattr(self, 'old_crit'):  return self.old_crit\n        elif hasattr(self, 'old_red'): \n            setattr(self.crit, 'reduction', self.old_red)\n            return self.crit\n\ndef mixup(learn:Learner, alpha:float=0.4, stack_x:bool=False, stack_y:bool=True) -> Learner:\n    \"Add mixup https://arxiv.org/abs/1710.09412 to `learn`.\"\n    learn.callback_fns.append(partial(MixUpCallback, alpha=alpha, stack_x=stack_x, stack_y=stack_y))\n    return learn\nLearner.mixup = mixup\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ConvBlock(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n        \n        self.conv1 = nn.Sequential(\n            nn.Conv2d(in_channels, out_channels, 3, 1, 1),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(),\n        )\n        self._init_weights()\n        \n    def _init_weights(self):\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight)\n                if m.bias is not None:\n                    nn.init.zeros_(m.bias)\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.constant_(m.weight, 1)\n                nn.init.zeros_(m.bias)\n        \n    def forward(self, x):\n        x = self.conv1(x)\n        x = F.avg_pool2d(x, 2)\n        return x\n    \nclass Classifier(nn.Module):\n    def __init__(self, num_classes=1000): # <======== modificaition to comply fast.ai\n        super().__init__()\n        \n        self.conv = nn.Sequential(\n            ConvBlock(in_channels=3, out_channels=64),\n            ConvBlock(in_channels=64, out_channels=128),\n            ConvBlock(in_channels=128, out_channels=256),\n            ConvBlock(in_channels=256, out_channels=512),\n        )\n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        self.fc = nn.Sequential(\n            nn.Dropout(0.2),\n            nn.Linear(512, 128),\n            nn.PReLU(),\n            nn.BatchNorm1d(128),\n            nn.Dropout(0.1),\n            nn.Linear(128, num_classes),\n        )\n\n    def forward(self, x):\n        x = self.conv(x)\n        x = self.avgpool(x)\n        x = self.fc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\n\ndef borrowed_model(pretrained=False, **kwargs):\n    return Classifier(**kwargs)\n\nn_fold = 2\nseed = 20\nfolds = KFold(n_splits=n_fold, shuffle=True, random_state=seed)\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(df['fname'],df['labels'])):\n    tfms = get_transforms(do_flip=False, max_rotate=0, max_lighting=0.1, max_zoom=0, max_warp=0.)\n    src = (ImageList.from_df(df, WORK)\n       .split_by_idxs(trn_idx, val_idx)\n       .label_from_df(label_delim=',')\n    )\n    data = (src.transform(tfms, size=128)\n            .databunch(bs=128).normalize(imagenet_stats)\n    )\n    learn = cnn_learner(data, borrowed_model, pretrained=False, metrics=[lwlrap]).mixup(stack_y=False)\n    learn.unfreeze()\n    learn.fit_one_cycle(2, 2e-2)\n    learn.fit_one_cycle(3, 2e-3,callbacks=[SaveModelCallback(learn, every='improvement', monitor='lwlrap', name='best')])\n    learn.export('FAT2019_Simple'+str(fold_)+'.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.export()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test prediction and making submission file simple\n- Switch to test data.\n- Overwrite results to sample submission; simple way to prepare submission file."},{"metadata":{"trusted":true},"cell_type":"markdown","source":"from fastai.core import *\nfrom fastai.basic_data import *\nfrom fastai.basic_train import *\nfrom fastai.torch_core import *\ndef _tta_only(learn:Learner, ds_type:DatasetType=DatasetType.Valid, num_pred:int=5) -> Iterator[List[Tensor]]:\n    \"Computes the outputs for several augmented inputs for TTA\"\n    dl = learn.dl(ds_type)\n    ds = dl.dataset\n    old = ds.tfms\n    aug_tfms = [o for o in learn.data.train_ds.tfms]\n    try:\n        pbar = master_bar(range(num_pred))\n        for i in pbar:\n            ds.tfms = aug_tfms\n            yield get_preds(learn.model, dl, pbar=pbar)[0]\n    finally: ds.tfms = old\n\nLearner.tta_only = _tta_only\n\ndef _TTA(learn:Learner, beta:float=0, ds_type:DatasetType=DatasetType.Valid, num_pred:int=5, with_loss:bool=False) -> Tensors:\n    \"Applies TTA to predict on `ds_type` dataset.\"\n    preds,y = learn.get_preds(ds_type)\n    all_preds = list(learn.tta_only(ds_type=ds_type, num_pred=num_pred))\n    avg_preds = torch.stack(all_preds).mean(0)\n    if beta is None: return preds,avg_preds,y\n    else:            \n        final_preds = preds*beta + avg_preds*(1-beta)\n        if with_loss: \n            with NoneReduceOnCPU(learn.loss_func) as lf: loss = lf(final_preds, y)\n            return final_preds, y, loss\n        return final_preds, y\n\nLearner.TTA = _TTA"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"del X_train\nX_test = pickle.load(open(MELS_TEST, 'rb'))\nCUR_X_FILES, CUR_X = list(test_df.fname.values), X_test\n\n\ntest = ImageList.from_csv(WORK, Path('..')/CSV_SUBMISSION, folder='test')\n\npredictions = torch.from_numpy(np.zeros((1120,80))).float()\nprint(predictions.shape)\nnum_folds = 3\nfor i in range(num_folds):\n    learn = load_learner(WORK, file='../input/simple-model-fat2019/FAT2019_Simple'+str(i)+'.pkl',test = test)\n    preds, _ = learn.TTA(ds_type=DatasetType.Test)\n    print(preds.shape)\n    predictions = predictions + preds\npredictions = predictions/num_folds"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"test_df[learn.data.classes] = predictions\ntest_df.to_csv('submission.csv', index=False)\ntest_df.head()"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}