{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook\nimport IPython\nimport IPython.display\nimport PIL\nimport time\nimport sklearn.metrics\nimport pickle\nimport random\nimport cv2\nimport librosa","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.parallel\nimport torch.backends.cudnn as cudnn\nimport torch.optim as optim\nimport torch.utils.data\nimport torchvision.transforms as transforms\nimport torchvision.datasets as datasets\nfrom torch.utils.data.dataset import Dataset\nfrom torch.utils.data import DataLoader\nimport torch.nn.functional as F\nfrom torch.optim.lr_scheduler import _LRScheduler\nimport torchvision.models as models\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_FOLD = 5\nNUM_CLASS = 80\nSEED = 42\nNUM_EPOCH = 64*7\nNUM_CYCLE = 64\nBATCH_SIZE = 32\nBATCH_SIZE_VALID = 32\nDO_TRAIN = True\nDO_EVALUATE = True\nDEBUG = False\nEXPERIMENT = False\nEXPERIMENT_FOLD = 1\nDO_FOLD = [5]\nNOISY_LABEL_RATE = 0.\nC_SEMI = 20 # 75\nTEMPERATURE = 2\nNORM = 'Rel'\nSLICE_RATE = 0.25\nDO_PSEUDO_NOISY = False\nNOISY_LAST = \"SIGMOID\"\nTRAIN_LAST = \"SIGMOID\"\nCONSISTENCY_LOSS = 'MSE'\nC_NOISY = 1\nNUM_MIX = 1\nLR = [1e-3, 1e-6]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### seed固定\n# torch.manual_seed(SEED)\n# random.seed(SEED)\n# np.random.seed(SEED)\n# torch.manual_seed(SEED)\n# torch.cuda.manual_seed(SEED)\n\n# cudnn speed up\ncudnn.benchmark = True\n# cudnn.benchmark = False  # CUDNN高速化OFF\n# torch.backends.cudnn.deterministic = True  # 再現性確保","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/freesound-audio-tagging-2019/train_curated.csv\")\ndf_test = pd.read_csv(\"../input/freesound-audio-tagging-2019/sample_submission.csv\")\ndf_noise = pd.read_csv(\"../input/freesound-audio-tagging-2019/train_noisy.csv\")\nlabels = df_test.columns[1:].tolist()\n\nfor label in labels:\n    df_train[label] = df_train['labels'].apply(lambda x: label in x)\n    df_noise[label] = df_noise['labels'].apply(lambda x: label in x)\n    \ndf_train['num_labels'] = df_train[labels].values.sum(axis=1)\ndf_noise['num_labels'] = df_noise[labels].values.sum(axis=1)\n\n# df_train[labels] = df_train[labels].values / df_train['num_labels'].values[:,np.newaxis]\n# if NOISY_LAST=='SOFTMAX':\n#     df_noise[labels] = df_noise[labels].values / df_noise['num_labels'].values[:,np.newaxis]\n\ndf_train['path'] = \"../input/mel128v3/train/\" + df_train['fname']\ndf_test['path'] = \"../input/mel128v3/test/\" + df_train['fname']\ndf_noise['path'] = \"../input/mel128v3n/noise/\" + df_noise['fname']\n\nprint(df_train.shape, df_noise.shape, df_test.shape)\ndf_train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['weight'] = 1\ndf_noise['weight'] = len(df_train)/len(df_noise)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mean_list = np.zeros(len(df_train), np.float32)\nstd_list = np.zeros(len(df_train), np.float32)\nfor i in range(len(df_train)):\n    if i%1000==0:\n        print(\"{}/{}\".format(i+1, len(df_train)))\n    path = \"../input/mel128v3/train/{}.npy\".format(df_train['fname'][i][:-4])\n    mel = np.load(path)\n    if NORM=='Abs2':\n        mel = librosa.power_to_db(mel, top_db=None, amin=1e-5)\n    else:\n        mel = librosa.power_to_db(mel)\n    mean_list[i] = mel.mean()\n    std_list[i] = mel.std()\nmel_mean = np.mean(mean_list)\nmel_std = np.mean(std_list)\nprint(mel_mean, mel_std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tmp = np.load(\"../input/freesound-submission/resnet34multi512_noisypred/preds_mel_noise.npy\")\n# print(tmp.shape)\n# tmp = tmp[0,0,0]\ntmp1 = np.load(\"../input/freesound-submission/preds_mel_noise_c1234.npy\")\ntmp2 = np.load(\"../input/freesound-submission/preds_mel_noise_c5678.npy\")\ntmp = np.concatenate([tmp1, tmp2], axis=1)\n# tmp = np.load(\"../input/freesound-submission/preds_mel_noise_c5678.npy\")\nprint(tmp.shape)\ntmp = tmp[DO_FOLD[0]-1].mean(axis=(0,1))\n# tmp = tmp.mean(axis=(0,1,2))\n\n\ntmp = tmp**TEMPERATURE\ntmp = tmp / tmp.sum(axis=1)[:,np.newaxis]\ndf_noise_pseudo = df_noise.copy()\ndf_noise_pseudo[labels] = df_noise[labels].values * NOISY_LABEL_RATE +  tmp * (1-NOISY_LABEL_RATE)\ndf_noise_pseudo.head()\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp = np.load(\"../input/freesound-submission/resnet34multi512_noisypred/preds_mel_valid_noisy.npy\").mean(axis=(0,1,2))\ntmp = tmp**TEMPERATURE\ntmp = tmp / tmp.sum(axis=1)[:,np.newaxis]\ndf_train_pseudo = df_train.copy()\ndf_train_pseudo[labels] = df_train_pseudo[labels].values * NOISY_LABEL_RATE +  tmp * (1-NOISY_LABEL_RATE)\ndf_train_pseudo.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def sharpen(pred, T):\n    pred = pred**T\n    pred = pred / pred.sum()\n    return pred\n\nimport IPython.display as ipd  # To play sound in the notebook\nidx = np.random.randint(0,len(df_noise_pseudo))\n# idx = 0\nfname = '../input/freesound-audio-tagging-2019/train_noisy//{}'.format(df_noise_pseudo['fname'][idx])\ntmp1 = np.load(\"../input/freesound-submission/preds_mel_noise_c1234.npy\")\ntmp2 = np.load(\"../input/freesound-submission/preds_mel_noise_c5678.npy\")\ntmp = np.concatenate([tmp1, tmp2], axis=1)\nprint(tmp.shape)\ndf_tmp = df_noise.copy()\ndf_tmp[labels] = tmp.mean(axis=(0,1,2))\nlabel_idx = df_tmp[labels].values[idx]\nlabel_idx_sort = np.argsort(label_idx)[::-1]\npred = df_tmp[labels].values[idx]\npred_shapen1 = sharpen(pred, 1)\npred_shapen2 = sharpen(pred, 2)\n\nfor i in range(5):\n    label = labels[label_idx_sort[i]] + \" \"*(40-len(labels[label_idx_sort[i]]))\n    print(\"{}, {} {:.3f} | {:.3f} | {:.3f}\".format(\n        i, label, label_idx[label_idx_sort[i]], pred_shapen1[label_idx_sort[i]], pred_shapen2[label_idx_sort[i]]))\nprint(\"true label: {}\".format(df_noise_pseudo['labels'][idx]))\nipd.Audio(fname)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pretrainedmodels\nimport pretrainedmodels\nimport pretrainedmodels.utils\nclass ResNet(nn.Module):\n    def __init__(self, num_classes=2):\n        super(ResNet, self).__init__()\n\n        self.num_classes = num_classes\n        self.mode = 'train'\n\n        self.base_model = pretrainedmodels.__dict__['resnet34'](num_classes=num_classes, pretrained=None)\n\n        self.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3,\n                               bias=False)\n        self.bn1 = self.base_model.bn1\n        self.relu = self.base_model.relu\n        self.maxpool = self.base_model.maxpool\n        self.layer1 = self.base_model.layer1\n        self.layer2 = self.base_model.layer2\n        self.layer3 = self.base_model.layer3\n        self.layer4 = self.base_model.layer4\n        self.avgpool = nn.AdaptiveMaxPool2d((1, 1))\n#         self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        self.last_linear = nn.Linear(self.base_model.layer4[1].conv1.in_channels, num_classes)\n        self.last_linear = nn.Sequential(\n            nn.Linear(self.base_model.layer4[1].conv1.in_channels, 1024),\n            nn.ReLU(),\n            nn.Dropout(p=0.2),\n            nn.Linear(1024, 1024),\n            nn.ReLU(),\n            nn.Dropout(p=0.1),\n            nn.Linear(1024, NUM_CLASS),\n        )\n        self.last_linear2 = nn.Sequential(\n            nn.Linear(self.base_model.layer4[1].conv1.in_channels, 1024),\n            nn.ReLU(),\n            nn.Dropout(p=0.2),\n            nn.Linear(1024, 1024),\n            nn.ReLU(),\n            nn.Dropout(p=0.1),\n            nn.Linear(1024, NUM_CLASS),\n        )\n\n\n    def feature(self, input):\n        x0 = self.conv1(input)  #; print('layer conv1 ',x.size()) # [8, 64, 112, 112]\n        x0 = self.bn1(x0)\n        x0 = self.relu(x0)\n        x1 = self.maxpool(x0)\n        x1 = self.layer1(x1) #  ; print('layer 1 ',x.size()) # [8, 1024, 28, 28])\n        x2 = self.layer2(x1) #  ; print('layer 2 ',x.size()) # [8, 1024, 28, 28])\n        x3 = self.layer3(x2) #  ; print('layer 3 ',x.size()) # [8, 1024, 28, 28])\n        # x4 = self.layer4(x3) #  ; print('layer 4 ',x.size()) # [8, 2048, 14, 14])\n        x = self.avgpool(x3) #  ; print('layer 4 ',x.size()) # [8, 2048, 14, 14])\n        return x\n\n    def forward(self, input):\n        bs, ch, h, w = input.size()\n        x0 = self.conv1(input)  #; print('layer conv1 ',x.size()) # [8, 64, 112, 112]\n        x0 = self.bn1(x0)\n        x0 = self.relu(x0)\n        x1 = self.maxpool(x0)\n        x1 = self.layer1(x1) #  ; print('layer 1 ',x.size()) # [8, 1024, 28, 28])\n        x2 = self.layer2(x1) #  ; print('layer 2 ',x.size()) # [8, 1024, 28, 28])\n        x3 = self.layer3(x2) #  ; print('layer 3 ',x.size()) # [8, 1024, 28, 28])\n        x4 = self.layer4(x3) #  ; print('layer 4 ',x.size()) # [8, 2048, 14, 14])\n        x = self.avgpool(x4).view(bs, -1) #  ; print('layer 4 ',x.size()) # [8, 2048, 14, 14])\n        x = self.last_linear(x) #  ; print('layer 4 ',x.size()) # [8, 2048, 14, 14])\n\n        return x\n    \n    def noisy(self, input):\n        bs, ch, h, w = input.size()\n        x0 = self.conv1(input)  #; print('layer conv1 ',x.size()) # [8, 64, 112, 112]\n        x0 = self.bn1(x0)\n        x0 = self.relu(x0)\n        x1 = self.maxpool(x0)\n        x1 = self.layer1(x1) #  ; print('layer 1 ',x.size()) # [8, 1024, 28, 28])\n        x2 = self.layer2(x1) #  ; print('layer 2 ',x.size()) # [8, 1024, 28, 28])\n        x3 = self.layer3(x2) #  ; print('layer 3 ',x.size()) # [8, 1024, 28, 28])\n        x4 = self.layer4(x3) #  ; print('layer 4 ',x.size()) # [8, 2048, 14, 14])\n        x = self.avgpool(x4).view(bs, -1) #  ; print('layer 4 ',x.size()) # [8, 2048, 14, 14])\n        x = self.last_linear2(x) #  ; print('layer 4 ',x.size()) # [8, 2048, 14, 14])\n\n        return x\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MfccDataset(Dataset):\n    \"\"\"Dataset wrapping images and target labels for Kaggle - Planet Amazon from Space competition.\n\n    Arguments:\n        A CSV file path\n        Path to image folder\n        Extension of images\n        PIL transforms\n    \"\"\"\n\n    def __init__(self, df, load_dir, slice=-1, mixup=False, \n                 cutout=False, cutout_h=False, cutout_w=False,\n                 gain=False, resize=False,\n                flip=False,\n                warp=False,\n                highpass=False,\n                white=False,\n                 slice_mode='original',\n                ):\n        self.X_train = df['path']\n        self.y_train = df[labels].values\n        self.slice = slice\n        self.mixup = mixup\n        self.cutout = cutout\n        self.cutout_h = cutout_h\n        self.cutout_w = cutout_w\n        self.gain = gain\n        self.resize = resize\n        self.highpass = highpass\n        self.flip = flip\n        self.warp = warp\n        self.load_dir = load_dir\n        # print(self.y_train.shape)\n        self.white = white\n        white_noise = (np.random.rand(44100*120)-0.5).astype(np.float32)/100\n        self.white_noise = librosa.feature.melspectrogram(\n                white_noise,\n                sr=44100,\n                n_mels=128,\n                hop_length=347*1, # 1sec -> 128\n                n_fft=128*20,\n                fmin=20,\n                fmax=44100//2,\n            ).astype(np.float32)\n        self.slice_mode = slice_mode\n\n    def do_rate_slice(self, img, min_rate=SLICE_RATE):\n        len_img = img.shape[1]\n        img_new = np.zeros([img.shape[0], self.slice], np.float32)\n        rate = np.random.random() * (1-min_rate) + min_rate\n        if np.random.random()<0.5: rate = 1\n            \n        if img.shape[1]<=self.slice:\n            len_slice = int(img.shape[1]*rate)\n            if img.shape[1]-len_slice==0:\n                shift_slice = 0\n            else:\n                shift_slice = np.random.randint(0, img.shape[1]-len_slice)\n            img = img[:, shift_slice:shift_slice+len_slice]\n            if self.slice - len_slice==0:\n                shift = 0\n            else:\n                shift = np.random.randint(0, self.slice - len_slice)\n            img_new[:, shift:shift + len_slice] =img\n        else:\n            shift = np.random.randint(0, img.shape[1]-self.slice)\n            img_new = img[:, shift:shift+self.slice]\n            len_slice = int(self.slice*rate)\n            if self.slice-len_slice==0:\n                shift_slice = 0\n            else:\n                shift_slice = np.random.randint(0, self.slice-len_slice)\n            img_new[:shift_slice] = 0\n            img_new[shift_slice+len_slice:] = 0\n        return img_new\n\n    def do_random_slice(self, img):\n        img_new = np.zeros([img.shape[0], self.slice], np.float32)\n        if img.shape[1]<self.slice:\n            shift = np.random.randint(0, self.slice - img.shape[1])\n            img_new[:, shift:shift + img.shape[1]] =img\n        elif img.shape[1]==self.slice:\n            img_new = img\n        else:\n            shift = np.random.randint(0, img.shape[1]-self.slice)\n            img_new = img[:, shift:shift+self.slice]\n        return img_new\n\n    def do_original_slice(self, img):\n        return img\n        \n        \n    def do_slice(self, img):\n        if self.slice_mode=='random':\n            return self.do_random_slice(img)\n        elif self.slice_mode=='rate':\n            return self.do_rate_slice(img)\n        elif self.slice_mode=='original':\n            return self.do_original_slice(img)\n    \n    def do_highpass(self, img):\n        coord = np.random.randint(0, img.shape[0])\n        img[coord:] = 0\n    \n    def do_mixup(self, img, label, alpha=1.):\n        idx = np.random.randint(0,len(self.X_train))\n        img2 = np.load(\"{}.npy\".format(self.X_train[idx][:-4]))\n        img2 = self.do_slice(img2)\n        \n        label2 = self.y_train[idx].astype(np.float32)\n\n        rate = np.random.beta(alpha,alpha)\n        img = img*rate + img2*(1-rate)\n        label = label*rate + label2*(1-rate)\n        return img, label\n    \n    def do_white(self, img):\n        shift = np.random.randint(0, self.white_noise.shape[1] - self.slice)\n        white_noise_slice = self.white_noise[:, shift:shift + self.slice] * np.random.rand()\n        img += white_noise_slice\n        return img\n        \n        \n        \n    def do_flip(self, img):\n        return img[:,::-1]\n    \n    \n    def do_cutout_h(self, img, max = 32):\n        coord = np.random.randint(0, img.shape[0])\n        width = np.random.randint(8, max)\n        cut = np.array([coord-width, coord+width])\n        cut = np.clip(cut, 0, img.shape[0])\n        img[cut[0]:cut[1]] = 0\n        return img\n    \n    \n    def do_cutout_w(self, img, max = 32):\n        coord = np.random.randint(0, img.shape[1])\n        width = np.random.randint(8, max)\n        cut = np.array([coord-width, coord+width])\n        cut = np.clip(cut, 0, img.shape[1])\n        img[:,cut[0]:cut[1]] = 0\n        return img\n    \n    def do_highpass(self, img):\n        th = np.random.randint(0, img.shape[0])\n        img[th:] = 0\n        return img\n    \n    def cutout_bug(self, img):\n        coordx = np.sort(np.random.randint(0, self.slice,2))\n        coordy = np.sort(np.random.randint(0, 128, 2))\n        img[coordx[0]:coordx[1]] = 0\n        return img\n        \n    def do_resize(self, img, max=0.1):\n        rate = 1- max + np.random.random() * max * 2\n        img_tmp = cv2.resize(img, (int(self.slice*rate), img.shape[0], ))\n        if rate>1:\n            img_new = img_tmp[:,:img.shape[1]]\n        else:\n            img_new = np.zeros_like(img)\n            img_new[:,:img_tmp.shape[1]] = img_tmp\n        return img_new\n\n    \n    def do_gain(self, img, max=0.1):\n        rate = 1- max + np.random.random() * max * 2\n        return img * rate\n    \n    def do_warp(self, img, max=64):\n        left = np.random.randint(0, img.shape[1])\n        right = np.min([img.shape[1], left+np.random.randint(8, max)])\n        tmp = img[:,left:img.shape[1]-(right-left)]\n        img[:,left:right] = 0\n        img[:, right:] = tmp\n#         print(left, right, tmp.shape)\n        return img\n        \n    def __getitem__(self, index):\n        img = np.load(\"{}.npy\".format(self.X_train[index][:-4]))\n        img = self.do_slice(img)\n        label = self.y_train[index].astype(np.float32)\n            \n        for i in range(NUM_MIX):\n            if self.mixup and np.random.random()<0.5:\n                img, label = self.do_mixup(img, label)\n        if self.gain and np.random.random()<0.5:\n             img = self.do_gain(img)\n        if self.resize and np.random.random()<0.5:\n             img = self.do_resize(img)\n        if self.white and np.random.random()<0.5:\n             img = self.do_white(img)\n                \n                \n        if self.cutout and np.random.random()<0.5:\n            img = self.cutout_bug(img)\n        if self.cutout_h and np.random.random()<0.5:\n            img = self.do_cutout_h(img)\n        if self.cutout_w and np.random.random()<0.5:\n            img = self.do_cutout_w(img)\n        if self.warp and np.random.random()<0.5:\n            img = self.do_warp(img)\n        if self.flip and np.random.random()<0.5:\n            img = self.do_flip(img)\n        if self.highpass and np.random.random()<0.5:\n            img = self.do_highpass(img)\n            \n        if NORM=='Abs' or NORM=='Abs2':\n            img = librosa.power_to_db(img, top_db=None, amin=1e-5)\n            img = (img - mel_mean) / (mel_std+1e-7)\n        elif NORM=='Rel':\n            img = librosa.power_to_db(img)\n            img = (img - img.mean()) / (img.std()+1e-7)\n        img = img.reshape([1, img.shape[0], img.shape[1]])\n        \n        return img, label\n\n    def __len__(self):\n        return len(self.X_train.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from math import cos, pi\n\n\ndef cycle(iterable):\n    \"\"\"\n    dataloaderをiteratorに変換\n    :param iterable:\n    :return:\n    \"\"\"\n    while True:\n        for x in iterable:\n            yield x\n            \ndef _one_sample_positive_class_precisions(scores, truth):\n    \"\"\"Calculate precisions for each true class for a single sample.\n\n    Args:\n      scores: np.array of (num_classes,) giving the individual classifier scores.\n      truth: np.array of (num_classes,) bools indicating which classes are true.\n\n    Returns:\n      pos_class_indices: np.array of indices of the true classes for this sample.\n      pos_class_precisions: np.array of precisions corresponding to each of those\n        classes.\n    \"\"\"\n    num_classes = scores.shape[0]\n    pos_class_indices = np.flatnonzero(truth > 0)\n    # Only calculate precisions if there are some true classes.\n    if not len(pos_class_indices):\n        return pos_class_indices, np.zeros(0)\n    # Retrieval list of classes for this sample.\n    retrieved_classes = np.argsort(scores)[::-1]\n    # class_rankings[top_scoring_class_index] == 0 etc.\n    class_rankings = np.zeros(num_classes, dtype=np.int)\n    class_rankings[retrieved_classes] = range(num_classes)\n    # Which of these is a true label?\n    retrieved_class_true = np.zeros(num_classes, dtype=np.bool)\n    retrieved_class_true[class_rankings[pos_class_indices]] = True\n    # Num hits for every truncated retrieval list.\n    retrieved_cumulative_hits = np.cumsum(retrieved_class_true)\n    # Precision of retrieval list truncated at each hit, in order of pos_labels.\n    precision_at_hits = (\n            retrieved_cumulative_hits[class_rankings[pos_class_indices]] /\n            (1 + class_rankings[pos_class_indices].astype(np.float)))\n    return pos_class_indices, precision_at_hits\n\n\n# All-in-one calculation of per-class lwlrap.\n\ndef calculate_per_class_lwlrap(truth, scores):\n    \"\"\"Calculate label-weighted label-ranking average precision.\n\n    Arguments:\n      truth: np.array of (num_samples, num_classes) giving boolean ground-truth\n        of presence of that class in that sample.\n      scores: np.array of (num_samples, num_classes) giving the classifier-under-\n        test's real-valued score for each class for each sample.\n\n    Returns:\n      per_class_lwlrap: np.array of (num_classes,) giving the lwlrap for each\n        class.\n      weight_per_class: np.array of (num_classes,) giving the prior of each\n        class within the truth labels.  Then the overall unbalanced lwlrap is\n        simply np.sum(per_class_lwlrap * weight_per_class)\n    \"\"\"\n    assert truth.shape == scores.shape\n    num_samples, num_classes = scores.shape\n    # Space to store a distinct precision value for each class on each sample.\n    # Only the classes that are true for each sample will be filled in.\n    precisions_for_samples_by_classes = np.zeros((num_samples, num_classes))\n    for sample_num in range(num_samples):\n        pos_class_indices, precision_at_hits = (\n            _one_sample_positive_class_precisions(scores[sample_num, :],\n                                                  truth[sample_num, :]))\n        precisions_for_samples_by_classes[sample_num, pos_class_indices] = (\n            precision_at_hits)\n    labels_per_class = np.sum(truth > 0, axis=0)\n    weight_per_class = labels_per_class / float(np.sum(labels_per_class))\n    # Form average of each column, i.e. all the precisions assigned to labels in\n    # a particular class.\n    per_class_lwlrap = (np.sum(precisions_for_samples_by_classes, axis=0) /\n                        np.maximum(1, labels_per_class))\n    # overall_lwlrap = simple average of all the actual per-class, per-sample precisions\n    #                = np.sum(precisions_for_samples_by_classes) / np.sum(precisions_for_samples_by_classes > 0)\n    #           also = weighted mean of per-class lwlraps, weighted by class label prior across samples\n    #                = np.sum(per_class_lwlrap * weight_per_class)\n    return per_class_lwlrap, weight_per_class\n\n\nclass CosineLR(_LRScheduler):\n    \"\"\"SGD with cosine annealing.\n    \"\"\"\n\n    def __init__(self, optimizer, step_size_min=1e-5, t0=100, tmult=2, curr_epoch=-1, last_epoch=-1):\n        self.step_size_min = step_size_min\n        self.t0 = t0\n        self.tmult = tmult\n        self.epochs_since_restart = curr_epoch\n        super(CosineLR, self).__init__(optimizer, last_epoch)\n\n    def get_lr(self):\n        self.epochs_since_restart += 1\n\n        if self.epochs_since_restart > self.t0:\n            self.t0 *= self.tmult\n            self.epochs_since_restart = 0\n\n        lrs = [self.step_size_min + (\n                    0.5 * (base_lr - self.step_size_min) * (1 + cos(self.epochs_since_restart * pi / self.t0)))\n               for base_lr in self.base_lrs]\n\n        # print(lrs)\n\n        return lrs\n\n    \nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(train_loaders, model, optimizer, scheduler, epoch):\n    train_loader, noise_itr, semi_itr, semi_itr2 = train_loaders\n    ce_avr   = AverageMeter()\n    ce_noise_avr   = AverageMeter()\n    mse_semi_avr    = AverageMeter()\n    if DO_PSEUDO_NOISY:\n        mse_semi2_avr = AverageMeter()\n    else:\n        mse_semi2_avr = 0\n    \n    criterion_bce = nn.BCELoss().cuda()\n    criterion_mse = nn.MSELoss().cuda()\n#     criterion_bce    = nn.KLDivLoss(reduction='mean').cuda()\n    sigmoid = torch.nn.Sigmoid().cuda()\n    softmax = torch.nn.Softmax().cuda()\n    if NOISY_LAST=='SIGMOID':\n        act_noisy = sigmoid\n    elif NOISY_LAST=='SOFTMAX':\n        act_noisy = softmax\n    if TRAIN_LAST=='SIGMOID':\n        act_train = sigmoid\n    elif TRAIN_LAST=='SOFTMAX':\n        act_train = softmax\n    if CONSISTENCY_LOSS=='MSE':\n        criterion_consistency = criterion_mse\n    elif CONSISTENCY_LOSS=='CE':\n        criterion_consistency = criterion_bce\n\n    # switch to train mode\n    model.train()\n\n    starttime = time.time()\n    preds = np.zeros([0, NUM_CLASS], np.float32)\n    y_true = np.zeros([0, NUM_CLASS], np.float32)\n    preds_noise = np.zeros([0, NUM_CLASS], np.float32)\n    y_true_noise = np.zeros([0, NUM_CLASS], np.float32)\n    for i, (input, target) in enumerate(train_loader):\n        # prepare batches\n        input = input.cuda(async=True)\n        target = target.cuda(async=True)\n        input_var = torch.autograd.Variable(input)\n        target_var = torch.autograd.Variable(target)\n\n        input_noise, target_noise = next(noise_itr)  # test dataのバッチ\n        input_noise = torch.autograd.Variable(input_noise.cuda(async=True))\n        target_noise = torch.autograd.Variable(target_noise.cuda(async=True))\n        \n        input_semi, target_semi = next(semi_itr)\n        input_semi = torch.autograd.Variable(input_semi.cuda(async=True))\n        target_semi = torch.autograd.Variable(target_semi.cuda(async=True))\n        \n        \n        \n        # get model outputs\n        output = model(input_var)\n        ce = criterion_bce(act_train(output), target_var)\n                           \n        output_noise = model.noisy(input_noise)\n        ce_noise = criterion_bce(act_noisy(output_noise), target_noise)\n        \n        output_semi = model(input_semi)\n        mse_semi = criterion_consistency(act_train(output_semi), target_semi)\n                           \n        if DO_PSEUDO_NOISY:\n            input_semi2, target_semi2 = next(semi_itr2)\n            input_semi2 = torch.autograd.Variable(input_semi2.cuda(async=True))\n            target_semi2 = torch.autograd.Variable(target_semi2.cuda(async=True))\n            output_semi2 = model.noisy(input_semi2)\n            mse_semi2 = criterion_consistency(softmax(output_semi2), target_semi2)\n        else:\n            mse_semi2 = 0\n                           \n        \n        # calc losses\n#         l1 = 0\n#         l2 = 0\n#         for param in model.parameters():\n#             l1 += param.abs().sum()\n#             l2 += param.norm(2)\n#         loss = bce + bce_noise + 1e-7*l1 + 1e-5*l2\n        loss = ce + ce_noise * C_NOISY + C_SEMI * mse_semi + C_SEMI * mse_semi2\n        pred = act_train(output)\n        pred = pred.data.cpu().numpy()\n        pred_noise = act_noisy(output_noise)\n        pred_noise = pred_noise.data.cpu().numpy()\n        ce_avr.update(ce.data, input.size(0))\n        ce_noise_avr.update(ce_noise.data, input.size(0))\n        mse_semi_avr.update(mse_semi.data, input.size(0))\n        if DO_PSEUDO_NOISY:\n            mse_semi2_avr.update(mse_semi2.data, input.size(0))\n\n        # compute gradient and do SGD step\n        optimizer.zero_grad()  # # 勾配の初期化\n        loss.backward()\n        optimizer.step()\n        scheduler.step()\n\n        preds = np.concatenate([preds, pred])\n        y_true = np.concatenate([y_true, target.data.cpu().numpy()])\n        preds_noise = np.concatenate([preds_noise, pred_noise])\n        y_true_noise = np.concatenate([y_true_noise, target_noise.data.cpu().numpy()])\n#         print(pred[0])\n#         print(\n#             \"CE: {:.4f} \".format(ce_avr.avg.item())\n#               + \"noise CE: {:.4f} \".format(ce_noise_avr.avg.item())\n#               +\"semi MSE: {:.4f} \".format(mse_semi_avr.avg.item())\n#               )\n        \n\n    # print(preds.shape, y_true.shape)\n    # print(y_true[:,:-1].shape, preds[:,:-1].shape)\n    per_class_lwlrap, weight_per_class = calculate_per_class_lwlrap(y_true, preds)\n    lwlrap = np.sum(per_class_lwlrap * weight_per_class)\n    # print(y_true_noise[:,:-1].shape, preds_noise[:,:-1].shape)\n    per_class_lwlrap, weight_per_class = calculate_per_class_lwlrap(y_true_noise, preds_noise)\n    lwlrap_noise = np.sum(per_class_lwlrap * weight_per_class)\n    if DO_PSEUDO_NOISY:\n        mse_semi2_avr = mse_semi2_avr.avg.item()\n    return ce_avr.avg.item(), lwlrap, ce_noise_avr.avg.item(), lwlrap_noise, mse_semi_avr.avg.item(), mse_semi2_avr\n\n\ndef validate(val_loader, model):\n    ce_avr   = AverageMeter()\n    criterion_bce = nn.BCELoss().cuda()\n    sigmoid = torch.nn.Sigmoid().cuda()\n    softmax = torch.nn.Softmax().cuda()\n    if TRAIN_LAST=='SIGMOID':\n        act_train = sigmoid\n    elif TRAIN_LAST=='SOFTMAX':\n        act_train = softmax\n    \n    # switch to train mode\n    model.eval()\n\n    starttime = time.time()\n    preds = np.zeros([0, NUM_CLASS], np.float32)\n    y_true = np.zeros([0, NUM_CLASS], np.float32)\n    for i, (input, target) in enumerate(val_loader):\n        input = input.cuda(async=True)\n        target = target.cuda(async=True)\n        input_var = torch.autograd.Variable(input)\n        target_var = torch.autograd.Variable(target)\n        # print(input.size())\n\n        # compute output\n        with torch.no_grad():\n            output = model(input_var)\n            ce = criterion_bce(act_train(output), target_var)\n            pred = softmax(output)\n            pred = pred.data.cpu().numpy()\n\n        # measure accuracy and record loss\n        ce_avr.update(ce.data, input.size(0))\n        preds = np.concatenate([preds, pred])\n        y_true = np.concatenate([y_true, target.data.cpu().numpy()])\n        \n        \n    per_class_lwlrap, weight_per_class = calculate_per_class_lwlrap(y_true, preds)\n    lwlrap = np.sum(per_class_lwlrap * weight_per_class)\n\n    return ce_avr.avg.item(), lwlrap","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold, StratifiedKFold\n# set log columns\nfolds = list(KFold(n_splits=NUM_FOLD, shuffle=True, random_state=SEED).split(np.arange(len(df_train))))\nfolds_noisy = list(KFold(n_splits=NUM_FOLD, shuffle=True, random_state=SEED).split(np.arange(len(df_noise))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# baseline\n\nlog_columns = ['epoch', 'bce', 'lwlrap', 'bce_noise', 'lwlrap_noise', 'semi_mse', 'semi_mse2', 'val_bce', 'val_lwlrap', 'time']  # 学習ログのカラム名\nfor fold, (ids_train_split, ids_valid_split) in enumerate(folds):\n    print(\"fold: {}\".format(fold + 1))\n    starttime = time.time()\n    if fold+1 not in DO_FOLD: continue\n    train_log = pd.DataFrame(columns=log_columns)\n\n    # build model\n    model = ResNet(NUM_CLASS).cuda()\n    model.load_state_dict(torch.load(\"../input/resnet34hardaug512/models/weight_fold_{}_epoch_512.pth\".format(fold+1)))\n\n    # set generator\n    df_train_fold = df_train.iloc[ids_train_split].reset_index(drop=True)\n    dataset_train = MfccDataset(df_train_fold, \"../input/mel128v3/train/\", \n                                slice=512, \n                                mixup=True, \n                                cutout_h=True,\n                                gain=True,\n#                                 resize=True,\n#                                 warp=True,\n#                                 white=True,\n                                slice_mode='random',\n                               )\n    train_loader = DataLoader(dataset_train,\n                              batch_size=BATCH_SIZE,\n                              shuffle=True,\n                              num_workers=1,  # 1 for CUDA?\n                              pin_memory=True,  # CUDA only\n                              )\n\n    df_valid = df_train.iloc[ids_valid_split].reset_index(drop=True)\n    dataset_valid = MfccDataset(df_valid, \"../input/mel128v3/train/\", slice_mode='original')\n    valid_loader = DataLoader(dataset_valid,\n                              batch_size=1,\n                              shuffle=False,\n                              num_workers=1,  # 1 for CUDA\n                              pin_memory=True,  # CUDA only\n                              )\n\n    df_noise_fold = df_noise.iloc[folds_noisy[fold][0]].reset_index(drop=True)\n    dataset_noise = MfccDataset(df_noise_fold, \"../input/mel128v3n/noise/\",\n                                slice=512, \n                                mixup=True, \n                                cutout_h=True,\n                                gain=True,\n                                slice_mode='random',\n                               )\n    noise_loader = DataLoader(dataset_noise,\n                              batch_size=BATCH_SIZE,\n                              shuffle=True,\n                              num_workers=1,  # 1 for CUDA?\n                              pin_memory=True,  # CUDA only\n                              )\n    noise_itr = cycle(noise_loader)  # dataloaderをgeneratorに変換\n    \n\n    df_semi = pd.concat([df_train.iloc[ids_train_split], df_noise_pseudo.iloc[folds_noisy[fold][0]]]).reset_index(drop=True)\n    semi_sampler = torch.utils.data.sampler.WeightedRandomSampler(df_semi['weight'].values, len(df_semi))\n    dataset_semi = MfccDataset(df_semi, \"../input/mel128v3/train/\", \n                                slice=512, \n                                mixup=True, \n                                cutout_h=True,\n                                gain=True,\n                                resize=True,\n                                slice_mode='random',\n                               )\n    semi_loader = DataLoader(dataset_semi,\n                              batch_size=BATCH_SIZE,\n                              shuffle=False,\n                              num_workers=1,  # 1 for CUDA?\n                              pin_memory=True,  # CUDA only\n                              sampler=semi_sampler,\n                              )\n    semi_itr = cycle(semi_loader)  # dataloaderをgeneratorに変換\n    \n    \n    df_semi2 = pd.concat([df_train_pseudo.iloc[ids_train_split], df_noise.iloc[folds_noisy[fold][0]]]).reset_index(drop=True)\n    semi_sampler2 = torch.utils.data.sampler.WeightedRandomSampler(df_semi2['weight'].values, len(df_semi))\n    dataset_semi2 = MfccDataset(df_semi, \"../input/mel128v3/train/\", \n                                slice=512, \n                                mixup=True, \n                                cutout_h=True,\n                                gain=True,\n                                resize=True,\n                                slice_mode='random',\n                               )\n    semi_loader2 = DataLoader(dataset_semi2,\n                              batch_size=BATCH_SIZE,\n                              shuffle=False,\n                              num_workers=1,  # 1 for CUDA?\n                              pin_memory=True,  # CUDA only\n                              sampler=semi_sampler2,\n                              )\n    semi_itr2 = cycle(semi_loader2)  # dataloaderをgeneratorに変換\n    \n\n    # set optimizer and loss\n    optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=LR[0])\n    scheduler = CosineLR(optimizer, step_size_min=LR[1], t0=len(train_loader) * NUM_CYCLE, tmult=1)\n\n    # training\n    for epoch in range(NUM_EPOCH):\n        # train for one epoch\n        bce, lwlrap, bce_noise, lwlrap_noise, mse_semi, mse_semi2 = train(\n            (train_loader, noise_itr, semi_itr, semi_itr2),\n             model, optimizer, scheduler, epoch\n        )\n\n        # evaluate on validation set\n        val_bce, val_lwlrap = validate(valid_loader, model)\n        \n        endtime = time.time() - starttime\n        print(\"Epoch: {}/{} \".format(epoch + 1, NUM_EPOCH)\n              + \"CE: {:.4f} \".format(bce)\n              + \"LwLRAP: {:.4f} \".format(lwlrap)\n              + \"noise CE: {:.4f} \".format(bce_noise)\n              + \"noise LwLRAP: {:.4f} \".format(lwlrap_noise)\n              + \"semi MSE: {:.4f} \".format(mse_semi)\n              + \"semi2 MSE: {:.4f} \".format(mse_semi2)\n              + \"Valid CE: {:.4f} \".format(val_bce)\n              + \"Valid LwLRAP: {:.4f} \".format(val_lwlrap)\n              + \"sec: {:.1f}\".format(endtime)\n              )\n        train_log_epoch = pd.DataFrame([[epoch+1, bce, lwlrap, bce_noise, lwlrap_noise, mse_semi, mse_semi2, val_bce, val_lwlrap, endtime]],\n                               columns=log_columns)\n        train_log = pd.concat([train_log, train_log_epoch])\n        train_log.to_csv(\"train_log_fold{}.csv\".format(fold+1), index=False)\n        if (epoch+1)%NUM_CYCLE==0:\n            torch.save(model.state_dict(), \"weight_fold_{}_epoch_{}.pth\".format(fold+1, epoch+1))\n    torch.save(optimizer.state_dict(), 'optimizer_fold_{}_epoch_{}.pth'.format(fold+1, epoch+1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_bce_last = train_log['val_bce'].values[-1]\nval_lwlrap_last = train_log['val_lwlrap'].values[-1]\nbest_lwlrap = train_log['val_lwlrap'].values.max()\nbest_epoch = train_log['epoch'][train_log['val_lwlrap']==best_lwlrap].values[0]\nbest_bce = train_log['val_bce'].values.min()\nprint(\"{:.8f}, {:.8f}, {}, {:.8f}, {:.8f}\".format(val_lwlrap_last, val_bce_last ,best_epoch, best_lwlrap, best_bce))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}