{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"FOLD = [0, 1]","execution_count":1,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## imports"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport sys\nimport pickle\nimport random\nimport time\nimport logging\n\nimport datetime as dt\nfrom collections import Counter, defaultdict\nfrom functools import partial\nfrom pathlib import Path\nfrom psutil import cpu_count\nfrom hashlib import sha1\n\nimport librosa\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\n# from sklearn.model_selection import train_test_split, KFold\nfrom sklearn.model_selection import train_test_split, KFold, StratifiedKFold\n#from skmultilearn.model_selection import iterative_train_test_split\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom fastprogress import master_bar, progress_bar\nfrom torch.optim import Adam, SGD\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.transforms import transforms\n\nfrom tqdm import tqdm_notebook\nimport matplotlib.pyplot as plt\n\nfrom imgaug import augmenters as iaa","execution_count":2,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir(\"../input\"))","execution_count":3,"outputs":[{"output_type":"stream","text":"['fat2019-new-1ch', 'freesound-audio-tagging-2019', 'freesound-sox-pitchup-equ-tremolo-tbup', 'freesound-all-curated-aug-1ch', 'freesound-convnet-noisy-pitch-pretrain-rrc']\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"## utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_logger(name=\"Main\", tag=\"exp\", log_dir=\"log/\"):\n log_path = Path(log_dir)\n path = log_path / tag\n path.mkdir(exist_ok=True, parents=True)\n\n logger = logging.getLogger(name)\n logger.setLevel(logging.INFO)\n\n fh = logging.FileHandler(\n path / (dt.datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\") + \".log\"))\n sh = logging.StreamHandler(sys.stdout)\n formatter = logging.Formatter(\n \"%(asctime)s %(name)s %(levelname)s %(message)s\")\n\n fh.setFormatter(formatter)\n sh.setFormatter(formatter)\n logger.addHandler(fh)\n logger.addHandler(sh)\n return logger","execution_count":4,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n random.seed(seed)\n os.environ['PYTHONHASHSEED'] = str(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.deterministic = True\n\nSEED = 520\nseed_everything(SEED)","execution_count":5,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N_JOBS = cpu_count()\nos.environ['MKL_NUM_THREADS'] = str(N_JOBS)\nos.environ['OMP_NUM_THREADS'] = str(N_JOBS)\nDataLoader = partial(DataLoader, num_workers=N_JOBS)","execution_count":6,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from official code https://colab.research.google.com/drive/1AgPdhSp7ttY18O3fEoHOQKlt_3HJDLi8#scrollTo=cRCaCIb9oguU\ndef _one_sample_positive_class_precisions(scores, truth):\n \"\"\"Calculate precisions for each true class for a single sample.\n\n Args:\n scores: np.array of (num_classes,) giving the individual classifier scores.\n truth: np.array of (num_classes,) bools indicating which classes are true.\n\n Returns:\n pos_class_indices: np.array of indices of the true classes for this sample.\n pos_class_precisions: np.array of precisions corresponding to each of those\n classes.\n \"\"\"\n num_classes = scores.shape[0]\n pos_class_indices = np.flatnonzero(truth > 0)\n # Only calculate precisions if there are some true classes.\n if not len(pos_class_indices):\n return pos_class_indices, np.zeros(0)\n # Retrieval list of classes for this sample.\n retrieved_classes = np.argsort(scores)[::-1]\n # class_rankings[top_scoring_class_index] == 0 etc.\n class_rankings = np.zeros(num_classes, dtype=np.int)\n class_rankings[retrieved_classes] = range(num_classes)\n # Which of these is a true label?\n retrieved_class_true = np.zeros(num_classes, dtype=np.bool)\n retrieved_class_true[class_rankings[pos_class_indices]] = True\n # Num hits for every truncated retrieval list.\n retrieved_cumulative_hits = np.cumsum(retrieved_class_true)\n # Precision of retrieval list truncated at each hit, in order of pos_labels.\n precision_at_hits = (\n retrieved_cumulative_hits[class_rankings[pos_class_indices]] /\n (1 + class_rankings[pos_class_indices].astype(np.float)))\n return pos_class_indices, precision_at_hits\n\n\ndef calculate_per_class_lwlrap(truth, scores):\n \"\"\"Calculate label-weighted label-ranking average precision.\n\n Arguments:\n truth: np.array of (num_samples, num_classes) giving boolean ground-truth\n of presence of that class in that sample.\n scores: np.array of (num_samples, num_classes) giving the classifier-under-\n test's real-valued score for each class for each sample.\n\n Returns:\n per_class_lwlrap: np.array of (num_classes,) giving the lwlrap for each\n class.\n weight_per_class: np.array of (num_classes,) giving the prior of each\n class within the truth labels. Then the overall unbalanced lwlrap is\n simply np.sum(per_class_lwlrap * weight_per_class)\n \"\"\"\n assert truth.shape == scores.shape\n num_samples, num_classes = scores.shape\n # Space to store a distinct precision value for each class on each sample.\n # Only the classes that are true for each sample will be filled in.\n precisions_for_samples_by_classes = np.zeros((num_samples, num_classes))\n for sample_num in range(num_samples):\n pos_class_indices, precision_at_hits = (\n _one_sample_positive_class_precisions(scores[sample_num, :],\n truth[sample_num, :]))\n precisions_for_samples_by_classes[sample_num, pos_class_indices] = (\n precision_at_hits)\n labels_per_class = np.sum(truth > 0, axis=0)\n weight_per_class = labels_per_class / float(np.sum(labels_per_class))\n # Form average of each column, i.e. all the precisions assigned to labels in\n # a particular class.\n per_class_lwlrap = (np.sum(precisions_for_samples_by_classes, axis=0) /\n np.maximum(1, labels_per_class))\n # overall_lwlrap = simple average of all the actual per-class, per-sample precisions\n # = np.sum(precisions_for_samples_by_classes) / np.sum(precisions_for_samples_by_classes > 0)\n # also = weighted mean of per-class lwlraps, weighted by class label prior across samples\n # = np.sum(per_class_lwlrap * weight_per_class)\n return per_class_lwlrap, weight_per_class","execution_count":7,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"dataset_dir = Path('../input/freesound-audio-tagging-2019')\npreprocessed_dir = Path('../input/fat2019-new-1ch/')\naugmented_dir = Path(\"../input/freesound-all-curated-aug-1ch/\")\naugmented_dir2 = Path(\"../input/freesound-sox-pitchup-equ-tremolo-tbup/\")","execution_count":8,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"csvs = {\n 'train_curated': dataset_dir / 'train_curated.csv',\n 'sample_submission': dataset_dir / 'sample_submission.csv'\n}\n\ndataset = {\n 'train_curated': dataset_dir / 'train_curated'\n}\n\nmels = {\n 'train_curated': preprocessed_dir / 'mels_train_curated.pkl',\n \"fade\": augmented_dir / \"mel_fade.pkl\",\n \"pitch\": augmented_dir / \"mel_pitch.pkl\",\n \"reverb\": augmented_dir / \"mel_reverb.pkl\",\n \"tb\": augmented_dir / \"mel_tb.pkl\",\n \"pitchup\": augmented_dir2 / \"mel_pitchup.pkl\",\n \"eq\": augmented_dir2 / \"mel_equalize.pkl\",\n \"tbup\": augmented_dir2 / \"mel_tbup.pkl\"\n}","execution_count":10,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_curated = pd.read_csv(csvs['train_curated'])\ntrain_df = train_curated\n\ntest_df = pd.read_csv(csvs['sample_submission'])\n\nlabels = test_df.columns[1:].tolist()\nnum_classes = len(labels)\nnum_classes","execution_count":11,"outputs":[{"output_type":"execute_result","execution_count":11,"data":{"text/plain":"80"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = np.zeros((len(train_df), num_classes)).astype(int)\nfor i, row in enumerate(train_df['labels'].str.split(',')):\n for label in row:\n idx = labels.index(label)\n y_train[i, idx] = 1\n\ny_train.shape","execution_count":12,"outputs":[{"output_type":"execute_result","execution_count":12,"data":{"text/plain":"(4970, 80)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(mels['train_curated'], 'rb') as curated:\n x_train = pickle.load(curated)\n \nwith open(mels[\"fade\"], \"rb\") as fade:\n x_fade = pickle.load(fade)\n \nwith open(mels[\"pitch\"], \"rb\") as pitch:\n x_pitch = pickle.load(pitch)\n \nwith open(mels[\"reverb\"], \"rb\") as reverb:\n x_reverb = pickle.load(reverb)\n \nwith open(mels[\"tb\"], \"rb\") as tb:\n x_tb = pickle.load(tb)\n \nwith open(mels[\"pitchup\"], \"rb\") as pu:\n x_pitchup = pickle.load(pu)\n \nwith open(mels[\"eq\"], \"rb\") as eq:\n x_eq = pickle.load(eq)\n \nwith open(mels[\"tbup\"], \"rb\") as tbup:\n x_tbup = pickle.load(tbup)\n \nlen(x_train), len(x_fade), len(x_pitch), len(x_reverb), len(x_tb), len(x_pitchup), len(x_eq), len(x_tbup)","execution_count":13,"outputs":[{"output_type":"execute_result","execution_count":13,"data":{"text/plain":"(4970, 4970, 4970, 4970, 4970, 4970, 4970, 4970)"},"metadata":{}}]},{"metadata":{},"cell_type":"markdown","source":"## Data Transformation"},{"metadata":{"trusted":true},"cell_type":"code","source":"#########################################################\nclass ImgAugTransform:\n def __init__(self):\n self.aug = iaa.Sequential([\n iaa.CoarseDropout(0.1,size_percent=0.02)\n ])\n \n def __call__(self, img):\n img = np.array(img)\n return self.aug.augment_image(img)\n#########################################################","execution_count":14,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision.transforms import *\nimport math\nfrom imgaug import augmenters as iaa\n\nclass RandomErasingOrCoaseDropout(object):\n\n def __init__(self, probability = 0.3, probability2 = 0.6, sl = 0.02, sh = 0.4, r1 = 0.3, mean=[0.4914, 0.4822, 0.4465]):\n self.probability = probability\n self.probability2 = probability2\n self.mean = mean\n self.sl = sl\n self.sh = sh\n self.r1 = r1\n \n self.aug = iaa.Sequential([\n iaa.CoarseDropout(0.1,size_percent=0.02)\n ])\n \n def __call__(self, img):\n \n th = random.uniform(0, 1)\n if th < self.probability:\n img = np.array(img)\n return torch.tensor(self.aug.augment_image(img))\n \n if th > self.probability2:\n return img\n\n for attempt in range(100):\n area = img.size()[1] * img.size()[2]\n \n target_area = random.uniform(self.sl, self.sh) * area\n aspect_ratio = random.uniform(self.r1, 1/self.r1)\n\n h = int(round(math.sqrt(target_area * aspect_ratio)))\n w = int(round(math.sqrt(target_area / aspect_ratio)))\n\n if w < img.size()[2] and h < img.size()[1]:\n x1 = random.randint(0, img.size()[1] - h)\n y1 = random.randint(0, img.size()[2] - w)\n if img.size()[0] == 3:\n img[0, x1:x1+h, y1:y1+w] = self.mean[0]\n img[1, x1:x1+h, y1:y1+w] = self.mean[1]\n img[2, x1:x1+h, y1:y1+w] = self.mean[2]\n else:\n img[0, x1:x1+h, y1:y1+w] = self.mean[0]\n return img\n return img","execution_count":15,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# blend image augmentation\n# https://pillow.readthedocs.io/en/stable/reference/Image.html\n\ndef array2img(array): \n image = Image.fromarray(array, mode='L') \n time_dim, base_dim = image.size\n crop = random.randint(0, time_dim - base_dim)\n image = image.crop([crop, 0, crop + base_dim, base_dim])\n return image\n","execution_count":16,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_dict = {\n 'train': transforms.Compose([\n transforms.RandomHorizontalFlip(0.5),\n #########################################\n # ImgAugTransform(),\n #########################################\n transforms.ToTensor(),\n # ===============\n RandomErasingOrCoaseDropout(),\n # ===============\n ]),\n #########################################\n 'valid': transforms.Compose([\n transforms.RandomHorizontalFlip(0.5),\n #ImgAugTransform(),\n transforms.ToTensor(),\n # RandomErasingOrCoaseDropout(),\n ]),\n #########################################\n 'test': transforms.Compose([\n transforms.RandomHorizontalFlip(0.5),\n #########################################\n # ImgAugTransform(),\n #########################################\n transforms.ToTensor(),\n ]),\n}","execution_count":17,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _get_strength_list(x_train):\n ##################################################################\n strength_list_train = []\n for i in progress_bar(range(len(x_train))):\n time_dim, base_dim = x_train[i].shape[1], x_train[i].shape[0]\n x = x_train[i]\n x_sum = []\n x_max = 0\n x_min = np.inf\n for j in range(0,time_dim-base_dim + 1):\n #x_temp = x[:,j:j + base_dim].sum()\n x_temp = x[0:84,j:j + base_dim].sum()\n x_sum.append(x_temp)\n if x_temp > x_max:\n x_max = x_temp\n if x_temp < x_min:\n x_min = x_temp\n if (x_max == x_min):\n height = x_max\n else:\n height = x_max - x_min\n strength_list_train.append([x_sum, height, x_max, x_min, time_dim, base_dim])\n ##################################################################\n return strength_list_train\n\nstrength_list_train = _get_strength_list(x_train)","execution_count":18,"outputs":[{"output_type":"display_data","data":{"text/plain":"","text/html":"\n
\n \n \n 100.00% [4970/4970 00:25<00:00]\n
\n "},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FATTrainDataset(Dataset):\n #def __init__(self, mels, labels, transforms):\n ############################################################\n def __init__(self, mels, labels, transforms, strength_list):\n ############################################################\n super().__init__()\n self.mels = mels\n self.labels = labels\n self.transforms = transforms\n ################################\n self.strength_list = strength_list\n ################################\n\n \n def __len__(self):\n return len(self.mels)\n \n def __getitem__(self, idx):\n # crop 2sec\n #image = Image.fromarray(self.mels[idx], mode='RGB')　#後でimage化 \n #time_dim, base_dim = image.size\n ##################################################################\n #time_dim, base_dim = self.mels[idx].shape[1], self.mels[idx].shape[0] \n x = self.mels[idx]\n x_sum = self.strength_list[idx][0]\n height = self.strength_list[idx][1]\n x_max = self.strength_list[idx][2]\n x_min = self.strength_list[idx][3]\n time_dim = self.strength_list[idx][4]\n base_dim = self.strength_list[idx][5]\n flag = True\n while flag:\n crop = random.randint(0, int(height * (time_dim- base_dim))) # 一様分布の乱数\n #オール0対策\n if height == 0:\n crop_idx = 0\n value = 0\n else:\n crop_idx = int(crop // height)\n value = int(crop % height) \n if ((x_sum[crop_idx] - x_min) - value) >= 0:\n flag = False\n crop = crop_idx\n image = Image.fromarray(x, mode='L') \n ##################################################################\n #crop = random.randint(0, time_dim - base_dim)\n image = image.crop([crop, 0, crop + base_dim, base_dim])\n image = self.transforms(image).div_(255)[0, :, :]\n \n label = self.labels[idx]\n label = torch.from_numpy(label).float()\n \n return image, label","execution_count":19,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FATValidDataset(Dataset):\n #def __init__(self, mels, labels, transforms):\n ############################################################\n def __init__(self, fnames, mels, labels, transforms, strength_list, tta=5):\n ############################################################\n super().__init__()\n self.fnames = fnames\n self.mels = mels\n self.labels = labels\n self.transforms = transforms\n ################################\n self.strength_list = strength_list\n ################################\n self.tta = tta\n\n \n def __len__(self):\n return len(self.fnames) * self.tta\n \n def __getitem__(self, idx): \n new_idx = idx % len(self.fnames)\n # crop 2sec\n #image = Image.fromarray(self.mels[idx], mode='RGB')　#後でimage化 \n #time_dim, base_dim = image.size\n ##################################################################\n #time_dim, base_dim = self.mels[idx].shape[1], self.mels[idx].shape[0] \n x = self.mels[new_idx]\n x_sum = self.strength_list[new_idx][0]\n height = self.strength_list[new_idx][1]\n x_max = self.strength_list[new_idx][2]\n x_min = self.strength_list[new_idx][3]\n time_dim = self.strength_list[new_idx][4]\n base_dim = self.strength_list[new_idx][5]\n flag = True\n while flag:\n crop = random.randint(0, int(height * (time_dim- base_dim))) # 一様分布の乱数\n #オール0対策\n if height == 0:\n crop_idx = 0\n value = 0\n else:\n crop_idx = int(crop // height)\n value = int(crop % height) \n if ((x_sum[crop_idx] - x_min) - value) >= 0:\n flag = False\n crop = crop_idx\n image = Image.fromarray(x, mode='L') \n ##################################################################\n #crop = random.randint(0, time_dim - base_dim)\n image = image.crop([crop, 0, crop + base_dim, base_dim])\n image = self.transforms(image).div_(255)[0, :, :]\n \n label = self.labels[new_idx]\n label = torch.from_numpy(label).float()\n \n fname = self.fnames[new_idx]\n \n return image, label, fname","execution_count":20,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def init_layer(layer, nonlinearity=\"leaky_relu\"):\n nn.init.kaiming_uniform_(layer.weight, nonlinearity=nonlinearity)\n \n if hasattr(layer, \"bias\"):\n if layer.bias is not None:\n layer.bias.data.fill_(0.0)\n \n \ndef init_bn(bn):\n bn.bias.data.fill_(0.0)\n bn.running_mean.data.fill_(0.0)\n bn.weight.data.fill_(1.0)\n bn.running_var.data.fill_(1.0)\n \n \nclass SpatialAttention2d(nn.Module):\n def __init__(self, channel):\n super(SpatialAttention2d, self).__init__()\n self.squeeze = nn.Conv2d(channel, 1, kernel_size=1, bias=False)\n self.sigmoid = nn.Sigmoid()\n\n def forward(self, x):\n z = self.squeeze(x)\n z = self.sigmoid(z)\n return x * z\n\n\nclass GAB(nn.Module):\n def __init__(self, input_dim, reduction=4):\n super(GAB, self).__init__()\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n self.conv1 = nn.Conv2d(input_dim, input_dim // reduction, kernel_size=1, stride=1)\n self.conv2 = nn.Conv2d(input_dim // reduction, input_dim, kernel_size=1, stride=1)\n self.relu = nn.ReLU(inplace=True)\n self.sigmoid = nn.Sigmoid()\n\n def forward(self, x):\n z = self.global_avgpool(x)\n z = self.relu(self.conv1(z))\n z = self.sigmoid(self.conv2(z))\n return x * z\n\n \nclass SCse(nn.Module):\n def __init__(self, dim):\n super(SCse, self).__init__()\n self.satt = SpatialAttention2d(dim)\n self.catt = GAB(dim)\n\n def forward(self, x):\n return self.satt(x) + self.catt(x)\n \n \nclass ConvBlock(nn.Module):\n def __init__(self, in_channels, out_channels, reduction=16):\n super(ConvBlock, self).__init__()\n \n self.conv1 = nn.Conv2d(in_channels=in_channels,\n out_channels=out_channels,\n kernel_size=(2, 2),\n stride=(1, 1),\n padding=(1, 1),\n bias=False)\n self.conv2 = nn.Conv2d(in_channels=out_channels,\n out_channels=out_channels,\n kernel_size=(2, 2),\n stride=(1, 1),\n padding=(1, 1),\n bias=False)\n self.bn1 = nn.BatchNorm2d(out_channels)\n self.bn2 = nn.BatchNorm2d(out_channels)\n self.scse = SCse(out_channels)\n # self.se = SELayer(out_channels)\n \n self.init_weights()\n \n def init_weights(self):\n init_layer(self.conv1)\n init_layer(self.conv2)\n init_bn(self.bn1)\n init_bn(self.bn2)\n \n def forward(self, inp, pool_size=(2, 2), pool_type=\"avg\"):\n x = inp\n x = F.relu_(self.bn1(self.conv1(x)))\n x = F.relu_(self.scse(self.bn2(self.conv2(x))))\n # x = F.relu_(self.se(self.bn2(self.conv2(x))))\n if pool_type == \"max\":\n x = F.max_pool2d(x, kernel_size=pool_size)\n elif pool_type == \"avg\":\n x = F.avg_pool2d(x, kernel_size=pool_size)\n elif pool_type == \"both\":\n x1 = F.max_pool2d(x, kernel_size=pool_size)\n x2 = F.avg_pool2d(x, kernel_size=pool_size)\n x = x1 + x2\n else:\n import pdb\n pdb.set_trace()\n return x\n \n \nclass ConvNet(nn.Module):\n def __init__(self, n_classes=80):\n super(ConvNet, self).__init__()\n self.conv1 = ConvBlock(1, 32)\n self.conv2 = ConvBlock(32, 64)\n self.conv3 = ConvBlock(64, 128)\n self.conv4 = ConvBlock(128, 256)\n self.conv5 = ConvBlock(256, 512)\n \n self.bn1 = nn.BatchNorm1d((1 + 4 + 20) * 512)\n self.drop1 = nn.Dropout(0.4)\n self.fc1 = nn.Linear((1 + 4 + 20) * 512, 512)\n self.prelu = nn.PReLU()\n self.bn2 = nn.BatchNorm1d(512)\n self.drop2 = nn.Dropout(0.2)\n self.fc2 = nn.Linear(512, n_classes)\n \n def init_weight(self):\n init_layer(self.fc1)\n init_layer(self.fc2)\n init_bn(self.bn1)\n init_bn(self.bn2)\n \n def forward(self, x):\n x = x.view(x.size(0), 1, x.size(1), x.size(2))\n x = self.conv1(x, pool_size=(1, 1), pool_type=\"both\")\n x = self.conv2(x, pool_size=(4, 1), pool_type=\"both\")\n x = self.conv3(x, pool_size=(1, 3), pool_type=\"both\")\n x = self.conv4(x, pool_size=(4, 1), pool_type=\"both\")\n x = self.conv5(x, pool_size=(1, 3), pool_type=\"both\")\n \n x1_max = F.max_pool2d(x, (5, 8))\n x1_mean = F.avg_pool2d(x, (5, 8))\n x1 = (x1_max + x1_mean).reshape(x.size(0), -1)\n \n x2_max = F.max_pool2d(x, (2, 4))\n x2_mean = F.avg_pool2d(x, (2, 4))\n x2 = (x2_max + x2_mean).reshape(x.size(0), -1)\n \n x = torch.mean(x, dim=3)\n x, _ = torch.max(x, dim=2)\n \n x = torch.cat([x, x1, x2], dim=1)\n x = self.drop1(self.bn1(x))\n x = self.prelu(self.fc1(x))\n x = self.drop2(self.bn2(x))\n x = self.fc2(x)\n \n return x","execution_count":21,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FineTune(nn.Module):\n def __init__(self, n_classes, weight_path):\n super(FineTune, self).__init__()\n\n self.convnet = ConvNet(n_classes)\n self.convnet.load_state_dict(torch.load(weight_path))\n \n self.bn1 = nn.BatchNorm1d((1 + 4 + 20) * 512)\n self.drop1 = nn.Dropout(0.4)\n self.fc1 = nn.Linear((1 + 4 + 20) * 512, 512)\n self.prelu = nn.PReLU()\n self.bn2 = nn.BatchNorm1d(512)\n self.drop2 = nn.Dropout(0.2)\n self.fc2 = nn.Linear(512, n_classes)\n \n def init_weight(self):\n init_layer(self.fc1)\n init_layer(self.fc2)\n init_bn(self.bn1)\n init_bn(self.bn2)\n \n def forward(self, x):\n x = x.view(x.size(0), 1, x.size(1), x.size(2))\n x = self.convnet.conv1(x, pool_size=(1, 1), pool_type=\"both\")\n x = self.convnet.conv2(x, pool_size=(4, 1), pool_type=\"both\")\n x = self.convnet.conv3(x, pool_size=(1, 3), pool_type=\"both\")\n x = self.convnet.conv4(x, pool_size=(4, 1), pool_type=\"both\")\n x = self.convnet.conv5(x, pool_size=(1, 3), pool_type=\"both\")\n \n x1_max = F.max_pool2d(x, (5, 8))\n x1_mean = F.avg_pool2d(x, (5, 8))\n x1 = (x1_max + x1_mean).reshape(x.size(0), -1)\n \n x2_max = F.max_pool2d(x, (2, 4))\n x2_mean = F.avg_pool2d(x, (2, 4))\n x2 = (x2_max + x2_mean).reshape(x.size(0), -1)\n \n x = torch.mean(x, dim=3)\n x, _ = torch.max(x, dim=2)\n \n x = torch.cat([x, x1, x2], dim=1)\n x = self.drop1(self.bn1(x))\n x = self.prelu(self.fc1(x))\n x = self.drop2(self.bn2(x))\n x = self.fc2(x)\n \n return x","execution_count":22,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## train"},{"metadata":{"trusted":true},"cell_type":"code","source":"#def train_model(x_train, y_train, train_transforms):\n#####################################################################\n#def train_model(x_train, y_train, train_transforms, strength_list):\ndef train_model(x_train, y_train, \n x_fade, x_pitch, x_reverb, x_tb,\n x_pitchup, x_eq, x_tbup,\n train_df, \n train_transforms, valid_transforms, \n strength_list):\n#def train_model(x_train, y_train, x_fade, x_pitch, x_reverb, \n# train_transforms, valid_transforms, strength_list, strength_list_blend_arr_x, blend_arr_x, blend_arr_y):\n#####################################################################\n num_epochs = 40\n batch_size = 128\n test_batch_size = 128\n lr = 1e-3\n eta_min = 1e-5\n t_max = 10\n \n num_classes = y_train.shape[1]\n \n idx = np.arange(len(x_train))\n #idx_ex = np.arange(len(blend_arr_x))\n \n lam = np.random.beta(1.0, 1.0)\n \n kfold = KFold(n_splits=5, shuffle=True, random_state=SEED)\n splits = list(kfold.split(idx))\n use_fold = []\n for i in FOLD:\n use_fold.append(splits[i])\n # =====\n # kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)\n # =====\n \n logger = get_logger(\"Main\", tag=\"train\", log_dir=\"log/\")\n bests = []\n loss_list = []\n lwlrap_list = []\n for fold_num, (trn_idx, val_idx) in enumerate(use_fold):\n # for fold_num, (trn_idx, val_idx) in enumerate(kfold.split(idx, fold_order)):\n logger.info(f\"Fold {fold_num + 1}\")\n \n x_trn, x_val = list(), list()\n for j in idx:\n if j in trn_idx:\n x_trn.append(x_train[j])\n else:\n x_val.append(x_train[j])\n \n y_trn, y_val = y_train[trn_idx], y_train[val_idx]\n val_fnames = set(train_df.loc[val_idx, \"fname\"].values.tolist())\n \n # get mix data\n blend_arr_x = []\n blend_arr_y = []\n for i in range(1000):\n r1, r2 = random.randint(0, len(x_trn)-1), random.randint(0, len(x_trn)-1)\n #img0 = array2img(x_train[r1])\n #img1 = array2img(x_train[r2])\n img0 = array2img(x_trn[r1])\n img1 = array2img(x_trn[r2])\n img_blend = Image.blend(im1=img0, im2=img1, alpha=lam)\n\n #y_blend = y_train[r1]*(1-lam) + y_train[r2]*lam\n y_blend = y_trn[r1]*(1-lam) + y_trn[r2]*lam\n img_blend_arr = np.asarray(img_blend)\n blend_arr_x.append(img_blend_arr)\n blend_arr_y.append(y_blend)\n \n idx_ex = np.arange(len(blend_arr_x))\n strength_list_blend_arr_x = _get_strength_list(blend_arr_x)\n \n for data_list in [x_fade, x_pitch, x_reverb, x_tb, x_pitchup, x_eq, x_tbup]:\n for j in idx:\n if j in trn_idx:\n x_trn.append(data_list[j])\n\n #y_trn, y_val = y_train[trn_idx], y_train[val_idx]\n y_trn = np.tile(y_trn, reps=(8, 1))\n \n # =====================================================\n y_trn = np.concatenate([y_trn, np.stack(blend_arr_y)])\n for k in idx_ex:\n x_trn.append(blend_arr_x[k])\n # =====================================================\n \n #####################################################################\n strength_list_trn = np.array(strength_list)[trn_idx]\n strength_list_trn = np.tile(strength_list_trn, reps=(8, 1))\n\n strength_list_val = np.array(strength_list)[val_idx]\n #####################################################################\n \n strength_list_trn = np.concatenate([strength_list_trn, strength_list_blend_arr_x])\n #train_dataset = FATTrainDataset(x_trn, y_trn, train_transforms)\n #valid_dataset = FATTrainDataset(x_val, y_val, train_transforms)\n \n #fname作成\n #####################################################################\n #train_df_trn = train_df[trn_idx]\n train_df_val = np.array(train_df['fname'])[val_idx]\n #####################################################################\n \n #####################################################################\n train_dataset = FATTrainDataset(x_trn, y_trn, train_transforms, strength_list_trn)\n #valid_dataset = FATTrainDataset(x_val, y_val, train_transforms, strength_list_val)\n #valid_dataset = FATValidDataset(train_df_val, x_val, y_val, train_transforms, strength_list_val, tta=20)\n valid_dataset = FATValidDataset(train_df_val, x_val, y_val, valid_transforms, strength_list_val, tta=20)\n #####################################################################\n\n train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n valid_loader = DataLoader(valid_dataset, batch_size=test_batch_size, shuffle=False)\n\n model = FineTune(n_classes=80, \n weight_path=Path(\"../input/freesound-convnet-noisy-pitch-pretrain-rrc/weight_best.pt\")).cuda()\n # model = ConvNet(n_classes=80).cuda()\n \n criterion = nn.BCEWithLogitsLoss().cuda()\n optimizer = Adam(params=model.parameters(), lr=lr, amsgrad=False)\n scheduler = CosineAnnealingLR(optimizer, T_max=t_max, eta_min=eta_min)\n\n best_epoch = -1\n best_lwlrap = 0.\n mb = master_bar(range(num_epochs))\n torch.cuda.empty_cache()\n\n for epoch in mb:\n start_time = time.time()\n model.train()\n avg_loss = 0.\n\n for x_batch, y_batch in progress_bar(train_loader, parent=mb):\n preds = model(x_batch.cuda())\n loss = criterion(preds, y_batch.cuda())\n\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n\n avg_loss += loss.item() / len(train_loader)\n\n model.eval()\n valid_preds = np.zeros((len(x_val), num_classes))\n avg_val_loss = 0.\n \n ########################################################\n all_outputs, all_y, all_fnames = [], [], []\n ########################################################\n\n #for i, (x_batch, y_batch) in enumerate(valid_loader):\n #############################################################\n for i, (x_batch, y_batch, fnames) in enumerate(valid_loader):\n #############################################################\n preds = model(x_batch.cuda()).detach()\n loss = criterion(preds, y_batch.cuda())\n\n #preds = torch.sigmoid(preds)\n #valid_preds[i * test_batch_size: (i+1) * test_batch_size] = preds.cpu().numpy()\n\n #avg_val_loss += loss.item() / len(valid_loader)\n #################################################\n all_outputs.append(preds.cpu().numpy())\n all_y.append(y_batch.cuda())\n all_fnames.extend(fnames)\n\n preds = torch.sigmoid(preds)\n #valid_preds[i * test_batch_size: (i+1) * test_batch_size] = preds.cpu().numpy()\n\n avg_val_loss += loss.item() / len(valid_loader)\n\n valid_preds = pd.DataFrame(data=np.concatenate(all_outputs),\n index=all_fnames,\n columns=map(str, range(num_classes)))\n valid_preds = valid_preds.groupby(level=0).mean()\n valid_preds = valid_preds.values\n ################################################# \n\n score, weight = calculate_per_class_lwlrap(y_val, valid_preds)\n lwlrap = (score * weight).sum()\n\n scheduler.step()\n\n if (epoch + 1) % 1 == 0:\n elapsed = time.time() - start_time\n mb.write(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f} avg_val_loss: {avg_val_loss:.4f} val_lwlrap: {lwlrap:.6f} time: {elapsed:.0f}s')\n logger.info(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f} avg_val_loss: {avg_val_loss:.4f} val_lwlrap: {lwlrap:.6f} time: {elapsed:.0f}s')\n\n if lwlrap > best_lwlrap:\n best_epoch = epoch + 1\n best_lwlrap = lwlrap\n torch.save(model.state_dict(), f'weight_best{fold_num}.pt')\n \n #loss, lwlrap 記録\n loss_list.append([fold_num, avg_loss, avg_val_loss])\n lwlrap_list.append([fold_num, lwlrap])\n \n bests.append((best_epoch, best_lwlrap))\n del x_trn, x_val, y_trn, y_val\n gc.collect()\n logger.info(f\"Best: {bests}\")\n \n return bests, loss_list, lwlrap_list\n","execution_count":23,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":24,"outputs":[{"output_type":"execute_result","execution_count":24,"data":{"text/plain":"0"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"#result = train_model(x_train, y_train, transforms_dict[\"train\"])\nresult, loss_list, lwlrap_list = train_model(x_train, y_train, \n x_fade, x_pitch, x_reverb, x_tb, \n x_pitchup, x_eq, x_tbup,\n train_df,\n transforms_dict[\"train\"], \n transforms_dict[\"valid\"], \n strength_list_train)","execution_count":null,"outputs":[{"output_type":"stream","text":"2019-06-07 12:16:55,681 Main INFO Fold 1\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"","text/html":"\n
\n \n \n 100.00% [1000/1000 00:00<00:00]\n
\n "},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"","text/html":"\n
\n \n \n 2.50% [1/40 03:10<2:03:47]\n
\n \nEpoch 1 - avg_train_loss: 0.1750 avg_val_loss: 0.0349 val_lwlrap: 0.782349 time: 190s

\n\n

\n \n \n 35.80% [92/257 00:57<01:42]\n
\n "},"metadata":{}},{"output_type":"stream","text":"2019-06-07 12:20:11,418 Main INFO Epoch 1 - avg_train_loss: 0.1750 avg_val_loss: 0.0349 val_lwlrap: 0.782349 time: 190s\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plotting"},{"metadata":{"trusted":true},"cell_type":"code","source":"num_fold = 5\n########################################################\n#可視化\nloss_list = pd.DataFrame(loss_list)\nlwlrap_list = pd.DataFrame(lwlrap_list)\nfor i in range(num_fold):\n\n loss = loss_list[loss_list.iloc[:,0] == i][1]\n val_loss = loss_list[loss_list.iloc[:,0] == i][2]\n full_lwlrap = lwlrap_list[lwlrap_list.iloc[:,0] == i][1]\n #curated_lwlrap = lwlrap_list[lwlrap_list.iloc[:,0] == i][2]\n\n epochs = range(1, len(loss) + 1)\n\n #lossとlwlrapをプロット\n plt.plot(epochs, loss, 'bo', label = \"Training loss\")\n plt.plot(epochs, val_loss, 'b', label = \"Validation loss\")\n plt.plot(epochs, full_lwlrap, 'bo',color='r', label = \"Full lwlrap\")\n #plt.plot(epochs, curated_lwlrap, 'b',color='r', label = \"Curated lwlrap\")\n plt.title('Fold {} Loss and lwlrap'.format(i+1))\n plt.legend()\n plt.show()\n\n########################################################","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Results"},{"metadata":{"trusted":true},"cell_type":"code","source":"result","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}