{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Configuration"},{"metadata":{},"cell_type":"markdown","source":"### imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport time\nimport pickle\nimport random\nimport itertools\n\nfrom collections import Counter, defaultdict, namedtuple\nfrom functools import partial\nfrom pathlib import Path\nfrom psutil import cpu_count\n\nimport librosa\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\n\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.transforms import transforms\nimport torchvision.models as models\n\nfrom tqdm import tqdm_notebook\nfrom fastprogress import master_bar, progress_bar\n\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nSEED = 1314\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N_JOBS = cpu_count()\nused = 4\nos.environ['MKL_NUM_THREADS'] = str(used)\nos.environ['OMP_NUM_THREADS'] = str(used)\nDataLoader = partial(DataLoader, num_workers=used)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from official code https://colab.research.google.com/drive/1AgPdhSp7ttY18O3fEoHOQKlt_3HJDLi8#scrollTo=cRCaCIb9oguU\ndef _one_sample_positive_class_precisions(scores, truth):\n\n    num_classes = scores.shape[0]\n    pos_class_indices = np.flatnonzero(truth > 0)\n    # Only calculate precisions if there are some true classes.\n    if not len(pos_class_indices):\n        return pos_class_indices, np.zeros(0)\n    # Retrieval list of classes for this sample.\n    retrieved_classes = np.argsort(scores)[::-1]\n    # class_rankings[top_scoring_class_index] == 0 etc.\n    class_rankings = np.zeros(num_classes, dtype=np.int)\n    class_rankings[retrieved_classes] = range(num_classes)\n    # Which of these is a true label?\n    retrieved_class_true = np.zeros(num_classes, dtype=np.bool)\n    retrieved_class_true[class_rankings[pos_class_indices]] = True\n    # Num hits for every truncated retrieval list.\n    retrieved_cumulative_hits = np.cumsum(retrieved_class_true)\n    # Precision of retrieval list truncated at each hit, in order of pos_labels.\n    precision_at_hits = (\n            retrieved_cumulative_hits[class_rankings[pos_class_indices]] /\n            (1 + class_rankings[pos_class_indices].astype(np.float)))\n    return pos_class_indices, precision_at_hits\n\n\ndef calculate_per_class_lwlrap(truth, scores):\n\n    assert truth.shape == scores.shape\n    num_samples, num_classes = scores.shape\n    # Space to store a distinct precision value for each class on each sample.\n    # Only the classes that are true for each sample will be filled in.\n    precisions_for_samples_by_classes = np.zeros((num_samples, num_classes))\n    for sample_num in range(num_samples):\n        pos_class_indices, precision_at_hits = (\n            _one_sample_positive_class_precisions(scores[sample_num, :],\n                                                  truth[sample_num, :]))\n        precisions_for_samples_by_classes[sample_num, pos_class_indices] = (\n            precision_at_hits)\n    labels_per_class = np.sum(truth > 0, axis=0)\n    weight_per_class = labels_per_class / float(np.sum(labels_per_class))\n    # Form average of each column, i.e. all the precisions assigned to labels in\n    # a particular class.\n    per_class_lwlrap = (np.sum(precisions_for_samples_by_classes, axis=0) /\n                        np.maximum(1, labels_per_class))\n    return per_class_lwlrap, weight_per_class","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### make dataset utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_audio(conf, pathname, trim_long_data):\n    y, sr = librosa.load(pathname, sr=conf.sampling_rate)\n    # trim silence\n    if 0 < len(y): # workaround: 0 length causes error\n        y, _ = librosa.effects.trim(y) # trim, top_db=default(60)\n    # make it unified length to conf.samples\n    if len(y) > conf.samples: # long enough\n        if trim_long_data:\n            y = y[0:0+conf.samples]\n    else: # pad blank\n        padding = conf.samples - len(y)    # add padding at both ends\n        offset = padding // 2\n        y = np.pad(y, (offset, conf.samples - len(y) - offset), conf.padmode)\n    return y\n\n\ndef audio_to_melspectrogram(conf, audio):\n    spectrogram = librosa.feature.melspectrogram(audio, \n                                                 sr=conf.sampling_rate,\n                                                 n_mels=conf.n_mels,\n                                                 hop_length=conf.hop_length,\n                                                 n_fft=conf.n_fft,\n                                                 fmin=conf.fmin,\n                                                 fmax=conf.fmax)\n    spectrogram = librosa.power_to_db(spectrogram)\n    spectrogram = spectrogram.astype(np.float32)\n    return spectrogram\n\ndef read_as_melspectrogram(conf, pathname, trim_long_data):\n    x = read_audio(conf, pathname, trim_long_data)\n    mels = audio_to_melspectrogram(conf, x)\n    return mels\n\nclass conf:\n    sampling_rate = 44100\n    duration = 2 # sec\n    hop_length = 347*duration # to make time steps 128\n    fmin = 20\n    fmax = sampling_rate // 2\n    n_mels = 128\n    n_fft = n_mels * 20\n    padmode = 'constant'\n    samples = sampling_rate * duration\n\n\ndef get_default_conf():\n    return conf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mono_to_color(X, mean=None, std=None, norm_max=None, norm_min=None, eps=1e-6):\n    # Stack X as [X,X,X]\n\n    # Standardize\n    mean = mean or X.mean()\n    X = X - mean\n    std = std or X.std()\n    Xstd = X / (std + eps)\n    _min, _max = Xstd.min(), Xstd.max()\n    norm_max = norm_max or _max\n    norm_min = norm_min or _min\n    if (_max - _min) > eps:\n        # Normalize to [0, 255]\n        V = Xstd\n        V[V < norm_min] = norm_min\n        V[V > norm_max] = norm_max\n        V = 255 * (V - norm_min) / (norm_max - norm_min)\n        V = V.astype(np.uint8)\n    else:\n        # Just zero\n        V = np.zeros_like(Xstd, dtype=np.uint8)\n    return V\n\n\ndef convert_wav_to_image(df, source):\n    X = []\n    for i, row in tqdm_notebook(df.iterrows()):\n        x = read_as_melspectrogram(conf, source/str(row.fname), trim_long_data=False)\n        x_color = mono_to_color(x)\n        X.append(x_color)\n    return X","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_dir = Path('../input/freesound-audio-tagging-2019')\npreprocessed_dir = Path('../input/fat2019_prep_mels1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"csvs = {\n    'train_curated': dataset_dir / 'train_curated.csv',\n    'train_noisy': dataset_dir / 'train_noisy.csv',\n    'sample_submission': dataset_dir / 'sample_submission.csv',\n}\n\ndataset = {\n    'train_curated': dataset_dir / 'train_curated',\n    'train_noisy': dataset_dir / 'train_noisy',\n    'test': dataset_dir / 'test',\n}\n\nmels = {\n    'train_curated': preprocessed_dir / 'mels_train_curated.pkl',\n    'train_noisy': preprocessed_dir / 'mels_trn_noisy_best50s.pkl',\n    'test': preprocessed_dir / 'mels_test.pkl',  # NOTE: this data doesn't work at 2nd stage\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_curated = pd.read_csv(csvs['train_curated'])\n# train_noisy = pd.read_csv(csvs['train_noisy'])\n# train_df = pd.concat([train_curated, train_noisy], sort=True, ignore_index=True)\ntrain_df = train_curated\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(csvs['sample_submission'])\n\nlabels = test_df.columns[1:].tolist()\nnum_classes = len(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"conf = get_default_conf()\n\ndef convert_dataset(df, source_folder):\n    X = convert_wav_to_image(df, source=source_folder)\n    return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = np.zeros((len(train_df), num_classes)).astype(int)\nfor i, row in enumerate(train_df['labels'].str.split(',')):\n    for label in row:\n        idx = labels.index(label)\n        y_train[i, idx] = 1\n\ny_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(mels['train_curated'], 'rb') as curated: #, open(mels['train_noisy'], 'rb') as noisy:\n    x_train = pickle.load(curated)\n    # x_train.extend(pickle.load(noisy))\n\nx_test = convert_dataset(test_df, dataset['test'])\n\nlen(x_train), len(x_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Simple CNN Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class ConvBlock(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n        \n        self.conv1 = nn.Sequential(\n            nn.Conv2d(in_channels, out_channels, 3, 1, 1),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(),\n        )\n        self.conv2 = nn.Sequential(\n            nn.Conv2d(out_channels, out_channels, 3, 1, 1),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(),\n        )\n\n        self._init_weights()\n        \n    def _init_weights(self):\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight)\n                if m.bias is not None:\n                    nn.init.zeros_(m.bias)\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.constant_(m.weight, 1)\n                nn.init.zeros_(m.bias)\n        \n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.conv2(x)\n        x = F.avg_pool2d(x, 2)\n        return x\n    \nclass Classifier(nn.Module):\n    def __init__(self, num_classes):\n        super().__init__()\n        \n        self.conv = nn.Sequential(\n            ConvBlock(in_channels=3, out_channels=64),\n            ConvBlock(in_channels=64, out_channels=128),\n            ConvBlock(in_channels=128, out_channels=256),\n            ConvBlock(in_channels=256, out_channels=512),\n        )\n        \n        self.fc = nn.Sequential(\n            nn.Dropout(0.2),\n            nn.Linear(512, 128),\n            nn.PReLU(),\n            nn.BatchNorm1d(128),\n            nn.Dropout(0.1),\n            nn.Linear(128, num_classes),\n        )\n\n    def forward(self, x):\n        x = self.conv(x)\n        x = torch.mean(x, dim=3)\n        x, _ = torch.max(x, dim=2)\n        x = self.fc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Inception 1ch"},{"metadata":{},"cell_type":"markdown","source":"### Inception pytorch models"},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_parameter_requires_grad(model, feature_extracting):\n    if feature_extracting:\n        for param in model.parameters():\n            param.requires_grad = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_InceptionOuputs = namedtuple('InceptionOuputs', ['logits', 'aux_logits'])\n\nclass Inception3(nn.Module):\n\n    def __init__(self, num_classes=1000, aux_logits=True, transform_input=False):\n        super(Inception3, self).__init__()\n        self.aux_logits = aux_logits\n        self.transform_input = transform_input\n        #self.Conv2d_1a_3x3 = BasicConv2d(3, 32, kernel_size=3, stride=2)\n        self.Conv2d_1a_3x3 = BasicConv2d(1, 32, kernel_size=3, stride=2)\n        self.Conv2d_2a_3x3 = BasicConv2d(32, 32, kernel_size=3)\n        self.Conv2d_2b_3x3 = BasicConv2d(32, 64, kernel_size=3, padding=1)\n        self.Conv2d_3b_1x1 = BasicConv2d(64, 80, kernel_size=1)\n        self.Conv2d_4a_3x3 = BasicConv2d(80, 192, kernel_size=3)\n        self.Mixed_5b = InceptionA(192, pool_features=32)\n        self.Mixed_5c = InceptionA(256, pool_features=64)\n        self.Mixed_5d = InceptionA(288, pool_features=64)\n        self.Mixed_6a = InceptionB(288)\n        self.Mixed_6b = InceptionC(768, channels_7x7=128)\n        self.Mixed_6c = InceptionC(768, channels_7x7=160)\n        self.Mixed_6d = InceptionC(768, channels_7x7=160)\n        self.Mixed_6e = InceptionC(768, channels_7x7=192)\n        if aux_logits:\n            self.AuxLogits = InceptionAux(768, num_classes)\n        self.Mixed_7a = InceptionD(768)\n        self.Mixed_7b = InceptionE(1280)\n        self.Mixed_7c = InceptionE(2048)\n        self.fc = nn.Linear(2048, num_classes)\n\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear):\n                import scipy.stats as stats\n                stddev = m.stddev if hasattr(m, 'stddev') else 0.1\n                X = stats.truncnorm(-2, 2, scale=stddev)\n                values = torch.as_tensor(X.rvs(m.weight.numel()), dtype=m.weight.dtype)\n                values = values.view(m.weight.size())\n                with torch.no_grad():\n                    m.weight.copy_(values)\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n\n    def forward(self, x):\n        if self.transform_input:\n            #x_ch0 = torch.unsqueeze(x[:, 0], 1) * (0.229 / 0.5) + (0.485 - 0.5) / 0.5\n            #x_ch1 = torch.unsqueeze(x[:, 1], 1) * (0.224 / 0.5) + (0.456 - 0.5) / 0.5\n            #x_ch2 = torch.unsqueeze(x[:, 2], 1) * (0.225 / 0.5) + (0.406 - 0.5) / 0.5\n            x_ch0 = torch.unsqueeze(x[:], 1) * (0.229 / 0.5) + (0.485 - 0.5) / 0.5\n            #x = torch.cat((x_ch0, x_ch1, x_ch2), 1)\n            x = torch.cat((x_ch0), 1)\n        # N x 3 x 299 x 299\n        x = self.Conv2d_1a_3x3(x)\n        # N x 32 x 149 x 149\n        x = self.Conv2d_2a_3x3(x)\n        # N x 32 x 147 x 147\n        x = self.Conv2d_2b_3x3(x)\n        # N x 64 x 147 x 147\n        x = F.max_pool2d(x, kernel_size=3, stride=2)\n        # N x 64 x 73 x 73\n        x = self.Conv2d_3b_1x1(x)\n        # N x 80 x 73 x 73\n        x = self.Conv2d_4a_3x3(x)\n        # N x 192 x 71 x 71\n        x = F.max_pool2d(x, kernel_size=3, stride=2)\n        # N x 192 x 35 x 35\n        x = self.Mixed_5b(x)\n        # N x 256 x 35 x 35\n        x = self.Mixed_5c(x)\n        # N x 288 x 35 x 35\n        x = self.Mixed_5d(x)\n        # N x 288 x 35 x 35\n        x = self.Mixed_6a(x)\n        # N x 768 x 17 x 17\n        x = self.Mixed_6b(x)\n        # N x 768 x 17 x 17\n        x = self.Mixed_6c(x)\n        # N x 768 x 17 x 17\n        x = self.Mixed_6d(x)\n        # N x 768 x 17 x 17\n        x = self.Mixed_6e(x)\n        # N x 768 x 17 x 17\n        if self.training and self.aux_logits:\n            aux = self.AuxLogits(x)\n        # N x 768 x 17 x 17\n        x = self.Mixed_7a(x)\n        # N x 1280 x 8 x 8\n        x = self.Mixed_7b(x)\n        # N x 2048 x 8 x 8\n        x = self.Mixed_7c(x)\n        # N x 2048 x 8 x 8\n        # Adaptive average pooling\n        x = F.adaptive_avg_pool2d(x, (1, 1))\n        # N x 2048 x 1 x 1\n        x = F.dropout(x, training=self.training)\n        # N x 2048 x 1 x 1\n        x = x.view(x.size(0), -1)\n        # N x 2048\n        x = self.fc(x)\n        # N x 1000 (num_classes)\n        if self.training and self.aux_logits:\n            return _InceptionOuputs(x, aux)\n        return x\n\nclass InceptionA(nn.Module):\n\n    def __init__(self, in_channels, pool_features):\n        super(InceptionA, self).__init__()\n        self.branch1x1 = BasicConv2d(in_channels, 64, kernel_size=1)\n\n        self.branch5x5_1 = BasicConv2d(in_channels, 48, kernel_size=1)\n        self.branch5x5_2 = BasicConv2d(48, 64, kernel_size=5, padding=2)\n\n        self.branch3x3dbl_1 = BasicConv2d(in_channels, 64, kernel_size=1)\n        self.branch3x3dbl_2 = BasicConv2d(64, 96, kernel_size=3, padding=1)\n        self.branch3x3dbl_3 = BasicConv2d(96, 96, kernel_size=3, padding=1)\n\n        self.branch_pool = BasicConv2d(in_channels, pool_features, kernel_size=1)\n\n    def forward(self, x):\n        branch1x1 = self.branch1x1(x)\n\n        branch5x5 = self.branch5x5_1(x)\n        branch5x5 = self.branch5x5_2(branch5x5)\n\n        branch3x3dbl = self.branch3x3dbl_1(x)\n        branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl)\n        branch3x3dbl = self.branch3x3dbl_3(branch3x3dbl)\n\n        branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1)\n        branch_pool = self.branch_pool(branch_pool)\n\n        outputs = [branch1x1, branch5x5, branch3x3dbl, branch_pool]\n        return torch.cat(outputs, 1)\n\n\nclass InceptionB(nn.Module):\n\n    def __init__(self, in_channels):\n        super(InceptionB, self).__init__()\n        self.branch3x3 = BasicConv2d(in_channels, 384, kernel_size=3, stride=2)\n\n        self.branch3x3dbl_1 = BasicConv2d(in_channels, 64, kernel_size=1)\n        self.branch3x3dbl_2 = BasicConv2d(64, 96, kernel_size=3, padding=1)\n        self.branch3x3dbl_3 = BasicConv2d(96, 96, kernel_size=3, stride=2)\n\n    def forward(self, x):\n        branch3x3 = self.branch3x3(x)\n\n        branch3x3dbl = self.branch3x3dbl_1(x)\n        branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl)\n        branch3x3dbl = self.branch3x3dbl_3(branch3x3dbl)\n\n        branch_pool = F.max_pool2d(x, kernel_size=3, stride=2)\n\n        outputs = [branch3x3, branch3x3dbl, branch_pool]\n        return torch.cat(outputs, 1)\n\n\nclass InceptionC(nn.Module):\n\n    def __init__(self, in_channels, channels_7x7):\n        super(InceptionC, self).__init__()\n        self.branch1x1 = BasicConv2d(in_channels, 192, kernel_size=1)\n\n        c7 = channels_7x7\n        self.branch7x7_1 = BasicConv2d(in_channels, c7, kernel_size=1)\n        self.branch7x7_2 = BasicConv2d(c7, c7, kernel_size=(1, 7), padding=(0, 3))\n        self.branch7x7_3 = BasicConv2d(c7, 192, kernel_size=(7, 1), padding=(3, 0))\n\n        self.branch7x7dbl_1 = BasicConv2d(in_channels, c7, kernel_size=1)\n        self.branch7x7dbl_2 = BasicConv2d(c7, c7, kernel_size=(7, 1), padding=(3, 0))\n        self.branch7x7dbl_3 = BasicConv2d(c7, c7, kernel_size=(1, 7), padding=(0, 3))\n        self.branch7x7dbl_4 = BasicConv2d(c7, c7, kernel_size=(7, 1), padding=(3, 0))\n        self.branch7x7dbl_5 = BasicConv2d(c7, 192, kernel_size=(1, 7), padding=(0, 3))\n\n        self.branch_pool = BasicConv2d(in_channels, 192, kernel_size=1)\n\n    def forward(self, x):\n        branch1x1 = self.branch1x1(x)\n\n        branch7x7 = self.branch7x7_1(x)\n        branch7x7 = self.branch7x7_2(branch7x7)\n        branch7x7 = self.branch7x7_3(branch7x7)\n\n        branch7x7dbl = self.branch7x7dbl_1(x)\n        branch7x7dbl = self.branch7x7dbl_2(branch7x7dbl)\n        branch7x7dbl = self.branch7x7dbl_3(branch7x7dbl)\n        branch7x7dbl = self.branch7x7dbl_4(branch7x7dbl)\n        branch7x7dbl = self.branch7x7dbl_5(branch7x7dbl)\n\n        branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1)\n        branch_pool = self.branch_pool(branch_pool)\n\n        outputs = [branch1x1, branch7x7, branch7x7dbl, branch_pool]\n        return torch.cat(outputs, 1)\n\n\nclass InceptionD(nn.Module):\n\n    def __init__(self, in_channels):\n        super(InceptionD, self).__init__()\n        self.branch3x3_1 = BasicConv2d(in_channels, 192, kernel_size=1)\n        self.branch3x3_2 = BasicConv2d(192, 320, kernel_size=3, stride=2)\n\n        self.branch7x7x3_1 = BasicConv2d(in_channels, 192, kernel_size=1)\n        self.branch7x7x3_2 = BasicConv2d(192, 192, kernel_size=(1, 7), padding=(0, 3))\n        self.branch7x7x3_3 = BasicConv2d(192, 192, kernel_size=(7, 1), padding=(3, 0))\n        self.branch7x7x3_4 = BasicConv2d(192, 192, kernel_size=3, stride=2)\n\n    def forward(self, x):\n        branch3x3 = self.branch3x3_1(x)\n        branch3x3 = self.branch3x3_2(branch3x3)\n\n        branch7x7x3 = self.branch7x7x3_1(x)\n        branch7x7x3 = self.branch7x7x3_2(branch7x7x3)\n        branch7x7x3 = self.branch7x7x3_3(branch7x7x3)\n        branch7x7x3 = self.branch7x7x3_4(branch7x7x3)\n\n        branch_pool = F.max_pool2d(x, kernel_size=3, stride=2)\n        outputs = [branch3x3, branch7x7x3, branch_pool]\n        return torch.cat(outputs, 1)\n\n\nclass InceptionE(nn.Module):\n\n    def __init__(self, in_channels):\n        super(InceptionE, self).__init__()\n        self.branch1x1 = BasicConv2d(in_channels, 320, kernel_size=1)\n\n        self.branch3x3_1 = BasicConv2d(in_channels, 384, kernel_size=1)\n        self.branch3x3_2a = BasicConv2d(384, 384, kernel_size=(1, 3), padding=(0, 1))\n        self.branch3x3_2b = BasicConv2d(384, 384, kernel_size=(3, 1), padding=(1, 0))\n\n        self.branch3x3dbl_1 = BasicConv2d(in_channels, 448, kernel_size=1)\n        self.branch3x3dbl_2 = BasicConv2d(448, 384, kernel_size=3, padding=1)\n        self.branch3x3dbl_3a = BasicConv2d(384, 384, kernel_size=(1, 3), padding=(0, 1))\n        self.branch3x3dbl_3b = BasicConv2d(384, 384, kernel_size=(3, 1), padding=(1, 0))\n\n        self.branch_pool = BasicConv2d(in_channels, 192, kernel_size=1)\n\n    def forward(self, x):\n        branch1x1 = self.branch1x1(x)\n\n        branch3x3 = self.branch3x3_1(x)\n        branch3x3 = [\n            self.branch3x3_2a(branch3x3),\n            self.branch3x3_2b(branch3x3),\n        ]\n        branch3x3 = torch.cat(branch3x3, 1)\n\n        branch3x3dbl = self.branch3x3dbl_1(x)\n        branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl)\n        branch3x3dbl = [\n            self.branch3x3dbl_3a(branch3x3dbl),\n            self.branch3x3dbl_3b(branch3x3dbl),\n        ]\n        branch3x3dbl = torch.cat(branch3x3dbl, 1)\n\n        branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1)\n        branch_pool = self.branch_pool(branch_pool)\n\n        outputs = [branch1x1, branch3x3, branch3x3dbl, branch_pool]\n        return torch.cat(outputs, 1)\n\n\nclass InceptionAux(nn.Module):\n\n    def __init__(self, in_channels, num_classes):\n        super(InceptionAux, self).__init__()\n        self.conv0 = BasicConv2d(in_channels, 128, kernel_size=1)\n        self.conv1 = BasicConv2d(128, 768, kernel_size=5)\n        self.conv1.stddev = 0.01\n        self.fc = nn.Linear(768, num_classes)\n        self.fc.stddev = 0.001\n\n    def forward(self, x):\n        # N x 768 x 17 x 17\n        x = F.avg_pool2d(x, kernel_size=5, stride=3)\n        # N x 768 x 5 x 5\n        x = self.conv0(x)\n        # N x 128 x 5 x 5\n        x = self.conv1(x)\n        # N x 768 x 1 x 1\n        # Adaptive average pooling\n        x = F.adaptive_avg_pool2d(x, (1, 1))\n        # N x 768 x 1 x 1\n        x = x.view(x.size(0), -1)\n        # N x 768\n        x = self.fc(x)\n        # N x 1000\n        return x\n\n\nclass BasicConv2d(nn.Module):\n\n    def __init__(self, in_channels, out_channels, **kwargs):\n        super(BasicConv2d, self).__init__()\n        self.conv = nn.Conv2d(in_channels, out_channels, bias=False, **kwargs)\n        self.bn = nn.BatchNorm2d(out_channels, eps=0.001)\n\n    def forward(self, x):\n        x = self.conv(x)\n        x = self.bn(x)\n        return F.relu(x, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def initialize_model(num_classes, feature_extract, use_pretrained=False):\n    model_ft = Inception3(num_classes=num_classes)\n    set_parameter_requires_grad(model_ft, feature_extract)\n    # Handle the auxilary net\n    num_ftrs = model_ft.AuxLogits.fc.in_features\n    model_ft.AuxLogits.fc = nn.Linear(num_ftrs, num_classes)\n    # Handle the primary net\n    num_ftrs = model_ft.fc.in_features\n    model_ft.fc = nn.Linear(num_ftrs,num_classes)\n    input_size = 299\n\n    return model_ft, input_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_1ch_incep, input_size = initialize_model(num_classes, False, use_pretrained=False)\n\n\ntransforms_dict = {\n    'train': transforms.Compose([\n        transforms.Resize(input_size),\n        transforms.RandomHorizontalFlip(0.5),\n        transforms.ToTensor(),\n    ]),\n    'test': transforms.Compose([\n        transforms.Resize(input_size),\n        transforms.RandomHorizontalFlip(0.5),\n        transforms.ToTensor(),\n    ]),\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Inception 3ch"},{"metadata":{"trusted":true},"cell_type":"code","source":"def initialize_model_3ch(num_classes, feature_extract, use_pretrained=False):\n    model_ft = models.inception_v3(pretrained=use_pretrained)\n    set_parameter_requires_grad(model_ft, feature_extract)\n    # Handle the auxilary net\n    num_ftrs = model_ft.AuxLogits.fc.in_features\n    model_ft.AuxLogits.fc = nn.Linear(num_ftrs, num_classes)\n    # Handle the primary net\n    num_ftrs = model_ft.fc.in_features\n    model_ft.fc = nn.Linear(num_ftrs,num_classes)\n    input_size = 299\n\n    return model_ft, input_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_3ch_incep, input_size = initialize_model_3ch(num_classes, False, use_pretrained=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data loader"},{"metadata":{"trusted":true},"cell_type":"code","source":"class FATTrainDataset(Dataset):\n    def __init__(self, mels, labels, transforms):\n        super().__init__()\n        self.mels = mels\n        self.labels = labels\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.mels)\n    \n    def __getitem__(self, idx):\n        # crop 1sec\n        image = Image.fromarray(self.mels[idx], mode='RGB')        \n        time_dim, base_dim = image.size\n        crop = random.randint(0, time_dim - base_dim)\n        image = image.crop([crop, 0, crop + base_dim, base_dim])\n        image = self.transforms(image).div_(255)\n        \n        label = self.labels[idx]\n        label = torch.from_numpy(label).float()\n        \n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(x_train, y_train, train_transforms):\n    num_epochs = 80\n    batch_size = 16\n    test_batch_size = 16\n    lr = 3e-3\n    eta_min = 1e-5\n    t_max = 10\n    \n    binary_number = 0\n    \n    num_classes = y_train.shape[1]\n\n    x_trn, x_val, y_trn, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=SEED)\n    \n    train_dataset = FATTrainDataset(x_trn, y_trn, train_transforms)\n    valid_dataset = FATTrainDataset(x_val, y_val, train_transforms)\n\n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n    valid_loader = DataLoader(valid_dataset, batch_size=test_batch_size, shuffle=False)\n\n    model = Classifier(num_classes=num_classes).cuda()\n    \n    model1 = model_1ch_incep\n    model1.load_state_dict(torch.load(f\"../input/inception-1ch-dameoshi-aug/weight_best{binary_number}.pt\"))\n    model1.cuda()\n    model1.eval()    \n    \n    model3 = model_3ch_incep\n    model3.load_state_dict(torch.load(f\"../input/inception-3ch-dameoshi-aug/weight_best{binary_number}.pt\"))\n    model3.cuda()\n    model3.eval()\n    \n    criterion = nn.BCEWithLogitsLoss().cuda()\n    optimizer = Adam(params=model.parameters(), lr=lr, amsgrad=False)\n    scheduler = CosineAnnealingLR(optimizer, T_max=t_max, eta_min=eta_min)\n    \n    temperature = 10\n    lambda_factor=1\n\n    best_epoch = -1\n    best_lwlrap = 0.\n    mb = master_bar(range(num_epochs))\n\n    for epoch in mb:\n        start_time = time.time()\n        model.train()\n        avg_loss = 0.\n\n        for x_batch, y_batch in progress_bar(train_loader, parent=mb):\n            \n            teacher1 = model1(x_batch[:, 0, :, :].view(-1, 1, 299, 299).cuda())\n            teacher3 = model3(x_batch.cuda())\n            lesson = ((teacher1 + teacher3) / 2)\n            \n            preds = model(x_batch.cuda())\n            \n            kl_loss = F.kl_div(F.log_softmax((preds / temperature)), F.softmax((lesson / temperature)),\n                               reduction=\"batchmean\")\n\n            # loss = criterion(preds, y_batch.cuda())\n            # loss = F.cross_entropy(preds, y_batch.cuda()) + lambda_factor * (temperature ** 2) * kl_loss\n            loss = criterion(preds, y_batch.cuda()) + lambda_factor * (temperature ** 2) * kl_loss\n\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n            avg_loss += loss.item() / len(train_loader)\n        \n        model.eval()\n        valid_preds = np.zeros((len(x_val), num_classes))\n        avg_val_loss = 0.\n\n        for i, (x_batch, y_batch) in enumerate(valid_loader):\n            preds = model(x_batch.cuda()).detach()\n            loss = criterion(preds, y_batch.cuda())\n\n            preds = torch.sigmoid(preds)\n            valid_preds[i * test_batch_size: (i+1) * test_batch_size] = preds.cpu().numpy()\n\n            avg_val_loss += loss.item() / len(valid_loader)\n            \n        score, weight = calculate_per_class_lwlrap(y_val, valid_preds)\n        lwlrap = (score * weight).sum()\n        \n        scheduler.step()\n\n        if (epoch + 1) % 2 == 0:\n            elapsed = time.time() - start_time\n            mb.write(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  val_lwlrap: {lwlrap:.6f}  time: {elapsed:.0f}s')\n    \n        if lwlrap > best_lwlrap:\n            best_epoch = epoch + 1\n            best_lwlrap = lwlrap\n            torch.save(model.state_dict(), 'weight_best.pt')\n            \n    return {\n        'best_epoch': best_epoch,\n        'best_lwlrap': best_lwlrap,\n    }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result = train_model(x_train, y_train, transforms_dict['train'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"class FATTestDataset(Dataset):\n    def __init__(self, fnames, mels, transforms, tta=5):\n        super().__init__()\n        self.fnames = fnames\n        self.mels = mels\n        self.transforms = transforms\n        self.tta = tta\n        \n    def __len__(self):\n        return len(self.fnames) * self.tta\n    \n    def __getitem__(self, idx):\n        new_idx = idx % len(self.fnames)\n        \n        x = np.stack([self.mels[new_idx], self.mels[new_idx], self.mels[new_idx]], axis=-1)\n        \n        image = Image.fromarray(x, mode='RGB')\n        time_dim, base_dim = image.size\n        crop = random.randint(0, time_dim - base_dim)\n        image = image.crop([crop, 0, crop + base_dim, base_dim])\n        image = self.transforms(image).div_(255)\n\n        fname = self.fnames[new_idx]\n        \n        return image, fname","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_model(test_fnames, x_test, test_transforms, num_classes, *, tta=5):\n    batch_size = 16\n\n    test_dataset = FATTestDataset(test_fnames, x_test, test_transforms, tta=tta)\n    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n\n    model = Classifier(num_classes=num_classes)\n    model.load_state_dict(torch.load('weight_best.pt'))\n    model.cuda()\n    model.eval()\n\n    all_outputs, all_fnames = [], []\n\n    pb = progress_bar(test_loader)\n    for images, fnames in pb:\n        preds = torch.sigmoid(model(images.cuda()).detach())\n        all_outputs.append(preds.cpu().numpy())\n        all_fnames.extend(fnames)\n\n    test_preds = pd.DataFrame(data=np.concatenate(all_outputs),\n                              index=all_fnames,\n                              columns=map(str, range(num_classes)))\n    test_preds = test_preds.groupby(level=0).mean()\n\n    return test_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = predict_model(test_df['fname'], x_test, transforms_dict['test'], num_classes, tta=20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[labels] = test_preds.values\ntest_df.to_csv('submission.csv', index=False)\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}