{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport gzip\nimport os\nimport glob2\nimport copy\nimport math\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.models as vision_models\n\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import SubsetRandomSampler, DataLoader","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2022-04-13T23:34:48.051321Z","iopub.execute_input":"2022-04-13T23:34:48.051801Z","iopub.status.idle":"2022-04-13T23:34:48.060303Z","shell.execute_reply.started":"2022-04-13T23:34:48.051719Z","shell.execute_reply":"2022-04-13T23:34:48.059353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_spectrogram(spectrogram_file, chroma=True):\n    with gzip.GzipFile(spectrogram_file, 'r') as f:\n        spectrograms = np.load(f)\n    # spectrograms contains a fused mel spectrogram and chromagram\n    # Decompose as follows\n    return spectrograms.T\n\n\nclass PaddingTransform(object):\n    def __init__(self, max_length, padding_value=0):\n        self.max_length = max_length\n        self.padding_value = padding_value\n\n    def __call__(self, s):\n        if len(s) == self.max_length:\n            return s\n\n        if len(s) > self.max_length:\n            return s[:self.max_length]\n\n        if len(s) < self.max_length:\n            s1 = copy.deepcopy(s)\n            pad = np.zeros((self.max_length - s.shape[0], s.shape[1]), dtype=np.float32)\n            s1 = np.vstack((s1, pad))\n            return s1\n\n\nclass SpectrogramDatasetTrain(Dataset):\n    def __init__(self, path, max_length=-1):\n        p = os.path.join(path, 'train')\n        self.index = os.path.join(path, 'train_labels.txt')\n        self.files, self.labels = self.get_files_labels(self.index)\n        self.feats = [read_spectrogram(os.path.join(p, f + '.fused.full.npy.gz')) \n                      for f in self.files]\n        self.feat_dim = self.feats[0].shape[1]\n        self.lengths = [len(i) for i in self.feats]\n        self.max_length = max(self.lengths) if max_length <= 0 else max_length\n        self.zero_pad_and_stack = PaddingTransform(self.max_length)\n\n    def get_files_labels(self, txt):\n        with open(txt, 'r') as fd:\n            lines = [l.rstrip().split(',') for l in fd.readlines()[1:]]\n        files, labels = [], []\n        for l in lines:\n            valence, energy, dance = float(l[1]), float(l[2]), float(l[3].strip())\n            files.append(l[0])\n            labels.append([valence, energy, dance])\n        return files, labels\n\n    def __getitem__(self, item):\n        l = min(self.lengths[item], self.max_length)\n        return self.zero_pad_and_stack(self.feats[item]), self.labels[item]\n\n    def __len__(self):\n        return len(self.labels)\n    \n\nclass SpectrogramDatasetTest(Dataset):\n    def __init__(self, max_length=-1):\n        self.files = glob2.glob('../input/data/data/multitask_dataset/test/*.gz')\n        self.feats = [read_spectrogram(f) for f in self.files]\n        self.feat_dim = self.feats[0].shape[1]\n        self.lengths = [len(i) for i in self.feats]\n        self.max_length = max(self.lengths) if max_length <= 0 else max_length\n        self.zero_pad_and_stack = PaddingTransform(self.max_length)\n\n    def __getitem__(self, item):\n        l = min(self.lengths[item], self.max_length)\n        return self.zero_pad_and_stack(self.feats[item]), self.files[item], l\n\n    def __len__(self):\n        return len(self.labels)","metadata":{"_uuid":"7a8ab1a43849b7c69e723538a75e2fa5eae5ff1e","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-04-13T23:34:48.087573Z","iopub.execute_input":"2022-04-13T23:34:48.087948Z","iopub.status.idle":"2022-04-13T23:34:48.10679Z","shell.execute_reply.started":"2022-04-13T23:34:48.087868Z","shell.execute_reply":"2022-04-13T23:34:48.105627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def torch_train_val_split(\n        dataset, batch_train, batch_eval,\n        val_size=.2, shuffle=True, seed=42):\n    # Creating data indices for training and validation splits:\n    dataset_size = len(dataset)\n    indices = list(range(dataset_size))\n    val_split = int(np.floor(val_size * dataset_size))\n    if shuffle:\n        np.random.seed(seed)\n        np.random.shuffle(indices)\n    train_indices = indices[val_split:]\n    val_indices = indices[:val_split]\n\n    # Creating PT data samplers and loaders:\n    train_sampler = SubsetRandomSampler(train_indices)\n    val_sampler = SubsetRandomSampler(val_indices)\n\n    train_loader = DataLoader(dataset,\n                              batch_size=batch_train,\n                              sampler=train_sampler)\n    val_loader = DataLoader(dataset,\n                            batch_size=batch_eval,\n                            sampler=val_sampler)\n    return train_loader, val_loader","metadata":{"_uuid":"e0542f8973174d86ee71b4070c1d8627afff921a","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-04-13T23:34:48.10881Z","iopub.execute_input":"2022-04-13T23:34:48.109806Z","iopub.status.idle":"2022-04-13T23:34:48.122237Z","shell.execute_reply.started":"2022-04-13T23:34:48.10974Z","shell.execute_reply":"2022-04-13T23:34:48.121113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EarlyStopping(object):\n    def __init__(self, patience, mode='min', base=None):\n        self.best = base\n        self.patience = patience\n        self.patience_left = patience\n        self.mode = mode\n    \n    def stop(self, value):\n        if self.has_improved(value):\n            self.patience_left = self.patience  # reset patience\n        else:\n            self.patience_left -= 1  # decrease patience\n        print(\"patience left:{}, best({})\".format(self.patience_left, self.best))\n\n        # if no more patience left, then stop training\n        return self.patience_left <= 0\n    \n    def has_improved(self, value):\n         # init best value\n        if self.best is None or math.isnan(self.best):\n            self.best = value\n            return True\n\n        if (\n                self.mode == \"min\" and value < self.best\n                or\n                self.mode == \"max\" and value > self.best\n        ):  # the performance of the model has been improved :)\n            self.best = value\n            return True\n        else:\n            # no improvement :(\n            return False ","metadata":{"_uuid":"5aa48c4b61999aa0aa748722406092508a5db565","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-04-13T23:34:48.124074Z","iopub.execute_input":"2022-04-13T23:34:48.124849Z","iopub.status.idle":"2022-04-13T23:34:48.137401Z","shell.execute_reply.started":"2022-04-13T23:34:48.124774Z","shell.execute_reply":"2022-04-13T23:34:48.136296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def spearman(x, y):\n    vx = x - torch.mean(x)\n    vy = y - torch.mean(y)\n    return torch.sum(vx * vy) / (torch.sqrt(torch.sum(vx ** 2)) * torch.sqrt(torch.sum(vy ** 2)))\n\n\ndef batch_metrics(model, criterion, batch):\n        features, labels = batch\n        val_gt = labels[0].float().to(device)\n        en_gt = labels[1].float().to(device)\n        dance_gt = labels[2].float().to(device)\n        features = features.float().to(device)\n        valence, energy, danceability = model(features)\n        loss1 = criterion(valence, val_gt)\n        loss2 = criterion(energy, en_gt)\n        loss3 = criterion(danceability, dance_gt)\n        loss = loss1 + loss2 + loss3\n        s1 = spearman(valence, val_gt)\n        s2 = spearman(energy, en_gt)\n        s3 = spearman(danceability, dance_gt)\n        s = (s1 + s2 + s3) / 3.0\n        return loss, s\n\n\ndef train_epoch(train_loader, model, criterion, optimizer, device='cuda'):\n    model.train()\n    running_loss = 0.0\n    running_corr = 0.0\n    for num_batch, batch in enumerate(train_loader):\n        optimizer.zero_grad()\n        loss, corr = batch_metrics(model, criterion, batch)\n        loss.backward()\n        optimizer.step()\n        running_corr += corr.item()\n        running_loss += loss.item()\n    train_loss = running_loss / num_batch\n    train_corr = running_corr / num_batch\n    return train_loss, train_corr\n\ndef val_epoch(val_loader, model, criterion):\n    model.eval()\n    running_loss = 0.0\n    running_corr = 0.0\n    with torch.no_grad():\n        y_pred = torch.empty(0, dtype=torch.int64)\n        y_true = torch.empty(0, dtype=torch.int64)\n        for num_batch, batch in enumerate(val_loader):\n            loss, corr = batch_metrics(model, criterion, batch)\n            running_corr += corr.item()\n            running_loss += loss.item()\n    valid_loss = running_loss / num_batch\n    valid_corr = running_corr / num_batch\n    return valid_loss, valid_corr","metadata":{"_uuid":"af4dc24ec9e3b79573e3646209005a1162c07c23","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-04-13T23:34:48.139361Z","iopub.execute_input":"2022-04-13T23:34:48.140118Z","iopub.status.idle":"2022-04-13T23:34:48.158299Z","shell.execute_reply.started":"2022-04-13T23:34:48.140055Z","shell.execute_reply":"2022-04-13T23:34:48.156869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Implementation\n\nWe use a combination of transfer and multitask learning for this model.\n\nFor the transfer learning we use a Resnet 50 architecture, pretrained on ImageNet (offered in the torchvision library).\n\nSurprisingly (?) CNNs trained on real images can also read spectrograms!!\nThis shows the power of transfer learning to reuse knowledge from other domains (with minimal effort).\n\nTake note that we drop the last 3 layers of the network (classification layer, Average Pooling + the last block of bottleneck layers).\n\nIf we try to keep the last block the network does not converge. The explanation is that this layer learns image specific features and\ncannot be used as a feature extractor in our case.\n\nAfterwards we use 3 linear layers to perform regression on valence, energy and danceability and train a single using multitask learning.\n\nAll of the code can be run in a Kaggle kernel. No tuning was performed on model parameters, learning rate etc.","metadata":{"_uuid":"ec196288e8856762b73d0aa04b2a9dd53ca5e860"}},{"cell_type":"code","source":"class PretrainedImagenet(nn.Module):\n    def __init__(self, original_model=None):\n        super(PretrainedImagenet, self).__init__()\n        if original_model is None:\n            original_model = vision_models.resnet50(pretrained=True)\n        self.features = nn.Sequential(*list(original_model.children())[:-3])\n        for i, p in enumerate(self.features.parameters()):\n            p.requires_grad = False\n        self.pool = nn.AvgPool2d(kernel_size=7, stride=2, padding=0)\n        self.val = nn.Linear(77824, 1)\n        self.en = nn.Linear(77824, 1)\n        self.danc = nn.Linear(77824, 1)\n\n    def forward(self, x):\n        out = self.features(x.unsqueeze(1).repeat(1, 3, 1, 1))\n        out = self.pool(out)\n        out = out.view(x.size(0),  -1)\n        val = torch.sigmoid(self.val(out))\n        en = torch.sigmoid(self.en(out))\n        danc = torch.sigmoid(self.danc(out))\n        return val.view(-1), en.view(-1), danc.view(-1)","metadata":{"_uuid":"1b2a0b0d11fefd6823eb572b5fefd40a158cca3c","execution":{"iopub.status.busy":"2022-04-13T23:34:48.160907Z","iopub.execute_input":"2022-04-13T23:34:48.161239Z","iopub.status.idle":"2022-04-13T23:34:48.174062Z","shell.execute_reply.started":"2022-04-13T23:34:48.161115Z","shell.execute_reply":"2022-04-13T23:34:48.172828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tar -xvzf ../input/fma_small_spectrograms.tar.gz","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../working/data/multitask_dataset","metadata":{"execution":{"iopub.status.busy":"2022-04-13T23:52:25.902922Z","iopub.execute_input":"2022-04-13T23:52:25.90325Z","iopub.status.idle":"2022-04-13T23:52:26.676636Z","shell.execute_reply.started":"2022-04-13T23:52:25.903189Z","shell.execute_reply":"2022-04-13T23:52:26.675494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-04-13T23:37:56.854323Z","iopub.execute_input":"2022-04-13T23:37:56.854686Z","iopub.status.idle":"2022-04-13T23:37:57.613711Z","shell.execute_reply.started":"2022-04-13T23:37:56.854612Z","shell.execute_reply":"2022-04-13T23:37:57.612742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"specs = SpectrogramDatasetTrain('../working/data/multitask_dataset/')\ntrain_loader, val_loader = torch_train_val_split(specs, 32 ,32, val_size=.33)","metadata":{"_uuid":"5d32a45a4d8c9c16b3bbca50423152045252c23e","execution":{"iopub.status.busy":"2022-04-13T23:52:07.047037Z","iopub.execute_input":"2022-04-13T23:52:07.047401Z","iopub.status.idle":"2022-04-13T23:52:07.080886Z","shell.execute_reply.started":"2022-04-13T23:52:07.047341Z","shell.execute_reply":"2022-04-13T23:52:07.079632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint('Running on {}'.format(device))\n\ncnn = PretrainedImagenet().to(device)  # PretrainedImagenet(resnet18(pretrained=True)).to(device)\nprint(cnn)\ntrainable_params = [p for p in cnn.parameters() if p.requires_grad]\nprint(trainable_params)","metadata":{"_uuid":"2d142e70b5f30146d781ce9e293f3cb49b04dc7a","execution":{"iopub.status.busy":"2022-04-13T23:34:48.369089Z","iopub.status.idle":"2022-04-13T23:34:48.369689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 60\n\ncriterion = torch.nn.MSELoss()\noptimizer = torch.optim.Adam(cnn.parameters(), lr=0.001)\nearly_stopping = EarlyStopping(patience=1)\nfor epoch in range(epochs):\n    # Training\n    train_loss, train_corr = train_epoch(train_loader, cnn, criterion, optimizer)\n    # Validation\n    valid_loss, valid_corr = val_epoch(val_loader, cnn, criterion)\n    print('Epoch {}: train loss = {}, valid loss = {}, train corr = {}, valid corr = {}'\n          .format(epoch, train_loss, valid_loss, train_corr, valid_corr))\n    if early_stopping.stop(valid_loss):\n        print('early stopping...')\n        break","metadata":{"_uuid":"24f46f37da6ee52e536e6e71c46d055550b16a39","scrolled":true,"execution":{"iopub.status.busy":"2022-04-13T23:34:48.372145Z","iopub.status.idle":"2022-04-13T23:34:48.373653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdata =  SpectrogramDatasetTest()\n\ncnn.eval()\npred = []\nwith torch.no_grad():\n    for num_batch, batch in enumerate(testdata):\n        features, fname, lengths = batch\n        fname = fname.split('/')[-1]\n        features = torch.as_tensor(features).float().to(device).unsqueeze(0)\n        valence, energy, danceability = cnn(features)\n        pred.append((fname, valence.item(), energy.item(), danceability.item()))","metadata":{"_uuid":"3c786a11560502999c8de6087fd866f1b2114129","execution":{"iopub.status.busy":"2022-04-13T23:34:48.374933Z","iopub.status.idle":"2022-04-13T23:34:48.37573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Now create a submission from pred.","metadata":{"_uuid":"7f63cee9ef623f67b934ebbf35bf18ed1b887318","execution":{"iopub.status.busy":"2022-04-13T23:34:48.37671Z","iopub.status.idle":"2022-04-13T23:34:48.377482Z"},"trusted":true},"execution_count":null,"outputs":[]}]}