{"cells":[{"metadata":{},"cell_type":"markdown","source":"### imports"},{"metadata":{},"cell_type":"markdown","source":"This kernel was forked from [https://www.kaggle.com/mhiro2/simple-2d-cnn-classifier-with-pytorch](http://www.kaggle.com/mhiro2/simple-2d-cnn-classifier-with-pytorch) simple changes in it significantly improved LB score to 0.630"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport pickle\nimport random\nimport time\nfrom collections import Counter, defaultdict\nfrom functools import partial\nfrom pathlib import Path\nfrom psutil import cpu_count\n\nimport librosa\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\n#from skmultilearn.model_selection import iterative_train_test_split\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom fastprogress import master_bar, progress_bar\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.transforms import transforms","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.cuda.is_available()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nSEED = 520\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N_JOBS = cpu_count()\nos.environ['MKL_NUM_THREADS'] = str(N_JOBS)\nos.environ['OMP_NUM_THREADS'] = str(N_JOBS)\nDataLoader = partial(DataLoader, num_workers=N_JOBS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from official code https://colab.research.google.com/drive/1AgPdhSp7ttY18O3fEoHOQKlt_3HJDLi8#scrollTo=cRCaCIb9oguU\ndef _one_sample_positive_class_precisions(scores, truth):\n    \"\"\"Calculate precisions for each true class for a single sample.\n\n    Args:\n      scores: np.array of (num_classes,) giving the individual classifier scores.\n      truth: np.array of (num_classes,) bools indicating which classes are true.\n\n    Returns:\n      pos_class_indices: np.array of indices of the true classes for this sample.\n      pos_class_precisions: np.array of precisions corresponding to each of those\n        classes.\n    \"\"\"\n    num_classes = scores.shape[0]\n    pos_class_indices = np.flatnonzero(truth > 0)\n    # Only calculate precisions if there are some true classes.\n    if not len(pos_class_indices):\n        return pos_class_indices, np.zeros(0)\n    # Retrieval list of classes for this sample.\n    retrieved_classes = np.argsort(scores)[::-1]\n    # class_rankings[top_scoring_class_index] == 0 etc.\n    class_rankings = np.zeros(num_classes, dtype=np.int)\n    class_rankings[retrieved_classes] = range(num_classes)\n    # Which of these is a true label?\n    retrieved_class_true = np.zeros(num_classes, dtype=np.bool)\n    retrieved_class_true[class_rankings[pos_class_indices]] = True\n    # Num hits for every truncated retrieval list.\n    retrieved_cumulative_hits = np.cumsum(retrieved_class_true)\n    # Precision of retrieval list truncated at each hit, in order of pos_labels.\n    precision_at_hits = (\n            retrieved_cumulative_hits[class_rankings[pos_class_indices]] /\n            (1 + class_rankings[pos_class_indices].astype(np.float)))\n    return pos_class_indices, precision_at_hits\n\n\ndef calculate_per_class_lwlrap(truth, scores):\n    \"\"\"Calculate label-weighted label-ranking average precision.\n\n    Arguments:\n      truth: np.array of (num_samples, num_classes) giving boolean ground-truth\n        of presence of that class in that sample.\n      scores: np.array of (num_samples, num_classes) giving the classifier-under-\n        test's real-valued score for each class for each sample.\n\n    Returns:\n      per_class_lwlrap: np.array of (num_classes,) giving the lwlrap for each\n        class.\n      weight_per_class: np.array of (num_classes,) giving the prior of each\n        class within the truth labels.  Then the overall unbalanced lwlrap is\n        simply np.sum(per_class_lwlrap * weight_per_class)\n    \"\"\"\n    assert truth.shape == scores.shape\n    num_samples, num_classes = scores.shape\n    # Space to store a distinct precision value for each class on each sample.\n    # Only the classes that are true for each sample will be filled in.\n    precisions_for_samples_by_classes = np.zeros((num_samples, num_classes))\n    for sample_num in range(num_samples):\n        pos_class_indices, precision_at_hits = (\n            _one_sample_positive_class_precisions(scores[sample_num, :],\n                                                  truth[sample_num, :]))\n        precisions_for_samples_by_classes[sample_num, pos_class_indices] = (\n            precision_at_hits)\n    labels_per_class = np.sum(truth > 0, axis=0)\n    weight_per_class = labels_per_class / float(np.sum(labels_per_class))\n    # Form average of each column, i.e. all the precisions assigned to labels in\n    # a particular class.\n    per_class_lwlrap = (np.sum(precisions_for_samples_by_classes, axis=0) /\n                        np.maximum(1, labels_per_class))\n    # overall_lwlrap = simple average of all the actual per-class, per-sample precisions\n    #                = np.sum(precisions_for_samples_by_classes) / np.sum(precisions_for_samples_by_classes > 0)\n    #           also = weighted mean of per-class lwlraps, weighted by class label prior across samples\n    #                = np.sum(per_class_lwlrap * weight_per_class)\n    return per_class_lwlrap, weight_per_class","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### dataset"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"dataset_dir = Path('../input/freesound-audio-tagging-2019')\npreprocessed_dir = Path('../input/fat2019_prep_mels1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"csvs = {\n    'train_curated': dataset_dir / 'train_curated.csv',\n    #'train_noisy': dataset_dir / 'train_noisy.csv',\n    'train_noisy': preprocessed_dir / 'trn_noisy_best50s.csv',\n    'sample_submission': dataset_dir / 'sample_submission.csv',\n}\n\ndataset = {\n    'train_curated': dataset_dir / 'train_curated',\n    'train_noisy': dataset_dir / 'train_noisy',\n    'test': dataset_dir / 'test',\n}\n\nmels = {\n    'train_curated': preprocessed_dir / 'mels_train_curated.pkl',\n    'train_noisy': preprocessed_dir / 'mels_trn_noisy_best50s.pkl',\n    'test': preprocessed_dir / 'mels_test.pkl',  # NOTE: this data doesn't work at 2nd stage\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_curated = pd.read_csv(csvs['train_curated'])\ntrain_noisy = pd.read_csv(csvs['train_noisy'])\ntrain_df = pd.concat([train_curated, train_noisy], sort=True, ignore_index=True)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(csvs['sample_submission'])\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = test_df.columns[1:].tolist()\nlabels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_classes = len(labels)\nnum_classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = np.zeros((len(train_df), num_classes)).astype(int)\nfor i, row in enumerate(train_df['labels'].str.split(',')):\n    for label in row:\n        idx = labels.index(label)\n        y_train[i, idx] = 1\n\ny_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(mels['train_curated'], 'rb') as curated, open(mels['train_noisy'], 'rb') as noisy:\n    x_train = pickle.load(curated)\n    x_train.extend(pickle.load(noisy))\n\nwith open(mels['test'], 'rb') as test:\n    x_test = pickle.load(test)\n    \nlen(x_train), len(x_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FATTrainDataset(Dataset):\n    def __init__(self, mels, labels, transforms):\n        super().__init__()\n        self.mels = mels\n        self.labels = labels\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.mels)\n    \n    def __getitem__(self, idx):\n        # crop 1sec\n        image = Image.fromarray(self.mels[idx], mode='RGB')        \n        time_dim, base_dim = image.size\n        crop = random.randint(0, time_dim - base_dim)\n        image = image.crop([crop, 0, crop + base_dim, base_dim])\n        image = self.transforms(image).div_(255)\n        \n        label = self.labels[idx]\n        label = torch.from_numpy(label).float()\n        \n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FATTestDataset(Dataset):\n    def __init__(self, fnames, mels, transforms, tta=5):\n        super().__init__()\n        self.fnames = fnames\n        self.mels = mels\n        self.transforms = transforms\n        self.tta = tta\n        \n    def __len__(self):\n        return len(self.fnames) * self.tta\n    \n    def __getitem__(self, idx):\n        new_idx = idx % len(self.fnames)\n        \n        image = Image.fromarray(self.mels[new_idx], mode='RGB')\n        time_dim, base_dim = image.size\n        crop = random.randint(0, time_dim - base_dim)\n        image = image.crop([crop, 0, crop + base_dim, base_dim])\n        image = self.transforms(image).div_(255)\n\n        fname = self.fnames[new_idx]\n        \n        return image, fname","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_dict = {\n    'train': transforms.Compose([\n        transforms.RandomHorizontalFlip(0.5),\n        transforms.ToTensor(),\n    ]),\n    'test': transforms.Compose([\n        transforms.RandomHorizontalFlip(0.5),\n        transforms.ToTensor(),\n    ]),\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class ConvBlock(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n        \n        self.conv1 = nn.Sequential(\n            nn.Conv2d(in_channels, out_channels, 3, 1, 1),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(),\n        )\n        self.conv2 = nn.Sequential(\n            nn.Conv2d(out_channels, out_channels, 3, 1, 1),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(),\n        )\n\n        self._init_weights()\n        \n    def _init_weights(self):\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight)\n                if m.bias is not None:\n                    nn.init.zeros_(m.bias)\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.constant_(m.weight, 1)\n                nn.init.zeros_(m.bias)\n        \n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.conv2(x)\n        x = F.avg_pool2d(x, 2)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Classifier(nn.Module):\n    def __init__(self, num_classes):\n        super().__init__()\n        \n        self.conv = nn.Sequential(\n            ConvBlock(in_channels=3, out_channels=64),\n            ConvBlock(in_channels=64, out_channels=128),\n            ConvBlock(in_channels=128, out_channels=256),\n            ConvBlock(in_channels=256, out_channels=512),\n        )\n        \n        self.fc = nn.Sequential(\n            nn.Dropout(0.2),\n            nn.Linear(512, 128),\n            nn.PReLU(),\n            nn.BatchNorm1d(128),\n            nn.Dropout(0.1),\n            nn.Linear(128, num_classes),\n        )\n\n    def forward(self, x):\n        x = self.conv(x)\n        x = torch.mean(x, dim=3)\n        x, _ = torch.max(x, dim=2)\n        x = self.fc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Classifier(num_classes=num_classes)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### train"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(x_train, y_train, train_transforms):\n    num_epochs = 118\n    batch_size = 128\n    test_batch_size = 256\n    lr = 1e-3\n    eta_min = 1e-5\n    t_max = 5\n    \n    num_classes = y_train.shape[1]\n\n    x_trn, x_val, y_trn, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=SEED)\n    \n    train_dataset = FATTrainDataset(x_trn, y_trn, train_transforms)\n    valid_dataset = FATTrainDataset(x_val, y_val, train_transforms)\n\n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n    valid_loader = DataLoader(valid_dataset, batch_size=test_batch_size, shuffle=False)\n\n    model = Classifier(num_classes=num_classes).cuda()\n    criterion = nn.BCEWithLogitsLoss().cuda()\n    optimizer = Adam(params=model.parameters(), lr=lr, amsgrad=False)\n    scheduler = CosineAnnealingLR(optimizer, T_max=t_max, eta_min=eta_min)\n\n    best_epoch = -1\n    best_lwlrap = 0.\n    mb = master_bar(range(num_epochs))\n\n    for epoch in mb:\n        start_time = time.time()\n        model.train()\n        avg_loss = 0.\n\n        for x_batch, y_batch in train_loader:\n            preds = model(x_batch.cuda())\n            loss = criterion(preds, y_batch.cuda())\n\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n            avg_loss += loss.item() / len(train_loader)\n\n        model.eval()\n        valid_preds = np.zeros((len(x_val), num_classes))\n        avg_val_loss = 0.\n\n        for i, (x_batch, y_batch) in enumerate(valid_loader):\n            preds = model(x_batch.cuda()).detach()\n            loss = criterion(preds, y_batch.cuda())\n\n            preds = torch.sigmoid(preds)\n            valid_preds[i * test_batch_size: (i+1) * test_batch_size] = preds.cpu().numpy()\n\n            avg_val_loss += loss.item() / len(valid_loader)\n            \n        score, weight = calculate_per_class_lwlrap(y_val, valid_preds)\n        lwlrap = (score * weight).sum()\n        \n        scheduler.step()\n\n        if (epoch + 1) % 1 == 0:\n            elapsed = time.time() - start_time\n            mb.write(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  val_lwlrap: {lwlrap:.6f}  time: {elapsed:.0f}s')\n    \n        if lwlrap > best_lwlrap:\n            best_epoch = epoch + 1\n            best_lwlrap = lwlrap\n            torch.save(model.state_dict(), 'weight_best.pt')\n            \n    return {\n        'best_epoch': best_epoch,\n        'best_lwlrap': best_lwlrap,\n    }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result = train_model(x_train, y_train, transforms_dict['train'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_model(test_fnames, x_test, test_transforms, num_classes, *, tta=5):\n    batch_size = 256\n\n    test_dataset = FATTestDataset(test_fnames, x_test, test_transforms, tta=tta)\n    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n\n    model = Classifier(num_classes=num_classes)\n    model.load_state_dict(torch.load('weight_best.pt'))\n    model.cuda()\n    model.eval()\n\n    all_outputs, all_fnames = [], []\n\n    pb = progress_bar(test_loader)\n    for images, fnames in pb:\n        preds = torch.sigmoid(model(images.cuda()).detach())\n        all_outputs.append(preds.cpu().numpy())\n        all_fnames.extend(fnames)\n\n    test_preds = pd.DataFrame(data=np.concatenate(all_outputs),\n                              index=all_fnames,\n                              columns=map(str, range(num_classes)))\n    test_preds = test_preds.groupby(level=0).mean()\n\n    return test_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = predict_model(test_df['fname'], x_test, transforms_dict['test'], num_classes, tta=35)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[labels] = test_preds.values\ntest_df.to_csv('submission.csv', index=False)\ntest_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}