{"cells":[{"metadata":{},"cell_type":"markdown","source":"Fork from mhiro, https://www.kaggle.com/mhiro2/simple-2d-cnn-classifier-with-pytorch .\n\nSingle model resnet18 with curated subset, to get the LB 0.634. commit V4 is for model training and commit V5 is for prediction and submission. \n\nAdded:  \n* Predefined NNs such as resnet18, resnet 34, alexnet, vgg;\n* Data augmentation mixup() added for the train data;\n* Optimizers, lr schedules (CyclicLR, ReduceLROnPlateau, CosineAnnealingLR) can be selected;\n* loss function weighted_BCEWithLogits added;\n\nModel change can increase the lb to 0.65+. Feel free to check the code, setup and discuss here.\n\nMy confusion:  \nSeveral guys report local lwlrap around 0.86 with LB around 0.7.  \nhttps://www.kaggle.com/c/freesound-audio-tagging-2019/discussion/91881#latest-535068  \nHowever for mine:  \nWith resnet18, local lwlrap is about 0.7324, while LB is 0.634.\nWhile with a self private model, almost same train setup, add \"k-fold averaging\", LB can be 0.7+ with local lwlrap around 0.76."},{"metadata":{},"cell_type":"markdown","source":"### imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport pickle\nimport random\nimport time\nfrom collections import Counter, defaultdict\nfrom functools import partial\nfrom pathlib import Path\nfrom psutil import cpu_count\n\nimport librosa\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\n#from skmultilearn.model_selection import iterative_train_test_split\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, ReduceLROnPlateau, MultiStepLR\nfrom fastprogress import master_bar, progress_bar\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.transforms import transforms\nimport torchvision.models as models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.cuda.is_available()\n# torch.cuda.set_device(0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nSEED = 1314\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N_JOBS = cpu_count()\nused = int(4)\nprint(N_JOBS, used)\nos.environ['MKL_NUM_THREADS'] = str(used)\nos.environ['OMP_NUM_THREADS'] = str(used)\nDataLoader = partial(DataLoader, num_workers=used)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### metric definition\ncalculate lwlrap for task2."},{"metadata":{"trusted":true},"cell_type":"code","source":"# from official code https://colab.research.google.com/drive/1AgPdhSp7ttY18O3fEoHOQKlt_3HJDLi8#scrollTo=cRCaCIb9oguU\ndef _one_sample_positive_class_precisions(scores, truth):\n    \"\"\"Calculate precisions for each true class for a single sample.\n\n    Args:\n      scores: np.array of (num_classes,) giving the individual classifier scores.\n      truth: np.array of (num_classes,) bools indicating which classes are true.\n\n    Returns:\n      pos_class_indices: np.array of indices of the true classes for this sample.\n      pos_class_precisions: np.array of precisions corresponding to each of those\n        classes.\n    \"\"\"\n    num_classes = scores.shape[0]\n    pos_class_indices = np.flatnonzero(truth > 0)\n    # Only calculate precisions if there are some true classes.\n    if not len(pos_class_indices):\n        return pos_class_indices, np.zeros(0)\n    # Retrieval list of classes for this sample.\n    retrieved_classes = np.argsort(scores)[::-1]\n    # class_rankings[top_scoring_class_index] == 0 etc.\n    class_rankings = np.zeros(num_classes, dtype=np.int)\n    class_rankings[retrieved_classes] = range(num_classes)\n    # Which of these is a true label?\n    retrieved_class_true = np.zeros(num_classes, dtype=np.bool)\n    retrieved_class_true[class_rankings[pos_class_indices]] = True\n    # Num hits for every truncated retrieval list.\n    retrieved_cumulative_hits = np.cumsum(retrieved_class_true)\n    # Precision of retrieval list truncated at each hit, in order of pos_labels.\n    precision_at_hits = (\n            retrieved_cumulative_hits[class_rankings[pos_class_indices]] /\n            (1 + class_rankings[pos_class_indices].astype(np.float)))\n    return pos_class_indices, precision_at_hits\n\n\ndef calculate_per_class_lwlrap(truth, scores):\n    \"\"\"Calculate label-weighted label-ranking average precision.\n\n    Arguments:\n      truth: np.array of (num_samples, num_classes) giving boolean ground-truth\n        of presence of that class in that sample.\n      scores: np.array of (num_samples, num_classes) giving the classifier-under-\n        test's real-valued score for each class for each sample.\n\n    Returns:\n      per_class_lwlrap: np.array of (num_classes,) giving the lwlrap for each\n        class.\n      weight_per_class: np.array of (num_classes,) giving the prior of each\n        class within the truth labels.  Then the overall unbalanced lwlrap is\n        simply np.sum(per_class_lwlrap * weight_per_class)\n    \"\"\"\n    assert truth.shape == scores.shape\n    num_samples, num_classes = scores.shape\n    # Space to store a distinct precision value for each class on each sample.\n    # Only the classes that are true for each sample will be filled in.\n    precisions_for_samples_by_classes = np.zeros((num_samples, num_classes))\n    for sample_num in range(num_samples):\n        pos_class_indices, precision_at_hits = (\n            _one_sample_positive_class_precisions(scores[sample_num, :],\n                                                  truth[sample_num, :]))\n        precisions_for_samples_by_classes[sample_num, pos_class_indices] = (\n            precision_at_hits)\n    labels_per_class = np.sum(truth > 0, axis=0)\n    weight_per_class = labels_per_class / float(np.sum(labels_per_class))\n    # Form average of each column, i.e. all the precisions assigned to labels in\n    # a particular class.\n    per_class_lwlrap = (np.sum(precisions_for_samples_by_classes, axis=0) /\n                        np.maximum(1, labels_per_class))\n    # overall_lwlrap = simple average of all the actual per-class, per-sample precisions\n    #                = np.sum(precisions_for_samples_by_classes) / np.sum(precisions_for_samples_by_classes > 0)\n    #           also = weighted mean of per-class lwlraps, weighted by class label prior across samples\n    #                = np.sum(per_class_lwlrap * weight_per_class)\n    return per_class_lwlrap, weight_per_class","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_dir = Path('../input/freesound-audio-tagging-2019')\nprint(os.listdir(dataset_dir))\npreprocessed_dir = Path('../input/fat2019_prep_mels1') # time duration of audio croped, 2s 4s1 4s2 2s_pcen;\nprint(os.listdir(preprocessed_dir))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"csvs = {\n    'train_curated': dataset_dir / 'train_curated.csv',\n    'train_noisy': dataset_dir / 'train_noisy.csv',\n#     'train_noisy': preprocessed_dir / 'trn_noisy_best50s.csv',\n    'sample_submission': dataset_dir / 'sample_submission.csv',\n}\n\ndataset = {\n    'train_curated': dataset_dir / 'train_curated',\n    'train_noisy': dataset_dir / 'train_noisy',\n    'test': dataset_dir / 'test',\n}\n\nmels = {\n    'train_curated': preprocessed_dir / 'mels_train_curated.pkl',\n    'train_noisy': preprocessed_dir / 'mels_train_noisy.pkl', # 'mels_trn_noisy_best50s.pkl'\n    'test': preprocessed_dir / 'mels_test.pkl',  # NOTE: this data doesn't work at 2nd stage\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_curated = pd.read_csv(csvs['train_curated'])\ntrain_noisy = pd.read_csv(csvs['train_noisy'])\ntrain_df = pd.concat([train_curated], sort=True, ignore_index=True)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(csvs['sample_submission'])\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = test_df.columns[1:].tolist()\n# print(labels)\nnum_classes = len(labels)\nnum_classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = np.zeros((len(train_df), num_classes)).astype(int)\nfor i, row in enumerate(train_df['labels'].str.split(',')):\n    for label in row:\n        idx = labels.index(label)\n        y_train[i, idx] = 1\n\ny_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(mels['train_curated'], 'rb') as curated, open(mels['train_noisy'], 'rb') as noisy:\n    x_train = pickle.load(curated)\n    print(len(x_train))\n#     x_train.extend(pickle.load(noisy))\n#     print(len(x_train))\n\nwith open(mels['test'], 'rb') as test:\n    x_test = pickle.load(test)\n    \nlen(x_train), len(x_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data loader"},{"metadata":{"trusted":true},"cell_type":"code","source":"class FATTrainDataset(Dataset):\n    def __init__(self, mels, labels, transforms):\n        super().__init__()\n        self.mels = mels\n        self.labels = labels\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.mels)\n    \n    def __getitem__(self, idx):\n        # crop 1sec\n        image = Image.fromarray(self.mels[idx], mode='RGB')        \n        time_dim, base_dim = image.size\n#         print('Shape: ',time_dim, base_dim)\n        crop = random.randint(0, time_dim - 1*base_dim)\n        image = image.crop([crop, 0, crop + 1*base_dim, base_dim])\n        image = self.transforms(image).div_(255)\n        \n        label = self.labels[idx]\n        label = torch.from_numpy(label).float()\n        \n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FATTestDataset(Dataset):\n    def __init__(self, fnames, mels, transforms, tta=5):\n        super().__init__()\n        self.fnames = fnames\n        self.mels = mels\n        self.transforms = transforms\n        self.tta = tta\n        \n    def __len__(self):\n        return len(self.fnames) * self.tta\n    \n    def __getitem__(self, idx):\n        new_idx = idx % len(self.fnames)\n        \n        image = Image.fromarray(self.mels[new_idx], mode='RGB')\n        time_dim, base_dim = image.size\n        crop = random.randint(0, time_dim - 1*base_dim)\n        image = image.crop([crop, 0, crop + 1*base_dim, base_dim])\n        image = self.transforms(image).div_(255)\n\n        fname = self.fnames[new_idx]\n        \n        return image, fname","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## pytorch models"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torchvision.models as models\ndef set_parameter_requires_grad(model, feature_extracting):\n    if feature_extracting:\n        for param in model.parameters():\n            param.requires_grad = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def initialize_model(model_name, num_classes, feature_extract, use_pretrained=False):\n    # Initialize these variables which will be set in this if statement. Each of these\n    #   variables is model specific.\n    model_ft = None\n    input_size = 0\n\n    if model_name == \"resnet18\":\n        \"\"\" Resnet18\n        \"\"\"\n        model_ft = models.resnet18(pretrained=use_pretrained)\n        set_parameter_requires_grad(model_ft, feature_extract)\n        num_ftrs = model_ft.fc.in_features\n        model_ft.fc = nn.Linear(num_ftrs, num_classes)\n        input_size = 224\n\n    elif model_name == \"resnet34\":\n        \"\"\" Resnet34\n        \"\"\"\n        model_ft = models.resnet34(pretrained=use_pretrained)\n        set_parameter_requires_grad(model_ft, feature_extract)\n        num_ftrs = model_ft.fc.in_features\n        model_ft.fc = nn.Linear(num_ftrs, num_classes)\n        input_size = 224\n\n    elif model_name == \"alexnet\":\n        \"\"\" Alexnet\n        \"\"\"\n        model_ft = models.alexnet(pretrained=use_pretrained)\n        set_parameter_requires_grad(model_ft, feature_extract)\n        num_ftrs = model_ft.classifier[6].in_features\n        model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)\n        input_size = 224\n\n    elif model_name == \"vgg\":\n        \"\"\" VGG11_bn\n        \"\"\"\n        model_ft = models.vgg11_bn(pretrained=use_pretrained)\n        set_parameter_requires_grad(model_ft, feature_extract)\n        num_ftrs = model_ft.classifier[6].in_features\n        model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)\n        input_size = 224\n\n    elif model_name == \"squeezenet\":\n        \"\"\" Squeezenet\n        \"\"\"\n        model_ft = models.squeezenet1_0(pretrained=use_pretrained)\n        set_parameter_requires_grad(model_ft, feature_extract)\n        model_ft.classifier[1] = nn.Conv2d(512, num_classes, kernel_size=(1,1), stride=(1,1))\n        model_ft.num_classes = num_classes\n        input_size = 224\n\n    elif model_name == \"inception\":\n        \"\"\" Inception v3\n        Be careful, expects (299,299) sized images and has auxiliary output\n        \"\"\"\n        model_ft = models.inception_v3(pretrained=use_pretrained)\n        set_parameter_requires_grad(model_ft, feature_extract)\n        # Handle the auxilary net\n        num_ftrs = model_ft.AuxLogits.fc.in_features\n        model_ft.AuxLogits.fc = nn.Linear(num_ftrs, num_classes)\n        # Handle the primary net\n        num_ftrs = model_ft.fc.in_features\n        model_ft.fc = nn.Linear(num_ftrs,num_classes)\n        input_size = 299\n\n    else:\n        print(\"Invalid model name, exiting...\")\n        exit()\n\n    return model_ft, input_size\n\n# # To tune or design nn here.\n# model_type = 'resnet18' # 'resnet18' # 'inception' ‘alexnet’ ‘vgg’\n# model_ft, input_size = initialize_model(model_type, num_classes, False, use_pretrained=False)\n\n# Print the model we just instantiated\n# print(model_ft)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ConvBlock(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n        \n        self.conv1 = nn.Sequential(\n            nn.Conv2d(in_channels, out_channels, 3, 1, 1),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(),\n        )\n        self.conv2 = nn.Sequential(\n            nn.Conv2d(out_channels, out_channels, 3, 1, 1),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(),\n        )\n\n        self._init_weights()\n        \n    def _init_weights(self):\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight)\n                if m.bias is not None:\n                    nn.init.zeros_(m.bias)\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.constant_(m.weight, 1)\n                nn.init.zeros_(m.bias)\n        \n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.conv2(x)\n        x = F.avg_pool2d(x, 2)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Classifier(nn.Module):\n    def __init__(self, num_classes):\n        super().__init__()\n        \n        self.conv = nn.Sequential(\n            ConvBlock(in_channels=3, out_channels=64),\n            ConvBlock(in_channels=64, out_channels=128),\n            ConvBlock(in_channels=128, out_channels=256),\n            ConvBlock(in_channels=256, out_channels=512),\n        )\n        \n        self.fc = nn.Sequential(\n            nn.Dropout(0.2),\n            nn.Linear(512, 128),\n            nn.LeakyReLU(0.2),\n            nn.BatchNorm1d(128),\n            nn.Dropout(0.1),\n            nn.Linear(128, num_classes),\n        )\n\n    def forward(self, x):\n        x = self.conv(x)\n        x = torch.mean(x, dim=3)\n        x, _ = torch.max(x, dim=2)\n        x = self.fc(x)\n        return x\n# Classifier(num_classes=num_classes)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### mixup and loss function defination"},{"metadata":{"trusted":true},"cell_type":"code","source":"def mixup_data(x, y, alpha=1.0, use_cuda=True):\n\n    '''Compute the mixup data. Return mixed inputs, pairs of targets, and lambda'''\n    if alpha > 0.:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1.\n    batch_size = x.size()[0]\n    if use_cuda:\n        index = torch.randperm(batch_size).cuda()\n    else:\n        index = torch.randperm(batch_size)\n#     lam = max(lam, 1 - lam)\n    mixed_x = lam * x + (1 - lam) * x[index,:]\n    mixed_y = lam * y + (1 - lam) * y[index]\n    return mixed_x, mixed_y\n\ndef mixup_criterion(y_a, y_b, lam):\n    return lambda criterion, pred: lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tf.nn.weighted_cross_entropy_with_logits \ndef weighted_BCEWithLogits(pos_weight):\n    def weighted_cross_entropy_with_logits(logits, targets):   # y_pred ,y_true\n        return (targets * -logits.sigmoid().log() * pos_weight + (1 - targets) * -(1 - logits.sigmoid()).log()).mean()\n    return weighted_cross_entropy_with_logits","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### train"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(x_train, y_train, train_transforms, model_path, num_epochs = 5, batch_size = 32, test_batch_size = 64):\n    \n    lr = 1e-3\n    eta_min = 1e-7\n    t_max = 10\n    num_classes = y_train.shape[1]\n\n    trn_loss = []\n    val_loss = []\n    val_lwlrap = []\n    lr_log = []\n    \n    x_trn, x_val, y_trn, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=SEED)\n    train_dataset = FATTrainDataset(x_trn, y_trn, train_transforms)\n    valid_dataset = FATTrainDataset(x_val, y_val, train_transforms)\n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n    valid_loader = DataLoader(valid_dataset, batch_size=test_batch_size, shuffle=False)\n\n#     model = Classifier(num_classes=num_classes).cuda()\n    model = model_ft.cuda()\n    # if continue to train\n    if model_path and False:\n        model.load_state_dict(torch.load(model_path))\n#     model.cuda()\n#    print(model.parameters())\n\n    # loss function\n    criterion = nn.BCEWithLogitsLoss().cuda()  # sigmoid 交叉熵\n#     criterion=weighted_BCEWithLogits(0.7)\n    # optimizer\n#     optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)\n    optimizer = Adam(params=model.parameters(), lr=lr, amsgrad=False)\n\n    # learning rate\n    scheduler = CosineAnnealingLR(optimizer, T_max=t_max, eta_min=eta_min)\n#     scheduler = CyclicLR(optimizer, 1e-7, lr, step_size_up=250, step_size_down=None,\\\n#                          mode='exp_range', gamma=1.0, scale_fn=None, scale_mode='cycle',\\\n#                          cycle_momentum=True, base_momentum=0.8, max_momentum=0.9, last_epoch=-1)\n#     scheduler = ReduceLROnPlateau(optimizer, 'min',factor=0.5, patience=6, verbose=True) \n\n    best_epoch = -1\n    best_lwlrap = 0.\n    mb = master_bar(range(num_epochs))\n    for epoch in mb:\n        start_time = time.time()\n        model.train()\n        avg_loss = 0.\n\n        # train process\n        for x_batch, y_batch in progress_bar(train_loader, parent=mb):\n            # generate mixed inputs, two one-hot label vectors and mixing coefficient\n            if mixup_on:\n                x_batch, y_batch = mixup_data(x_batch, y_batch, alpha=1, use_cuda=True)\n                \n            if model_type != 'inception':\n                preds = model(x_batch.cuda())\n                loss = criterion(preds, y_batch.cuda())\n            else:\n                outputs, aux_outputs = model(x_batch.cuda())\n                loss1 = criterion(outputs, y_batch.cuda())\n                loss2 = criterion(aux_outputs, y_batch.cuda())\n                loss = loss1 + 0.4*loss2\n\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n            avg_loss += loss.item() / len(train_loader)\n        \n        # validation process\n        model.eval()\n        valid_preds = np.zeros((len(x_val), num_classes))\n        avg_val_loss = 0.\n        for i, (x_batch, y_batch) in enumerate(valid_loader):\n            preds = model(x_batch.cuda()).detach()\n            loss = criterion(preds, y_batch.cuda())\n            preds = torch.sigmoid(preds)\n            valid_preds[i * test_batch_size: (i+1) * test_batch_size] = preds.cpu().numpy()\n            avg_val_loss += loss.item() / len(valid_loader)\n            \n        score, weight = calculate_per_class_lwlrap(y_val, valid_preds)\n        lwlrap = (score * weight).sum()\n        \n        scheduler.step() # for CosineAnnealingLR, CLR\n#         scheduler.step(avg_val_loss) # for reduceLR\n        \n        # record the metrics\n        for param_group in optimizer.param_groups:\n            lr_temp = param_group['lr']\n        lr_log.append(lr_temp)\n\n        if (epoch + 1) % 1 == 0:\n            elapsed = time.time() - start_time\n            mb.write(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  val_lwlrap: {lwlrap:.6f}  time: {elapsed:.0f}s')\n            trn_loss.append(avg_loss)\n            val_loss.append(avg_val_loss)\n            val_lwlrap.append(lwlrap)\n    \n        if lwlrap > best_lwlrap:\n            best_epoch = epoch + 1\n            best_lwlrap = lwlrap\n            best_train_loss = avg_loss\n            best_val_loss = avg_val_loss\n            torch.save(model.state_dict(), model_path)\n            \n    return {\n        'best_epoch': best_epoch,\n        'best_lwlrap': best_lwlrap,\n        'best_train_loss':best_train_loss,\n        'best_val_loss':best_val_loss,\n        'trn_loss':trn_loss,\n        'val_loss':val_loss,\n        'val_lwlrap':val_lwlrap,\n        'lr_log':lr_log\n    }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_epochs = 400\nbatch_size = 64\ntest_batch_size = 64\n# to change it in the train_model function now, avoiding too long code in train_model function.\nlr_method = 'cosine' # 'CLR' 'reduceLR' \naug_flip = 'flip' # 'noflip'\nmixup_on = True\n# Initialize the model for this run\nmodel_type = 'inception' # 'inception' # 'resnet18' # 'inception' ‘alexnet’ ‘vgg’\nmodel_ft, input_size = initialize_model(model_type, num_classes, False, use_pretrained=False)\n# model_ft = Classifier(num_classes=num_classes)\n# input_size = 128\n\n\nif aug_flip == 'flip':\n    transforms_dict = {\n        'train': transforms.Compose([\n            transforms.RandomResizedCrop(input_size),\n            transforms.RandomHorizontalFlip(0.5),\n            transforms.ToTensor(),\n        ]),\n        'test': transforms.Compose([\n            transforms.RandomResizedCrop(input_size),\n            transforms.RandomHorizontalFlip(0.5),\n            transforms.ToTensor(),\n        ]),\n    }\nelif aug_flip == 'noflip':\n    transforms_dict = {\n        'train': transforms.Compose([\n            transforms.RandomResizedCrop(input_size),\n            transforms.ToTensor(),\n        ]),\n        'test': transforms.Compose([\n            transforms.RandomResizedCrop(input_size),\n            transforms.ToTensor(),\n        ]),\n    }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_path = '../working/weight_best_2s_%(m)s_%(lr)s_%(aug)s.pt'% \\\n{'m':model_type, 'lr':lr_method, 'aug':aug_flip}\noutput_path = '../working/submission_2s_%(m)s_%(lr)s_%(aug)s.csv'%\\\n{'m':model_type, 'lr':lr_method, 'aug':aug_flip}\nprint(model_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# result = train_model(x_train, y_train, transforms_dict['train'], model_path,\n#                      num_epochs, batch_size, test_batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print('best_epoch:',result['best_epoch'])\n# print('best_lwlrap:',result['best_lwlrap'])\n# print('best_trn_loss:',result['best_train_loss'])\n# print('best_val_loss:',result['best_val_loss'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %matplotlib inline\n# import matplotlib.pyplot as plt\n# # plt.title(\"train loss, validation loss\")\n# fig, ax1 = plt.subplots()\n# ax1.plot(result['trn_loss'], label='trn_loss', color='b')\n# ax1.plot(result['val_loss'], label='val_loss', color='k')\n# ax1.set_ylabel('loss', color='k')\n# ax1.legend(loc='lower left')\n# # ax1.ylim((0,1.))\n# ax2 = ax1.twinx()\n# ax2.plot(result['lr_log'], label='lr_log', color='y')\n# ax2.set_ylabel('lr', color = 'k')\n# ax2.legend(loc='upper right')\n\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %matplotlib inline\n# import matplotlib.pyplot as plt\n# # plt.title(\"train loss, validation loss\")\n# fig, ax1 = plt.subplots()\n# ax1.plot(result['val_lwlrap'], label='val_lwlrap', color='b')\n# ax1.set_ylabel('val_lwlrap', color='k')\n# ax1.legend(loc='lower left')\n# # ax1.ylim((0,1.))\n# ax2 = ax1.twinx()\n# ax2.plot(result['lr_log'], label='lr_log', color='y')\n# ax2.set_ylabel('lr', color = 'k')\n# ax2.legend(loc='upper right')\n\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"# to load model and upload submission.csv\nmodel_path = '../input/outputmodels2/weight_best_2s_inception_cosine_flip.pt'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_model(test_fnames, x_test, test_transforms, num_classes, tta=5):\n    batch_size = 32\n\n    test_dataset = FATTestDataset(test_fnames, x_test, test_transforms, tta=tta)\n    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n#    model = Classifier(num_classes=num_classes)\n\n    model = model_ft\n    model.load_state_dict(torch.load(model_path))\n    model.cuda()\n    model.eval()\n\n    all_outputs, all_fnames = [], []\n\n    pb = progress_bar(test_loader)\n    for images, fnames in pb:\n        preds = torch.sigmoid(model(images.cuda()).detach())\n        all_outputs.append(preds.cpu().numpy())\n        all_fnames.extend(fnames)\n\n    test_preds = pd.DataFrame(data=np.concatenate(all_outputs),\n                              index=all_fnames,\n                              columns=map(str, range(num_classes)))\n    test_preds = test_preds.groupby(level=0).mean()\n\n    return test_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# torch.cuda.set_device(0)\nstart_time = time.time()\ntest_preds = predict_model(test_df['fname'], x_test, transforms_dict['test'], num_classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[labels] = test_preds.values\ntest_df.to_csv(output_path, index=False)\ntest_df.to_csv('submission.csv', index=False)\n\nelapsed = time.time() - start_time\nprint(f'Prediction time: {elapsed:.0f}s')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}