{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Yandex Praktikum PyTorch train baseline - LB 0.699"},{"metadata":{},"cell_type":"markdown","source":"A small guide prepared for students of [Yandex Praktikum](https://praktikum.yandex.ru/) Data Science Track"},{"metadata":{},"cell_type":"markdown","source":"> ## Make Folds\n\nThe first thing we will do is split the train into folds for cross-validation.\n\nFor example, divide by 5, for validation we will use zero fold."},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_ROOT = '../input/imet-2020-fgvc7/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict, Counter\nimport random\n\nimport pandas as pd\nimport tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_folds(n_folds: int) -> pd.DataFrame:\n    df = pd.read_csv(DATA_ROOT + 'train.csv')\n    cls_counts = Counter(cls for classes in df['attribute_ids'].str.split()\n                         for cls in classes)\n    fold_cls_counts = defaultdict(int)\n    folds = [-1] * len(df)\n    for item in tqdm.tqdm(df.sample(frac=1, random_state=42).itertuples(),\n                          total=len(df)):\n        cls = min(item.attribute_ids.split(), key=lambda cls: cls_counts[cls])\n        fold_counts = [(f, fold_cls_counts[f, cls]) for f in range(n_folds)]\n        min_count = min([count for _, count in fold_counts])\n        random.seed(item.Index)\n        fold = random.choice([f for f, count in fold_counts\n                              if count == min_count])\n        folds[item.Index] = fold\n        for cls in item.attribute_ids.split():\n            fold_cls_counts[fold, cls] += 1\n    df['fold'] = folds\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = make_folds(n_folds=5)\ndf.to_csv('folds.csv', index=None)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"You can also use ready-made already folded folds.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"folds = pd.read_csv('../input/imet2002folds/folds.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"folds.head(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Implement your dataset to load data\n\nWe implement our own Dataset class for loading data. Its purpose is to load data from the disk and issue a tensor on it with the network input, label and picture identifier.\n\nHere is a link where it’s well explained how to do this with an example: https://pytorch.org/tutorials/beginner/data_loading_tutorial.html"},{"metadata":{"trusted":true},"cell_type":"code","source":"from pathlib import Path\nfrom typing import Callable, List\n\nimport cv2\nimport pandas as pd\nfrom PIL import Image\nimport torch\nfrom torch.utils.data import Dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N_CLASSES = 3474","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainDataset(Dataset):\n    def __init__(self, root: Path, df: pd.DataFrame,\n                 image_transform: Callable, debug: bool = True):\n        super().__init__()\n        self._root = root\n        self._df = df\n        self._image_transform = image_transform\n        self._debug = debug\n\n    def __len__(self):\n        return len(self._df)\n\n    def __getitem__(self, idx: int):\n        item = self._df.iloc[idx]\n\n        image = load_transform_image(\n            item, self._root, self._image_transform, debug=self._debug)\n        target = torch.zeros(N_CLASSES)\n        for cls in item.attribute_ids.split():\n            target[int(cls)] = 1\n        return image, target\n\n\nclass TTADataset:\n    def __init__(self, root: Path, df: pd.DataFrame,\n                 image_transform: Callable, tta: int):\n        self._root = root\n        self._df = df\n        self._image_transform = image_transform\n        self._tta = tta\n\n    def __len__(self):\n        return len(self._df) * self._tta\n\n    def __getitem__(self, idx):\n        item = self._df.iloc[idx % len(self._df)]\n        #print(item)\n        image = load_transform_image(item, self._root, self._image_transform)\n        return image, item.id","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nimport math\n\nfrom PIL import Image\nfrom torchvision.transforms import (\n    ToTensor, Normalize, Compose, Resize, CenterCrop, RandomCrop,\n    RandomHorizontalFlip)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transform = Compose([\n    RandomCrop(288),\n    RandomHorizontalFlip(),\n])\n\ntest_transform = Compose([\n    #RandomCrop(288),\n    RandomCrop(256),\n    RandomHorizontalFlip(),\n])\n\ntensor_transform = Compose([\n    ToTensor(),\n    Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_transform_image(\n        item, root: Path, image_transform: Callable, debug: bool = False):\n    image = load_image(item, root)\n    image = image_transform(image)\n    if debug:\n        image.save('_debug.png')\n    return tensor_transform(image)\n\n\ndef load_image(item, root: Path) -> Image.Image:\n    image = cv2.imread(str(root + '/' + f'{item.id}.png'))\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    return Image.fromarray(image)\n\n\ndef get_ids(root: Path) -> List[str]:\n    return sorted({p.name.split('_')[0] for p in root.glob('*.png')})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load data "},{"metadata":{"trusted":true},"cell_type":"code","source":"folds = pd.read_csv('../input/imet2002folds/folds.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fold = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_fold = folds[folds['fold'] != 0]\nvalid_fold = folds[folds['fold'] == 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from itertools import islice\nimport json\nfrom pathlib import Path\nimport shutil\nimport warnings\nfrom typing import Dict\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import fbeta_score\nfrom sklearn.exceptions import UndefinedMetricWarning\nimport torch\nfrom torch import nn, cuda\nfrom torch.optim import Adam\nimport tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import DataLoader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_root = DATA_ROOT + 'train'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_workers = 4\nbatch_size = 64","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_loader(df: pd.DataFrame, image_transform) -> DataLoader:\n        return DataLoader(\n            TrainDataset(train_root, df, image_transform, debug=0),\n            shuffle=True,\n            batch_size=batch_size,\n            num_workers=num_workers,\n        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loader = make_loader(train_fold, train_transform)\nvalid_loader = make_loader(valid_fold, test_transform)\nprint(f'{len(train_loader.dataset):,} items in train, '\n      f'{len(valid_loader.dataset):,} in valid')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Models"},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.nn import functional as F\nimport torchvision.models as M\nfrom functools import partial","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AvgPool(nn.Module):\n    def forward(self, x):\n        return F.avg_pool2d(x, x.shape[2:])\n\n\ndef create_net(net_cls, pretrained: bool):\n    if pretrained:\n        net = net_cls()\n        model_name = net_cls.__name__\n        weights_path = f'../input/{model_name}/{model_name}.pth'\n        net.load_state_dict(torch.load(weights_path))\n    else:\n        net = net_cls(pretrained=pretrained)\n    return net\n\n\nclass ResNet(nn.Module):\n    def __init__(self, num_classes,\n                 pretrained=False, net_cls=M.resnet50, dropout=False):\n        super().__init__()\n        self.net = create_net(net_cls, pretrained=pretrained)\n        self.net.avgpool = AvgPool()\n        if dropout:\n            self.net.fc = nn.Sequential(\n                nn.Dropout(),\n                nn.Linear(self.net.fc.in_features, num_classes),\n            )\n        else:\n            self.net.fc = nn.Linear(self.net.fc.in_features, num_classes)\n\n    def fresh_params(self):\n        return self.net.fc.parameters()\n\n    def forward(self, x):\n        return self.net(x)\n\n\nclass DenseNet(nn.Module):\n    def __init__(self, num_classes,\n                 pretrained=False, net_cls=M.densenet121):\n        super().__init__()\n        self.net = create_net(net_cls, pretrained=pretrained)\n        self.avg_pool = AvgPool()\n        self.net.classifier = nn.Linear(\n            self.net.classifier.in_features, num_classes)\n\n    def fresh_params(self):\n        return self.net.classifier.parameters()\n\n    def forward(self, x):\n        out = self.net.features(x)\n        out = F.relu(out, inplace=True)\n        out = self.avg_pool(out).view(out.size(0), -1)\n        out = self.net.classifier(out)\n        return out\n\n\nresnet18 = partial(ResNet, net_cls=M.resnet18)\nresnet34 = partial(ResNet, net_cls=M.resnet34)\nresnet50 = partial(ResNet, net_cls=M.resnet50)\nresnet101 = partial(ResNet, net_cls=M.resnet101)\nresnet152 = partial(ResNet, net_cls=M.resnet152)\n\ndensenet121 = partial(DenseNet, net_cls=M.densenet121)\ndensenet169 = partial(DenseNet, net_cls=M.densenet169)\ndensenet201 = partial(DenseNet, net_cls=M.densenet201)\ndensenet161 = partial(DenseNet, net_cls=M.densenet161)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train model\n\nFor 0.699 score you nedd train 20+ epochs"},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.BCEWithLogitsLoss(reduction='none')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"use_cuda = cuda.is_available()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = resnet50(num_classes=N_CLASSES, pretrained=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fresh_params = list(model.fresh_params())\nall_params = list(model.parameters())\nif use_cuda:\n    model = model.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" train_kwargs = dict(\n            model=model,\n            criterion=criterion,\n            train_loader=train_loader,\n            valid_loader=valid_loader,\n            patience=4,\n            init_optimizer=lambda params, lr: Adam(params, lr),\n            use_cuda=use_cuda,\n        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_model(model: nn.Module, path: Path) -> Dict:\n    state = torch.load(str(path))\n    model.load_state_dict(state['model'])\n    print('Loaded model from epoch {epoch}, step {step:,}'.format(**state))\n    return state","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _reduce_loss(loss):\n    return loss.sum() / loss.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def binarize_prediction(probabilities, threshold: float, argsorted=None,\n                        min_labels=1, max_labels=10):\n    \"\"\" Return matrix of 0/1 predictions, same shape as probabilities.\n    \"\"\"\n    assert probabilities.shape[1] == N_CLASSES\n    if argsorted is None:\n        argsorted = probabilities.argsort(axis=1)\n    max_mask = _make_mask(argsorted, max_labels)\n    min_mask = _make_mask(argsorted, min_labels)\n    prob_mask = probabilities > threshold\n    return (max_mask & prob_mask) | min_mask\n\n\ndef _make_mask(argsorted, top_n: int):\n    mask = np.zeros_like(argsorted, dtype=np.uint8)\n    col_indices = argsorted[:, -top_n:].reshape(-1)\n    row_indices = [i // top_n for i in range(len(col_indices))]\n    mask[row_indices, col_indices] = 1\n    return mask\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def validation(\n        model: nn.Module, criterion, valid_loader, use_cuda,\n        ) -> Dict[str, float]:\n    model.eval()\n    all_losses, all_predictions, all_targets = [], [], []\n    with torch.no_grad():\n        for inputs, targets in valid_loader:\n            all_targets.append(targets.numpy().copy())\n            if use_cuda:\n                inputs, targets = inputs.cuda(), targets.cuda()\n            outputs = model(inputs)\n            loss = criterion(outputs, targets)\n            all_losses.append(_reduce_loss(loss).item())\n            predictions = torch.sigmoid(outputs)\n            all_predictions.append(predictions.cpu().numpy())\n    all_predictions = np.concatenate(all_predictions)\n    all_targets = np.concatenate(all_targets)\n\n    def get_score(y_pred):\n        with warnings.catch_warnings():\n            warnings.simplefilter('ignore', category=UndefinedMetricWarning)\n            return fbeta_score(\n                all_targets, y_pred, beta=2, average='samples')\n\n    metrics = {}\n    argsorted = all_predictions.argsort(axis=1)\n    for threshold in [0.10, 0.20]:\n        metrics[f'valid_f2_th_{threshold:.2f}'] = get_score(\n            binarize_prediction(all_predictions, threshold, argsorted))\n    metrics['valid_loss'] = np.mean(all_losses)\n    print(' | '.join(f'{k} {v:.3f}' for k, v in sorted(\n        metrics.items(), key=lambda kv: -kv[1])))\n\n    return metrics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train( model: nn.Module, criterion, *, params,\n          train_loader, valid_loader, init_optimizer, use_cuda,\n          n_epochs=None, patience=2, max_lr_changes=2) -> bool:\n    \n    lr = 1e-4\n    batch_size = 64\n    n_epochs = 1\n    params = list(params)\n    optimizer = init_optimizer(params, lr)\n\n    model_path = 'model.pt'\n    best_model_path = 'best-model.pt'\n    uptrain = False\n    if uptrain:\n        state = load_model(model, model_path)\n        epoch = state['epoch']\n        step = state['step']\n        best_valid_loss = state['best_valid_loss']\n    else:\n        epoch = 1\n        step = 0\n        best_valid_loss = float('inf')\n    lr_changes = 0\n\n    save = lambda ep: torch.save({\n        'model': model.state_dict(),\n        'epoch': ep,\n        'step': step,\n        'best_valid_loss': best_valid_loss\n    }, str(model_path))\n\n    report_each = 100\n    valid_losses = []\n    lr_reset_epoch = epoch\n    for epoch in range(epoch, n_epochs + 1):\n        model.train()\n        tq = tqdm.tqdm(total=(len(train_loader) * batch_size))\n        tq.set_description(f'Epoch {epoch}, lr {lr}')\n        losses = []\n        tl = train_loader\n        try:\n            mean_loss = 0\n            for i, (inputs, targets) in enumerate(tl):\n                if use_cuda:\n                    inputs, targets = inputs.cuda(), targets.cuda()\n                outputs = model(inputs)\n                loss = _reduce_loss(criterion(outputs, targets))\n                batch_size = inputs.size(0)\n                (batch_size * loss).backward()\n                if (i + 1) % 1 == 0:\n                    optimizer.step()\n                    optimizer.zero_grad()\n                    step += 1\n                tq.update(batch_size)\n                losses.append(loss.item())\n                mean_loss = np.mean(losses[-report_each:])\n                tq.set_postfix(loss=f'{mean_loss:.3f}')\n            tq.close()\n            save(epoch + 1)\n            valid_metrics = validation(model, criterion, valid_loader, use_cuda)\n            \n            valid_loss = valid_metrics['valid_loss']\n            valid_losses.append(valid_loss)\n            if valid_loss < best_valid_loss:\n                best_valid_loss = valid_loss\n                shutil.copy(str(model_path), str(best_model_path))\n            elif (patience and epoch - lr_reset_epoch > patience and\n                  min(valid_losses[-patience:]) > best_valid_loss):\n                lr_changes +=1\n                if lr_changes > max_lr_changes:\n                    break\n                lr /= 5\n                print(f'lr updated to {lr}')\n                lr_reset_epoch = epoch\n                optimizer = init_optimizer(params, lr)\n        except KeyboardInterrupt:\n            tq.close()\n            print('Ctrl+C, saving snapshot')\n            save(epoch)\n            print('done.')\n            return False\n    return True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train(params=all_params, **train_kwargs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}