{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Using 6 Pretrained Models \n## (ResNet, SqueezeNet, DenseNet, MobileNet, ShuffleNet, EfficientNet)\n\nReference: https://www.kaggle.com/alimbekovkz/yandex-praktikum-pytorch-train-baseline-lb-0-699","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import tqdm\nimport random\nimport json\nimport math\nimport cv2\nimport shutil\nimport warnings\nimport numpy as np\nimport pandas as pd\n\nfrom collections import defaultdict, Counter\nfrom pathlib import Path\nfrom itertools import islice\nfrom typing import Callable, List, Dict\nfrom PIL import Image\nfrom functools import partial\nfrom efficientnet_pytorch import EfficientNet\n\nimport torch\nimport torchvision.models as M\nfrom torch import nn, cuda\nfrom torch.nn import functional as F\nfrom torch.optim import Adam\nfrom torch.utils import model_zoo\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.transforms import (\n    ToTensor, Normalize, Compose, Resize, CenterCrop, RandomCrop,\n    RandomHorizontalFlip)\nfrom sklearn.metrics import fbeta_score\nfrom sklearn.exceptions import UndefinedMetricWarning\n\nN_CLASSES = 3474\n\n\nDATA_ROOT = '../input/imet-2020-fgvc7/'\ntrain = pd.read_csv('../input/imet-2020-fgvc7/train.csv')\ndevice = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\ndevice","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"folds = pd.read_csv('../input/imet2002folds/folds.csv') \nfolds.head(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Implement your dataset to load data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_image(item, root):\n    image = cv2.imread(str(root + '/' + f'{item.id}.png'))\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    return Image.fromarray(image)\n\ndef load_transform_image(item, root, image_transform, debug=False):\n    image = load_image(item, root)\n    image = image_transform(image)\n    if debug:\n        image.save('_debug.png')\n    return tensor_transform(image)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainDataset(Dataset):\n    def __init__(self, root, df, image_transform, debug=True):\n        super().__init__()\n        self.root = root\n        self.df = df\n        self.image_transform = image_transform\n        self.debug = debug\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        item = self.df.iloc[idx]\n\n        image = load_transform_image(item, self.root, self.image_transform, debug=self.debug)\n        target = torch.zeros(N_CLASSES)\n        for cls in item.attribute_ids.split():\n            target[int(cls)] = 1\n        return image, target\n\n\nclass TTADataset:\n    def __init__(self, root, df, image_transform, tta):\n        self.root = root\n        self.df = df\n        self.image_transform = image_transform\n        self.tta = tta\n\n    def __len__(self):\n        return len(self.df) * self.tta #tta周分する\n\n    def __getitem__(self, idx):\n        item = self.df.iloc[idx % len(self.df)]\n        #print(item)\n        image = load_transform_image(item, self.root, self.image_transform)\n        return image, item.id","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Transforms","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transform = Compose([\n    RandomCrop(288),\n    RandomHorizontalFlip(),\n])\n\ntest_transform = Compose([\n    #RandomCrop(288),\n    RandomCrop(256),\n    RandomHorizontalFlip(),\n])\n\ntensor_transform = Compose([\n    ToTensor(),\n    Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load data ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"folds = pd.read_csv('../input/imet2002folds/folds.csv')\n\ntrain_fold = folds[folds['fold'] != 0]\nvalid_fold = folds[folds['fold'] == 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_root = DATA_ROOT + 'train'\nnum_workers = 4\nbatch_size = 64","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_loader(df, image_transform):\n        return DataLoader(\n            TrainDataset(train_root, df, image_transform, debug=0),\n            shuffle=True,\n            batch_size=batch_size,\n            num_workers=num_workers,\n        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loader = make_loader(train_fold, train_transform)\nvalid_loader = make_loader(valid_fold, test_transform)\nprint(f'{len(train_loader.dataset):,} items in train, '\n      f'{len(valid_loader.dataset):,} in valid')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Models","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class AvgPool(nn.Module):\n    def forward(self, x):\n        return F.avg_pool2d(x, x.shape[2:])\n\nclass ResNet(nn.Module):\n    def __init__(self, num_classes, pretrained=False, net_cls=M.resnet50, dropout=False):\n        super().__init__()\n        self.net = net_cls(pretrained=pretrained)\n        self.net.avgpool = AvgPool()\n        if dropout:\n            self.net.fc = nn.Sequential(\n                nn.Dropout(),\n                nn.Linear(self.net.fc.in_features, num_classes),\n            )\n        else:\n            self.net.fc = nn.Linear(self.net.fc.in_features, num_classes)\n\n    def fresh_params(self):\n        return self.net.fc.parameters()\n\n    def forward(self, x):\n        return self.net(x)\n\n\nclass DenseNet(nn.Module):\n    def __init__(self, num_classes, pretrained=False, net_cls=M.densenet121):\n        super().__init__()\n        self.net = net_cls(pretrained=pretrained)\n        self.avg_pool = AvgPool()\n        self.net.classifier = nn.Linear(\n            self.net.classifier.in_features, num_classes)\n\n    def fresh_params(self):\n        return self.net.classifier.parameters()\n\n    def forward(self, x):\n        out = self.net.features(x)\n        out = F.relu(out, inplace=True)\n        out = self.avg_pool(out).view(out.size(0), -1)\n        out = self.net.classifier(out)\n        return out\n\n    \nclass ShuffleNet(nn.Module):\n    def __init__(self, num_classes, pretrained=False, net_cls=M.shufflenet_v2_x1_0, dropout=False):\n        super().__init__()\n        self.net = net_cls(pretrained=pretrained)\n        if dropout:\n            self.net.fc = nn.Sequential(\n                nn.Dropout(),\n                nn.Linear(self.net.fc.in_features, num_classes),\n            )\n        else:\n            self.net.fc = nn.Linear(self.net.fc.in_features, num_classes)\n\n    def fresh_params(self):\n        return self.net.classifier.parameters()\n\n    def forward(self, x):\n        return self.net(x)\n    \n    \nclass SqueezeNet(nn.Module):\n    def __init__(self, num_classes, pretrained=False, net_cls=M.squeezenet1_0, dropout=False):\n        super().__init__()\n        self.net = net_cls(pretrained=pretrained)\n    \n        fin_conv = nn.Conv2d(512, num_classes, kernel_size=1)\n        self.net.classifier = nn.Sequential(\n            nn.Dropout(p=0.5),\n            fin_conv,\n            nn.ReLU(inplace=True),\n            nn.AdaptiveAvgPool2d((1, 1))\n        )\n    def fresh_params(self):\n        return self.net.classifier.parameters()\n\n    def forward(self, x):\n        out = self.net.features(x)\n        out = self.net.classifier(out)\n        out = out.view(-1, 3474)\n        return out\n    \n    \nclass MobileNet(nn.Module):\n    def __init__(self, num_classes, pretrained=False, net_cls=M.shufflenet_v2_x1_0, dropout=False):\n        super().__init__()\n        self.net = net_cls(pretrained=pretrained)\n        \n        self.net.classifier = nn.Sequential(\n            nn.Dropout(0.2),\n            nn.Linear(self.net.last_channel, num_classes),\n        )\n\n    def fresh_params(self):\n        return self.net.classifier.parameters()\n\n    def forward(self, x):\n        return self.net(x)\n        \n\nresnet50 = partial(ResNet, net_cls=M.resnet50)\ndensenet121 = partial(DenseNet, net_cls=M.densenet121)\nshufflenet = partial(ShuffleNet, net_cls=M.shufflenet_v2_x1_0)\nsqueezenet = partial(SqueezeNet, net_cls=M.squeezenet1_0)\nmobilenet = partial(MobileNet, net_cls=M.mobilenet_v2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Preparation for EfficientNet\nfrom efficientnet_pytorch import * \nDIR_WEIGHTS = '/kaggle/input/efficientnet-pytorch'\nWEIGHTS_FILE = f'{DIR_WEIGHTS}/efficientnet-b0-08094119.pth'\n\ndef load_pretrained_weights2(model, model_name, weights_path=None, load_fc=True, advprop=False):\n    \"\"\"Loads pretrained weights from weights path or download using url.\n    Args:\n        model (Module): The whole model of efficientnet.\n        model_name (str): Model name of efficientnet.\n        weights_path (None or str): \n            str: path to pretrained weights file on the local disk.\n            None: use pretrained weights downloaded from the Internet.\n        load_fc (bool): Whether to load pretrained weights for fc layer at the end of the model.\n        advprop (bool): Whether to load pretrained weights\n                        trained with advprop (valid when weights_path is None).\n    \"\"\"\n    if isinstance(weights_path,str):\n        state_dict = torch.load(weights_path)\n    else:\n        # AutoAugment or Advprop (different preprocessing)\n        url_map_ = url_map_advprop if advprop else url_map\n        state_dict = model_zoo.load_url(url_map_[model_name])\n    \n    if load_fc:\n        ret = model.load_state_dict(state_dict, strict=False)\n        assert not ret.missing_keys, f'Missing keys when loading pretrained weights: {ret.missing_keys}'\n    else:\n        state_dict.pop('_fc.weight')\n        state_dict.pop('_fc.bias')\n        ret = model.load_state_dict(state_dict, strict=False)\n        assert set(ret.missing_keys) == set(\n            ['_fc.weight', '_fc.bias']), f'Missing keys when loading pretrained weights: {ret.missing_keys}'\n    assert not ret.unexpected_keys, f'Missing keys when loading pretrained weights: {ret.unexpected_keys}'\n\n    print('Loaded pretrained weights for {}'.format(model_name))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train model\n\nChoose one model (Loading function is different in the case of EfficientNet)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Use one of them below.\n\n# model = resnet50(num_classes=N_CLASSES, pretrained=True)\n\n# model = squeezenet(num_classes=N_CLASSES, pretrained=True)\n\n# model = densenet121(num_classes=N_CLASSES, pretrained=True)\n\n# model = mobilenet(num_classes=N_CLASSES, pretrained=True)\n\n# model = shufflenet(num_classes=N_CLASSES, pretrained=True)\n\nmodel = EfficientNet.from_name('efficientnet-b0', override_params={'num_classes': 3474})\nload_pretrained_weights2(model, 'efficientnet-b0', weights_path=WEIGHTS_FILE, load_fc=(N_CLASSES == 1000), advprop=False)\n\n\nmodel.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.BCEWithLogitsLoss(reduction='none')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_loss(loss):\n    return loss.sum() / loss.shape[0]\n\ndef make_mask(argsorted, top_n):\n    mask = np.zeros_like(argsorted, dtype=np.uint8)\n    col_indices = argsorted[:, -top_n:].reshape(-1)\n    row_indices = [i // top_n for i in range(len(col_indices))]\n    mask[row_indices, col_indices] = 1\n    return mask\n\ndef binarize_prediction(probabilities, threshold, argsorted=None, min_labels=1, max_labels=10):\n    #3474個それぞれが当てはまるか(1)当てはまらないか(0)を出力\n    assert probabilities.shape[1] == N_CLASSES\n    if argsorted is None:\n        argsorted = probabilities.argsort(axis=1)\n    max_mask = make_mask(argsorted, max_labels)\n    min_mask = make_mask(argsorted, min_labels)\n    prob_mask = probabilities > threshold\n    return (max_mask & prob_mask) | min_mask\n\ndef get_score(target, y_pred):\n    return fbeta_score(target , y_pred, beta=2, average='samples') #βで重み付けしてるF2score?","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = 1e-4\nn_epochs = 1\n\noptimizer = Adam(model.parameters(), lr)\n\nmodel_path = 'model.pth'\nbest_model_path = 'best-model.pth'\n\nvalid_losses = []\nbest_valid_loss = float('inf')\n\nstep = 0\ncount = []\nloss_all = []\nscore_all = []\nfor epoch in range(n_epochs):\n    model.train()\n    losses = []\n    mean_loss = 0\n    for i, (inputs, targets) in enumerate(train_loader):\n        #print(step)\n        inputs = inputs.to(device)\n        targets = targets.to(device)\n        \n        outputs = model(inputs)\n        loss = reduce_loss(criterion(outputs, targets))\n        batch_size = inputs.size(0)\n        (batch_size*loss).backward()\n        optimizer.step()\n        optimizer.zero_grad()\n        \n        step += 1\n\n        losses.append(loss.item())\n        mean_loss = np.mean(losses[-100:])\n        if step % 100 == 0:\n            print(f\"step:{step}/{len(train_fold)*n_epochs//batch_size}\")\n            print(f\"mean_loss:{mean_loss}\")\n            \n            count.append(step)\n            loss_all.append(loss.item())\n            \n            current_score = get_score(targets.cpu().detach().numpy(), binarize_prediction(outputs.cpu().detach().numpy(), 0.1))\n            score_all.append(current_score)\n            print(f\"current_score:{current_score}\")\n\ntorch.save(model.state_dict(), model_path)\nprint(\"Finished.\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Graphs for the loss and the score ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n#Loss\nplt.title(model.__class__.__name__ +\"-Loss\")\nplt.plot(count, loss_all, linewidth=2)\nplt.grid()\nplt.xlabel(\"Step\")\nplt.ylabel(\"Loss\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Score\nplt.title(model.__class__.__name__ +\"-Score\")\nplt.plot(count, score_all, linewidth=2, color=\"red\")\nplt.grid()\nplt.xlabel(\"Step\")\nplt.ylabel(\"Loss\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}