{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"kaggle = True\nshould_train = False\ninput_dir = '../input/' if kaggle else './'\nbase_dir = '../input/imet-2020-fgvc7/' if kaggle else '/macierz/home/s165138/UGWK/'\nmodel_path_for_test = f\"../input/yestte/best-model1.pt\" if kaggle else './best-model.pt'\ntrain_root = base_dir + 'train'\nnumber_of_classes = 3474\nnum_workers = 4\nbatch_size = 32","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:10.877568Z","iopub.execute_input":"2021-06-14T14:19:10.877926Z","iopub.status.idle":"2021-06-14T14:19:10.884082Z","shell.execute_reply.started":"2021-06-14T14:19:10.877872Z","shell.execute_reply":"2021-06-14T14:19:10.883260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import defaultdict, Counter\nfrom typing import Callable, List, Dict\nfrom pathlib import Path\nfrom itertools import islice\nfrom functools import partial\nfrom PIL import Image\nfrom sklearn.metrics import fbeta_score\nfrom sklearn.exceptions import UndefinedMetricWarning\nfrom torch import nn, cuda\nfrom torch.nn import functional as F\nfrom torch.optim import Adam\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.transforms import ToTensor, Normalize, Compose, Resize, CenterCrop, RandomCrop, RandomHorizontalFlip\nimport pandas as pd\nimport numpy as np\nimport torchvision.models as M\nimport os\nimport random\nimport cv2\nimport torch\nimport math\nimport json\nimport shutil\nimport warnings\nimport torch","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:10.885750Z","iopub.execute_input":"2021-06-14T14:19:10.886263Z","iopub.status.idle":"2021-06-14T14:19:10.907371Z","shell.execute_reply.started":"2021-06-14T14:19:10.886228Z","shell.execute_reply":"2021-06-14T14:19:10.906712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_folds(n_folds: int, labels_file_path: str) -> pd.DataFrame:\n    df = pd.read_csv(labels_file_path)\n    cls_counts = Counter(cls for classes in df['attribute_ids'].str.split() for cls in classes)\n    fold_cls_counts = defaultdict(int)\n    folds = [-1] * len(df)\n    for item in df.sample(frac=1, random_state=42).itertuples():\n        cls = min(item.attribute_ids.split(), key=lambda cls: cls_counts[cls])\n        fold_counts = [(f, fold_cls_counts[f, cls]) for f in range(n_folds)]\n        min_count = min([count for _, count in fold_counts])\n        random.seed(item.Index)\n        fold = random.choice([f for f, count in fold_counts\n                              if count == min_count])\n        folds[item.Index] = fold\n        for cls in item.attribute_ids.split():\n            fold_cls_counts[fold, cls] += 1\n    df['fold'] = folds\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:10.909076Z","iopub.execute_input":"2021-06-14T14:19:10.909836Z","iopub.status.idle":"2021-06-14T14:19:10.919973Z","shell.execute_reply.started":"2021-06-14T14:19:10.909784Z","shell.execute_reply":"2021-06-14T14:19:10.919050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_folds(n_folds: int, labels_file_path: str, save_path: str):\n    df = make_folds(n_folds=5, labels_file_path=base_dir + 'train.csv')\n    df.to_csv(save_path, index=None)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:10.921239Z","iopub.execute_input":"2021-06-14T14:19:10.921794Z","iopub.status.idle":"2021-06-14T14:19:10.931392Z","shell.execute_reply.started":"2021-06-14T14:19:10.921757Z","shell.execute_reply":"2021-06-14T14:19:10.930521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TrainDataset(Dataset):\n    def __init__(self, root_dir: Path, number_of_classes: int, df: pd.DataFrame, image_transform: Callable, tensor_transform: Callable):\n        self.root_dir = root_dir\n        self.df = df\n        self.image_transform = image_transform\n        self.tensor_transform = tensor_transform\n        self.number_of_classes = number_of_classes\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx: int):\n        item = self.df.iloc[idx]\n        image = load_transform_image(item.id, self.root_dir, self.image_transform, self.tensor_transform)\n        target = torch.zeros(self.number_of_classes)\n        \n        for attribute in item.attribute_ids.split():\n            target[int(attribute)] = 1\n\n        return image, target\n\nclass PredictImageDataset(Dataset):\n    def __init__(self, root_dir: Path, image_transform, tensor_transform):\n        self.root_dir = root_dir\n        self.files = os.listdir(root_dir)\n        self.tensor_transform = tensor_transform\n        self.image_transform = image_transform\n        \n    def __len__(self):\n        return len(self.files)\n    \n    def __getitem__(self, idx):\n        img_path = os.path.join(self.root_dir, self.files[idx])\n        img_name = os.path.splitext(self.files[idx])[0]\n        image = load_transform_image(img_name, self.root_dir, self.image_transform, self.tensor_transform)\n        return image\n    \n    def filenames(self):\n        return [os.path.splitext(x)[0] for x in self.files]","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:10.935015Z","iopub.execute_input":"2021-06-14T14:19:10.935354Z","iopub.status.idle":"2021-06-14T14:19:10.948575Z","shell.execute_reply.started":"2021-06-14T14:19:10.935329Z","shell.execute_reply":"2021-06-14T14:19:10.947827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_transform_image(item_id, root_dir: Path, image_transform: Callable, tensor_transform: Callable):\n    image = load_image(item_id, root_dir)\n    image = image_transform(image)\n    return tensor_transform(image)\n\n\ndef load_image(item_id, root_dir: Path) -> Image.Image:\n    path = root_dir + f'/{item_id}.png'\n    image = cv2.imread(path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    return Image.fromarray(image)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:10.949859Z","iopub.execute_input":"2021-06-14T14:19:10.950280Z","iopub.status.idle":"2021-06-14T14:19:10.962310Z","shell.execute_reply.started":"2021-06-14T14:19:10.950242Z","shell.execute_reply":"2021-06-14T14:19:10.961617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_loader(df: pd.DataFrame, image_transform: Callable, tensor_transform: Callable) -> DataLoader:\n    dataset = TrainDataset(train_root, number_of_classes, df, image_transform, tensor_transform)\n    return DataLoader(dataset, shuffle=True, batch_size=batch_size, num_workers=num_workers)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:10.963717Z","iopub.execute_input":"2021-06-14T14:19:10.964071Z","iopub.status.idle":"2021-06-14T14:19:10.977772Z","shell.execute_reply.started":"2021-06-14T14:19:10.964033Z","shell.execute_reply":"2021-06-14T14:19:10.976859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AvgPool(nn.Module):\n    def forward(self, x):\n        return F.avg_pool2d(x, x.shape[2:])\n\n\ndef create_net(net_cls, pretrained: bool):\n    if pretrained:\n        net = net_cls()\n        model_name = net_cls.__name__\n        weights_path = input_dir + f'{model_name}/{model_name}.pth'\n        net.load_state_dict(torch.load(weights_path))\n    else:\n        net = net_cls(pretrained=pretrained)\n    return net\n\n\nclass ResNet(nn.Module):\n    def __init__(self, num_classes, pretrained=False, net_cls=M.resnet50, dropout=False):\n        super().__init__()\n        self.net = create_net(net_cls, pretrained=pretrained)\n        self.net.avgpool = AvgPool()\n        if dropout:\n            self.net.fc = nn.Sequential(\n                nn.Dropout(),\n                nn.Linear(self.net.fc.in_features, num_classes),\n            )\n        else:\n            self.net.fc = nn.Linear(self.net.fc.in_features, num_classes)\n\n    def forward(self, x):\n        return self.net(x)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:10.979884Z","iopub.execute_input":"2021-06-14T14:19:10.980436Z","iopub.status.idle":"2021-06-14T14:19:10.992241Z","shell.execute_reply.started":"2021-06-14T14:19:10.980391Z","shell.execute_reply":"2021-06-14T14:19:10.991227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model(model: nn.Module, path: Path) -> Dict:\n    state = torch.load(str(path))\n    model.load_state_dict(state['model'])\n    epoch = state['epoch']\n    step = state['step']\n    print(f'Loaded model from epoch {epoch}, step {step}')\n    return state","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:10.994416Z","iopub.execute_input":"2021-06-14T14:19:10.995015Z","iopub.status.idle":"2021-06-14T14:19:11.002201Z","shell.execute_reply.started":"2021-06-14T14:19:10.994980Z","shell.execute_reply":"2021-06-14T14:19:11.001541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_loss(loss):\n    return loss.sum() / loss.shape[0]","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:11.003360Z","iopub.execute_input":"2021-06-14T14:19:11.004246Z","iopub.status.idle":"2021-06-14T14:19:11.013171Z","shell.execute_reply.started":"2021-06-14T14:19:11.004209Z","shell.execute_reply":"2021-06-14T14:19:11.012384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def binarize_prediction(probabilities, threshold: float, argsorted=None,\n                        min_labels=1, max_labels=10):\n    \"\"\" Return matrix of 0/1 predictions, same shape as probabilities.\n    \"\"\"\n    assert probabilities.shape[1] == number_of_classes\n    if argsorted is None:\n        argsorted = probabilities.argsort(axis=1)\n    max_mask = make_mask(argsorted, max_labels)\n    min_mask = make_mask(argsorted, min_labels)\n    prob_mask = probabilities > threshold\n    return (max_mask & prob_mask) | min_mask\n\n\ndef make_mask(argsorted, top_n: int):\n    mask = np.zeros_like(argsorted, dtype=np.uint8)\n    col_indices = argsorted[:, -top_n:].reshape(-1)\n    row_indices = [i // top_n for i in range(len(col_indices))]\n    mask[row_indices, col_indices] = 1\n    return mask\n","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:11.014523Z","iopub.execute_input":"2021-06-14T14:19:11.015148Z","iopub.status.idle":"2021-06-14T14:19:11.024421Z","shell.execute_reply.started":"2021-06-14T14:19:11.015093Z","shell.execute_reply":"2021-06-14T14:19:11.023725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validation(\n        model: nn.Module, criterion, valid_loader, use_cuda,\n        ) -> Dict[str, float]:\n    model.eval()\n    all_losses, all_predictions, all_targets = [], [], []\n    with torch.no_grad():\n        for inputs, targets in valid_loader:\n            all_targets.append(targets.numpy().copy())\n            if use_cuda:\n                inputs, targets = inputs.cuda(), targets.cuda()\n            outputs = model(inputs)\n            loss = criterion(outputs, targets)\n            all_losses.append(reduce_loss(loss).item())\n            predictions = torch.sigmoid(outputs)\n            all_predictions.append(predictions.cpu().numpy())\n    all_predictions = np.concatenate(all_predictions)\n    all_targets = np.concatenate(all_targets)\n\n    def get_score(y_pred):\n        with warnings.catch_warnings():\n            warnings.simplefilter('ignore', category=UndefinedMetricWarning)\n            return fbeta_score(\n                all_targets, y_pred, beta=2, average='samples')\n\n    metrics = {}\n    argsorted = all_predictions.argsort(axis=1)\n    for threshold in [0.10, 0.20]:\n        metrics[f'valid_f2_th_{threshold:.2f}'] = get_score(\n            binarize_prediction(all_predictions, threshold, argsorted))\n    metrics['valid_loss'] = np.mean(all_losses)\n    print(' | '.join(f'{k} {v:.3f}' for k, v in sorted(\n        metrics.items(), key=lambda kv: -kv[1])))\n\n    return metrics","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:11.025846Z","iopub.execute_input":"2021-06-14T14:19:11.026483Z","iopub.status.idle":"2021-06-14T14:19:11.038097Z","shell.execute_reply.started":"2021-06-14T14:19:11.026447Z","shell.execute_reply":"2021-06-14T14:19:11.037398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model: nn.Module, criterion, *, params,\n          train_loader, valid_loader, init_optimizer, use_cuda,\n          n_epochs=None, patience=2, max_lr_changes=2) -> bool:\n    \n    lr = 1e-4\n    n_epochs = 100\n    params = list(params)\n    optimizer = init_optimizer(params, lr)\n\n    model_path = 'model.pt'\n    best_model_path = 'best-model.pt'\n    #we could make it more automatic...\n    uptrain = True\n    if uptrain:\n        state = load_model(model, model_path)\n        epoch = state['epoch']\n        step = state['step']\n        best_valid_loss = state['best_valid_loss']\n    else:\n        epoch = 1\n        step = 0\n        best_valid_loss = float('inf')\n    lr_changes = 0\n\n    save = lambda ep: torch.save({\n        'model': model.state_dict(),\n        'epoch': ep,\n        'step': step,\n        'best_valid_loss': best_valid_loss\n    }, str(model_path))\n\n    report_each = 100\n    valid_losses = []\n    lr_reset_epoch = epoch\n    for epoch in range(epoch, n_epochs + 1):\n        model.train()\n        #tq = tqdm.tqdm(total=(len(train_loader) * batch_size))\n        #tq.set_description(f'Epoch {epoch}, lr {lr}')\n        losses = []\n        tl = train_loader\n        mean_loss = 0\n        for i, (inputs, targets) in enumerate(tl):\n            if use_cuda:\n                inputs, targets = inputs.cuda(), targets.cuda()\n            outputs = model(inputs)\n            loss = reduce_loss(criterion(outputs, targets))\n            batch_size = inputs.size(0)\n            (batch_size * loss).backward()\n            if (i + 1) % 1 == 0:\n                optimizer.step()\n                optimizer.zero_grad()\n                step += 1\n            #tq.update(batch_size)\n            losses.append(loss.item())\n            mean_loss = np.mean(losses[-report_each:])\n            print(f'mean_loss: {mean_loss:.3f}')\n            #tq.set_postfix(loss=f'{mean_loss:.3f}')\n        #tq.close()\n        save(epoch + 1)\n        valid_metrics = validation(model, criterion, valid_loader, use_cuda)\n        \n        valid_loss = valid_metrics['valid_loss']\n        valid_losses.append(valid_loss)\n        if valid_loss < best_valid_loss:\n            best_valid_loss = valid_loss\n            shutil.copy(str(model_path), str(best_model_path))\n        elif (patience and epoch - lr_reset_epoch > patience and\n              min(valid_losses[-patience:]) > best_valid_loss):\n            lr_changes +=1\n            if lr_changes > max_lr_changes:\n                break\n            lr /= 5\n            print(f'lr updated to {lr}')\n            lr_reset_epoch = epoch\n            optimizer = init_optimizer(params, lr)\n    return True","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:11.040640Z","iopub.execute_input":"2021-06-14T14:19:11.041128Z","iopub.status.idle":"2021-06-14T14:19:11.056036Z","shell.execute_reply.started":"2021-06-14T14:19:11.041101Z","shell.execute_reply":"2021-06-14T14:19:11.055026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform = Compose(\n[\n    RandomCrop(288),\n    RandomHorizontalFlip(),\n])\n\ntest_transform = Compose(\n[\n    RandomCrop(256),\n    RandomHorizontalFlip(),\n])\n\neval_transform = Compose(\n[\n    Resize((288, 288))\n])\n\ntensor_transform = Compose(\n[\n    ToTensor(),\n    Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:11.058731Z","iopub.execute_input":"2021-06-14T14:19:11.059144Z","iopub.status.idle":"2021-06-14T14:19:11.068591Z","shell.execute_reply.started":"2021-06-14T14:19:11.059109Z","shell.execute_reply":"2021-06-14T14:19:11.067700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_folds():\n    folds = make_folds(n_folds=5, labels_file_path=base_dir + 'train.csv')\n\n    #save_path = '../input/imet2002folds/folds.csv'\n    #save_folds(n_folds=5, labels_file_path=base_dir + 'train.csv', save_path=save_path)\n    #read pre-made\n    #pd.read_csv(save_path)\n    return folds","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:11.069980Z","iopub.execute_input":"2021-06-14T14:19:11.070233Z","iopub.status.idle":"2021-06-14T14:19:11.080517Z","shell.execute_reply.started":"2021-06-14T14:19:11.070203Z","shell.execute_reply":"2021-06-14T14:19:11.079636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def start_train():\n    folds = prepare_folds()\n    train_fold = folds[folds['fold'] != 0]\n    valid_fold = folds[folds['fold'] == 0]\n\n    train_loader = make_loader(train_fold, train_transform, tensor_transform)\n    valid_loader = make_loader(valid_fold, test_transform, tensor_transform)\n\n    criterion = nn.BCEWithLogitsLoss(reduction='none')\n\n    model = ResNet(num_classes=number_of_classes, pretrained=True, net_cls=M.resnet50)\n\n    use_cuda = cuda.is_available()\n    if use_cuda:\n        model = model.cuda()\n\n    train(\n        params=model.parameters(),\n        model=model,\n        criterion=criterion,\n        train_loader=train_loader,\n        valid_loader=valid_loader,\n        patience=4,\n        init_optimizer=lambda params, lr: Adam(params, lr),\n        use_cuda=use_cuda)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:11.081746Z","iopub.execute_input":"2021-06-14T14:19:11.082256Z","iopub.status.idle":"2021-06-14T14:19:11.090686Z","shell.execute_reply.started":"2021-06-14T14:19:11.082222Z","shell.execute_reply":"2021-06-14T14:19:11.089770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(model, test_dataset, batch_size):\n        loader = torch.utils.data.DataLoader(test_dataset, \n                                 batch_size=batch_size, \n                                 shuffle=False)\n        \n        preds = np.zeros((len(test_dataset), 1103))\n        model.cuda()\n        model.eval()\n        temp = np.zeros_like(preds)\n         \n        model_result = []\n        \n        it = 0\n        for i_batch in loader:\n            print(it)\n            inputs = i_batch.cuda()\n            with torch.no_grad():\n                model_batch_result = model(inputs)\n                model_result.extend(model_batch_result.cpu().numpy())\n            it = it + batch_size\n        return model_result","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:11.093459Z","iopub.execute_input":"2021-06-14T14:19:11.093754Z","iopub.status.idle":"2021-06-14T14:19:11.101762Z","shell.execute_reply.started":"2021-06-14T14:19:11.093724Z","shell.execute_reply":"2021-06-14T14:19:11.100885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_predictions(preds, test_dataset):\n    preds_fin = (np.array(preds) > 0.2).astype(int)\n    prediction = []\n    for i in range(preds_fin.shape[0]):\n        pred1 = np.argwhere(preds_fin[i] == 1.0).reshape(-1).tolist()\n        pred_str = \" \".join(list(map(str, pred1)))\n        prediction.append(pred_str)\n\n    sample_path = os.path.join(base_dir, \"sample_submission.csv\")\n    sample = pd.read_csv(sample_path)\n    sample.head()\n    sample.id = test_dataset.filenames()\n    sample.attribute_ids = prediction\n    sample.to_csv(\"submission.csv\", index=False)\n    sample.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:11.104451Z","iopub.execute_input":"2021-06-14T14:19:11.105345Z","iopub.status.idle":"2021-06-14T14:19:11.114317Z","shell.execute_reply.started":"2021-06-14T14:19:11.105315Z","shell.execute_reply":"2021-06-14T14:19:11.113523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test():\n    batch_size = 200\n    test_img_dir = os.path.join(base_dir, \"test\")\n    test_dataset = PredictImageDataset(test_img_dir, eval_transform, tensor_transform)\n    new_model = ResNet(num_classes=number_of_classes)\n    load_model(new_model, model_path_for_test)\n    new_model.eval()\n    preds = predict(new_model, test_dataset, batch_size)\n    save_predictions(preds, test_dataset)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:19:11.116187Z","iopub.execute_input":"2021-06-14T14:19:11.116460Z","iopub.status.idle":"2021-06-14T14:19:11.125171Z","shell.execute_reply.started":"2021-06-14T14:19:11.116437Z","shell.execute_reply":"2021-06-14T14:19:11.124235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if should_train:\n    start_train()\nelse:\n    test()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-06-14T14:19:11.128523Z","iopub.execute_input":"2021-06-14T14:19:11.128862Z","iopub.status.idle":"2021-06-14T14:26:21.361322Z","shell.execute_reply.started":"2021-06-14T14:19:11.128837Z","shell.execute_reply":"2021-06-14T14:26:21.360466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}