{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"kaggle = True\nshould_train = False\ninput_dir = '../input/'\nbase_dir = '../input/imet-2020-fgvc7/'\nmodel_path_for_test = f\"../input/resnext-mod/best-model_resnext_mod.pt\"\ntrain_root = base_dir + 'train'\nnumber_of_classes = 3474\nnum_workers = 4\nbatch_size = 32","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-18T08:16:38.730418Z","iopub.execute_input":"2021-06-18T08:16:38.730782Z","iopub.status.idle":"2021-06-18T08:16:38.740186Z","shell.execute_reply.started":"2021-06-18T08:16:38.730708Z","shell.execute_reply":"2021-06-18T08:16:38.739228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom collections import defaultdict, Counter\nfrom typing import Callable, List, Dict\nfrom pathlib import Path\nfrom itertools import islice\nfrom functools import partial\nfrom PIL import Image\nfrom sklearn.metrics import fbeta_score\nfrom sklearn.exceptions import UndefinedMetricWarning\nfrom torch import nn, cuda\nfrom torch.nn import functional as F\nfrom torch.optim import Adam\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.transforms import ToTensor, Normalize, Compose, Resize, CenterCrop, RandomCrop, RandomHorizontalFlip\nimport pandas as pd\nimport numpy as np\nimport torchvision.models as M\nimport os\nimport random\nimport cv2\nimport torch\nimport math\nimport json\nimport shutil\nimport warnings\nimport torch","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:38.749745Z","iopub.execute_input":"2021-06-18T08:16:38.750047Z","iopub.status.idle":"2021-06-18T08:16:41.083065Z","shell.execute_reply.started":"2021-06-18T08:16:38.750012Z","shell.execute_reply":"2021-06-18T08:16:41.082168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_folds(n_folds: int, labels_file_path: str) -> pd.DataFrame:\n    df = pd.read_csv(labels_file_path)\n    cls_counts = Counter(cls for classes in df['attribute_ids'].str.split() for cls in classes)\n    fold_cls_counts = defaultdict(int)\n    folds = [-1] * len(df)\n    for item in df.sample(frac=1, random_state=42).itertuples():\n        cls = min(item.attribute_ids.split(), key=lambda cls: cls_counts[cls])\n        fold_counts = [(f, fold_cls_counts[f, cls]) for f in range(n_folds)]\n        min_count = min([count for _, count in fold_counts])\n        random.seed(item.Index)\n        fold = random.choice([f for f, count in fold_counts\n                              if count == min_count])\n        folds[item.Index] = fold\n        for cls in item.attribute_ids.split():\n            fold_cls_counts[fold, cls] += 1\n    df['fold'] = folds\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:41.084443Z","iopub.execute_input":"2021-06-18T08:16:41.084758Z","iopub.status.idle":"2021-06-18T08:16:41.096792Z","shell.execute_reply.started":"2021-06-18T08:16:41.084725Z","shell.execute_reply":"2021-06-18T08:16:41.095924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_folds(n_folds: int, labels_file_path: str, save_path: str):\n    df = make_folds(n_folds=5, labels_file_path=base_dir + 'train.csv')\n    df.to_csv(save_path, index=None)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:41.100491Z","iopub.execute_input":"2021-06-18T08:16:41.10075Z","iopub.status.idle":"2021-06-18T08:16:41.106859Z","shell.execute_reply.started":"2021-06-18T08:16:41.100724Z","shell.execute_reply":"2021-06-18T08:16:41.10608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TrainDataset(Dataset):\n    def __init__(self, root_dir: Path, number_of_classes: int, df: pd.DataFrame, image_transform: Callable, tensor_transform: Callable):\n        self.root_dir = root_dir\n        self.df = df\n        self.image_transform = image_transform\n        self.tensor_transform = tensor_transform\n        self.number_of_classes = number_of_classes\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx: int):\n        item = self.df.iloc[idx]\n        image = load_transform_image(item.id, self.root_dir, self.image_transform, self.tensor_transform)\n        target = torch.zeros(self.number_of_classes)\n        \n        for attribute in item.attribute_ids.split():\n            target[int(attribute)] = 1\n\n        return image, target\n\nclass PredictImageDataset(Dataset):\n    def __init__(self, root_dir: Path, image_transform, tensor_transform):\n        self.root_dir = root_dir\n        self.files = os.listdir(root_dir)\n        self.tensor_transform = tensor_transform\n        self.image_transform = image_transform\n        \n    def __len__(self):\n        return len(self.files)\n    \n    def __getitem__(self, idx):\n        img_path = os.path.join(self.root_dir, self.files[idx])\n        img_name = os.path.splitext(self.files[idx])[0]\n        image = load_transform_image(img_name, self.root_dir, self.image_transform, self.tensor_transform)\n        return image\n    \n    def filenames(self):\n        return [os.path.splitext(x)[0] for x in self.files]","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:41.110082Z","iopub.execute_input":"2021-06-18T08:16:41.110379Z","iopub.status.idle":"2021-06-18T08:16:41.124206Z","shell.execute_reply.started":"2021-06-18T08:16:41.110354Z","shell.execute_reply":"2021-06-18T08:16:41.123334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef load_transform_image(item_id, root_dir: Path, image_transform: Callable, tensor_transform: Callable):\n    image = load_image(item_id, root_dir)\n    image = image_transform(image)\n    return tensor_transform(image)\n\n\ndef load_image(item_id, root_dir: Path) -> Image.Image:\n    path = root_dir + f'/{item_id}.png'\n    image = cv2.imread(path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    return Image.fromarray(image)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:41.125649Z","iopub.execute_input":"2021-06-18T08:16:41.125988Z","iopub.status.idle":"2021-06-18T08:16:41.133265Z","shell.execute_reply.started":"2021-06-18T08:16:41.12595Z","shell.execute_reply":"2021-06-18T08:16:41.132351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_loader(df: pd.DataFrame, image_transform: Callable, tensor_transform: Callable) -> DataLoader:\n    dataset = TrainDataset(train_root, number_of_classes, df, image_transform, tensor_transform)\n    return DataLoader(dataset, shuffle=True, batch_size=batch_size, num_workers=num_workers)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:41.134751Z","iopub.execute_input":"2021-06-18T08:16:41.135323Z","iopub.status.idle":"2021-06-18T08:16:41.14569Z","shell.execute_reply.started":"2021-06-18T08:16:41.135286Z","shell.execute_reply":"2021-06-18T08:16:41.144778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AvgPool(nn.Module):\n    def forward(self, x):\n        return F.avg_pool2d(x, x.shape[2:])\n\n\ndef create_net(net_cls, pretrained: bool):\n    if pretrained:\n        net = net_cls()\n        model_name = net_cls.__name__\n        weights_path = input_dir + f'{model_name}/{model_name}.pth'\n        net.load_state_dict(torch.load(weights_path))\n    else:\n        net = net_cls(pretrained=pretrained)\n    return net\n\n\nclass ResNet(nn.Module):\n    #def __init__(self, num_classes, pretrained=False, net_cls=M.resnext50_32x4d, dropout=False):\n    def __init__(self, num_classes, pretrained=False, net_cls=M.resnext50_32x4d, dropout=False):\n        super().__init__()\n        self.net = create_net(net_cls, pretrained=pretrained)\n        self.net.avgpool = AvgPool()\n        if dropout:\n            self.net.fc = nn.Sequential(\n                nn.Dropout(),\n                nn.Linear(self.net.fc.in_features, num_classes),\n            )\n        else:\n            self.net.fc = nn.Linear(self.net.fc.in_features, num_classes)\n\n    def forward(self, x):\n        return self.net(x)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:41.147698Z","iopub.execute_input":"2021-06-18T08:16:41.147946Z","iopub.status.idle":"2021-06-18T08:16:41.160043Z","shell.execute_reply.started":"2021-06-18T08:16:41.147921Z","shell.execute_reply":"2021-06-18T08:16:41.159261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model(model: nn.Module, path: Path) -> Dict:\n    state = torch.load(str(path))\n    model.load_state_dict(state['model'])\n    epoch = state['epoch']\n    step = state['step']\n    print(f'Loaded model from epoch {epoch}, step {step}')\n    return state","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:41.164209Z","iopub.execute_input":"2021-06-18T08:16:41.164493Z","iopub.status.idle":"2021-06-18T08:16:41.171359Z","shell.execute_reply.started":"2021-06-18T08:16:41.164467Z","shell.execute_reply":"2021-06-18T08:16:41.170431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_loss(loss):\n    return loss.sum() / loss.shape[0]","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:41.173168Z","iopub.execute_input":"2021-06-18T08:16:41.173763Z","iopub.status.idle":"2021-06-18T08:16:41.18608Z","shell.execute_reply.started":"2021-06-18T08:16:41.173722Z","shell.execute_reply":"2021-06-18T08:16:41.185212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def binarize_prediction(probabilities, threshold: float, argsorted=None,\n                        min_labels=1, max_labels=10):\n    \"\"\" Return matrix of 0/1 predictions, same shape as probabilities.\n    \"\"\"\n    assert probabilities.shape[1] == number_of_classes\n    if argsorted is None:\n        argsorted = probabilities.argsort(axis=1)\n    max_mask = make_mask(argsorted, max_labels)\n    min_mask = make_mask(argsorted, min_labels)\n    prob_mask = probabilities > threshold\n    return (max_mask & prob_mask) | min_mask\n\n\ndef make_mask(argsorted, top_n: int):\n    mask = np.zeros_like(argsorted, dtype=np.uint8)\n    col_indices = argsorted[:, -top_n:].reshape(-1)\n    row_indices = [i // top_n for i in range(len(col_indices))]\n    mask[row_indices, col_indices] = 1\n    return mask","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:41.18729Z","iopub.execute_input":"2021-06-18T08:16:41.187543Z","iopub.status.idle":"2021-06-18T08:16:41.198034Z","shell.execute_reply.started":"2021-06-18T08:16:41.187518Z","shell.execute_reply":"2021-06-18T08:16:41.196971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validation(\n        model: nn.Module, criterion, valid_loader, use_cuda,\n        ) -> Dict[str, float]:\n    model.eval()\n    all_losses, all_predictions, all_targets = [], [], []\n    with torch.no_grad():\n        for inputs, targets in valid_loader:\n            all_targets.append(targets.numpy().copy())\n            if use_cuda:\n                inputs, targets = inputs.cuda(), targets.cuda()\n            outputs = model(inputs)\n            loss = criterion(outputs, targets)\n            all_losses.append(reduce_loss(loss).item())\n            predictions = torch.sigmoid(outputs)\n            all_predictions.append(predictions.cpu().numpy())\n    all_predictions = np.concatenate(all_predictions)\n    all_targets = np.concatenate(all_targets)\n\n    def get_score(y_pred):\n        with warnings.catch_warnings():\n            warnings.simplefilter('ignore', category=UndefinedMetricWarning)\n            return fbeta_score(\n                all_targets, y_pred, beta=2, average='samples')\n\n    metrics = {}\n    argsorted = all_predictions.argsort(axis=1)\n    for threshold in [0.10, 0.20]:\n        metrics[f'valid_f2_th_{threshold:.2f}'] = get_score(\n            binarize_prediction(all_predictions, threshold, argsorted))\n    metrics['valid_loss'] = np.mean(all_losses)\n    print(' | '.join(f'{k} {v:.3f}' for k, v in sorted(\n        metrics.items(), key=lambda kv: -kv[1])))\n\n    return metrics","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:41.199523Z","iopub.execute_input":"2021-06-18T08:16:41.200153Z","iopub.status.idle":"2021-06-18T08:16:41.212067Z","shell.execute_reply.started":"2021-06-18T08:16:41.20011Z","shell.execute_reply":"2021-06-18T08:16:41.211273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model: nn.Module, criterion, *, params,\n          train_loader, valid_loader, init_optimizer, use_cuda,\n          n_epochs=None, patience=2, max_lr_changes=2) -> bool:\n    \n    lr = 1e-4\n    n_epochs = 100\n    params = list(params)\n    optimizer = init_optimizer(params, lr)\n\n    model_path = 'model.pt'\n    best_model_path = 'best-model.pt'\n    #we could make it more automatic...\n    uptrain = True\n    if uptrain:\n        state = load_model(model, model_path)\n        epoch = state['epoch']\n        step = state['step']\n        best_valid_loss = state['best_valid_loss']\n    else:\n        epoch = 1\n        step = 0\n        best_valid_loss = float('inf')\n    lr_changes = 0\n\n    save = lambda ep: torch.save({\n        'model': model.state_dict(),\n        'epoch': ep,\n        'step': step,\n        'best_valid_loss': best_valid_loss\n    }, str(model_path))\n\n    report_each = 100\n    valid_losses = []\n    lr_reset_epoch = epoch\n    for epoch in range(epoch, n_epochs + 1):\n        model.train()\n        #tq = tqdm.tqdm(total=(len(train_loader) * batch_size))\n        #tq.set_description(f'Epoch {epoch}, lr {lr}')\n        losses = []\n        tl = train_loader\n        mean_loss = 0\n        for i, (inputs, targets) in enumerate(tl):\n            if use_cuda:\n                inputs, targets = inputs.cuda(), targets.cuda()\n            outputs = model(inputs)\n            loss = reduce_loss(criterion(outputs, targets))\n            batch_size = inputs.size(0)\n            (batch_size * loss).backward()\n            if (i + 1) % 1 == 0:\n                optimizer.step()\n                optimizer.zero_grad()\n                step += 1\n            #tq.update(batch_size)\n            losses.append(loss.item())\n            mean_loss = np.mean(losses[-report_each:])\n            print(f'mean_loss: {mean_loss:.3f}')\n            #tq.set_postfix(loss=f'{mean_loss:.3f}')\n        #tq.close()\n        save(epoch + 1)\n        valid_metrics = validation(model, criterion, valid_loader, use_cuda)\n        \n        valid_loss = valid_metrics['valid_loss']\n        valid_losses.append(valid_loss)\n        if valid_loss < best_valid_loss:\n            best_valid_loss = valid_loss\n            shutil.copy(str(model_path), str(best_model_path))\n        elif (patience and epoch - lr_reset_epoch > patience and\n              min(valid_losses[-patience:]) > best_valid_loss):\n            lr_changes +=1\n            if lr_changes > max_lr_changes:\n                break\n            lr /= 5\n            print(f'lr updated to {lr}')\n            lr_reset_epoch = epoch\n            optimizer = init_optimizer(params, lr)\n    return True","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:41.213393Z","iopub.execute_input":"2021-06-18T08:16:41.21382Z","iopub.status.idle":"2021-06-18T08:16:41.229524Z","shell.execute_reply.started":"2021-06-18T08:16:41.213785Z","shell.execute_reply":"2021-06-18T08:16:41.228571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform = Compose(\n[\n    Resize((288,288)),\n    #RandomCrop(288),\n    RandomHorizontalFlip(),\n])\n\ntest_transform = Compose(\n[\n    Resize((288,288)),\n    #RandomCrop(256),\n    RandomHorizontalFlip(),\n])\n\neval_transform = Compose(\n[\n    Resize((288,288))\n    #CenterCrop(288)\n])\n\ntensor_transform = Compose([\n    ToTensor(),\n    Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:41.231038Z","iopub.execute_input":"2021-06-18T08:16:41.231561Z","iopub.status.idle":"2021-06-18T08:16:41.240749Z","shell.execute_reply.started":"2021-06-18T08:16:41.231521Z","shell.execute_reply":"2021-06-18T08:16:41.239971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_folds():\n    folds = make_folds(n_folds=5, labels_file_path=base_dir + 'train.csv')\n\n    #save_path = '../input/imet2002folds/folds.csv'\n    #save_folds(n_folds=5, labels_file_path=base_dir + 'train.csv', save_path=save_path)\n    #read pre-made\n    #pd.read_csv(save_path)\n    return folds","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:41.241987Z","iopub.execute_input":"2021-06-18T08:16:41.242482Z","iopub.status.idle":"2021-06-18T08:16:41.253263Z","shell.execute_reply.started":"2021-06-18T08:16:41.242444Z","shell.execute_reply":"2021-06-18T08:16:41.25243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def start_train():\n    folds = prepare_folds()\n    train_fold = folds[folds['fold'] != 0]\n    valid_fold = folds[folds['fold'] == 0]\n\n    train_loader = make_loader(train_fold, train_transform, tensor_transform)\n    valid_loader = make_loader(valid_fold, test_transform, tensor_transform)\n\n    criterion = nn.BCEWithLogitsLoss(reduction='none')\n\n    model = ResNet(num_classes=number_of_classes, pretrained=True, net_cls=M.resnext50_32x4d)\n\n    use_cuda = cuda.is_available()\n    if use_cuda:\n        model = model.cuda()\n\n    train(\n        params=model.parameters(),\n        model=model,\n        criterion=criterion,\n        train_loader=train_loader,\n        valid_loader=valid_loader,\n        patience=4,\n        init_optimizer=lambda params, lr: Adam(params, lr),\n        use_cuda=use_cuda)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:41.254579Z","iopub.execute_input":"2021-06-18T08:16:41.254922Z","iopub.status.idle":"2021-06-18T08:16:41.264572Z","shell.execute_reply.started":"2021-06-18T08:16:41.254893Z","shell.execute_reply":"2021-06-18T08:16:41.263666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(model, test_dataset, batch_size):\n        loader = torch.utils.data.DataLoader(test_dataset, \n                                 batch_size=batch_size, \n                                 shuffle=False)\n        \n        preds = np.zeros((len(test_dataset), 1103))\n        model.cuda()\n        model.eval()\n        temp = np.zeros_like(preds)\n         \n        model_result = []\n        \n        it = 0\n        for i_batch in loader:\n            print(it)\n            inputs = i_batch.cuda()\n            with torch.no_grad():\n                model_batch_result = model(inputs)\n                model_result.extend(model_batch_result.cpu().numpy())\n            it = it + batch_size\n        return model_result","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:41.26579Z","iopub.execute_input":"2021-06-18T08:16:41.266222Z","iopub.status.idle":"2021-06-18T08:16:41.275051Z","shell.execute_reply.started":"2021-06-18T08:16:41.266184Z","shell.execute_reply":"2021-06-18T08:16:41.274198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_predictions(preds, test_dataset):\n    preds_fin = (np.array(preds) > 0.01).astype(int)\n    prediction = []\n    for i in range(preds_fin.shape[0]):\n        pred1 = np.argwhere(preds_fin[i] == 1.0).reshape(-1).tolist()\n        pred_str = \" \".join(list(map(str, pred1)))\n        prediction.append(pred_str)\n\n    sample_path = os.path.join(base_dir, \"sample_submission.csv\")\n    sample = pd.read_csv(sample_path)\n    sample.head()\n    sample.id = test_dataset.filenames()\n    sample.attribute_ids = prediction\n    sample.to_csv(\"submission.csv\", index=False)\n    sample.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:41.276581Z","iopub.execute_input":"2021-06-18T08:16:41.276938Z","iopub.status.idle":"2021-06-18T08:16:41.287101Z","shell.execute_reply.started":"2021-06-18T08:16:41.276901Z","shell.execute_reply":"2021-06-18T08:16:41.286168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test():\n    batch_size = 200\n    test_img_dir = os.path.join(base_dir, \"test\")\n    test_dataset = PredictImageDataset(test_img_dir, eval_transform, tensor_transform)\n    new_model = ResNet(num_classes=number_of_classes)\n    load_model(new_model, model_path_for_test)\n    new_model.eval()\n    preds = predict(new_model, test_dataset, batch_size)\n    save_predictions(preds, test_dataset)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:41.290492Z","iopub.execute_input":"2021-06-18T08:16:41.290745Z","iopub.status.idle":"2021-06-18T08:16:41.297643Z","shell.execute_reply.started":"2021-06-18T08:16:41.290721Z","shell.execute_reply":"2021-06-18T08:16:41.296875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if should_train:\n#     start_train()\n# else:\ntest()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:16:41.29916Z","iopub.execute_input":"2021-06-18T08:16:41.299587Z","iopub.status.idle":"2021-06-18T08:26:16.316643Z","shell.execute_reply.started":"2021-06-18T08:16:41.29955Z","shell.execute_reply":"2021-06-18T08:26:16.315753Z"},"trusted":true},"execution_count":null,"outputs":[]}]}