{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Problem description\n\nIn this kernel, we are going to use Densenet121 pretrained model with Pytorch."},{"metadata":{},"cell_type":"markdown","source":"## Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport sys\nimport time\nimport random\nimport logging\nimport datetime as dt\n\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.utils.data as data\nimport torch.nn.functional as F\nimport torchvision as vision\n\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\n\nfrom pathlib import Path\nfrom PIL import Image\nfrom contextlib import contextmanager\n\nfrom joblib import Parallel, delayed\nfrom tqdm import tqdm\nfrom fastprogress import master_bar, progress_bar\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import fbeta_score\n\ntorch.multiprocessing.set_start_method(\"spawn\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Utilities"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"@contextmanager\ndef timer(name=\"Main\", logger=None):\n    t0 = time.time()\n    yield\n    msg = f\"[{name}] done in {time.time() - t0} s\"\n    if logger is not None:\n        logger.info(msg)\n    else:\n        print(msg)\n        \n\ndef get_logger(name=\"Main\", tag=\"exp\", log_dir=\"log/\"):\n    log_path = Path(log_dir)\n    path = log_path / tag\n    path.mkdir(exist_ok=True, parents=True)\n\n    logger = logging.getLogger(name)\n    logger.setLevel(logging.INFO)\n\n    fh = logging.FileHandler(\n        path / (dt.datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\") + \".log\"))\n    sh = logging.StreamHandler(sys.stdout)\n    formatter = logging.Formatter(\n        \"%(asctime)s %(name)s %(levelname)s %(message)s\")\n\n    fh.setFormatter(formatter)\n    sh.setFormatter(formatter)\n    logger.addHandler(fh)\n    logger.addHandler(sh)\n    return logger\n\n\ndef seed_torch(seed=1029):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logger = get_logger(name=\"Main\", tag=\"Pytorch-VGG16\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/imet-2019-fgvc6/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = pd.read_csv(\"../input/imet-2019-fgvc6/labels.csv\")\ntrain = pd.read_csv(\"../input/imet-2019-fgvc6/train.csv\")\nsample = pd.read_csv(\"../input/imet-2019-fgvc6/sample_submission.csv\")\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cp ../input/pytorch-pretrained-image-models/* ./\n!ls","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## DataLoader"},{"metadata":{"trusted":true},"cell_type":"code","source":"# This loader is to extract 1024d features from the images.\nclass ImageDataLoader(data.DataLoader):\n    def __init__(self, root_dir: Path, \n                 df: pd.DataFrame, \n                 mode=\"train\", \n                 transforms=None):\n        self._root = root_dir\n        self.transform = transforms[mode]\n        self._img_id = (df[\"id\"] + \".png\").values\n        \n    def __len__(self):\n        return len(self._img_id)\n    \n    def __getitem__(self, idx):\n        img_id = self._img_id[idx]\n        file_name = self._root / img_id\n        img = Image.open(file_name)\n        \n        if self.transform:\n            img = self.transform(img)\n            \n        return [img]\n    \n    \ndata_transforms = {\n    'train': vision.transforms.Compose([\n        vision.transforms.RandomResizedCrop(224),\n        vision.transforms.RandomHorizontalFlip(),\n        vision.transforms.ToTensor(),\n        vision.transforms.Normalize(\n            [0.485, 0.456, 0.406], \n            [0.229, 0.224, 0.225])\n    ]),\n    'val': vision.transforms.Compose([\n        vision.transforms.Resize(256),\n        vision.transforms.CenterCrop(224),\n        vision.transforms.ToTensor(),\n        vision.transforms.Normalize(\n            [0.485, 0.456, 0.406], \n            [0.229, 0.224, 0.225])\n    ]),\n}\n\ndata_transforms[\"test\"] = data_transforms[\"val\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This loader is to be used for serving image tensors for the MLP.\nclass IMetDataset(data.Dataset):\n    def __init__(self, tensor, device=\"cuda:0\", labels=None):\n        self.tensor = tensor\n        self.labels = labels\n        self.device= device\n        \n    def __len__(self):\n        return self.tensor.size(0)\n    \n    def __getitem__(self, idx):\n        tensor = self.tensor[idx, :]\n        if self.labels is not None:\n            label = self.labels[idx]\n            label_tensor = torch.zeros((1, 1103))\n            for i in label:\n                label_tensor[0, int(i)] = 1\n            label_tensor = label_tensor.to(self.device)\n            return [tensor, label_tensor]\n        else:\n            return [tensor]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{},"cell_type":"markdown","source":"Since passing images through the Densenet121 takes a lot of time, I don't want to take that time per epoch.\n\nI will calculate the output of the Densenet121 beforehand so that converting raw images to processed feature vectors happens once."},{"metadata":{"trusted":true},"cell_type":"code","source":"class Classifier(nn.Module):\n    def __init__(self):\n        super(Classifier, self).__init__()\n        \n    def forward(self, x):\n        return x\n\n\nclass Densenet121(nn.Module):\n    def __init__(self, pretrained: Path):\n        super(Densenet121, self).__init__()\n        self.densenet121 = vision.models.densenet121()\n        self.densenet121.load_state_dict(torch.load(pretrained))\n        self.densenet121.classifier = Classifier()\n        \n        dense = nn.Sequential(*list(self.densenet121.children())[:-1])\n        for param in dense.parameters():\n            param.requires_grad = False\n        \n    def forward(self, x):\n        return self.densenet121(x)\n    \n    \nclass MultiLayerPerceptron(nn.Module):\n    def __init__(self):\n        super(MultiLayerPerceptron, self).__init__()\n        self.linear1 = nn.Linear(1024, 1024)\n        self.relu = nn.ReLU()\n        self.linear2 = nn.Linear(1024, 1103)\n        self.dropout = nn.Dropout(0.5)\n        self.sigmoid = nn.Sigmoid()\n        \n    def forward(self, x):\n        x = self.relu(self.linear1(x))\n        x = self.dropout(x)\n        return self.sigmoid(self.linear2(x))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Preprocessing with Densenet121"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = ImageDataLoader(\n    root_dir=Path(\"../input/imet-2019-fgvc6/train/\"),\n    df=train,\n    mode=\"train\",\n    transforms=data_transforms)\ntrain_loader = data.DataLoader(dataset=train_dataset,\n                               shuffle=False,\n                               batch_size=128)\ntest_dataset = ImageDataLoader(\n    root_dir=Path(\"../input/imet-2019-fgvc6/test/\"),\n    df=sample,\n    mode=\"test\",\n    transforms=data_transforms)\ntest_loader = data.DataLoader(dataset=test_dataset,\n                              shuffle=False,\n                              batch_size=128)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_feature_vector(df, loader, device):\n    matrix = torch.zeros((df.shape[0], 1024)).to(device)\n    model = Densenet121(\"densenet121.pth\")\n    model.to(device)\n    batch = loader.batch_size\n    for i, (i_batch,) in tqdm(enumerate(loader)):\n        i_batch = i_batch.to(device)\n        pred = model(i_batch).detach()\n        matrix[i * batch:(i + 1) * batch] = pred\n    return matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_tensor = get_feature_vector(train, train_loader, \"cuda:0\")\ntest_tensor = get_feature_vector(sample, test_loader, \"cuda:0\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_dataset, train_loader\ndel test_dataset, test_loader\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Utilities"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Trainer:\n    def __init__(self, \n                 model, \n                 logger,\n                 n_splits=5,\n                 seed=42,\n                 device=\"cuda:0\",\n                 train_batch=32,\n                 valid_batch=128,\n                 kwargs={}):\n        self.model = model\n        self.logger = logger\n        self.device = device\n        self.n_splits = n_splits\n        self.seed = seed\n        self.train_batch = train_batch\n        self.valid_batch = valid_batch\n        self.kwargs = kwargs\n        \n        self.best_score = None\n        self.tag = dt.datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n        self.loss_fn = nn.BCELoss(reduction=\"mean\").to(self.device)\n        \n        path = Path(f\"bin/{self.tag}\")\n        path.mkdir(exist_ok=True, parents=True)\n        self.path = path\n        \n    def fit(self, X, y, n_epochs=10):\n        train_preds = np.zeros((len(X), 1103))\n        fold = KFold(n_splits=self.n_splits, random_state=self.seed)\n        for i, (trn_idx, val_idx) in enumerate(fold.split(X)):\n            self.fold_num = i\n            self.logger.info(f\"Fold {i + 1}\")\n            X_train, X_val = X[trn_idx, :], X[val_idx, :]\n            y_train, y_val = y[trn_idx], y[val_idx]\n            \n            valid_preds = self._fit(X_train, y_train, X_val, y_val, n_epochs)\n            train_preds[val_idx] = valid_preds\n        return train_preds\n    \n    def _fit(self, X_train, y_train, X_val, y_val, n_epochs):\n        seed_torch(self.seed)\n        train_dataset = IMetDataset(X_train, labels=y_train, device=self.device)\n        train_loader = data.DataLoader(train_dataset, \n                                       batch_size=self.train_batch,\n                                       shuffle=True)\n\n        valid_dataset = IMetDataset(X_val, labels=y_val, device=self.device)\n        valid_loader = data.DataLoader(valid_dataset,\n                                       batch_size=self.valid_batch,\n                                       shuffle=False)\n        \n        model = self.model(**self.kwargs)\n        model.to(self.device)\n        \n        optimizer = optim.Adam(params=model.parameters(), \n                                lr=0.0001)\n        scheduler = CosineAnnealingLR(optimizer, T_max=n_epochs)\n        best_score = np.inf\n        mb = master_bar(range(n_epochs))\n        for epoch in mb:\n            model.train()\n            avg_loss = 0.0\n            for i_batch, y_batch in progress_bar(train_loader, parent=mb):\n                y_pred = model(i_batch)\n                loss = self.loss_fn(y_pred, y_batch)\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n                avg_loss += loss.item() / len(train_loader)\n            valid_preds, avg_val_loss = self._val(valid_loader, model)\n            scheduler.step()\n\n            self.logger.info(\"=========================================\")\n            self.logger.info(f\"Epoch {epoch + 1} / {n_epochs}\")\n            self.logger.info(\"=========================================\")\n            self.logger.info(f\"avg_loss: {avg_loss:.8f}\")\n            self.logger.info(f\"avg_val_loss: {avg_val_loss:.8f}\")\n            \n            if best_score > avg_val_loss:\n                torch.save(model.state_dict(),\n                           self.path / f\"best{self.fold_num}.pth\")\n                self.logger.info(f\"Save model at Epoch {epoch + 1}\")\n                best_score = avg_val_loss\n        model.load_state_dict(torch.load(self.path / f\"best{self.fold_num}.pth\"))\n        valid_preds, avg_val_loss = self._val(valid_loader, model)\n        self.logger.info(f\"Best Validation Loss: {avg_val_loss:.8f}\")\n        return valid_preds\n    \n    def _val(self, loader, model):\n        model.eval()\n        valid_preds = np.zeros((len(loader.dataset), 1103))\n        avg_val_loss = 0.0\n        for i, (i_batch, y_batch) in enumerate(loader):\n            with torch.no_grad():\n                y_pred = model(i_batch).detach()\n                avg_val_loss += self.loss_fn(y_pred, y_batch).item() / len(loader)\n                valid_preds[i * self.valid_batch:(i + 1) * self.valid_batch] = \\\n                    y_pred.cpu().numpy()\n        return valid_preds, avg_val_loss\n    \n    def predict(self, X):\n        dataset = IMetDataset(X, labels=None)\n        loader = data.DataLoader(dataset, \n                                 batch_size=self.valid_batch, \n                                 shuffle=False)\n        model = self.model(**self.kwargs)\n        preds = np.zeros((X.size(0), 1103))\n        for path in self.path.iterdir():\n            with timer(f\"Using {str(path)}\", self.logger):\n                model.load_state_dict(torch.load(path))\n                model.to(self.device)\n                model.eval()\n                temp = np.zeros_like(preds)\n                for i, (i_batch, ) in enumerate(loader):\n                    with torch.no_grad():\n                        y_pred = model(i_batch).detach()\n                        temp[i * self.valid_batch:(i + 1) * self.valid_batch] = \\\n                            y_pred.cpu().numpy()\n                preds += temp / self.n_splits\n        return preds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer = Trainer(MultiLayerPerceptron, logger, train_batch=64, kwargs={})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train.attribute_ids.map(lambda x: x.split()).values\nvalid_preds = trainer.fit(train_tensor, y, n_epochs=40)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Post process - threshold search -"},{"metadata":{},"cell_type":"markdown","source":"Since I used sigmoid for the activation, I've got the 1103 probability output for each data row.\n\nI need to decide threshold for this.There are two ways to deal with this.\n\n- Class-wise threshold search\n  - Takes some time but it's natural.\n- One threshold for all the class\n  - Low cost way.\n\n**UPDATE**\nI will use the first -> second one."},{"metadata":{"trusted":true},"cell_type":"code","source":"def threshold_search(y_pred, y_true):\n    score = []\n    candidates = np.arange(0, 1.0, 0.01)\n    for th in progress_bar(candidates):\n        yp = (y_pred > th).astype(int)\n        score.append(fbeta_score(y_pred=yp, y_true=y_true, beta=2, average=\"samples\"))\n    score = np.array(score)\n    pm = score.argmax()\n    best_th, best_score = candidates[pm], score[pm]\n    return best_th, best_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_true = np.zeros((train.shape[0], 1103)).astype(int)\nfor i, row in enumerate(y):\n    for idx in row:\n        y_true[i, int(idx)] = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_threshold, best_score = threshold_search(valid_preds, y_true)\nbest_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction for test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = trainer.predict(test_tensor)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = (test_preds > best_threshold).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction = []\nfor i in range(preds.shape[0]):\n    pred1 = np.argwhere(preds[i] == 1.0).reshape(-1).tolist()\n    pred_str = \" \".join(list(map(str, pred1)))\n    prediction.append(pred_str)\n    \nsample.attribute_ids = prediction\nsample.to_csv(\"submission.csv\", index=False)\nsample.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}