{"cells":[{"metadata":{},"cell_type":"markdown","source":"# About this notebook\n- PyTorch Resnet18 starter code\n- single fold\n- 1 epochs\n\nIf this notebook is helpful, feel free to upvote :)"},{"metadata":{},"cell_type":"markdown","source":"# Library"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd \nimport json","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nwith open('../input/herbarium-2020-fgvc7/nybg2020/train/metadata.json', \"r\", encoding=\"ISO-8859-1\") as file:\n    train = json.load(file)\n\ntrain_img = pd.DataFrame(train['images'])\ntrain_ann = pd.DataFrame(train['annotations']).drop(columns='image_id')\ntrain_df = train_img.merge(train_ann, on='id')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nwith open('../input/herbarium-2020-fgvc7/nybg2020/test/metadata.json', \"r\", encoding=\"ISO-8859-1\") as file:\n    test = json.load(file)\n\ntest_df = pd.DataFrame(test['images'])\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TARGET (\"category_id\")"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['category_id'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- imbalance\n- 32093 targets"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import preprocessing\n\nle = preprocessing.LabelEncoder()\nle.fit(train_df['category_id'])\ntrain_df['category_id'] = le.transform(train_df['category_id'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Library"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\n\nimport sys\n\nimport gc\nimport os\nimport random\nimport time\nfrom contextlib import contextmanager\nfrom pathlib import Path\n\nimport cv2\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\n\nimport sklearn.metrics\n\nfrom functools import partial\n\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam, SGD\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torch.utils.data import DataLoader, Dataset\n\nfrom albumentations import Compose, Normalize, Resize\nfrom albumentations.pytorch import ToTensorV2\n\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n\n    \ndef init_logger(log_file='train.log'):\n    from logging import getLogger, DEBUG, FileHandler,  Formatter,  StreamHandler\n    \n    log_format = '%(asctime)s %(levelname)s %(message)s'\n    \n    stream_handler = StreamHandler()\n    stream_handler.setLevel(DEBUG)\n    stream_handler.setFormatter(Formatter(log_format))\n    \n    file_handler = FileHandler(log_file)\n    file_handler.setFormatter(Formatter(log_format))\n    \n    logger = getLogger('Herbarium')\n    logger.setLevel(DEBUG)\n    logger.addHandler(stream_handler)\n    logger.addHandler(file_handler)\n    \n    return logger\n\nLOG_FILE = 'train.log'\nLOGGER = init_logger(LOG_FILE)\n\n\ndef seed_torch(seed=777):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nSEED = 777\nseed_torch(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"N_CLASSES = 32093\n\n\nclass TrainDataset(Dataset):\n    def __init__(self, df, labels, transform=None):\n        self.df = df\n        self.labels = labels\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.df['file_name'].values[idx]\n        file_path = f'../input/herbarium-2020-fgvc7/nybg2020/train/{file_name}'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        label = self.labels.values[idx]\n        \n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        \n        return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"HEIGHT = 128\nWIDTH = 128\n\n\ndef get_transforms(*, data):\n    \n    assert data in ('train', 'valid')\n    \n    if data == 'train':\n        return Compose([\n            Resize(HEIGHT, WIDTH),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])\n    \n    elif data == 'valid':\n        return Compose([\n            Resize(HEIGHT, WIDTH),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# train valid split"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\nDEBUG = False\n\nif DEBUG:\n    folds = train_df.sample(n=10000, random_state=0).reset_index(drop=True).copy()\nelse:\n    folds = train_df.copy()\ntrain_labels = folds['category_id'].values\nkf = StratifiedKFold(n_splits=2)\nfor fold, (train_index, val_index) in enumerate(kf.split(folds.values, train_labels)):\n    folds.loc[val_index, 'fold'] = int(fold)\nfolds['fold'] = folds['fold'].astype(int)\nfolds.to_csv('folds.csv', index=None)\nfolds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FOLD = 0\ntrn_idx = folds[folds['fold'] != FOLD].index\nval_idx = folds[folds['fold'] == FOLD].index\nprint(trn_idx.shape, val_idx.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = TrainDataset(folds.loc[trn_idx].reset_index(drop=True), \n                             folds.loc[trn_idx]['category_id'], \n                             transform=get_transforms(data='train'))\nvalid_dataset = TrainDataset(folds.loc[val_idx].reset_index(drop=True), \n                             folds.loc[val_idx]['category_id'], \n                             transform=get_transforms(data='valid'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 512\n\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\nvalid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torchvision.models as models\n\nmodel = models.resnet18(pretrained=True)\nmodel.avgpool = nn.AdaptiveAvgPool2d(1)\nmodel.fc = nn.Linear(model.fc.in_features, N_CLASSES)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.metrics import f1_score\nfrom tqdm import tqdm\n\n\nwith timer('Train model'):\n    \n    n_epochs = 1\n    lr = 4e-4\n    \n    model.to(device)\n    \n    optimizer = Adam(model.parameters(), lr=lr, amsgrad=False)\n    scheduler = ReduceLROnPlateau(optimizer, 'min', factor=0.75, patience=5, verbose=True, eps=1e-6)\n    \n    criterion = nn.CrossEntropyLoss()\n    best_score = 0.\n    best_loss = np.inf\n    \n    for epoch in range(n_epochs):\n        \n        start_time = time.time()\n\n        model.train()\n        avg_loss = 0.\n\n        optimizer.zero_grad()\n\n        for i, (images, labels) in tqdm(enumerate(train_loader)):\n\n            images = images.to(device)\n            labels = labels.to(device)\n            \n            y_preds = model(images)\n            loss = criterion(y_preds, labels)\n            \n            loss.backward()\n            optimizer.step()\n            optimizer.zero_grad()\n\n            avg_loss += loss.item() / len(train_loader)\n            \n        model.eval()\n        avg_val_loss = 0.\n        preds = np.zeros((len(valid_dataset)))\n\n        for i, (images, labels) in enumerate(valid_loader):\n            \n            images = images.to(device)\n            labels = labels.to(device)\n            \n            with torch.no_grad():\n                y_preds = model(images)\n            \n            preds[i * batch_size: (i+1) * batch_size] = y_preds.argmax(1).to('cpu').numpy()\n\n            loss = criterion(y_preds, labels)\n            avg_val_loss += loss.item() / len(valid_loader)\n        \n        scheduler.step(avg_val_loss)\n            \n        score = f1_score(folds.loc[val_idx]['category_id'].values, preds, average='macro')\n\n        elapsed = time.time() - start_time\n\n        LOGGER.debug(f'  Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  F1: {score:.6f}  time: {elapsed:.0f}s')\n\n        if score>best_score:\n            best_score = score\n            LOGGER.debug(f'  Epoch {epoch+1} - Save Best Score: {best_score:.6f} Model')\n            torch.save(model.state_dict(), f'fold{FOLD}_best_score.pth')\n\n        if avg_val_loss<best_loss:\n            best_loss = avg_val_loss\n            LOGGER.debug(f'  Epoch {epoch+1} - Save Best Loss: {best_loss:.4f} Model')\n            torch.save(model.state_dict(), f'fold{FOLD}_best_loss.pth')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference\n- Check [inference kernel](https://www.kaggle.com/yasufuminakama/herbarium-2020-pytorch-resnet18-inference) :)"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}