{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install git+https://github.com/ildoonet/pytorch-gradual-warmup-lr.git\n!pip install efficientnet_pytorch\n!pip install albumentations\n!pip install git+https://github.com/NVIDIA/apex","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport sys\nimport json\nimport time\nimport random\nimport cv2\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score\nfrom tqdm.notebook import tqdm\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.utils.data as data\nimport torchvision\nfrom torchvision import models, transforms\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom efficientnet_pytorch import model as enet\nimport albumentations as albu\nfrom apex import amp\nimport warnings\nwarnings.filterwarnings('ignore', category=UserWarning) \nos.environ['CUDA_VISIBLE_DEVICES'] = '0'\nDEVICE = torch.device('cuda')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VER = 'v0'\nDEBUG = False\nPARAMS = {\n    'version': VER,\n    'folds': 4,\n    'folds_train': 1,\n    'img_size': 300, # 224=B0 240=B1 260=B2 300=B3 380=B4 456=B5 528=B6 600=B7\n    'batch_size': 24,\n    'workers': 4,\n    'epochs': 2 if DEBUG else 10,\n    'warmup': False,\n    'dropout': .4,\n    'backbone': 'efficientnet-b3', # 'efficientnet-bX' or 'resnext'\n    'seed': 2021,\n    'aughard': True,\n    'lr': .0005,\n    'average': 'macro', # 'micro', 'macro' or 'samples'\n    'apex': True,\n    'comments': 'f1 score'\n}\nDATA_PATH = '../input/plant-pathology-2021-fgvc8'\nIMGS_PATH = f'{DATA_PATH}/train_images/'\nMDLS_PATH = f'./models_{VER}'\nif not os.path.exists(MDLS_PATH):\n    os.mkdir(MDLS_PATH)\n    \ndef seed_all(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n\nseed_all(PARAMS['seed'])\n\nstart_time = time.time()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEBUG:\n    df_train = pd.read_csv(f'{DATA_PATH}/train.csv').sample(100).reset_index(drop=True)\nelse:\n    df_train = pd.read_csv(f'{DATA_PATH}/train.csv')\ndf_sub = pd.read_csv(f'{DATA_PATH}/sample_submission.csv')\ndisplay(df_train.head())\ndisplay(df_train.labels.value_counts())\nlabels = []\nfor lbl in list(set(df_train.labels)):\n    labels.extend(lbl.split())\nlabels = list(set(labels))\nLABELS = {i: x for i, x in enumerate(sorted(labels))}\nLABELS_ = {x: i for i, x in enumerate(sorted(labels))}\nPARAMS['labels'] = LABELS\nPARAMS['labels_'] = LABELS_\nwith open(f'{MDLS_PATH}/params.json', 'w') as file:\n    json.dump(PARAMS, file)\nprint('labels:', LABELS)\nprint('labels_:', LABELS_)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(PARAMS['folds'], shuffle=True, random_state=PARAMS['seed'])\ndf_train['fold'] = -1\nfor i, (train_idx, valid_idx) in enumerate(skf.split(df_train, df_train['labels'])):\n    df_train.loc[valid_idx, 'fold'] = i\ndisplay(df_train.head())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_occurence = pd.DataFrame({\n    'origin': df_train.labels.value_counts(normalize=True),\n    'fold_0': df_train[df_train.fold == 0].labels.value_counts(normalize=True),\n    'fold_1': df_train[df_train.fold == 1].labels.value_counts(normalize=True),\n    'fold_2': df_train[df_train.fold == 2].labels.value_counts(normalize=True),\n    'fold_3': df_train[df_train.fold == 3].labels.value_counts(normalize=True),\n    'fold_4': df_train[df_train.fold == 4].labels.value_counts(normalize=True)})\ndf_occurence.plot.barh(figsize=[12, 6], colormap='plasma')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!L\nif PARAMS['aughard']:\n    aug = albu.Compose([\n        albu.OneOf([\n            albu.RandomBrightness(limit=.2, p=1), \n            albu.RandomContrast(limit=.2, p=1), \n            albu.RandomGamma(p=1)\n        ], p=.5),\n        albu.OneOf([\n            albu.Blur(blur_limit=3, p=1),\n            albu.MedianBlur(blur_limit=3, p=1)\n        ], p=.25),\n        albu.OneOf([\n            albu.GaussNoise(0.002, p=.5),\n            albu.IAAAffine(p=.5),\n        ], p=.25),\n        albu.RandomRotate90(p=.5),\n        albu.HorizontalFlip(p=.5),\n        albu.VerticalFlip(p=.5),\n        albu.Transpose(p=.5),\n        albu.Cutout(\n            num_holes=10, \n            max_h_size=int(.1 * PARAMS['img_size']), \n            max_w_size=int(.1 * PARAMS['img_size']), \n            p=.25),\n        albu.ShiftScaleRotate(p=.5)\n    ])\nelse:\n    aug = albu.Compose([\n        albu.OneOf([\n            albu.RandomBrightness(limit=.2, p=1), \n            albu.RandomContrast(limit=.2, p=1), \n            albu.RandomGamma(p=1)\n        ], p=.5),\n        albu.RandomRotate90(p=.25),\n        albu.HorizontalFlip(p=.25),\n        albu.VerticalFlip(p=.25)\n    ])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!L\ndef flip(img, axis=0):\n    if axis == 1:\n        return img[::-1, :, ]\n    elif axis == 2:\n        return img[:, ::-1, ]\n    elif axis == 3:\n        return img[::-1, ::-1, ]\n    else:\n        return img\n\nclass PlantDataset(data.Dataset):\n    \n    def __init__(self, df, size, labels, transform=None, tta=0):\n        self.df = df.reset_index(drop=True)\n        self.size = size\n        self.labels = labels\n        self.transform = transform\n        self.tta = tta\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        img_name = row.image\n        img_path = f'{IMGS_PATH}/{img_name}'\n        img = cv2.imread(img_path)\n        if not np.any(img):\n            print('no img file read:', img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = cv2.resize(img, (self.size, self.size))\n        img = img.astype(np.float32) / 255\n        if self.transform is not None:\n            img = self.transform(image=img)['image']\n        if self.labels:\n            img = img.transpose(2, 0, 1)\n            label = np.zeros(len(self.labels)).astype(np.float32)\n            for lbl in row.labels.split():\n                label[self.labels[lbl]] = 1\n            return torch.tensor(img), torch.tensor(label)\n        else:\n            img = flip(img, axis=self.tta)\n            img = img.transpose(2, 0, 1)\n            return torch.tensor(img.copy())\n\ndataset_show = PlantDataset(\n    df=df_train,\n    size=PARAMS['img_size'],\n    labels=LABELS_,\n    transform=aug\n)\nimg_test, lbl_test = dataset_show.__getitem__(7)\nimg_test = img_test.numpy().transpose([1, 2, 0])\nimg_test = np.clip(img_test, 0, 1)\nplt.imshow(img_test)\nplt.title(lbl_test)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!L\nclass EffNet(nn.Module):\n    \n    def __init__(self, params, out_dim):\n        super(EffNet, self).__init__()\n        self.enet = enet.EfficientNet.from_name(params['backbone'])\n        nc = self.enet._fc.in_features\n        self.enet._fc = nn.Identity()\n        self.myfc = nn.Sequential(\n            nn.Dropout(params['dropout']),\n            nn.Linear(nc, int(nc / 4)),\n            nn.Dropout(params['dropout']),\n            nn.Linear(int(nc / 4), out_dim)\n        )\n        \n    def extract(self, x):\n        return self.enet(x)\n    \n    def forward(self, x):\n        x = self.extract(x)\n        x = self.myfc(x)\n        return x\n    \nclass ResNext(nn.Module):\n    \n    def __init__(self, params, out_dim):\n        super(ResNext, self).__init__()\n        self.rsnxt = torchvision.models.resnext50_32x4d(pretrained=True)\n        nc = self.rsnxt.fc.in_features\n        self.rsnxt.fc = nn.Sequential(\n            nn.Flatten(),\n            nn.Linear(nc, int(nc / 4)),\n            nn.ReLU(),\n            nn.Dropout(params['dropout']),\n            nn.Linear(int(nc / 4), out_dim)\n        )\n        self.rsnxt = nn.DataParallel(self.rsnxt)\n        \n    def forward(self, x):\n        x = self.rsnxt(x)\n        return x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!L\ncriterion = nn.BCEWithLogitsLoss()\n\ndef train_epoch(loader, optimizer):\n    model.train()\n    train_loss = []\n    bar = tqdm(loader, desc='ep')\n    for (data, target) in bar:\n        data, target = data.to(DEVICE), target.to(DEVICE)\n        loss_func = criterion\n        optimizer.zero_grad()\n        logits = model(data)\n        loss = loss_func(logits, target)\n        if PARAMS['apex']:\n            with amp.scale_loss(loss, optimizer) as scaled_loss:\n                scaled_loss.backward()\n        else:\n            loss.backward()\n        optimizer.step()\n        loss_np = loss.detach().cpu().numpy()\n        train_loss.append(loss_np)\n        smooth_loss = sum(train_loss[-100:]) / min(len(train_loss), 100)\n        bar.set_description('loss: {:.4f}, smth: {:.4f}'.format(loss_np, smooth_loss))\n    return train_loss\n\ndef val_epoch(loader, get_output=False, verbose=False):\n    model.eval()\n    val_loss = []\n    val_logits = []\n    val_preds = []\n    val_targets = []\n    with torch.no_grad():\n        for (data, target) in tqdm(loader):\n            data, target = data.to(DEVICE), target.to(DEVICE)\n            logits = model(data)\n            loss = criterion(logits, target)\n            pred = logits.sigmoid().detach().round()\n            val_logits.append(logits)\n            val_preds.append(pred)\n            val_targets.append(target)\n            val_loss.append(loss.detach().cpu().numpy())\n        val_loss = np.mean(val_loss)\n    val_logits = torch.cat(val_logits).cpu().numpy()\n    val_preds = torch.cat(val_preds).cpu().numpy()\n    val_targets = torch.cat(val_targets).cpu().numpy()\n    val_acc = (val_preds == val_targets).mean() * 100\n    val_f1 = f1_score(val_targets, val_preds, average=PARAMS['average'])\n    if verbose:\n        print('val acc: {:.2f} | val loss: {:.4f} | val f1: {:.4f}'.format(val_acc, val_loss, val_f1))\n    if get_output:\n        return val_logits\n    else:\n        return val_loss, val_acc, val_f1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!L\npred, target = [], []\npreds_val, target_val = [], []\n\nif DEBUG:\n    n_folds_train = 2\nelse:\n    n_folds_train = PARAMS['folds'] if not PARAMS['folds_train'] else PARAMS['folds_train']\nstart_folds_train = 0\n\nfor fold_num in range(start_folds_train, n_folds_train):\n    print('=' * 20, 'FOLD:', fold_num, '=' * 20)\n    train_idxs = np.where((df_train['fold'] != fold_num))[0]\n    val_idxs = np.where((df_train['fold'] == fold_num))[0]\n    df_fold  = df_train.loc[train_idxs]\n    df_val = df_train.loc[val_idxs]\n    dataset_train = PlantDataset(\n        df=df_fold,\n        size=PARAMS['img_size'],\n        labels=LABELS_,\n        transform=aug\n    )\n    dataset_val = PlantDataset(\n        df=df_val,\n        size=PARAMS['img_size'],\n        labels=LABELS_,\n        transform=None\n    )\n    train_loader = torch.utils.data.DataLoader(\n        dataset_train, \n        batch_size=PARAMS['batch_size'], \n        sampler=RandomSampler(dataset_train), \n        num_workers=PARAMS['workers']\n    )\n    val_loader = torch.utils.data.DataLoader(\n        dataset_val, \n        batch_size=PARAMS['batch_size'], \n        sampler=SequentialSampler(dataset_val), \n        num_workers=PARAMS['workers']\n    )\n    if PARAMS['backbone'] == 'resnext':\n        model = ResNext(params=PARAMS, out_dim=len(LABELS_)) \n    else:\n        model = EffNet(params=PARAMS, out_dim=len(LABELS_)) \n    model = model.to(DEVICE)\n    optimizer = optim.Adam(model.parameters(), lr=PARAMS['lr'])\n    if PARAMS['apex']:\n        model, optimizer = amp.initialize(model, optimizer, opt_level='O1')\n    if PARAMS['warmup']:\n        scheduler = torch.optim.lr_scheduler.OneCycleLR(\n            optimizer, \n            max_lr=PARAMS['lr'], \n            total_steps=PARAMS['epochs'],\n            div_factor=(PARAMS['lr'] / 1e-5), \n            final_div_factor=1000,\n            pct_start=(int(.1 * PARAMS['epochs']) / PARAMS['epochs']),\n        )\n    else:\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, PARAMS['epochs'])\n    print('train len:', len(dataset_train),'| val len:', len(dataset_val))\n    best_file = '{}/model_best_{}.pth'.format(MDLS_PATH, fold_num)\n    acc_max = 0\n    f1_max = 0\n    for epoch in tqdm(range(PARAMS['epochs']), desc='epochs'):\n        print(time.ctime(), 'epoch:', epoch)\n        train_loss = train_epoch(train_loader, optimizer)\n        val_loss, acc, f1 = val_epoch(val_loader)\n        scheduler.step(epoch)\n        content = '{} epoch {}, lr: {:.8f}, train loss: {:.4f}, val loss: {:.4f}, acc: {:.2f}, val f1: {:.4f}'.format(\n            time.ctime(),\n            epoch, \n            optimizer.param_groups[0]['lr'], \n            np.mean(train_loss),\n            np.mean(val_loss),\n            acc,\n            f1\n        )\n        print(content)\n        with open('{}/log_{}.txt'.format(MDLS_PATH, fold_num), 'a') as appender:\n            appender.write(content + '\\n')\n        if f1 > f1_max:\n            torch.save(model.state_dict(), best_file)\n            print('f1 improved {:.2f} --> {:.2f} model saved'.format(f1_max, f1))\n            f1_max = f1\n            preds_best, target_best = [], []\n            with torch.no_grad():\n                for img_data, img_lbls in tqdm(val_loader):\n                    img_data = img_data.to(DEVICE)\n                    preds = np.squeeze(model(img_data).sigmoid().cpu().numpy())\n                    preds_best.extend(preds)\n                    target_best.extend(img_lbls.cpu().numpy())\n            print('val preds done:', len(preds_best), len(target_best))\n    preds_val.extend(preds_best)\n    target_val.extend(target_best)\n    with open('log_total.txt', 'a') as appender:\n        appender.write('{} | fold: {} | max f1: {:.2f}\\n'.format(PARAMS, fold_num, f1_max))\n    torch.save(\n        model.state_dict(), \n        os.path.join('{}/model_final_{}.pth'.format(MDLS_PATH, fold_num))\n    )\n    del model, dataset_train, dataset_val, train_loader, val_loader\n    torch.cuda.empty_cache()\n    gc.collect()\npreds_val = np.array(preds_val)\ntarget_val = np.array(target_val)\n\nelapsed_time = time.time() - start_time\nprint(f'time elapsed: {elapsed_time // 60:.0f} min {elapsed_time % 60:.0f} sec')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"th_dict = {}\nfor i, lbl in LABELS.items():\n    f1_max = 0\n    for th in np.linspace(.1, 1, 100):\n        f1 = f1_score(preds_val[:, i] > th, target_val[:, i])\n        if f1 > f1_max:\n            f1_max = f1\n            th_max = th\n    th_dict[i] = th_max\n    print(lbl, '| f1 max:', np.round(f1_max, 2), '| th max:', np.round(th_max, 2))\nwith open(f'{MDLS_PATH}/ths.json', 'w') as file:\n    json.dump(th_dict, file)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}