{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore')\n\nimport gc\nimport os\nimport random\n\nimport pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\n\nfrom torch import Tensor\n\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom torchvision import models\nfrom torchvision import transforms as T\nfrom torchvision.io import read_image, ImageReadMode\nfrom torchvision.utils import make_grid\n\nimport cv2\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nfrom torchinfo import summary\n\nimport timm\n\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\n\nfrom tqdm.auto import tqdm, trange\n\nfrom pprint import pprint\n\nfrom kaggle_secrets import UserSecretsClient\n\nimport wandb\n\nSEED = 2023\npd.set_option('display.max_colwidth', None)\n\n%config InlineBackend.figure_format = 'retina'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed(seed: int):\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    \nseed(SEED)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_secrets = UserSecretsClient()\nkey = user_secrets.get_secret(\"wandb_key\")\n\n!wandb login $key","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = dict(\n    num_ops = 2,\n    magnitude = 10,\n    batch_size = 256, # 512,\n    learning_rate = 1e-3,\n    epochs = 120,\n    T_0 = 30,  ## 40,\n    eta_min = 1e-6,\n    eps = 1e-2,  # 1e-05,\n    dropout = 0.0,    \n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DatasetGender(Dataset):\n    \n    def __init__(self, data_frame: pd.DataFrame, transforms: list = None):\n        self.data_frame = data_frame\n        self.img_path = data_frame['file_path'].values\n        self.labels = data_frame['gender'].values\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.data_frame)\n    \n    def __getitem__(self, index):\n        \n        try:            \n            img_path = self.img_path[index]\n\n            image = read_image(img_path, mode=ImageReadMode.RGB)\n            label = self.labels[index]\n            \n            if self.transforms is not None:\n                image = self.transforms(image)\n                \n            return image, label\n        \n        except Exception as e:\n            print(e, index, self.img_path[index], sep='\\n')\n            ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run = wandb.init(project='spr_x-ray_gender', reinit=True, config=config, save_code=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\ndevice","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    \n    def __init__(self, model, num_classes: int, feature_extracting: bool = True):\n        super().__init__()\n        self.model = model\n        \n        if feature_extracting:\n            for name, param in self.model.named_parameters():\n                \n                unfreeze_conditions = (('blocks.9'  in name) or\n                                       ('blocks.10' in name) or\n                                       ('blocks.11' in name) or \n                                       ('norm.bias' in name) or \n                                       ('norm.weight' in name))\n                if not unfreeze_conditions:\n                    param.requires_grad = False\n        \n        self.model.head = nn.Sequential(\n            nn.Linear(768, 512),\n            nn.BatchNorm1d(512, eps=config['eps']),\n            nn.ReLU(),\n            nn.Dropout1d(p=config['dropout']),\n            nn.Linear(512, num_classes)\n        )\n    \n    @torch.autocast(device_type=device)\n    def forward(self, x):\n        \n        logits = self.model(x)\n        \n        return logits","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_fn(model, loader, optimizer, criterion, epoch, scheduler, scaler):\n    \n    model.train()\n    \n    running_loss = 0\n    y_true = []\n    y_pred = []\n    \n    iters = len(loader)\n    for i, (images, labels) in tqdm(enumerate(loader), total = len(loader)):\n        \n        images = images.to(device)\n        labels = labels.to(device, dtype=torch.float)\n        \n        optimizer.zero_grad()        \n        \n        # Forward pass\n        output = model(images)\n        output = output.squeeze()\n        loss = criterion(output, labels)\n        \n        scaler.scale(loss).backward()  # scale gradients\n        scaler.step(optimizer)         # update optimizer\n        scaler.update()\n        \n        scheduler.step(epoch + i / iters)  # update scheduler\n        \n        # Compute loss\n        running_loss += loss.item()        \n        y_true.extend(labels.detach().cpu().numpy())\n        y_pred.extend(output.detach().cpu().numpy())\n        \n        # Garbage Collection\n        torch.cuda.empty_cache()\n        _ = gc.collect()\n        \n    final_loss = running_loss / len(loader)\n    aucroc = roc_auc_score(y_true, y_pred)\n    \n    return final_loss, aucroc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@torch.no_grad()\ndef valid_fn(model, loader, criterion):\n    \n    model.eval()\n    \n    running_loss = 0\n    \n    y_true = []\n    y_pred = []\n    \n    for images, labels in tqdm(loader):\n        \n        images = images.to(device)\n        labels = labels.to(device, dtype=torch.float)\n        \n        output = model(images)\n        output = output.squeeze()\n        loss = criterion(output, labels)\n        running_loss += loss.item()\n        \n        y_true.extend(labels.detach().cpu().numpy())\n        y_pred.extend(output.detach().cpu().numpy())\n        \n        # Garbage Collection\n        torch.cuda.empty_cache()\n        _ = gc.collect()\n        \n    final_loss = running_loss / len(loader)\n    aucroc = roc_auc_score(y_true, y_pred)\n    \n    return final_loss, aucroc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_eval_fn(train_loader: Tensor, valid_loader: Tensor, fold: int) -> None:\n    \n    timm_model = timm.create_model('vit_base_patch16_224', pretrained=True)\n\n    model = Model(timm_model, num_classes=1)\n    model= nn.DataParallel(model)\n    model.to(device)\n\n    criterion = nn.BCEWithLogitsLoss().to(device)\n    optimizer = optim.Adam(model.parameters(), lr=config['learning_rate'])\n\n    scheduler = optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, \n                                                           T_0=config['T_0'], \n                                                           eta_min=config['eta_min'])\n    scaler = torch.cuda.amp.GradScaler()\n    \n    wandb.watch(model, criterion, log='all', log_freq=10)  # WANDB WATCH\n\n    early_stopping = 10\n    early_stoppíng_count = 0\n    valid_loss_min = np.Inf\n    \n    for epoch in range(config['epochs']):\n\n        train_loss, train_auc = train_fn(model, train_loader, optimizer, criterion, epoch, scheduler, scaler)\n        valid_loss, valid_auc = valid_fn(model, valid_loader, criterion)\n\n        print(f'epoch: {epoch + 1:03d} | train loss: {train_loss:.5f} | train auc: {train_auc:.2%}')\n        print(f'epoch: {epoch + 1:03d} | valid loss: {valid_loss:.5f} | valid auc: {valid_auc:.2%}')\n\n        if valid_loss <= valid_loss_min:\n            print(f'Validation loss decresead: {valid_loss_min:.5f} --> {valid_loss:.5f}. Saving model...')\n            valid_loss_min = valid_loss\n            early_stoppíng_count = 0\n            best_model  = torch.save(model.state_dict(), f'model{fold}.pt')            \n            best_auc = valid_auc\n\n        # early stopping    \n        else:\n            early_stoppíng_count += 1\n            if early_stoppíng_count > early_stopping:\n                print(f'Early stopping after {epoch + 1:03d} epochs.')\n                break\n\n        # Wandb tracking metrics\n        wandb.log({'train loss': train_loss, 'train auc': train_auc,\n                   'valid loss': valid_loss, 'valid auc': valid_auc,\n                   'learning rate': scheduler.optimizer.param_groups[0]['lr'],\n                   'best valid loss': valid_loss_min}, step=epoch)\n    \n    return best_auc\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loading data","metadata":{}},{"cell_type":"code","source":"TRAIN_FILE_PATH = '/kaggle/input/spr-x-ray-gender/train_gender.csv'\nTRAIN_IMAGES_PATH = '/kaggle/input/spr-x-ray-gender-224v2/train'\n\nTEST_FILE_PATH = '/kaggle/input/spr-x-ray-gender/sample_submission_gender.csv'\nTEST_IMAGES_PATH = '/kaggle/input/spr-x-ray-gender-224v2/test'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_file_path(image_id, image_folder_path):\n    return os.path.join(image_folder_path, f'{str(image_id).zfill(6)}.png')\n\ndf = pd.read_csv(TRAIN_FILE_PATH)\ndf['file_path'] = df['imageId'].apply(get_file_path, image_folder_path=TRAIN_IMAGES_PATH)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(TEST_FILE_PATH)\ndf_test['file_path'] = df_test['imageId'].apply(get_file_path, image_folder_path=TEST_IMAGES_PATH)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Cross-Validation","metadata":{}},{"cell_type":"code","source":"transforms = {\n    'train': T.Compose([\n        T.ToPILImage(),\n        T.CenterCrop(224),\n        T.RandAugment(num_ops=config['num_ops'], \n                      magnitude=config['magnitude']),\n        T.ToTensor(),\n        T.Normalize(mean=(0.5, 0.5, 0.5), \n                    std=(0.5, 0.5, 0.5))\n    ]),\n    'valid': T.Compose([\n        T.ToPILImage(),\n        T.CenterCrop(224),\n        T.ToTensor(),\n        T.Normalize(mean=(0.5, 0.5, 0.5), \n                    std=(0.5, 0.5, 0.5)),\n    ])\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n\n\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)\n\nbest_auc_list = []\nfor fold, (train_indices, valid_indices) in enumerate(skf.split(X=df, y=df['gender']), start=1):\n    df_train = df.iloc[train_indices]\n    df_valid = df.iloc[valid_indices]\n    \n    train_data = DatasetGender(df_train, transforms=transforms['train'])\n    valid_data = DatasetGender(df_valid, transforms=transforms['valid'])\n    \n    train_loader = DataLoader(train_data, config['batch_size'], shuffle=True,  num_workers=2, pin_memory=True)\n    valid_loader = DataLoader(valid_data, config['batch_size'], shuffle=False, num_workers=2, pin_memory=True)\n    \n    print(f'\\nFOLD {fold}', sep='\\n')\n    \n    best_auc = train_eval_fn(train_loader, valid_loader, fold)\n    \n    best_auc_list.append(best_auc)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'{np.mean(best_auc_list):.5f} +/- {np.std(best_auc_list):.5f}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.eval()\n\n# final_preds = []\n# with torch.no_grad():\n#     for images, _ in tqdm(test_loader):\n#         images = images.to(device)\n        \n#         output = model(images)\n#         y_pred = torch.sigmoid(output)\n#         y_pred = y_pred.squeeze()\n        \n#         final_preds.extend(y_pred.detach().cpu().numpy())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission = df_test[['imageId']].copy()\n# submission['gender'] = final_preds\n# submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(submission['gender'].min(), submission['gender'].max(), end='\\n\\n')\n# display(submission)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Finish tracking\nrun.finish()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}