{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore')\n\nimport gc\nimport os\nimport random\n\nimport pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\n\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom torchvision import models\nfrom torchvision import transforms as T\nfrom torchvision.io import read_image, ImageReadMode\nfrom torchvision.utils import make_grid\n\nimport cv2\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nfrom torchinfo import summary\n\nimport timm\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\n\nfrom tqdm.auto import tqdm, trange\n\nfrom pprint import pprint\n\nfrom kaggle_secrets import UserSecretsClient\n\nSEED = 2023\npd.set_option('display.max_colwidth', None)\n\n%config InlineBackend.figure_format = 'retina'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed(seed: int):\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    \nseed(SEED)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_FILE_PATH = '/kaggle/input/spr-x-ray-gender/sample_submission_gender.csv'\nTEST_IMAGES_PATH = '/kaggle/input/spr-x-ray-gender-224v2/test'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_file_path(image_id, image_folder_path):\n    return os.path.join(image_folder_path, f'{str(image_id).zfill(6)}.png')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(TEST_FILE_PATH)\ndf_test['file_path'] = df_test['imageId'].apply(get_file_path, image_folder_path=TEST_IMAGES_PATH)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_test.shape)\ndisplay(df_test.head(3))\ndisplay(df_test.tail(3))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DatasetGender(Dataset):\n    \n    def __init__(self, data_frame: pd.DataFrame, transforms: list = None):\n        self.data_frame = data_frame\n        self.img_path = data_frame['file_path'].values\n        self.labels = data_frame['gender'].values\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.data_frame)\n    \n    def __getitem__(self, index):\n        \n        try:            \n            img_path = self.img_path[index]\n\n            image = read_image(img_path, mode=ImageReadMode.RGB)\n            label = self.labels[index]\n            \n            if self.transforms is not None:\n                image = self.transforms(image)\n                \n            return image, label\n        \n        except Exception as e:\n            print(e, index, self.img_path[index], sep='\\n')\n            ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = dict(\n    num_ops = 2,\n    magnitude = 10,\n    batch_size = 128,\n    learning_rate = 1e-3,\n    epochs = 120,\n    T_0 = 30,  ## 40,\n    eta_min = 1e-6,\n    eps = 1e-2,  # 1e-05,\n    dropout = 0.0,    \n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transforms = {\n    'test': T.Compose([\n        T.ToPILImage(),\n        T.CenterCrop(224),\n        T.ToTensor(),\n        T.Normalize(mean=(0.5, 0.5, 0.5), \n                    std=(0.5, 0.5, 0.5)),\n    ])\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data  = DatasetGender(df_test,  transforms=transforms['test'])\ntest_loader = DataLoader(test_data, \n                         config['batch_size'], \n                         shuffle=False, \n                         num_workers=2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\ndevice","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    \n    def __init__(self, model, num_classes: int, feature_extracting: bool = True):\n        super().__init__()\n        self.model = model\n        \n        if feature_extracting:\n            for name, param in self.model.named_parameters():\n                \n                unfreeze_conditions = (('blocks.9'  in name) or\n                                       ('blocks.10' in name) or\n                                       ('blocks.11' in name) or \n                                       ('norm.bias' in name) or \n                                       ('norm.weight' in name))\n                if not unfreeze_conditions:\n                    param.requires_grad = False\n        \n        self.model.head = nn.Sequential(\n            nn.Linear(768, 512),\n            nn.BatchNorm1d(512, eps=config['eps']),\n            nn.ReLU(),\n            nn.Dropout1d(p=config['dropout']),\n            nn.Linear(512, num_classes)\n        )\n    \n#     @torch.autocast(device_type=device)\n    def forward(self, x):\n        \n        logits = self.model(x)\n        \n        return logits","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"timm_model = timm.create_model('vit_base_patch16_224', pretrained=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Model(timm_model, num_classes=1)\nmodel= nn.DataParallel(model)\nmodel.to(device);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_PATH = '/kaggle/input/spr-x-ray-gender-cross-validation-training'\n\nfinal_preds = []\nfor ii in range(1, 6):\n    \n    checkpoint = torch.load(os.path.join(MODEL_PATH, f'model{ii}.pt'))\n    model.load_state_dict(checkpoint) # load model\n\n    # Evaluation\n    model.eval()\n\n    preds = []\n    with torch.no_grad():\n        for images, _ in tqdm(test_loader):\n            images = images.to(device)\n\n            output = model(images)\n            y_pred = torch.sigmoid(output)\n            y_pred = y_pred.squeeze()\n\n            preds.extend(y_pred.detach().cpu().numpy())\n            \n            # Garbage Collection\n            torch.cuda.empty_cache()\n            _ = gc.collect()\n            \n    final_preds.append(preds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.column_stack(final_preds).mean(axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = df_test[['imageId']].copy()\nsubmission['gender'] = np.column_stack(final_preds).mean(axis=1)\nsubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(submission['gender'].min(), submission['gender'].max(), end='\\n\\n')\ndisplay(submission)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}