{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"!pip install python-box timm pytorch-lightning==1.4.0 grad-cam ttach","metadata":{"execution":{"iopub.status.busy":"2021-09-29T17:07:04.708077Z","iopub.execute_input":"2021-09-29T17:07:04.708972Z","iopub.status.idle":"2021-09-29T17:07:31.945245Z","shell.execute_reply.started":"2021-09-29T17:07:04.708831Z","shell.execute_reply":"2021-09-29T17:07:31.944106Z"},"_kg_hide-input":false}},{"cell_type":"code","source":"\"\"\"\n%%python\n    \nimport os\nimport sys\n\n\nimport warnings\nfrom pprint import pprint\nfrom glob import glob\nfrom tqdm import tqdm\nimport pickle\n\nfrom sklearn.linear_model import Ridge\nimport torch\nimport torch.optim as optim\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torchvision.transforms as T\nsys.path.append('../input/pythonbox')\nfrom box import Box\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nfrom timm import create_model\nfrom sklearn.model_selection import StratifiedKFold\nfrom torchvision.io import read_image\nfrom torch.utils.data import DataLoader, Dataset\n#from pytorch_grad_cam import GradCAMPlusPlus\n#from pytorch_grad_cam.utils.image import show_cam_on_image\nfrom PIL import Image\nfrom fastai.vision.all import *\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning.utilities.seed import seed_everything\nfrom pytorch_lightning import callbacks\nfrom pytorch_lightning.callbacks.progress import ProgressBarBase\nfrom pytorch_lightning.callbacks.early_stopping import EarlyStopping\nfrom pytorch_lightning.loggers import TensorBoardLogger\nfrom pytorch_lightning import LightningDataModule, LightningModule\n\nsys.path.append('../input/poolformer-master')\nimport models as PoolFormerModels #poolformer\n\nprint(pl.__version__)\nwarnings.filterwarnings(\"ignore\")\n\n\n\nconfig_ensemble = {'exp_name':'exp073',\n            'seed': 2021,\n              'n_splits': 5,\n              'oof_expname':{\n                  1:'exp049',\n                  2:'exp054',\n                  3:'exp060',\n                  4:'exp065',\n                  5:'exp069',\n                  6:'exp074',\n              },\n              'model':'Ridge'\n}\n\nconfig_ensemble = Box(config_ensemble)\n\n#049\nconfig_exp049 = {'exp_name':'exp049',\n            'seed': 2021,\n          'root': '/kaggle/input/petfinder-pawpularity-score/', \n          'n_splits': 5,\n          'epoch': 20,\n          'transform':{\n              'name': 'get_default_transforms',\n              'image_size': 384\n          },\n          'test_loader': {\n              'batch_size': 32,\n              'shuffle': False,\n              'num_workers': os.cpu_count(),\n              'pin_memory': False,\n              'drop_last': False\n          },\n          'model':{\n              'name': 'swin_large_patch4_window12_384',\n              'output_dim': 1\n          },\n          'loss': 'nn.BCEWithLogitsLoss',\n}\n\nconfig_exp049 = Box(config_exp049)\n\n#054\nconfig_exp054 = {'exp_name':'exp054',\n          'seed': 2021,\n          'root': '/kaggle/input/petfinder-pawpularity-score/', \n          'n_splits': 5,\n          'transform':{\n              'name': 'get_default_transforms',\n              'image_size': 224\n          },\n          'test_loader': {\n              'batch_size': 32,\n              'shuffle': False,\n              'num_workers': os.cpu_count(),\n              'pin_memory': False,\n              'drop_last': False\n          },\n          'model':{\n              'name': 'swin_large_patch4_window7_224',\n              'output_dim': 1\n          },\n          'optimizer':{\n              'name': 'optim.AdamW',\n              'params':{\n                  'lr': 1e-5\n              },\n          },\n          'scheduler':{\n              'name': 'optim.lr_scheduler.CosineAnnealingWarmRestarts',\n              'params':{\n                  'T_0': 20,\n                  'eta_min': 1e-4,\n              }\n          },\n          'loss': 'nn.BCEWithLogitsLoss',\n}\n\nconfig_exp054 = Box(config_exp054)\n\n#060\nconfig_exp060 = {'exp_name':'exp060',\n          'seed': 2021,\n          'root': '/kaggle/input/petfinder-pawpularity-score/', \n          'n_splits': 5,\n          'transform':{\n              'name': 'get_default_transforms',\n              'image_size': 224\n          },\n          'test_loader': {\n              'batch_size': 32,\n              'shuffle': False,\n              'num_workers': os.cpu_count(),\n              'pin_memory': False,\n              'drop_last': False\n          },\n          'model':{\n              'name': 'swin_large_patch4_window7_224_in22k',\n              'output_dim': 1\n          },\n          'optimizer':{\n              'name': 'optim.AdamW',\n              'params':{\n                  'lr': 1e-5\n              },\n          },\n          'scheduler':{\n              'name': 'optim.lr_scheduler.CosineAnnealingWarmRestarts',\n              'params':{\n                  'T_0': 20,\n                  'eta_min': 1e-4,\n              }\n          },\n          'loss': 'nn.BCEWithLogitsLoss',\n}\n\nconfig_exp060 = Box(config_exp060)\n\n#065\nconfig_exp065 = {'exp_name':'exp065',\n          'seed': 2021,\n          'root': '/kaggle/input/petfinder-pawpularity-score/', \n          'n_splits': 5,\n          'transform':{\n              'name': 'get_default_transforms',\n              'image_size': 384\n          },\n          'test_loader': {\n              'batch_size': 32,\n              'shuffle': False,\n              'num_workers': os.cpu_count(),\n              'pin_memory': False,\n              'drop_last': False\n          },\n          'model':{\n              'name': 'swin_large_patch4_window12_384',\n              'output_dim': 1\n          },\n          'optimizer':{\n              'name': 'optim.AdamW',\n              'params':{\n                  'lr': 1e-5\n              },\n          },\n          'scheduler':{\n              'name': 'optim.lr_scheduler.CosineAnnealingWarmRestarts',\n              'params':{\n                  'T_0': 20,\n                  'eta_min': 1e-4,\n              }\n          },\n          'loss': 'nn.BCEWithLogitsLoss',\n}\n\nconfig_exp065 = Box(config_exp065)\n\n#069\nconfig_exp069 = {'exp_name':'exp069',\n          'seed': 2021,\n          'root': '/kaggle/input/petfinder-pawpularity-score/', \n          'n_splits': 5,\n          'transform':{\n              'name': 'get_default_transforms',\n              'image_size': 224\n          },\n          'test_loader': {\n              'batch_size': 32,\n              'shuffle': False,\n              'num_workers': os.cpu_count(),\n              'pin_memory': False,\n              'drop_last': False\n          },\n          'model':{\n              'name': 'swin_large_patch4_window7_224',\n              'output_dim': 1\n          },\n          'optimizer':{\n              'name': 'optim.AdamW',\n              'params':{\n                  'lr': 1e-5\n              },\n          },\n          'scheduler':{\n              'name': 'optim.lr_scheduler.CosineAnnealingWarmRestarts',\n              'params':{\n                  'T_0': 20,\n                  'eta_min': 1e-4,\n              }\n          },\n          'loss': 'nn.BCEWithLogitsLoss',\n}\n\nconfig_exp069 = Box(config_exp069)\n\n#074\nconfig_exp074 = {'exp_name':'exp074',\n          'seed': 2021,\n          'root': '/kaggle/input/petfinder-pawpularity-score/', \n          'n_splits': 5,\n          'transform':{\n              'name': 'get_default_transforms',\n              'image_size': 224\n          },\n          'test_loader': {\n              'batch_size': 32,\n              'shuffle': False,\n              'num_workers': os.cpu_count(),\n              'pin_memory': False,\n              'drop_last': False\n          },\n          'model':{\n              'name': 'tf_efficientnetv2_b1',\n              'output_dim': 1\n          },\n          'optimizer':{\n              'name': 'optim.AdamW',\n              'params':{\n                  'lr': 1e-5\n              },\n          },\n          'scheduler':{\n              'name': 'optim.lr_scheduler.CosineAnnealingWarmRestarts',\n              'params':{\n                  'T_0': 20,\n                  'eta_min': 1e-4,\n              }\n          },\n          'loss': 'nn.BCEWithLogitsLoss',\n}\n\nconfig_exp074 = Box(config_exp074)\n\nclass PetfinderDataset_exp049(Dataset):\n    def __init__(self, df, image_size=224):\n        self._X = df[\"Id\"].values\n        self._y = None\n        if \"Pawpularity\" in df.keys():\n            self._y = df[\"Pawpularity\"].values\n        self._transform = T.Compose([\n                                        T.Resize(image_size),  # 1\n                                        T.CenterCrop([image_size, image_size]),  # 2\n                                    ]\n                                    )\n\n    def __len__(self):\n        return len(self._X)\n\n    def __getitem__(self, idx):\n        image_path = self._X[idx]\n        image = read_image(image_path)\n        image = self._transform(image)\n        if self._y is not None:\n            label = self._y[idx]\n            return image, label\n        return image\n\nclass PetfinderDataset_exp054(Dataset):\n    def __init__(self, df, image_size=224):\n        dense_feature_cols = [\n            'Subject Focus', 'Eyes', 'Face', 'Near', 'Action', 'Accessory',\n            'Group', 'Collage', 'Human', 'Occlusion', 'Info', 'Blur',\n            ]\n        self._X = df[\"Id\"].values\n        self.dense_features = df[dense_feature_cols].values\n        self._y = None\n        if \"Pawpularity\" in df.keys():\n            self._y = df[\"Pawpularity\"].values\n        self._transform = T.Compose([\n                                        T.Resize(image_size),  # 1\n                                        T.CenterCrop([image_size, image_size]),  # 2\n                                    ]\n                                    )\n\n    def __len__(self):\n        return len(self._X)\n\n    def __getitem__(self, idx):\n        image_path = self._X[idx]\n        image = read_image(image_path)\n        image = self._transform(image)\n\n        features = self.dense_features[idx, :]\n        if self._y is not None:\n            label = self._y[idx]\n            return image, features, label\n        return image, features\n\nclass PetfinderDataset_exp060(Dataset):\n    def __init__(self, df, image_size=224):\n        dense_feature_cols = [\n            'Subject Focus', 'Eyes', 'Face', 'Near', 'Action', 'Accessory',\n            'Group', 'Collage', 'Human', 'Occlusion', 'Info', 'Blur',\n            'height', 'width', 'aspect'\n            ]\n        self._X = df[\"Id\"].values\n        self.dense_features = df[dense_feature_cols].values\n        self._y = None\n        if \"Pawpularity\" in df.keys():\n            self._y = df[\"Pawpularity\"].values\n        self._transform = T.Compose([\n                                        T.Resize(image_size),  # 1\n                                        T.CenterCrop([image_size, image_size]),  # 2\n                                    ]\n                                    )\n\n    def __len__(self):\n        return len(self._X)\n\n    def __getitem__(self, idx):\n        image_path = self._X[idx]\n        image = read_image(image_path)\n        image = self._transform(image)\n\n        features = self.dense_features[idx, :]\n        if self._y is not None:\n            label = self._y[idx]\n            return image, features, label\n        return image, features    \n\nclass PetfinderDataset_exp065(Dataset):\n    def __init__(self, df, image_size=224):\n        self._X = df[\"Id\"].values\n        self._y = None\n        if \"Pawpularity\" in df.keys():\n            self._y = df[\"Pawpularity\"].values\n        self._transform = T.Compose([\n                                        T.Resize(image_size),  # 1\n                                        T.CenterCrop([image_size, image_size]),  # 2\n                                    ]\n                                    )\n\n    def __len__(self):\n        return len(self._X)\n\n    def __getitem__(self, idx):\n        image_path = self._X[idx]\n        image = read_image(image_path)\n        image = self._transform(image)\n        if self._y is not None:\n            label = self._y[idx]\n            return image, label\n        return image\n\nclass PetfinderDataset_exp069(Dataset):\n    def __init__(self, df, image_size=224):\n        dense_feature_cols = [\n            'height', 'width', 'aspect'\n            ]\n        self._X = df[\"Id\"].values\n        self.dense_features = df[dense_feature_cols].values\n        self._y = None\n        if \"Pawpularity\" in df.keys():\n            self._y = df[\"Pawpularity\"].values\n        self._transform = T.Compose([\n                                        T.Resize(image_size),  # 1\n                                        T.CenterCrop([image_size, image_size]),  # 2\n                                    ]\n                                    )\n\n    def __len__(self):\n        return len(self._X)\n\n    def __getitem__(self, idx):\n        image_path = self._X[idx]\n        image = read_image(image_path)\n        image = self._transform(image)\n\n        features = self.dense_features[idx, :]\n        if self._y is not None:\n            label = self._y[idx]\n            return image, features, label\n        return image, features\n\nclass PetfinderDataset_exp074(Dataset):\n    def __init__(self, df, image_size=224):\n        dense_feature_cols = [\n            'Subject Focus', 'Eyes', 'Face', 'Near', 'Action', 'Accessory',\n            'Group', 'Collage', 'Human', 'Occlusion', 'Info', 'Blur',\n            'height', 'width', 'aspect'\n            ]\n        self._X = df[\"Id\"].values\n        self.dense_features = df[dense_feature_cols].values\n        self._y = None\n        if \"Pawpularity\" in df.keys():\n            self._y = df[\"Pawpularity\"].values\n        self._transform = T.Compose([\n                                        T.Resize(image_size),  # 1\n                                        T.CenterCrop([image_size, image_size]),  # 2\n                                    ]\n                                    )\n\n    def __len__(self):\n        return len(self._X)\n\n    def __getitem__(self, idx):\n        image_path = self._X[idx]\n        image = read_image(image_path)\n        image = self._transform(image)\n\n        features = self.dense_features[idx, :]\n        if self._y is not None:\n            label = self._y[idx]\n            return image, features, label\n        return image, features\n    \nclass PetfinderDataModule(LightningDataModule):\n    def __init__(\n        self,\n        test_df,\n        cfg,\n    ):\n        super().__init__()\n        self._test_df = test_df\n        self._cfg = cfg\n\n    def __create_dataset(self, train=True):\n        \n        if self._cfg.exp_name=='exp049':\n            return (\n                PetfinderDataset_exp049(self._test_df, self._cfg.transform.image_size)\n                if train\n                else PetfinderDataset_exp049(self._test_df, self._cfg.transform.image_size)\n            )\n        if self._cfg.exp_name=='exp054':\n            return (\n                PetfinderDataset_exp054(self._test_df, self._cfg.transform.image_size)\n                if train\n                else PetfinderDataset_exp054(self._test_df, self._cfg.transform.image_size)\n            )\n        if self._cfg.exp_name=='exp060':\n            return (\n                PetfinderDataset_exp060(self._test_df, self._cfg.transform.image_size)\n                if train\n                else PetfinderDataset_exp060(self._test_df, self._cfg.transform.image_size)\n            )\n        if self._cfg.exp_name=='exp065':\n            return (\n                PetfinderDataset_exp065(self._test_df, self._cfg.transform.image_size)\n                if train\n                else PetfinderDataset_exp065(self._test_df, self._cfg.transform.image_size)\n            )\n        if self._cfg.exp_name=='exp069':\n            return (\n                PetfinderDataset_exp069(self._test_df, self._cfg.transform.image_size)\n                if train\n                else PetfinderDataset_exp069(self._test_df, self._cfg.transform.image_size)\n            )\n        if self._cfg.exp_name=='exp074':\n            return (\n                PetfinderDataset_exp074(self._test_df, self._cfg.transform.image_size)\n                if train\n                else PetfinderDataset_exp074(self._test_df, self._cfg.transform.image_size)\n            )\n\n    def test_dataloader(self):\n        dataset = self.__create_dataset(True)\n        return DataLoader(dataset, **self._cfg.test_loader)\n    \nIMAGENET_MEAN = [0.485, 0.456, 0.406]  # RGB\nIMAGENET_STD = [0.229, 0.224, 0.225]  # RGB\n\ndef get_default_transforms():\n    transform = {\n        \"train\": T.Compose(\n            [\n                T.RandomHorizontalFlip(),\n                T.RandomVerticalFlip(),\n                T.RandomAffine(15, translate=(0.1, 0.1), scale=(0.9, 1.1)),\n                T.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1),\n                T.ConvertImageDtype(torch.float),\n                T.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD),\n            ]\n        ),\n        \"val\": T.Compose(\n            [\n                T.ConvertImageDtype(torch.float),\n                T.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD),\n            ]\n        ),\n    }\n    return transform\n\ndef mixup(x: torch.Tensor, y: torch.Tensor, alpha: float = 1.0):\n    assert alpha > 0, \"alpha should be larger than 0\"\n    assert x.size(0) > 1, \"Mixup cannot be applied to a single instance.\"\n\n    lam = np.random.beta(alpha, alpha)\n    rand_index = torch.randperm(x.size()[0])\n    mixed_x = lam * x + (1 - lam) * x[rand_index, :]\n    target_a, target_b = y, y[rand_index]\n    return mixed_x, target_a, target_b, lam\n    \nclass Model_exp049(pl.LightningModule):\n    def __init__(self, cfg, val_losses=None):\n        super().__init__()\n        self.cfg = cfg\n        self.__build_model()\n        self._criterion = eval(self.cfg.loss)()\n        self.transform = get_default_transforms()\n        self.save_hyperparameters(cfg)\n        self.val_losses = val_losses \n\n    def __build_model(self):\n        self.backbone = create_model(\n            self.cfg.model.name, pretrained=False, num_classes=0, in_chans=3\n        )\n        num_features = self.backbone.num_features\n        self.fc = nn.Sequential(\n            nn.Dropout(0.5), nn.Linear(num_features, self.cfg.model.output_dim)\n        )\n\n    def forward(self, x):\n        f = self.backbone(x)\n        out = self.fc(f)\n        return out\n\n    def training_step(self, batch, batch_idx):\n        loss, pred, labels = self.__share_step(batch, 'train')\n        self.log(\"train/loss\", loss)\n        return {'loss': loss, 'pred': pred, 'labels': labels}\n        \n    def validation_step(self, batch, batch_idx):\n        loss, pred, labels = self.__share_step(batch, 'val')\n        self.log(\"val/loss\", loss)\n        return {'pred': pred, 'labels': labels}\n    \n    def __share_step(self, batch, mode):\n        images, labels = batch\n        labels = labels.float() / 100.0\n        images = self.transform[mode](images)\n        \n        if torch.rand(1)[0] < 0.5 and mode == 'train':\n            mix_images, target_a, target_b, lam = mixup(images, labels, alpha=0.5)\n            logits = self.forward(mix_images).squeeze(1)\n            loss = self._criterion(logits, target_a) * lam +                 (1 - lam) * self._criterion(logits, target_b)\n        else:\n            logits = self.forward(images).squeeze(1)\n            loss = self._criterion(logits, labels)\n        \n        pred = logits.sigmoid().detach().cpu() * 100.\n        labels = labels.detach().cpu() * 100.\n        \n        return loss, pred, labels\n\nclass Model_exp054(pl.LightningModule):\n    def __init__(self, cfg, val_losses=None):\n        super().__init__()\n        self.cfg = cfg\n        self.__build_model()\n        self._criterion = eval(self.cfg.loss)()\n        self.transform = get_default_transforms()\n        self.save_hyperparameters(cfg)\n        self.val_losses = val_losses \n\n    def __build_model(self):\n        self.backbone = self.backbone = PoolFormerModels.poolformer_m36(pretrained=False)\n        self.backbone.head = nn.Linear(self.backbone.head.in_features, 128)\n        self.dropout = nn.Dropout(0.1)\n        self.dense1 = nn.Linear(128+12, 64)\n        self.dense2 = nn.Linear(64, 1)\n\n    def forward(self, image, features):\n        x = self.get_feature(image)\n        x = self.dropout(x)\n        x = torch.cat([x, features], dim=1)\n        x = self.dense1(x)\n        out = self.dense2(x)\n        return out\n\n    def get_feature(self, image):\n        x = self.backbone(image)\n        return x\n\n    def training_step(self, batch, batch_idx):\n        loss, pred, labels = self.__share_step(batch, 'train')\n        self.log(\"train/loss\", loss)\n        return {'loss': loss, 'pred': pred, 'labels': labels}\n        \n    def validation_step(self, batch, batch_idx):\n        loss, pred, labels = self.__share_step(batch, 'val')\n        self.log(\"val/loss\", loss)\n        return {'pred': pred, 'labels': labels}\n    \n    def __share_step(self, batch, mode):\n        images, feature, labels = batch\n        labels = labels.float() / 100.0\n        images = self.transform[mode](images)\n        \n        if torch.rand(1)[0] < 0.5 and mode == 'train':\n            mix_images, target_a, target_b, lam = mixup(images, labels, alpha=self.cfg.mixup.alpha)\n            logits = self.forward(mix_images, feature).squeeze(1)\n            loss = self._criterion(logits, target_a) * lam + (1 - lam) * self._criterion(logits, target_b)\n        else:\n            logits = self.forward(images, feature).squeeze(1)\n            loss = self._criterion(logits, labels)\n        \n        pred = logits.sigmoid().detach().cpu() * 100.\n        labels = labels.detach().cpu() * 100.\n        \n        return loss, pred, labels\n\nclass Model_exp060(pl.LightningModule):\n    def __init__(self, cfg, val_losses=None):\n        super().__init__()\n        self.cfg = cfg\n        self.__build_model()\n        self._criterion = eval(self.cfg.loss)()\n        self.transform = get_default_transforms()\n        self.save_hyperparameters(cfg)\n        self.val_losses = val_losses \n\n    def __build_model(self):\n        self.backbone = create_model(\n            self.cfg.model.name, pretrained=False, num_classes=128, in_chans=3\n        )\n        self.dropout = nn.Dropout(0.1)\n        self.dense1 = nn.Linear(128+15, 64)\n        self.dense2 = nn.Linear(64, 1)\n\n    def forward(self, image, features):\n        x = self.get_feature(image)\n        x = self.dropout(x)\n        x = torch.cat([x, features], dim=1)\n        x = self.dense1(x)\n        out = self.dense2(x)\n        return out\n\n    def get_feature(self, image):\n        x = self.backbone(image)\n        return x\n\n    def training_step(self, batch, batch_idx):\n        loss, pred, labels = self.__share_step(batch, 'train')\n        self.log(\"train/loss\", loss)\n        return {'loss': loss, 'pred': pred, 'labels': labels}\n        \n    def validation_step(self, batch, batch_idx):\n        loss, pred, labels = self.__share_step(batch, 'val')\n        self.log(\"val/loss\", loss)\n        return {'pred': pred, 'labels': labels}\n    \n    def __share_step(self, batch, mode):\n        images, feature, labels = batch\n        feature = feature.float()\n        labels = labels.float() / 100.0\n        images = self.transform[mode](images)\n        \n        if torch.rand(1)[0] < 0.5 and mode == 'train':\n            mix_images, target_a, target_b, lam = mixup(images, labels, alpha=self.cfg.mixup.alpha)\n            logits = self.forward(mix_images, feature).squeeze(1)\n            loss = self._criterion(logits, target_a) * lam + (1 - lam) * self._criterion(logits, target_b)\n        else:\n            logits = self.forward(images, feature).squeeze(1)\n            loss = self._criterion(logits, labels)\n        \n        pred = logits.sigmoid().detach().cpu() * 100.\n        labels = labels.detach().cpu() * 100.\n        \n        return loss, pred, labels\n\nclass Model_exp065(pl.LightningModule):\n    def __init__(self, cfg, val_losses=None):\n        super().__init__()\n        self.cfg = cfg\n        self.__build_model()\n        self._criterion = eval(self.cfg.loss)()\n        self.transform = get_default_transforms()\n        self.save_hyperparameters(cfg)\n        self.val_losses = val_losses \n\n    def __build_model(self):\n        self.backbone = create_model(\n            self.cfg.model.name, pretrained=False, num_classes=0, in_chans=3\n        )\n        num_features = self.backbone.num_features\n        self.fc = nn.Sequential(\n            nn.Dropout(0.5), nn.Linear(num_features, self.cfg.model.output_dim)\n        )\n\n    def forward(self, x):\n        f = self.backbone(x)\n        out = self.fc(f)\n        return out\n\n    def training_step(self, batch, batch_idx):\n        loss, pred, labels = self.__share_step(batch, 'train')\n        self.log(\"train/loss\", loss)\n        return {'loss': loss, 'pred': pred, 'labels': labels}\n        \n    def validation_step(self, batch, batch_idx):\n        loss, pred, labels = self.__share_step(batch, 'val')\n        self.log(\"val/loss\", loss)\n        return {'pred': pred, 'labels': labels}\n    \n    def __share_step(self, batch, mode):\n        images, feature, labels = batch\n        feature = feature.float()\n        labels = labels.float() / 100.0\n        images = self.transform[mode](images)\n        \n        if torch.rand(1)[0] < 0.5 and mode == 'train':\n            mix_images, target_a, target_b, lam = mixup(images, labels, alpha=self.cfg.mixup.alpha)\n            logits = self.forward(mix_images, feature).squeeze(1)\n            loss = self._criterion(logits, target_a) * lam + (1 - lam) * self._criterion(logits, target_b)\n        else:\n            logits = self.forward(images, feature).squeeze(1)\n            loss = self._criterion(logits, labels)\n        \n        pred = logits.sigmoid().detach().cpu() * 100.\n        labels = labels.detach().cpu() * 100.\n        \n        return loss, pred, labels\n    \nclass Model_exp069(pl.LightningModule):\n    def __init__(self, cfg, val_losses=None):\n        super().__init__()\n        self.cfg = cfg\n        self.__build_model()\n        self._criterion = eval(self.cfg.loss)()\n        self.transform = get_default_transforms()\n        self.save_hyperparameters(cfg)\n        self.val_losses = val_losses \n\n    def __build_model(self):\n        self.backbone = create_model(\n            self.cfg.model.name, pretrained=False, num_classes=128, in_chans=3\n        )\n        self.dropout1 = nn.Dropout(0.5)\n        self.dense1 = nn.Linear(128+3, 64)\n        self.dropout2 = nn.Dropout(0.5)\n        self.dense2 = nn.Linear(64, 1)\n        num_features = self.backbone.num_features\n\n    def forward(self, image, features):\n        x = self.backbone(image)\n        x = self.dropout1(x)\n        x = torch.cat([x, features], dim=1)\n        x = self.dense1(x)\n        x = self.dropout2(x)\n        out = self.dense2(x)\n        return out\n\n    def training_step(self, batch, batch_idx):\n        loss, pred, labels = self.__share_step(batch, 'train')\n        self.log(\"train/loss\", loss)\n        return {'loss': loss, 'pred': pred, 'labels': labels}\n        \n    def validation_step(self, batch, batch_idx):\n        loss, pred, labels = self.__share_step(batch, 'val')\n        self.log(\"val/loss\", loss)\n        return {'pred': pred, 'labels': labels}\n    \n    def __share_step(self, batch, mode):\n        images, feature, labels = batch\n        feature = feature.float()\n        labels = labels.float() / 100.0\n        images = self.transform[mode](images)\n        \n        if torch.rand(1)[0] < 0.5 and mode == 'train':\n            mix_images, target_a, target_b, lam = mixup(images, labels, alpha=self.cfg.mixup.alpha)\n            logits = self.forward(mix_images, feature).squeeze(1)\n            loss = self._criterion(logits, target_a) * lam + (1 - lam) * self._criterion(logits, target_b)\n        else:\n            logits = self.forward(images, feature).squeeze(1)\n            loss = self._criterion(logits, labels)\n        \n        pred = logits.sigmoid().detach().cpu() * 100.\n        labels = labels.detach().cpu() * 100.\n        \n        return loss, pred, labels\n    \nclass Model_exp074(pl.LightningModule):\n    def __init__(self, cfg, val_losses=None):\n        super().__init__()\n        self.cfg = cfg\n        self.__build_model()\n        self._criterion = eval(self.cfg.loss)()\n        self.transform = get_default_transforms()\n        self.save_hyperparameters(cfg)\n        self.val_losses = val_losses \n\n    def __build_model(self):\n        self.backbone = create_model(\n            self.cfg.model.name, pretrained=False, num_classes=128, in_chans=3\n        )\n        self.dropout1 = nn.Dropout(0.5)\n        self.dense1 = nn.Linear(128+15, 64)\n        self.dropout2 = nn.Dropout(0.5)\n        self.dense2 = nn.Linear(64, 1)\n        num_features = self.backbone.num_features\n\n    def forward(self, image, features):\n        x = self.get_feature(image)\n        x = self.dropout1(x)\n        x = torch.cat([x, features], dim=1)\n        x = self.dense1(x)\n        x = self.dropout2(x)\n        out = self.dense2(x)\n        return out\n\n    def get_feature(self, image):\n        x = self.backbone(image)\n        return x\n\n    def training_step(self, batch, batch_idx):\n        loss, pred, labels = self.__share_step(batch, 'train')\n        self.log(\"train/loss\", loss)\n        return {'loss': loss, 'pred': pred, 'labels': labels}\n        \n    def validation_step(self, batch, batch_idx):\n        loss, pred, labels = self.__share_step(batch, 'val')\n        self.log(\"val/loss\", loss)\n        return {'pred': pred, 'labels': labels}\n    \n    def __share_step(self, batch, mode):\n        images, feature, labels = batch\n        feature = feature.float()\n        labels = labels.float() / 100.0\n        images = self.transform[mode](images)\n        \n        if torch.rand(1)[0] < 0.5 and mode == 'train':\n            mix_images, target_a, target_b, lam = mixup(images, labels, alpha=self.cfg.mixup.alpha)\n            logits = self.forward(mix_images, feature).squeeze(1)\n            loss = self._criterion(logits, target_a) * lam + (1 - lam) * self._criterion(logits, target_b)\n        else:\n            logits = self.forward(images, feature).squeeze(1)\n            loss = self._criterion(logits, labels)\n        \n        pred = logits.sigmoid().detach().cpu() * 100.\n        labels = labels.detach().cpu() * 100.\n        \n        return loss, pred, labels\n    \ndef transform_get_image_shape_describe(input_df): \n    pathes = input_df['Id'].values\n    shapes = []\n    for path in tqdm(pathes):\n        img = Image.open(path)\n        height, width = img.height, img.width\n        shapes.append([height, width])\n    input_df[[\"height\", \"width\"]] = shapes\n    input_df[\"aspect\"] = input_df[\"height\"] / input_df[\"width\"]\n    height_mean = 904.2843018563358\n    height_std = 156.90598049629264\n    width_mean = 804.4262510088781\n    width_std = 270.21192072081044\n    if len(input_df)==8:#testだと全部同じ大きさで標準化ミスるからエラー除去でいれる。\n        input_df['height'] = input_df['height'].fillna(0)\n        input_df['width'] = input_df['width'].fillna(0)\n    else:\n        input_df['height'] = input_df['height'].apply(lambda x: (x-height_mean)/ height_std)\n        input_df['width'] = input_df['width'].apply(lambda x: (x-width_mean)/ width_std)\n\n    print(f'height_mean{height_mean}_std{height_std}')\n    print(f'width_mean{width_mean}_std{width_std}')\n    return input_df\n\ntest_df = pd.read_csv(os.path.join(config_exp049.root, \"test.csv\"))\ntest = test_df.copy()\ntest_df[\"Id\"] = test_df[\"Id\"].apply(lambda x: os.path.join(config_exp049.root, \"test\", x + \".jpg\"))\ntest_df = transform_get_image_shape_describe(test_df)\ndataloader_exp049 = PetfinderDataModule(test_df, config_exp049).test_dataloader()\ndataloader_exp054 = PetfinderDataModule(test_df, config_exp054).test_dataloader()\ndataloader_exp060 = PetfinderDataModule(test_df, config_exp060).test_dataloader()\ndataloader_exp065 = PetfinderDataModule(test_df, config_exp065).test_dataloader()\ndataloader_exp069 = PetfinderDataModule(test_df, config_exp069).test_dataloader()\ndataloader_exp074 = PetfinderDataModule(test_df, config_exp074).test_dataloader()\n\n\ndef get_predict_feature(test_loader, model, device):\n    model = model.to(device)\n    model = model.eval()\n    predicts = []\n    for images, features in test_loader:\n        images = images.to(device)\n        features = features.to(device).float()\n        images = get_default_transforms()['val'](images)\n        batch_size = images.size(0)\n        with torch.no_grad():\n            predict = model(images, features).sigmoid().detach().cpu().numpy() * 100\n        predicts.append(predict)\n    predicts = np.concatenate(predicts)\n    return predicts\n\ndef get_predict(test_loader, model, device):\n    model = model.to(device)\n    model = model.eval()\n    predicts = []\n    for images in test_loader:\n        images = images.to(device)\n        images = get_default_transforms()['val'](images)\n        batch_size = images.size(0)\n        with torch.no_grad():\n            predict = model(images).sigmoid().detach().cpu().numpy() * 100\n        predicts.append(predict)\n    predicts = np.concatenate(predicts)\n    return predicts\n\nIMG_PREDICTS = []\ntmp = []\nfor fold in tqdm(range(config_exp049.n_splits)):\n    model = Model_exp049(config_exp049)\n    model_path = \"../input/petfinder-334model/exp049\"\n    model.load_state_dict(torch.load(f'{model_path}/fold{fold}/best_loss_fold{fold}.pth'))\n    predicts = get_predict(dataloader_exp049, model, 'cuda')\n    tmp.append(predicts)\n    \n    del model\n    torch.cuda.empty_cache()\nIMG_PREDICTS.append(tmp)\n    \ntmp = []\nfor fold in tqdm(range(config_exp054.n_splits)):\n    model = Model_exp054(config_exp054)\n    model_path = \"../input/petfinder-224model/exp054\"\n    model.load_state_dict(torch.load(f'{model_path}/fold{fold}/best_loss_fold{fold}.pth'))\n    predicts = get_predict_feature(dataloader_exp054, model, 'cuda')\n    tmp.append(predicts)\n    \n    del model\n    torch.cuda.empty_cache()\nIMG_PREDICTS.append(tmp)\n\ntmp = []\nfor fold in tqdm(range(config_exp060.n_splits)):\n    model = Model_exp060(config_exp060)\n    model_path = \"../input/petfinder-224model/exp060\"\n    model.load_state_dict(torch.load(f'{model_path}/fold{fold}/best_loss_fold{fold}.pth'))\n    predicts = get_predict_feature(dataloader_exp060, model, 'cuda')\n    tmp.append(predicts)\n    \n    del model\n    torch.cuda.empty_cache()\nIMG_PREDICTS.append(tmp)\n\ntmp = []\nfor fold in tqdm(range(config_exp065.n_splits)):\n    model = Model_exp065(config_exp065)\n    model_path = \"../input/petfinder-334model/exp065\"\n    model.load_state_dict(torch.load(f'{model_path}/fold{fold}/best_loss_fold{fold}.pth'))\n    predicts = get_predict(dataloader_exp065, model, 'cuda')\n    tmp.append(predicts)\n    \n    del model\n    torch.cuda.empty_cache()\nIMG_PREDICTS.append(tmp)\n\ntmp = []\nfor fold in tqdm(range(config_exp069.n_splits)):\n    model = Model_exp069(config_exp069)\n    model_path = \"../input/petfinder-224model/exp069\"\n    model.load_state_dict(torch.load(f'{model_path}/fold{fold}/best_loss_fold{fold}.pth'))\n    predicts = get_predict_feature(dataloader_exp069, model, 'cuda')\n    tmp.append(predicts)\n    \n    del model\n    torch.cuda.empty_cache()\nIMG_PREDICTS.append(tmp)\n\ntmp = []\nfor fold in tqdm(range(config_exp074.n_splits)):\n    model = Model_exp074(config_exp074)\n    model_path = \"../input/petfinder-224model/exp074\"\n    model.load_state_dict(torch.load(f'{model_path}/fold{fold}/best_loss_fold{fold}.pth'))\n    predicts = get_predict_feature(dataloader_exp074, model, 'cuda')\n    tmp.append(predicts)\n    \n    del model\n    torch.cuda.empty_cache()\nIMG_PREDICTS.append(tmp)\n\ndef load_ensemble_model(path):\n    with open(path, mode='rb') as fp:\n        clf = pickle.load(fp)\n    return clf\n\nensemble_predict = []\nensemble_model_exp_path = '../input/d/ktakita/petfinder-ensumble/exp085_ensumble'\nfor fold in range(5):\n    X = np.concatenate([IMG_PREDICTS[0][fold],\n                        IMG_PREDICTS[1][fold],\n                        IMG_PREDICTS[2][fold],\n                        IMG_PREDICTS[3][fold],\n                        IMG_PREDICTS[4][fold],\n                        IMG_PREDICTS[5][fold]],  \n                       axis=1)\n    ensemble_model_path = f'{ensemble_model_exp_path}/fold{fold}_{config_ensemble.model}.pickle'\n    model = load_ensemble_model(ensemble_model_path)\n    pred = model.predict(X)\n    ensemble_predict.append(pred)\n    \npd.DataFrame(ensemble_predict)\n\npredict = np.mean(ensemble_predict, 0)\n\npredictions = predict\ntest['Pawpularity'] = predictions\ntest[['Id', 'Pawpularity']].to_csv('submission_at.csv', index=False)\n#display(test[['Id', 'Pawpularity']].head(8))\n\"\"\"\n","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:54:00.968943Z","iopub.execute_input":"2022-01-13T13:54:00.969374Z","iopub.status.idle":"2022-01-13T13:58:26.041722Z","shell.execute_reply.started":"2022-01-13T13:54:00.969285Z","shell.execute_reply":"2022-01-13T13:58:26.040882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\n\nimport os\nimport sys\nimport warnings\nfrom pprint import pprint\nfrom glob import glob\nfrom tqdm.notebook import tqdm\nimport pickle\nimport glob\n\nfrom sklearn.linear_model import Ridge\nimport torch\nimport torch.optim as optim\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torchvision.transforms as T\nsys.path.append('../input/pythonbox')\nfrom box import Box\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nfrom timm import create_model\nfrom sklearn.model_selection import StratifiedKFold\nfrom torchvision.io import read_image\nfrom torch.utils.data import DataLoader, Dataset\n#from pytorch_grad_cam import GradCAMPlusPlus\n#from pytorch_grad_cam.utils.image import show_cam_on_image\nfrom PIL import Image\nfrom fastai.vision.all import *\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning.utilities.seed import seed_everything\nfrom pytorch_lightning import callbacks\nfrom pytorch_lightning.callbacks.progress import ProgressBarBase\nfrom pytorch_lightning.callbacks.early_stopping import EarlyStopping\nfrom pytorch_lightning.loggers import TensorBoardLogger\nfrom pytorch_lightning import LightningDataModule, LightningModule\n\n#sys.path.append('../input/poolformer-master')\n#import models as PoolFormerModels #poolformer\n\nprint(pl.__version__)\nwarnings.filterwarnings(\"ignore\")\nIMAGENET_MEAN = [0.485, 0.456, 0.406]  # RGB\nIMAGENET_STD = [0.229, 0.224, 0.225]  # RGB\n\nsys.path.append(f'../input/petfinder-scripts')\n\n\nclass PetfinderDataModule(LightningDataModule):\n    def __init__(\n        self,\n        test_df,\n        dataset,\n        cfg,\n    ):\n        super().__init__()\n        self._test_df = test_df\n        self._dataset = dataset\n        self._cfg = cfg\n\n    def __create_dataset(self, train=True):\n        return (\n            self._dataset(self._test_df, self._cfg.transform.image_size)\n            if train\n            else self._dataset(self._test_df, self._cfg.transform.image_size)\n        )\n\n    def test_dataloader(self):\n        dataset = self.__create_dataset(True)\n        self._cfg.val_loader.batch_size = 32\n        self._cfg.val_loader.num_workers = os.cpu_count()\n        return DataLoader(dataset, **self._cfg.val_loader)\n\ndef transform_get_image_shape_describe(input_df): \n    pathes = input_df['Id'].values\n    shapes = []\n    for path in tqdm(pathes):\n        img = Image.open(path)\n        height, width = img.height, img.width\n        shapes.append([height, width])\n    input_df[[\"height\", \"width\"]] = shapes\n    input_df[\"aspect\"] = input_df[\"height\"] / input_df[\"width\"]\n    height_mean = 904.2843018563358\n    height_std = 156.90598049629264\n    width_mean = 804.4262510088781\n    width_std = 270.21192072081044\n    if len(test_df)==8:#testだと全部同じ大きさで標準化ミスるからエラー除去でいれる。\n        input_df['height'] = input_df['height'].fillna(0)\n        input_df['width'] = input_df['width'].fillna(0)\n    else:\n        input_df['height'] = input_df['height'].apply(lambda x: (x-height_mean)/ height_std)\n        input_df['width'] = input_df['width'].apply(lambda x: (x-width_mean)/ width_std)\n\n    print(f'height_mean{height_mean}_std{height_std}')\n    print(f'width_mean{width_mean}_std{width_std}')\n    return input_df\n    \ndef get_predict_feature(model, test_loader, transforms,device):\n    model = model.to(device)\n    model = model.eval()\n    predicts = []\n    for images, features in test_loader:\n        images = images.to(device)\n        features = features.to(device).float()\n        images = transforms()['val'](images)\n        batch_size = images.size(0)\n        with torch.no_grad():\n            predict = model(images, features).sigmoid().detach().cpu().numpy() * 100\n        predicts.append(predict)\n    predicts = np.concatenate(predicts)\n    return predicts\n\ndef get_predict(model, test_loader, transforms,device):\n    model = model.to(device)\n    model = model.eval()\n    predicts = []\n    for images in test_loader:\n        images = images.to(device)\n        images = transforms()['val'](images)\n        batch_size = images.size(0)\n        with torch.no_grad():\n            predict = model(images).sigmoid().detach().cpu().numpy() * 100\n        predicts.append(predict)\n    predicts = np.concatenate(predicts)\n    return predicts\n\ndef model_load(model, exp_name, model_path):\n    if exp_name == 'exp092' or exp_name == 'exp100':\n        model.load_state_dict(torch.load(model_path)['state_dict'])\n    else:\n        model.load_state_dict(torch.load(model_path))\n    return model\n\nimport exp108\n\ntest_df = pd.read_csv('../input/petfinder-pawpularity-score/test.csv')\ntest_df[\"Id\"] = test_df[\"Id\"].apply(lambda x: os.path.join('../input/petfinder-pawpularity-score', \"test\", x + \".jpg\"))\ntest_df = transform_get_image_shape_describe(test_df)\n\nIMG_PREDICTS = []\nexp_name = 'exp108'\nprint(exp_name)\nconfig = eval(exp_name).config\nconfig.model.pretrain = False\ndataloader = PetfinderDataModule(test_df, \n                                 eval(exp_name).PetfinderDataset, \n                                 config).test_dataloader()\n\ntmp = []\nfor fold in tqdm(range(config.n_splits)):\n    model = eval(exp_name).Model(config)\n    model_path = glob.glob(os.path.join('../input/petfinder-exp108', f'{exp_name}/fold{fold}/best_loss_fold{fold}.*'))[0]\n    model.load_state_dict(torch.load(model_path)['state_dict'])\n    predicts = get_predict_feature(model, dataloader, eval(exp_name).get_default_transforms, 'cuda')\n\n    tmp.append(predicts)\n\n    del model\n    torch.cuda.empty_cache()\nIMG_PREDICTS = tmp\n\n#predict make\npreds = []\nfold_num = len(IMG_PREDICTS)\nfor fold in range(fold_num):\n    preds.append(IMG_PREDICTS[fold] / fold_num)\npredict = np.sum(preds, 0)\n\npredictions = predict\n\ntest = pd.read_csv('../input/petfinder-pawpularity-score/sample_submission.csv')\ntest['Pawpularity'] = predictions\ntest[['Id', 'Pawpularity']].to_csv('submission_exp108.csv', index=False)\n#display(test[['Id', 'Pawpularity']].head(8))","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:58:26.044008Z","iopub.execute_input":"2022-01-13T13:58:26.044237Z","iopub.status.idle":"2022-01-13T14:01:08.776095Z","shell.execute_reply.started":"2022-01-13T13:58:26.044203Z","shell.execute_reply":"2022-01-13T14:01:08.77526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n%%python\nimport sys\nsys.path.append('/kaggle/input/timm-pytorch-image-models/pytorch-image-models-master')\n\n# Python Libraries\nimport os\nimport gc\n\n# Third party\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm\n\n# Pytorch \nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\n\n# Pytorch Image Models\nfrom timm import create_model\nfrom torchvision.io import read_image\nimport torchvision.transforms as T\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# ====================================================\n# Dataset\n# ====================================================\nclass PetfinderDataset(Dataset):\n    def __init__(self, df, feature_cols, image_size=224):\n        self._X = df[\"Id\"].values\n        self.meta = df[feature_cols].values\n        self._y = None\n#         if \"Pawpularity\" in df.keys():\n#             self._y = df[\"Pawpularity\"].values\n        self._transform = T.Compose([\n                                        T.Resize(image_size),  # 1\n                                        T.CenterCrop([image_size, image_size]),  # 2\n                                    ]\n                                    )\n\n    def __len__(self):\n        return len(self._X)\n\n    def __getitem__(self, idx):\n        image_path = self._X[idx]\n        image = read_image(image_path)\n        image = self._transform(image)\n        features = torch.FloatTensor(self.meta[idx, :])\n        \n        return image, features \n# ====================================================\n# Transforms\n# ====================================================\nIMAGENET_MEAN = [0.485, 0.456, 0.406]  # RGB\nIMAGENET_STD = [0.229, 0.224, 0.225]  # RGB\ndef get_default_transforms():\n    transform = {\n        \"val\": T.Compose(\n            [\n                T.ConvertImageDtype(torch.float),\n                T.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD),\n            ]\n        ),\n    }\n    return transform   \n# ====================================================\n# model\n# ====================================================\nclass CustomModel(nn.Module):\n    def __init__(self, cfg, pretrained=False):\n        super().__init__()\n        self.cfg = cfg\n        self.backbone = create_model(model_name=self.cfg.model_name,\n                                     pretrained=pretrained,\n                                     in_chans=self.cfg.in_chans,\n                                     num_classes=0)\n        self.dropout1 = nn.Dropout(p=0.5)\n        self.dropout2 = nn.Dropout(p=0.5)\n        self.fc = nn.Linear(3084, self.cfg.target_size)\n        \n#         self.fc = nn.Sequential(\n#             nn.Dropout(0.5),\n#             nn.LazyLinear(self.cfg.target_size)\n#         )\n    def get_feature(self, x, features):\n        return self.backbone(x)\n        \n    def forward(self, x, features):\n        f = self.backbone(x) # (bs, embedding_size)\n        f = self.dropout1(f)\n        if features.shape[1] != 0:\n            f = torch.cat([f, features],dim=1)\n            f =  self.dropout2(f)\n        out = self.fc(f)\n        return out\n    \ndef get_model(cfg):\n    model = CustomModel(cfg, pretrained=False)\n    return model\n\ndef predict(model, loader):\n    model = model.to(device)\n    model = model.eval()\n    predicts = []\n    for batch in tqdm(loader):\n        images, features = batch\n        images = images.to(device)\n        features = features.to(device)\n        images = get_default_transforms()['val'](images)\n        batch_size = images.size(0)\n        with torch.no_grad():\n            predict = model(images, features).sigmoid().detach().cpu().numpy() * 100\n        predicts.append(predict)\n    predicts = np.concatenate(predicts)\n    return predicts\n\n# ====================================================\n# Config\n# ====================================================\nclass CFG_029:\n    debug = False\n    model_name = 'dm_nfnet_f3'\n    img_size = 512\n    in_chans = 3\n    n_fold = 5\n    target_size = 1\n    # DataLoader\n    loader = {\n        \"valid\": {\n            \"batch_size\": 32,\n            \"num_workers\": 4,\n            \"shuffle\": False,\n            \"pin_memory\": True,\n            \"drop_last\": False\n        }\n    }\n    # exp020\n    feature_cols = ['Subject Focus', 'Eyes', 'Face', 'Near', 'Action', 'Accessory',\n        'Group', 'Collage', 'Human', 'Occlusion', 'Info', 'Blur',\n        ]\n\ndf_test = pd.read_csv('../input/petfinder-pawpularity-score/test.csv')\ndf_test[\"Id\"] = df_test[\"Id\"].apply(lambda x: os.path.join('../input/petfinder-pawpularity-score', \"test\", x + \".jpg\"))\n    \nloader = DataLoader(PetfinderDataset(df_test, CFG_029.feature_cols, CFG_029.img_size),**CFG_029.loader['valid'])\n\ntmp = []\nfor i in tqdm(range(CFG_029.n_fold)):\n    model = get_model(CFG_029)\n    model.load_state_dict(torch.load(f'../input/pet2teyomodel1/exp029_1759169/exp029_fold{i}_best.pth', map_location=device))\n    predicts = predict(model, loader)\n    tmp.append(predicts)\n    del model\n    torch.cuda.empty_cache()\n    gc.collect()\n\ntest = pd.read_csv('../input/petfinder-pawpularity-score/sample_submission.csv')\ntest['Pawpularity'] = np.mean(tmp,axis=0)\ntest[['Id', 'Pawpularity']].to_csv('submission_exp029.csv',index=False)\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:01:08.778031Z","iopub.execute_input":"2022-01-13T14:01:08.778311Z","iopub.status.idle":"2022-01-13T14:03:03.934014Z","shell.execute_reply.started":"2022-01-13T14:01:08.778272Z","shell.execute_reply":"2022-01-13T14:03:03.933184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\n#import pandas as pd\n#if pd.read_csv('../input/petfinder-pawpularity-score/test.csv').shape[0] == 8:\n#    exit()\n    \nimport sys\nimport gc\nfrom tqdm.auto import tqdm\nsys.path = [\"../input/pytorch-1-10-1/\"] + sys.path\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nfrom timm import create_model\n\nfrom fastai.vision.all import *\nprint(torch.__version__)\n\nset_seed(999, reproducible=True)\nBATCH_SIZE = 16\n\ndataset_path = Path('../input/petfinder-pawpularity-score/')\n\nseed = 999\nset_seed(seed, reproducible=True)\ntorch.manual_seed(seed)\ntorch.cuda.manual_seed(seed)\ntorch.backends.cudnn.deterministic = True\ntorch.use_deterministic_algorithms = True\n\ndef petfinder_rmse(input,target):\n    return 100*torch.sqrt(F.mse_loss(F.sigmoid(input.flatten()), target))\n\ntest_df = pd.read_csv(dataset_path/'test.csv')\ntest_df.head()\n\ntest_df['Pawpularity'] = [1]*len(test_df)\ntest_df['path'] = test_df['Id'].map(lambda x:str(dataset_path/'test'/x)+'.jpg')\ntest_df = test_df.drop(columns=['Id'])\ntest_df['norm_score'] = test_df['Pawpularity'] / 100\n\ndef get_data(img_size):\n    dls = ImageDataLoaders.from_df(test_df, #pass in train DataFrame\n                               #valid_col='is_valid', #\n                               seed=999, #seed\n                               fn_col='path', #filename/path is in the second column of the DataFrame\n                               label_col='norm_score', #label is in the first column of the DataFrame\n                               y_block=RegressionBlock, #The type of target\n                               bs=BATCH_SIZE, #pass in batch size\n                               num_workers=8,\n                               item_tfms=Resize(img_size), #pass in item_tfms\n                               batch_tfms=setup_aug_tfms([Brightness(), Contrast(), Hue(), Saturation()])) #pass in batch_tfms\n    \n    return dls\n\n\n\nmodel_weights = {'exp7_cait_m36_384': 0.10573687177566875,\n 'exp4_xcit_small_24_p16_384_dist': 0.09017084075938603,\n 'exp4_crossvit_18_dagger_408': 0.0864555998708494,\n 'exp4_xcit_small_24_p8_384_dist': 0.08478909740799544,\n 'exp15_vit_base_patch16_224_miil_in21k': 0.08388133976905879,\n 'exp7_swin_large_patch4_window12_384_in22k': 0.08162600824050978,\n 'exp9_cait_m36_384': 0.07951668818830163,\n 'exp7_vit_base_patch16_224_miil_in21k': 0.07448271317999128,\n 'exp7_swin_base_patch4_window12_384': 0.07014647726439097,\n 'exp7_jx_nest_base': 0.06841255969223342,\n 'exp8_vit_base_patch16_224_miil_in21k': 0.06359775694462852,\n 'exp7_vit_base_r50_s16_384': 0.061862705948120524,\n 'exp15_vit_large_patch16_224': 0.0546458052004623}\n\n\nN_FOLDS = 5\nall_preds = []\n\nfor path, w in tqdm(model_weights.items()):\n    model_name = path.split('_', 1)[-1]\n    try:\n        img_size = int(model_name.split('_')[-1])\n    except:\n        try:\n            img_size = int(model_name.split('_')[-2])\n        except:\n            img_size = 224\n    dls = get_data(img_size)\n    \n    def proc(pred):\n        return pred\n    num_classes = 1\n    if 'exp4_' in path:\n        model_dir = '../input/model-exp4-0106/pet'\n        loss = BCEWithLogitsLossFlat()\n    elif 'exp7_' in path:\n        model_dir = '../input/model-exp7-0106/pet'\n        loss = BCEWithLogitsLossFlat()\n    elif 'exp8_' in path:\n        model_dir = '../input/model-exp8-9-15-0106/pet'\n        loss = MSELossFlat()\n    elif 'exp9_' in path:\n        model_dir = '../input/model-exp8-9-15-0106/pet' \n        loss = CrossEntropyLossFlat()\n        num_classes = 100\n        def proc(pred):\n            return (pred.argmax(axis=1) + 1) / 100\n    elif 'exp15_' in path:\n        model_dir = '../input/model-exp8-9-15-0106/pet'\n        loss = MSELossFlat()\n        def proc(pred):\n            return (np.exp(pred) / 100).clip(0.01, 1)\n    else:\n        raise\n            \n    for i in range(N_FOLDS):\n        model = create_model(model_name, pretrained=False, num_classes=num_classes)\n        model.load_state_dict(torch.load(f'{model_dir}/{path}/{model_name}_{i}.pth'))\n\n        learn = Learner(dls, model, \n                        loss_func=loss,\n                        metrics=petfinder_rmse).to_fp16()\n\n        test_dl = dls.test_dl(test_df)\n\n        preds, _ = learn.get_preds(dl=test_dl)\n        #preds, _ = learn.tta(dl=test_dl, n=2, beta=0)\n        \n        preds = proc(preds).flatten()\n        \n        all_preds.append(preds * w / N_FOLDS)\n\n        del learn\n        torch.cuda.empty_cache()\n        gc.collect()\n    del dls\n    torch.cuda.empty_cache()\n    gc.collect()\n\nsample_df = pd.read_csv(dataset_path/'sample_submission.csv')\npreds = np.sum(np.stack(all_preds), axis=0)\nsample_df['Pawpularity'] = preds*100\nsample_df.to_csv('submission_tkm.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:03:03.936779Z","iopub.execute_input":"2022-01-13T14:03:03.93705Z","iopub.status.idle":"2022-01-13T14:13:11.163692Z","shell.execute_reply.started":"2022-01-13T14:03:03.93702Z","shell.execute_reply":"2022-01-13T14:13:11.162907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n#sub_at = pd.read_csv('submission_at.csv', index_col='Id')\nsub_tkm = pd.read_csv('submission_tkm.csv', index_col='Id')\nsub_exp108 = pd.read_csv('submission_exp108.csv', index_col='Id')\n#sub_exp029 = pd.read_csv('submission_exp029.csv', index_col='Id')\n\n#[0.502754   0.18612928 0.31658946]\n#[0.51461806 0.16874229 0.3218239 ]\n#['exp108', 'exp029', 'oof0(tkmsan)', 'oof1(at)']\n#[0.29673496 0.17641786 0.41963073 0.11312639]\n#sub = (0.29673496 * sub_exp108 + 0.17641786 * sub_exp029 + 0.41963073 * sub_tkm + 0.11312639 * sub_at)\nsub=(0.57142857* sub_tkm+0.42857143*sub_exp108 )\nsub.to_csv('submission_ens.csv')\nsub\n","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:13:11.165247Z","iopub.execute_input":"2022-01-13T14:13:11.165542Z","iopub.status.idle":"2022-01-13T14:13:11.291156Z","shell.execute_reply.started":"2022-01-13T14:13:11.165502Z","shell.execute_reply":"2022-01-13T14:13:11.29043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pickle\nimport numpy as np\nfrom PIL import Image\nimport imagehash\nfrom tqdm.auto import tqdm\n\nDEBUG = pd.read_csv('../input/petfinder-pawpularity-score/test.csv').shape[0] == 8\n\ndf1 = pd.read_csv('../input/pet-data/df1_train_test_v5.csv')\n\nif DEBUG:\n    df2 = pd.read_csv('../input/petfinder-pawpularity-score/train.csv')\n    df2['pred'] = 0.5\n    df2['path'] = df2['Id'].map(lambda x:str(f'../input/petfinder-pawpularity-score/train/{x}.jpg'))\nelse:\n    df2 = pd.read_csv('../input/petfinder-pawpularity-score/test.csv')\n    df2['pred'] = df2['Id'].map(pd.read_csv('submission_ens.csv', index_col='Id')['Pawpularity']) / 100\n    df2['path'] = df2['Id'].map(lambda x:str(f'../input/petfinder-pawpularity-score/test/{x}.jpg'))\n    \ndf2['hash'] = [imagehash.average_hash(Image.open(x))\n                    for x in tqdm(df2['path'].values)]","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:13:11.292442Z","iopub.execute_input":"2022-01-13T14:13:11.29285Z","iopub.status.idle":"2022-01-13T14:16:14.899522Z","shell.execute_reply.started":"2022-01-13T14:13:11.292814Z","shell.execute_reply":"2022-01-13T14:16:14.898774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1['hash'] = df1['hash'].astype(str)\ndf2['hash'] = df2['hash'].astype(str)\ndf = pd.merge(df2, df1, how='left', on='hash')","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:16:14.900891Z","iopub.execute_input":"2022-01-13T14:16:14.901362Z","iopub.status.idle":"2022-01-13T14:16:15.730758Z","shell.execute_reply.started":"2022-01-13T14:16:14.901319Z","shell.execute_reply":"2022-01-13T14:16:15.730024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if df.shape[0] > 0:\n    with open('../input/0113-2nd-stage-gbdt-fixed-w-newmodels3/list_model_lgb_last_comp_all.pkl', 'rb') as f:\n        models = pickle.load(f)\n    df['len_desc'] = df['Description'].fillna('').map(len)\n    df['Name'] = df['Name'].fillna('').replace('No Name Yet', '')\n    df['len_name'] = df['Name'].map(len)\n    \n    pred = 0\n    for model in models:\n        pred += model.predict(df[model.feature_name()])\n    pred /= len(models)\n    \n    df_pred = pd.DataFrame()\n    df_pred['Id'] = df['Id'].values\n    df_pred['Pawpularity'] = (pred * 100).clip(1, 100)\n    \n    sub = df_pred.groupby('Id')[['Pawpularity']].mean()\n    #sub = pd.read_csv('submission_ens.csv')\n    #sub = sub[~sub['Id'].isin(df_pred['Id'])].append(df_pred)\n    #sub = sub.set_index('Id').sort_index()\n    #sub.to_csv('submission.csv')\nelse:\n    sub = pd.read_csv('submission_ens.csv', index_col='Id')\n    #sub.to_csv('submission.csv', index=False)\n    \n\npred = sub['Pawpularity'].values / 100\n#mu = pred.mean()\n#pred = (pred - mu) + 0.38362035973750004\npred = pred.clip(0.01, 1)\n\nsub['Pawpularity'] = pred * 100\nsub.to_csv('submission.csv')\nsub","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:16:15.731884Z","iopub.execute_input":"2022-01-13T14:16:15.732142Z","iopub.status.idle":"2022-01-13T14:16:18.972447Z","shell.execute_reply.started":"2022-01-13T14:16:15.732112Z","shell.execute_reply":"2022-01-13T14:16:18.97165Z"},"trusted":true},"execution_count":null,"outputs":[]}]}