{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":65332,"databundleVersionId":7183351,"sourceType":"competition"}],"dockerImageVersionId":30559,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"PANDAコンペのデータのサブセット　重複画像は除去済み<br>\n画像はWSI→タイルに処理済み<br>\nsave version>save &run all でnotebook全体が実行され結果が保存される<br>\noutputのtest*.csvを提出<br>\nbaselineはefnetB0+concat tile pooling(GAP前特徴をタイルごとに求めてconcat→横長特徴に対してGAPかけてlinearに通す)<br>\nPANDAコンペと同様にタイルの枚数,backboneのネットワーク,集約層の構造,augmentationなどをいじれば良いかも<br>\nラベルなし画像が与えられているので疑似ラベルつけてtrainに追加or疑似ラベルデータだけでpretrain→trainで学習or他の半教師あり(e.g:MPL)<br>\nテストデータは画像が見える状態で与えられているのでスコアあげるには疑似ラベル必須<br>\n\n\nPANDAコンペではデータセットの施設が違う/アノテーションの方法が違う　などがあってここをクリアするのが鍵だった、が今回はこれらの点には触れなくても取り組むことができるようになっている<br>\n","metadata":{}},{"cell_type":"code","source":"!git clone https://github.com/JamesQFreeman/LoRA-ViT.git","metadata":{"execution":{"iopub.status.busy":"2024-01-06T02:06:15.501449Z","iopub.execute_input":"2024-01-06T02:06:15.501687Z","iopub.status.idle":"2024-01-06T02:06:17.199062Z","shell.execute_reply.started":"2024-01-06T02:06:15.501665Z","shell.execute_reply":"2024-01-06T02:06:17.197833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append(\"/kaggle/working/LoRA-ViT/\")","metadata":{"execution":{"iopub.status.busy":"2024-01-06T02:06:17.201769Z","iopub.execute_input":"2024-01-06T02:06:17.202164Z","iopub.status.idle":"2024-01-06T02:06:17.207144Z","shell.execute_reply.started":"2024-01-06T02:06:17.202129Z","shell.execute_reply":"2024-01-06T02:06:17.206120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install einops\n!pip install open_clip_torch","metadata":{"execution":{"iopub.status.busy":"2024-01-06T02:06:17.208511Z","iopub.execute_input":"2024-01-06T02:06:17.208829Z","iopub.status.idle":"2024-01-06T02:06:42.152880Z","shell.execute_reply.started":"2024-01-06T02:06:17.208798Z","shell.execute_reply":"2024-01-06T02:06:42.151971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2,glob\nimport matplotlib.pyplot as plt\nimport pandas as pd\n\ntrain = pd.read_csv(\"/kaggle/input/copy-of-pathologyaidemo2/train.csv\")\nplt.imshow(cv2.imread(glob.glob(\"/kaggle/input/copy-of-pathologyaidemo2/train/*\")[0])[:,:,::-1])\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-01-06T02:06:42.155167Z","iopub.execute_input":"2024-01-06T02:06:42.155476Z","iopub.status.idle":"2024-01-06T02:06:44.956206Z","shell.execute_reply.started":"2024-01-06T02:06:42.155449Z","shell.execute_reply":"2024-01-06T02:06:44.955200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install hydra-core --upgrade","metadata":{"execution":{"iopub.status.busy":"2024-01-06T02:06:44.957429Z","iopub.execute_input":"2024-01-06T02:06:44.957699Z","iopub.status.idle":"2024-01-06T02:06:59.428202Z","shell.execute_reply.started":"2024-01-06T02:06:44.957674Z","shell.execute_reply":"2024-01-06T02:06:59.426969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import transformers","metadata":{"execution":{"iopub.status.busy":"2024-01-06T02:06:59.429937Z","iopub.execute_input":"2024-01-06T02:06:59.430349Z","iopub.status.idle":"2024-01-06T02:07:00.920245Z","shell.execute_reply.started":"2024-01-06T02:06:59.430298Z","shell.execute_reply":"2024-01-06T02:07:00.919366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import yaml\n\nyaml_str = \"\"\"\ngeneral:\n  debug: false\n  exp_num: 000\n  seed: 42\n  num_folds: 5\nN_patch: 16\nloss:\n  name: CE\npreprocess:\n  size: 256\naug:\n  HorizontalFlip:\n    p: 0.5\n  VerticalFlip:\n    p: 0.5\n  RandomRotate90:\n    p: 0.5\n  ShiftScaleRotate:\n    p: 0.7\n    shift_limit: 0.1\n    scale_limit: 0.1\n    rotate_limit: 15\n  RandomBrightnessContrast:\n    p: 0.75\n    brightness_limit: 0.1\n    contrast_limit: 0.1\n  CLAHE:\n    p: 0.1\n  one_of_Distortion:\n    p: 0\n  one_of_Blur_Gnoise:\n    p: 0\n  compress:\n    p: 0\n  CoarseDropout:\n    p: 0.5\n    max_holes: 4\n    max_height: 6\n    max_width: 6\nmodel:\n  name: vit_base_patch32_224\n  pooling: avg\n  n_classes: 3\n  features_num: 1280\n  stride: 2\npsuedo_label: 0\ntrain:\n  amp: true\n  amp_inf: false\n  optim: adamw\n  lr: 0.0005\n  epochs: 10\n  without_hesitate: 15\n  batch_size: 4\n  ga_accum: 1\n  scheduler:\n    name: cosine_warmup\n    min_lr: 1.0e-07\n    t_0: 3\n    warmup: 3\n\n\"\"\"\n\ndata = yaml.load(yaml_str, Loader=yaml.FullLoader)\nyaml_str = yaml.dump(data)\nwith open('config.yaml', 'w') as f:\n    f.write(yaml_str)","metadata":{"execution":{"iopub.status.busy":"2024-01-06T02:31:04.507177Z","iopub.execute_input":"2024-01-06T02:31:04.508075Z","iopub.status.idle":"2024-01-06T02:31:04.528915Z","shell.execute_reply.started":"2024-01-06T02:31:04.508043Z","shell.execute_reply":"2024-01-06T02:31:04.527839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import hydra\nfrom omegaconf import DictConfig, OmegaConf\nimport sys,gc,os,random,time,math,glob\nimport matplotlib.pyplot as plt\nfrom contextlib import contextmanager\nfrom pathlib import Path\nfrom collections import defaultdict, Counter\nfrom  torch.cuda.amp import autocast, GradScaler \nimport cv2,timm\nfrom sklearn.metrics import roc_auc_score\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\nimport sklearn.metrics as metrics\nfrom sklearn.model_selection import StratifiedKFold,GroupKFold\nfrom sklearn.metrics import log_loss\nfrom functools import partial\nfrom tqdm import tqdm\nfrom sklearn.metrics import precision_score,recall_score,f1_score,log_loss\nfrom  sklearn.metrics import accuracy_score as acc\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam, SGD,AdamW\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, ReduceLROnPlateau,CosineAnnealingWarmRestarts\nfrom torch.utils.data import DataLoader, Dataset\nfrom albumentations import Compose, Normalize, HorizontalFlip, VerticalFlip,RandomGamma, RandomRotate90,GaussNoise,Cutout,RandomBrightnessContrast,RandomContrast,Resize\nfrom albumentations.pytorch import ToTensorV2\nimport transformers as T\n\nimport albumentations as A","metadata":{"execution":{"iopub.status.busy":"2024-01-06T02:31:04.854793Z","iopub.execute_input":"2024-01-06T02:31:04.855175Z","iopub.status.idle":"2024-01-06T02:31:04.865660Z","shell.execute_reply.started":"2024-01-06T02:31:04.855144Z","shell.execute_reply":"2024-01-06T02:31:04.864645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(timm.list_models(\"*max*rmlp*\"))","metadata":{"execution":{"iopub.status.busy":"2024-01-06T02:31:05.302611Z","iopub.execute_input":"2024-01-06T02:31:05.304477Z","iopub.status.idle":"2024-01-06T02:31:05.315452Z","shell.execute_reply.started":"2024-01-06T02:31:05.304388Z","shell.execute_reply":"2024-01-06T02:31:05.313399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import logging\nfrom lora import LoRA_ViT,LoRA_ViT_timm\n\n#from mylib.\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nclass TrainDataset(Dataset):\n    def __init__(self, df,CFG,train=True,transform1=None):\n        self.df = df\n        self.transform = transform1\n        self.CFG = CFG\n        self.train = train\n        self.labels = self.df['target'].to_numpy()\n        self.ids = self.df[\"ID\"].values\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        image_id = self.ids[idx]\n        imgs = [cv2.imread(f\"/kaggle/input/copy-of-pathologyaidemo2/train/{image_id}_{i}.png\")[:,:,::-1] for i in range(self.CFG.N_patch)]\n\n        imgs =  np.stack([self.transform(image=img)['image']  for img in imgs])\n        image = torch.from_numpy(imgs.transpose(0,3,1,2)).float()\n\n        label = self.labels[idx]\n\n        \n        return image, torch.tensor(label).long()\n    \nclass TestDataset(Dataset):\n    def __init__(self, df,CFG,train=True,transform1=None):\n        self.df = df\n        self.transform = transform1\n        self.CFG = CFG\n        self.ids = self.df[\"ID\"].values\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        image_id = self.ids[idx]\n        imgs = [cv2.imread(f\"/kaggle/input/copy-of-pathologyaidemo2/test (1)/{image_id}_{i}.png\")[:,:,::-1] for i in range(self.CFG.N_patch)]\n\n        imgs =  np.stack([self.transform(image=img)['image']  for img in imgs])\n        image = torch.from_numpy(imgs.transpose(0,3,1,2)).float()\n\n\n        \n        return image\n    \n\ndef get_transforms(*, data,CFG):\n    if data == 'train':\n        return Compose([\n            Resize(CFG.preprocess.size,CFG.preprocess.size),\n            #A.augmentations.crops.transforms.CenterCrop(CFG.preprocess.size*0.9,CFG.preprocess.size*0.9),\n            #A.crops.transforms.RandomResizedCrop(CFG.preprocess.size,CFG.preprocess.size,scale=(0.5, 1.0)),\n            #A.crops.transforms.RandomCrop(CFG.preprocess.size,CFG.preprocess.size),\n            A.HorizontalFlip(p=CFG.aug.HorizontalFlip.p),\n            A.VerticalFlip(p=CFG.aug.VerticalFlip.p),\n            A.RandomRotate90(p=CFG.aug.RandomRotate90.p),\n            A.ShiftScaleRotate(\n                shift_limit=CFG.aug.ShiftScaleRotate.shift_limit,\n                scale_limit=CFG.aug.ShiftScaleRotate.scale_limit,\n                rotate_limit=CFG.aug.ShiftScaleRotate.rotate_limit,\n                p=CFG.aug.ShiftScaleRotate.p),\n            A.RandomBrightnessContrast(\n                brightness_limit=CFG.aug.RandomBrightnessContrast.brightness_limit,\n                contrast_limit=CFG.aug.RandomBrightnessContrast.contrast_limit,\n                p=CFG.aug.RandomBrightnessContrast.p),\n            A.CLAHE(\n                clip_limit=(1,4),\n                p=CFG.aug.CLAHE.p),\n            A.OneOf([\n                A.JpegCompression(),\n                A.Downscale(scale_min=0.1, scale_max=0.15),\n                ], p=CFG.aug.compress.p),\n            A.CoarseDropout(max_holes=CFG.aug.CoarseDropout.max_holes, max_height=CFG.aug.CoarseDropout.max_height, max_width=CFG.aug.CoarseDropout.max_width, p=CFG.aug.CoarseDropout.p),\n            Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n            ])\n    elif data == 'valid':\n        return Compose([\n            Resize(CFG.preprocess.size,CFG.preprocess.size),\n            #A.augmentations.crops.transforms.CenterCrop(int(CFG.preprocess.size*0.9),int(CFG.preprocess.size*0.9)),\n            Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n            ])\n\n\n#### augmentation ==============\n#import open_clip\nfrom transformers import AutoProcessor, CLIPVisionModel\n#### model ================\nSEQ_POOLING = {\n    'avg': nn.AdaptiveAvgPool2d(1),\n    'max': nn.AdaptiveMaxPool2d(1)\n}\n\nclass Model_iafoss(nn.Module):\n    def __init__(self,CFG, base_model='tf_efficientnet_b0_ns',pool=\"avg\",pretrain=True):\n        super(Model_iafoss, self).__init__()\n        self.base_model = base_model \n        #\"\"\"\n        if self.base_model in [\"hipt\",\"plip\",\"qnet\",\"ibot\"]:\n            \"\"\"\n            if self.base_model==\"ibot\":\n                checkpoint_key = \"teacher\"\n                pretrained_weights = \"checkpoint.pth\"\n                self.model = vits.__dict__[\"vit_base\"](patch_size=16, num_classes=0)\n                state_dict = torch.load(pretrained_weights, map_location=\"cpu\")\n                if checkpoint_key is not None and checkpoint_key in state_dict:\n                    print(f\"Take key {checkpoint_key} in provided checkpoint dict\")\n                    state_dict = state_dict[checkpoint_key]\n                state_dict = {k.replace(\"module.\", \"\"): v for k, v in state_dict.items()}\n                # remove `backbone.` prefix induced by multicrop wrapper\n                state_dict = {k.replace(\"backbone.\", \"\"): v for k, v in state_dict.items()}\n                self.model.load_state_dict(state_dict, strict=False)\n                for _, p in self.model.named_parameters():\n                    p.requires_grad = False\n                for _, p in self.model.head.named_parameters():\n                    p.requires_grad = True\n\n                freeze =9\n                for n, p in self.model.blocks.named_parameters():\n                    if int(n.split(\".\")[0])>=(12-freeze):\n                        p.requires_grad = True\n                        \n                self.n_last_blocks  = 4\n                avgpool_patchtokens = 0\n                \n                nc = self.model.embed_dim * (self.n_last_blocks + int(avgpool_patchtokens))\n            \"\"\"\n            \n            if self.base_model==\"plip\":\n                self.model = CLIPVisionModel.from_pretrained(\"vinid/plip\")\n                nc = 768\n            \"\"\"\n            elif self.base_model==\"qnet\":\n                self.model = open_clip.create_model_and_transforms('hf-hub:wisdomik/QuiltNet-B-32')[0]\n                \n                nc = 512\n            \"\"\"\n            self.gru = nn.GRU(nc, 512, bidirectional=True, batch_first=True, num_layers=2)\n            nc*=CFG.N_patch\n            self.head = nn.Sequential(nn.Linear(nc,512),\n                            nn.ReLU(), nn.Dropout(0.5),nn.Linear(512,3))\n            self.exam_predictor = nn.Linear(512*2, 3)\n            self.pool = nn.AdaptiveAvgPool1d(1)\n\n        else:\n            self.model = timm.create_model(self.base_model, pretrained=True, num_classes=0,in_chans=3,dynamic_img_size=True)\n            nc = self.model.num_features\n            if \"vit_base\" in self.base_model:\n                self.model = LoRA_ViT_timm(vit_model=self.model, r=4, num_classes=0)\n                \n            \n            self.head = nn.Sequential(nn.AdaptiveAvgPool2d(1),nn.Flatten(),nn.Linear(nc,512),\n                            nn.ReLU(), nn.Dropout(0.5),nn.Linear(512,3))\n\n            self.gru = nn.GRU(nc, 512, bidirectional=True, batch_first=True, num_layers=2)\n            self.exam_predictor = nn.Linear(512*2, 3)\n            self.pool = nn.AdaptiveAvgPool1d(1)\n\n        \n    def forward(self, input1):\n        shape = input1.size()\n        batch_size = shape[0]\n        n = shape[1]\n\n        input1 = input1.view(-1,shape[2],shape[3],shape[4])\n\n        if \"ibot\" in self.base_model:\n            intermediate_output = self.model.get_intermediate_layers(input1, self.n_last_blocks)\n            x = torch.cat([x[:, 0] for x in intermediate_output], dim=-1)\n            embeds, _ = self.gru(x.view(batch_size,n,x.shape[1]))\n            embeds = self.pool(embeds.permute(0,2,1))[:,:,0]\n            y = self.exam_predictor(embeds)\n            #x = x.view(batch_size,x.shape[1]*n)\n            #y = self.head(x)\n           \n           #python base.py model.name=\"ibot\" train.lr=0.0001\n            return y\n        elif self.base_model==\"plip\":\n            x = self.model(input1)[\"pooler_output\"]\n            embeds, _ = self.gru(x.view(batch_size,n,x.shape[1]))\n            embeds = self.pool(embeds.permute(0,2,1))[:,:,0]\n            y = self.exam_predictor(embeds)\n           \n            return y\n        elif self.base_model==\"qnet\":\n            x = self.model.encode_image(input1)\n            embeds, _ = self.gru(x.view(batch_size,n,x.shape[1]))\n            embeds = self.pool(embeds.permute(0,2,1))[:,:,0]\n            y = self.exam_predictor(embeds)\n           \n            return y\n        elif \"vit_base\" in self.base_model:\n            x = self.model(input1)\n            embeds, _ = self.gru(x.view(batch_size,n,x.shape[1]))\n            embeds = self.pool(embeds.permute(0,2,1))[:,:,0]\n            y = self.exam_predictor(embeds)\n           \n            return y                \n        else:\n\n            x = self.model.forward_features(input1)#bs*num_tile,embed_dim,h,w\n\n                \n            shape = x.size()\n            x = x.view(-1,n,shape[1],shape[2],shape[3])\n            x = x.permute(0,2,1,3,4).contiguous().view(-1,shape[1],shape[2]*n,shape[3])\n            y = self.head(x)\n            \"\"\"\n            x =  self.model(input1)\n            \n            embeds, _ = self.gru(x.view(batch_size,n,x.shape[1]))\n            embeds = self.pool(embeds.permute(0,2,1))[:,:,0]\n            y = self.exam_predictor(embeds)\n            \"\"\"\n            return y\n        \n","metadata":{"execution":{"iopub.status.busy":"2024-01-06T02:31:05.690173Z","iopub.execute_input":"2024-01-06T02:31:05.690782Z","iopub.status.idle":"2024-01-06T02:31:05.736127Z","shell.execute_reply.started":"2024-01-06T02:31:05.690748Z","shell.execute_reply":"2024-01-06T02:31:05.734879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_fn(CFG,fold,folds,test_pl=0):\n\n\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    print(f\"### fold: {fold} ###\")\n    trn_idx = folds[folds['fold'] != fold].index\n    val_idx = folds[folds['fold'] == fold].index\n    val_folds = folds.loc[val_idx].reset_index(drop=True)\n    tra_folds = folds.loc[trn_idx]\n    print(val_folds.shape,tra_folds.shape)\n    if type(test_pl)!=type(0):\n        tra_folds = pd.concat([tra_folds,test_pl]).reset_index(drop=True)\n\n    train_dataset = TrainDataset(tra_folds,train=True, transform1=get_transforms(data='train',CFG=CFG),CFG=CFG)#get_transforms(data='train',CFG=CFG)\n    valid_dataset = TrainDataset(val_folds,train=False,transform1=get_transforms(data='valid',CFG=CFG),CFG=CFG)#\n\n\n    train_loader = DataLoader(train_dataset, batch_size=CFG.train.batch_size, shuffle=True, num_workers=8,pin_memory=True)\n    valid_loader = DataLoader(valid_dataset, batch_size=CFG.train.batch_size, shuffle=False, num_workers=8,pin_memory=True)\n\n    ###  model select ============\n    model = Model_iafoss(CFG,base_model=CFG.model.name).to(device)\n    # ============\n\n\n    ###  optim select ============\n    if CFG.train.optim==\"adam\":\n        optimizer = Adam(model.parameters(), lr=CFG.train.lr, amsgrad=False)\n    elif CFG.train.optim==\"adamw\":\n        optimizer = AdamW(model.parameters(), lr=CFG.train.lr,weight_decay=5e-5)\n    # ============\n\n    ###  scheduler select ============\n    if CFG.train.scheduler.name==\"cosine\":\n        scheduler = CosineAnnealingLR(optimizer, T_max=CFG.train.epochs, eta_min=CFG.train.scheduler.min_lr)\n    elif CFG.train.scheduler.name==\"cosine_warmup\":\n        scheduler =T.get_cosine_schedule_with_warmup(optimizer,\n        num_warmup_steps=len(train_loader)*CFG.train.scheduler.warmup,\n        num_training_steps=len(train_loader)*CFG.train.epochs)\n\n    # ============\n\n    ###  loss select ============\n    criterion=nn.CrossEntropyLoss()\n    print(criterion)\n    ###  loss select ============\n\n    softmax = nn.Softmax(dim=-1)\n    scaler = torch.cuda.amp.GradScaler()\n    best_score = 0\n    best_loss = np.inf\n    best_preds = None\n        \n    for epoch in range(CFG.train.epochs):\n        start_time = time.time()\n        model.train()\n        avg_loss = 0.\n\n        tk0 = tqdm(enumerate(train_loader), total=len(train_loader))\n\n        for i, (images, labels) in tk0:\n            optimizer.zero_grad()\n            \n            images = images.to(device)\n            labels = labels.to(device)\n            \n\n\n            if CFG.train.amp:\n                with autocast():\n                    y_preds = model(images)\n                    loss_ = criterion(y_preds,labels)\n\n                    loss=loss_\n\n                scaler.scale(loss).backward()\n\n                if (i+1)%CFG.train.ga_accum==0 or i==-1:\n                    scaler.step(optimizer)\n                    scaler.update()\n                    \n\n                        \n                    if CFG.train.scheduler.name==\"cosine_warmup\":\n                        \n                        scheduler.step()\n                        \n\n       \n\n            if CFG.train.scheduler.name==\"cosine\":\n                scheduler.step()\n\n\n            avg_loss += loss.item() / len(train_loader)\n        model.eval()\n        avg_val_loss = 0.\n        LOGITS = []\n        valid_labels = []\n        tk1 = tqdm(enumerate(valid_loader), total=len(valid_loader))\n\n        for i, (images, labels) in tk1:\n            images = images.to(device)\n            labels = labels.to(device)\n            with torch.no_grad():\n                with autocast(enabled=False):\n                    logits = model(images)\n                    loss =  criterion(logits,labels)\n\n            valid_labels.append(labels)\n            LOGITS.append(logits.detach())\n            avg_val_loss += loss.item() / len(valid_loader)\n        preds = softmax(torch.cat(LOGITS)).cpu().numpy().squeeze() \n        valid_labels = torch.cat(valid_labels).cpu().numpy()\n\n        print(preds.shape,valid_labels.shape)\n        print(valid_labels.mean(axis=0))\n\n        #each_auc,score =AUC(true=valid_labels,predict=preds)\n        AUC_score = roc_auc_score(valid_labels, preds,multi_class=\"ovr\")\n\n\n        elapsed = time.time() - start_time\n        print(f\"AUC_score  {AUC_score}\")\n\n\n        print(f'  Epoch {epoch+1} - avg_train_loss: {avg_loss:.6f}  avg_val_loss: {avg_val_loss:.6f}  time: {elapsed:.0f}s')\n\n        if best_loss>avg_val_loss:#pr_auc best\n            best_loss = avg_val_loss\n            print(f'  Epoch {epoch+1} - Save Best loss: {best_loss:.4f}')\n            torch.save(model.state_dict(), f'fold{fold}_{CFG.general.exp_num}_best_loss.pth')\n\n        if AUC_score>best_score:#pr_auc best\n            best_score = AUC_score\n            print(f'  Epoch {epoch+1} - Save Best AUC: {best_score:.4f}')\n            best_preds = preds\n            torch.save(model.state_dict(), f'fold{fold}_{CFG.general.exp_num}_best_AUC.pth')\n    for i in range(3):\n        col = f\"pred_{i}\"\n        val_folds[col]=best_preds[:,i]\n\n    return best_preds, valid_labels,val_folds\n\n\n\ndef eval_func(model, valid_loader, device,CFG):\n    model.to(device) \n    model.eval()\n\n    valid_labels = []\n    preds = []\n\n    tk1 = tqdm(enumerate(valid_loader), total=len(valid_loader))\n\n    for i, (images, labels,_) in tk1:\n        images = images.to(device)\n        labels = labels.to(device)\n        with torch.no_grad():\n            with autocast():\n                y_preds = model(images.float())\n                y_preds = y_preds.sigmoid()\n\n        valid_labels.append(labels.to('cpu').numpy())\n        preds.append(y_preds.to('cpu').numpy())\n    preds = np.concatenate(preds)\n    valid_labels = np.concatenate(valid_labels)\n\n    return preds,valid_labels\n\n\n\ndef inf_func(models, valid_loader, device):\n    softmax = nn.Softmax(dim=-1)\n    for model in models:\n        model.eval()\n\n    preds = []\n    \n\n    tk1 = tqdm(enumerate(valid_loader), total=len(valid_loader))\n\n    for i, images in tk1:\n        images = images.to(device,non_blocking=True)\n        with torch.no_grad():\n            with autocast():\n                y_preds = [softmax(m(images.float())) for m  in models]\n                y_preds  = torch.stack(y_preds).mean(0)\n\n        preds.append(y_preds)\n        \n    preds = torch.cat(preds).to('cpu').numpy()\n\n    return preds\n        \n\n    \ndef submit(CFG,test,DIR):\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    \n    models = []\n    for fold in   range(4):\n        model = Model_iafoss(CFG,base_model=CFG.model.name).to(device)\n        #if i==3:continue #AI049\n        model.load_state_dict(torch.load(f\"{DIR}/fold{fold}_{CFG.general.exp_num}_best_AUC.pth\", map_location=\"cpu\"))\n        models.append(model)\n        \n    valid_dataset = TestDataset(test,train=False,\n                                 transform1=get_transforms(data='valid',CFG=CFG),CFG=CFG)# \n    valid_loader = DataLoader(valid_dataset, batch_size=8, shuffle=False, num_workers=12,pin_memory=True)\n    tets_preds = inf_func(models, valid_loader, device)\n    \n    for i in range(3):\n        col = f\"pred_{i}\"\n        test[col]=tets_preds[:,i]    \n    \n    \n    return test","metadata":{"execution":{"iopub.status.busy":"2024-01-06T02:31:06.019445Z","iopub.execute_input":"2024-01-06T02:31:06.019794Z","iopub.status.idle":"2024-01-06T02:31:06.056336Z","shell.execute_reply.started":"2024-01-06T02:31:06.019764Z","shell.execute_reply":"2024-01-06T02:31:06.055286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold,StratifiedGroupKFold\nfolds = pd.read_csv(\"/kaggle/input/copy-of-pathologyaidemo2/train.csv\")\nskf = StratifiedKFold(n_splits=4,random_state=2023,shuffle=True)\nfor fold, ( _, val_) in enumerate(skf.split(X=folds, y=folds.target.to_numpy())):\n    folds.loc[val_ , \"fold\"] = fold\n\n    val_df = folds[folds[\"fold\"]==fold]\n    print(val_df[\"target\"].value_counts())\n    \nfolds.to_csv(\"fold.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-06T02:31:06.462100Z","iopub.execute_input":"2024-01-06T02:31:06.462781Z","iopub.status.idle":"2024-01-06T02:31:06.497809Z","shell.execute_reply.started":"2024-01-06T02:31:06.462746Z","shell.execute_reply":"2024-01-06T02:31:06.496784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from hydra import compose, initialize_config_dir\n\n\nclass CFG():\n    \"\"\"\n    hydraによる設定値の取得 (conf)\n    \"\"\"\n    @staticmethod\n    def get_cnf():\n        \"\"\"\n        設定値の辞書を取得\n        @return\n            cnf: OmegaDict\n        \"\"\"\n        conf_dir =\"/kaggle/working/\"\n        if not os.path.isdir(conf_dir):\n            print(f\"Can not find file: {conf_dir}.\")\n            sys.exit(-1)\n        with initialize_config_dir(version_base=None,config_dir=conf_dir):\n            cnf = compose(config_name=\"config.yaml\")\n            return cnf\n        \nc = CFG().get_cnf()\n","metadata":{"execution":{"iopub.status.busy":"2024-01-06T02:31:06.918728Z","iopub.execute_input":"2024-01-06T02:31:06.919084Z","iopub.status.idle":"2024-01-06T02:31:07.057382Z","shell.execute_reply.started":"2024-01-06T02:31:06.919055Z","shell.execute_reply":"2024-01-06T02:31:07.056271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nCFG = CFG().get_cnf()\nfolds = pd.read_csv(\"fold.csv\")\n\npreds = []\nvalid_labels = []\noof = pd.DataFrame()\n\ntest_pl = 0\n\n\n\nfor fold in range(4):\n    seed_torch(seed=CFG.general.seed)\n    _preds, _valid_labels,_oof_val = train_fn(CFG,fold,folds,test_pl)\n    preds.append(_preds)\n    valid_labels.append(_valid_labels)\n    oof = pd.concat([oof,_oof_val])\npreds = np.concatenate(preds)\nvalid_labels = np.concatenate(valid_labels)\n\nprint(f\"OOF\")\nAUC_score = roc_auc_score(valid_labels, preds,multi_class=\"ovr\")\nprint(f\"AUC_score  {AUC_score}\")\n\noof.to_csv(f\"oof_{CFG.general.exp_num}.csv\",index=False)\n\n\n\ntest = pd.read_csv(\"/kaggle/input/copy-of-pathologyaidemo2/sample_submission.csv\")\ntest = submit(CFG,test,DIR=\".\")\n\ntest.to_csv(f\"test_{CFG.general.exp_num}.csv\",index=False)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-01-06T02:31:07.345669Z","iopub.execute_input":"2024-01-06T02:31:07.346029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}