{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":61446,"databundleVersionId":6962461,"sourceType":"competition"},{"sourceId":1807973,"sourceType":"datasetVersion","datasetId":1074109},{"sourceId":150248402,"sourceType":"kernelVersion"}],"dockerImageVersionId":30588,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 6 version of 2.5d Cutting model baseline [training]\n#  author YOYOBAR\nhttps://www.kaggle.com/code/yoyobar/2-5d-cutting-model-baseline-training","metadata":{}},{"cell_type":"markdown","source":"**This code is base on [2.5d segmentaion baseline [training]](https://www.kaggle.com/code/tanakar/2-5d-segmentaion-baseline-training)**\nIf you think my code is useful,please upvote it ^w^.\n* Version6:\n1. *     using kidney_1_dense for training and kidney_3_dense for val\n2. *     image_size = 512\n3. *     useing DiceLoss\n4. *     norm_with_clip\n5. *     fix some bug\n\n\n* This version is correspond with [2.5d Cutting model baseline [inference]](https://www.kaggle.com/code/yoyobar/2-5d-cutting-model-baseline-inference) version3\n\n\n\nAccording to my experiments, using kidney_1_dense for training and kidney_3_dense for val is the best. You can even get 0.757, but using 2d model(se_resnext50_32x4d), you can set CFG.in_chans=1 to make this notebook as a 2d model training notebook.'","metadata":{}},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"!mkdir -p /root/.cache/torch/hub/checkpoints/\n!cp /kaggle/input/se-net-pretrained-imagenet-weights/* /root/.cache/torch/hub/checkpoints/\nimport torch as tc \nimport torch.nn as nn  \nimport numpy as np\nfrom tqdm import tqdm\nimport os,sys,cv2\nfrom torch.cuda.amp import autocast\nimport matplotlib.pyplot as plt\nimport albumentations as A\n!python -m pip install --no-index --find-links=/kaggle/input/pip-download-for-segmentation-models-pytorch segmentation-models-pytorch\nimport segmentation_models_pytorch as smp\nfrom albumentations.pytorch import ToTensorV2\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn.parallel import DataParallel\nfrom glob import glob","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-01-02T03:20:53.012427Z","iopub.execute_input":"2024-01-02T03:20:53.012977Z","iopub.status.idle":"2024-01-02T03:21:36.059446Z","shell.execute_reply.started":"2024-01-02T03:20:53.012948Z","shell.execute_reply":"2024-01-02T03:21:36.058300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# config","metadata":{}},{"cell_type":"code","source":"p_augm = 0.05 #0.5\n#add rotate.  less p_augm\n\nclass CFG:\n    # ============== pred target =============\n    target_size = 1\n\n    # ============== model CFG =============\n    model_name = 'Unet'\n    backbone = 'se_resnext50_32x4d'\n\n    in_chans = 1  #5 # 65\n    # ============== training CFG =============\n    image_size = 512\n    input_size=512\n\n    train_batch_size = 16\n    valid_batch_size = train_batch_size * 2\n\n    epochs = 50   #. 45 #30 #25\n    lr = 6e-5\n    chopping_percentile=1e-3\n    # ============== fold =============\n    valid_id = 1\n\n    rotate_p = 0.02 #0.25 #0.5\n    # ============== augmentation =============\n    train_aug_list = [\n        A.Rotate(limit=270, p= rotate_p),\n        A.RandomScale(scale_limit=(0.8,1.25),interpolation=cv2.INTER_CUBIC,p=p_augm),\n        A.RandomCrop(input_size, input_size,p=1),\n        A.RandomGamma(p=p_augm*2/3),\n        A.RandomBrightnessContrast(p=p_augm,),\n        A.GaussianBlur(p=p_augm),\n        A.MotionBlur(p=p_augm),\n        A.GridDistortion(num_steps=5, distort_limit=0.3, p=p_augm),\n        ToTensorV2(transpose_mask=True),\n    ]\n    train_aug = A.Compose(train_aug_list)\n    valid_aug_list = [\n        ToTensorV2(transpose_mask=True),\n    ]\n    valid_aug = A.Compose(valid_aug_list)","metadata":{"execution":{"iopub.status.busy":"2024-01-02T03:21:36.061301Z","iopub.execute_input":"2024-01-02T03:21:36.061626Z","iopub.status.idle":"2024-01-02T03:21:36.071515Z","shell.execute_reply.started":"2024-01-02T03:21:36.061598Z","shell.execute_reply":"2024-01-02T03:21:36.070548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class CustomModel(nn.Module):\n    def __init__(self, CFG, weight=None):\n        super().__init__()\n        self.model = smp.Unet(\n            encoder_name=CFG.backbone, \n            encoder_weights=weight,\n            in_channels=CFG.in_chans,\n            classes=CFG.target_size,\n            activation=None,\n        )\n\n    def forward(self, image):\n        output = self.model(image)\n        # output = output.squeeze(-1)\n        return output[:,0]#.sigmoid()\n\n\ndef build_model(weight=\"imagenet\"):\n    from dotenv import load_dotenv\n    load_dotenv()\n\n    print('model_name', CFG.model_name)\n    print('backbone', CFG.backbone)\n\n    model = CustomModel(CFG, weight)\n\n    return model.cuda()","metadata":{"execution":{"iopub.status.busy":"2024-01-02T03:21:36.073146Z","iopub.execute_input":"2024-01-02T03:21:36.073444Z","iopub.status.idle":"2024-01-02T03:21:36.084665Z","shell.execute_reply.started":"2024-01-02T03:21:36.073418Z","shell.execute_reply":"2024-01-02T03:21:36.083765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions","metadata":{}},{"cell_type":"code","source":"def min_max_normalization(x:tc.Tensor)->tc.Tensor:\n    \"\"\"input.shape=(batch,f1,...)\"\"\"\n    shape=x.shape\n    if x.ndim>2:\n        x=x.reshape(x.shape[0],-1)\n    \n    min_=x.min(dim=-1,keepdim=True)[0]\n    max_=x.max(dim=-1,keepdim=True)[0]\n    if min_.mean()==0 and max_.mean()==1:\n        return x.reshape(shape)\n    \n    x=(x-min_)/(max_-min_+1e-9)\n    return x.reshape(shape)\n\ndef norm_with_clip(x:tc.Tensor,smooth=1e-5):\n    dim=list(range(1,x.ndim))\n    mean=x.mean(dim=dim,keepdim=True)\n    std=x.std(dim=dim,keepdim=True)\n    x=(x-mean)/(std+smooth)\n    x[x>5]=(x[x>5]-5)*1e-3 +5\n    x[x<-3]=(x[x<-3]+3)*1e-3-3\n    return x\n\ndef add_noise(x:tc.Tensor,max_randn_rate=0.1,randn_rate=None,x_already_normed=False):\n    \"\"\"input.shape=(batch,f1,f2,...) output's var will be normalizate  \"\"\"\n    ndim=x.ndim-1\n    if x_already_normed:\n        x_std=tc.ones([x.shape[0]]+[1]*ndim,device=x.device,dtype=x.dtype)\n        x_mean=tc.zeros([x.shape[0]]+[1]*ndim,device=x.device,dtype=x.dtype)\n    else: \n        dim=list(range(1,x.ndim))\n        x_std=x.std(dim=dim,keepdim=True)\n        x_mean=x.mean(dim=dim,keepdim=True)\n    if randn_rate is None:\n        randn_rate=max_randn_rate*np.random.rand()*tc.rand(x_mean.shape,device=x.device,dtype=x.dtype)\n    cache=(x_std**2+(x_std*randn_rate)**2)**0.5\n    #https://blog.csdn.net/chaosir1991/article/details/106960408\n    \n    return (x-x_mean+tc.randn(size=x.shape,device=x.device,dtype=x.dtype)*randn_rate*x_std)/(cache+1e-7)\n\nclass Data_loader(Dataset):\n    def __init__(self,paths,is_label):\n        self.paths=paths\n        self.paths.sort()\n        self.is_label=is_label\n    \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self,index):\n        img=cv2.imread(self.paths[index],cv2.IMREAD_GRAYSCALE)\n        img=tc.from_numpy(img)\n        if self.is_label:\n            img=(img!=0).to(tc.uint8)*255\n        else:\n            img=img.to(tc.uint8)\n        return img\n\ndef load_data(paths,is_label=False):\n    data_loader=Data_loader(paths,is_label)\n    data_loader=DataLoader(data_loader, batch_size=16, num_workers=2)\n    data=[]\n    for x in tqdm(data_loader):\n        data.append(x)\n    x=tc.cat(data,dim=0)\n    del data\n    if not is_label:\n        ########################################################################\n        TH=x.reshape(-1).numpy()\n        index = -int(len(TH) * CFG.chopping_percentile)\n        TH:int = np.partition(TH, index)[index]\n        x[x>TH]=int(TH)\n        ########################################################################\n        TH=x.reshape(-1).numpy()\n        index = -int(len(TH) * CFG.chopping_percentile)\n        TH:int = np.partition(TH, -index)[-index]\n        x[x<TH]=int(TH)\n        ########################################################################\n        x=(min_max_normalization(x.to(tc.float16)[None])[0]*255).to(tc.uint8)\n    return x\n\n\n#https://www.kaggle.com/code/kashiwaba/sennet-hoa-train-unet-simple-baseline\ndef dice_coef(y_pred:tc.Tensor,y_true:tc.Tensor, thr=0.5, dim=(-1,-2), epsilon=0.001):\n    y_pred=y_pred.sigmoid()\n    y_true = y_true.to(tc.float32)\n    y_pred = (y_pred>thr).to(tc.float32)\n    inter = (y_true*y_pred).sum(dim=dim)\n    den = y_true.sum(dim=dim) + y_pred.sum(dim=dim)\n    dice = ((2*inter+epsilon)/(den+epsilon)).mean()\n    return dice\n\nclass DiceLoss(nn.Module):\n    def __init__(self, weight=None, size_average=True):\n        super(DiceLoss, self).__init__()\n\n    def forward(self, inputs, targets, smooth=1):\n        \n        #comment out if your model contains a sigmoid or equivalent activation layer\n        inputs = inputs.sigmoid()   \n        \n        #flatten label and prediction tensors\n        inputs = inputs.view(-1)\n        targets = targets.view(-1)\n        \n        intersection = (inputs * targets).sum()                            \n        dice = (2.*intersection + smooth)/(inputs.sum() + targets.sum() + smooth)  \n        \n        return 1 - dice\n\nclass Kaggld_Dataset(Dataset):\n    def __init__(self,x:list,y:list,arg=False):\n        super(Dataset,self).__init__()\n        self.x=x#list[(C,H,W),...]\n        self.y=y#list[(C,H,W),...]\n        self.image_size=CFG.image_size\n        self.in_chans=CFG.in_chans\n        self.arg=arg\n        if arg:\n            self.transform=CFG.train_aug\n        else: \n            self.transform=CFG.valid_aug\n\n    def __len__(self) -> int:\n        return sum([y.shape[0]-self.in_chans for y in self.y])\n    \n    def __getitem__(self,index):\n        i=0\n        for x in self.x:\n            if index>x.shape[0]-self.in_chans:\n                index-=x.shape[0]-self.in_chans\n                i+=1\n            else:\n                break\n        x=self.x[i]\n        y=self.y[i]\n        \n        print (f'x.shape[1] ={x.shape[1]}    x.shape[2]={x.shape[2]}')\n        \n        x_index= (x.shape[1]-self.image_size)//2 #np.random.randint(0,x.shape[1]-self.image_size)\n        y_index= (x.shape[2]-self.image_size)//2 # np.random.randint(0,x.shape[2]-self.image_size)\n        # i i+5 \n        x=x[index:index+self.in_chans   ,   x_index:x_index+self.image_size,   y_index:y_index+self.image_size]\n        # i+2\n        y=y[index+self.in_chans//2   ,      x_index:x_index+self.image_size,   y_index:y_index+self.image_size]\n\n        data = self.transform(image=x.numpy().transpose(1,2,0), mask=y.numpy())\n        x = data['image']\n        y = data['mask']>=127\n        if self.arg:\n            i=np.random.randint(4)\n            x=x.rot90(i,dims=(1,2))\n            y=y.rot90(i,dims=(0,1))\n            for i in range(3):\n                if np.random.randint(2):\n                    x=x.flip(dims=(i,))\n                    if i>=1:\n                        y=y.flip(dims=(i-1,))\n        return x,y#(uint8,uint8)\n","metadata":{"execution":{"iopub.status.busy":"2024-01-02T03:21:36.086937Z","iopub.execute_input":"2024-01-02T03:21:36.087241Z","iopub.status.idle":"2024-01-02T03:21:36.124624Z","shell.execute_reply.started":"2024-01-02T03:21:36.087216Z","shell.execute_reply":"2024-01-02T03:21:36.123567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data ","metadata":{}},{"cell_type":"code","source":"train_x=[]\ntrain_y=[]\n\nroot_path=\"/kaggle/input/blood-vessel-segmentation/\"\nparhs=[\"/kaggle/input/blood-vessel-segmentation/train/kidney_1_dense\"]\nfor i,path in enumerate(parhs):\n    if path==\"/kaggle/input/blood-vessel-segmentation/train/kidney_3_dense\":\n        continue\n    x=load_data(glob(f\"{path}/images/*\"),is_label=False)\n    print(x.shape)\n    y=load_data(glob(f\"{path}/labels/*\"),is_label=True)\n    print(y.shape)\n    train_x.append(x)\n    train_y.append(y)\n\n    #(C,H,W)\n\n    #aug\n    train_x.append(x.permute(1,2,0))\n    train_y.append(y.permute(1,2,0))\n    train_x.append(x.permute(2,0,1))\n    train_y.append(y.permute(2,0,1))\npath1=\"/kaggle/input/blood-vessel-segmentation/train/kidney_3_sparse\"\npath2=\"/kaggle/input/blood-vessel-segmentation/train/kidney_3_dense\"\npaths_y=glob(f\"{path2}/labels/*\")\npaths_x=[x.replace(\"labels\",\"images\").replace(\"dense\",\"sparse\") for x in paths_y]\n\nval_x=load_data(paths_x,is_label=False)\nprint(val_x.shape)\nval_y=load_data(paths_y,is_label=True)\nprint(val_y.shape)","metadata":{"execution":{"iopub.status.busy":"2024-01-02T03:21:36.125919Z","iopub.execute_input":"2024-01-02T03:21:36.126273Z","iopub.status.idle":"2024-01-02T03:25:46.819552Z","shell.execute_reply.started":"2024-01-02T03:21:36.126244Z","shell.execute_reply":"2024-01-02T03:25:46.817550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"tc.backends.cudnn.enabled = True\ntc.backends.cudnn.benchmark = True\n    \ntrain_dataset=Kaggld_Dataset(train_x,train_y,arg=True)\ntrain_dataset = DataLoader(train_dataset, batch_size=CFG.train_batch_size ,num_workers=2, shuffle=True, pin_memory=True)\nval_dataset=Kaggld_Dataset([val_x],[val_y])\nval_dataset = DataLoader(val_dataset, batch_size=CFG.valid_batch_size, num_workers=2, shuffle=False, pin_memory=True)\n\nmodel=build_model()\nmodel=DataParallel(model)\n\nloss_fc=DiceLoss()\n#loss_fn=nn.BCEWithLogitsLoss()\noptimizer=tc.optim.AdamW(model.parameters(),lr=CFG.lr)\nscaler=tc.cuda.amp.GradScaler()\nscheduler = tc.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=CFG.lr,\n                                                steps_per_epoch=len(train_dataset), epochs=CFG.epochs+1,\n                                                pct_start=0.1,)\nfor epoch in range(CFG.epochs):\n    model.train()\n    time=tqdm(range(len(train_dataset)))\n    losss=0\n    scores=0\n    for i,(x,y) in enumerate(train_dataset):\n        x=x.cuda().to(tc.float32)\n        y=y.cuda().to(tc.float32)\n        x=norm_with_clip(x.reshape(-1,*x.shape[2:])).reshape(x.shape)\n        x=add_noise(x,max_randn_rate=0.5,x_already_normed=True)\n        \n        with autocast():\n            pred=model(x)\n            loss=loss_fc(pred,y)\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        optimizer.zero_grad()\n        scheduler.step()\n        score=dice_coef(pred.detach(),y)\n        losss=(losss*i+loss.item())/(i+1)\n        scores=(scores*i+score)/(i+1)\n        time.set_description(f\"epoch:{epoch},loss:{losss:.4f},score:{scores:.4f},lr{optimizer.param_groups[0]['lr']:.4e}\")\n        time.update()\n        del loss,pred\n    time.close()\n    \n    model.eval()\n    time=tqdm(range(len(val_dataset)))\n    val_losss=0\n    val_scores=0\n    for i,(x,y) in enumerate(val_dataset):\n        x=x.cuda().to(tc.float32)\n        y=y.cuda().to(tc.float32)\n        x=norm_with_clip(x.reshape(-1,*x.shape[2:])).reshape(x.shape)\n\n        with autocast():\n            with tc.no_grad():\n                pred=model(x)\n                loss=loss_fc(pred,y)\n        score=dice_coef(pred.detach(),y)\n        val_losss=(val_losss*i+loss.item())/(i+1)\n        val_scores=(val_scores*i+score)/(i+1)\n        time.set_description(f\"val-->loss:{val_losss:.4f},score:{val_scores:.4f}\")\n        time.update()\n\n    time.close()\n    tc.save(model.module.state_dict(),f\"./{CFG.backbone}_{epoch}_loss{losss:.2f}_score{scores:.2f}_val_loss{val_losss:.2f}_val_score{val_scores:.2f}_midd_rot002.pt\")\n\ntime.close()","metadata":{"execution":{"iopub.status.busy":"2024-01-02T03:26:54.326913Z","iopub.execute_input":"2024-01-02T03:26:54.327784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}