{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Approach \n\n## Before version 11 is hengk's effb3 with aux loss training with 1e-3 for 10eps and 1e-4 for 5eps\n\n    I quickly get  (3.76 + 3.92 + 3.85 + 3.7 + 3.6)/5 avg 3.766 LB: 4.44\n\n\n\n## After Version 11 is efficientnet v2\n\n### I keep getting CV 3.6 - 3.7 and no improve for a whole month \n\n    Because at first event I notice efficientnetv2 strongly overfitting, But due to lack of experiment ,\n\n    I keep thinking use current data augments and dropout is enough for regularization , \n\n    I just keeping suspect that I have made some implement detail bugs in using Timm's efficientnetv2 \n\n\n1. First I try attach aux loss to different layers like block 4 or block1 , add more layer or units in aux loss\n\n    Not improve\n\n2. Then I try different pretraining weight like tf_efficientv2_rw_* or img21k \n\n    Have a impact on converge speed but no improve on map\n\n3. Then I take a divide on efficientnetv2 papers and source code for detail \n\n    like drop_path_rate, input stem, and efficientnetv2 new features like progress regulizer\n    \n4. I also try multi output like 6 classes output or freeze layers finetune\n    \n    \n### I keep observe efficient model is keeping overfitting and very very unstable\n\n1. I made some post in discuss then try gradient accumulate to increase batch size to make trainning more stable but not improve on map\n\n2. So I think I made some \"Simple but basicly Mistake\" becasue so many people achieve better socres with efficientnetv2 seem \"not many change in hengck's code \"\n\n3. So Finanly I put my attation back to overfitting , I add some data augmentation to make overfitting later\n\n    \n     So finnaly I achive about (0.392 + 0.392 + 0.385 + 0.381 + 0.374)  LB score 0.452\n\n# Refferences:\n\n1. hengck's efficientnetb3a with aux loss https://www.kaggle.com/c/siim-covid19-detection/discussion/240233\n2. pytorch Amp with gradiant accumulate  https://pytorch.org/docs/stable/notes/amp_examples.html#gradient-accumulation\n3. lovasz loss https://github.com/bermanmaxim/LovaszSoftmax\n4. Data augment for effB7  https://www.kaggle.com/southsakura/kaggle\n\n# Furthur Improvements\n\n1. tune data augment , now data augment seems too much make model a little damping\n\n2. shall use group stratify k-fold to avoid leaky rather than only stratify k-fold\n\n3. a more efficient way to use aux loss or multi output\n\n4. Transfer learning or unsupervise pretraning","metadata":{}},{"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nimport os\nimport sys\n\nimport pandas as pd\nimport math\nimport csv\n\nfrom timeit import default_timer as timer\n\nfrom datetime import datetime","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:32.603254Z","iopub.execute_input":"2021-08-10T10:09:32.603636Z","iopub.status.idle":"2021-08-10T10:09:32.611926Z","shell.execute_reply.started":"2021-08-10T10:09:32.603596Z","shell.execute_reply":"2021-08-10T10:09:32.611036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# madgrad\n# https://github.com/facebookresearch/madgrad\n\n!pip install madgrad\n\nfrom madgrad import MADGRAD","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:32.613553Z","iopub.execute_input":"2021-08-10T10:09:32.614188Z","iopub.status.idle":"2021-08-10T10:09:39.336902Z","shell.execute_reply.started":"2021-08-10T10:09:32.614139Z","shell.execute_reply":"2021-08-10T10:09:39.335737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip install wandb --upgrade","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:39.341373Z","iopub.execute_input":"2021-08-10T10:09:39.341697Z","iopub.status.idle":"2021-08-10T10:09:39.348340Z","shell.execute_reply.started":"2021-08-10T10:09:39.341663Z","shell.execute_reply":"2021-08-10T10:09:39.347349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import wandb\n#api_key = 'e5260514cd613164ad5c3fd66ad7e909beaf820f'\n#wandb.login(key=api_key)","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:39.350354Z","iopub.execute_input":"2021-08-10T10:09:39.350872Z","iopub.status.idle":"2021-08-10T10:09:39.357495Z","shell.execute_reply.started":"2021-08-10T10:09:39.350808Z","shell.execute_reply":"2021-08-10T10:09:39.356586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install timm","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:39.358943Z","iopub.execute_input":"2021-08-10T10:09:39.359456Z","iopub.status.idle":"2021-08-10T10:09:45.769304Z","shell.execute_reply.started":"2021-08-10T10:09:39.359413Z","shell.execute_reply":"2021-08-10T10:09:45.767927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport torch.optim as optim\nfrom torch.optim import Adam\n\nfrom torch.nn.parallel.data_parallel import data_parallel\n\nfrom torch.utils.data.dataset import Dataset\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data.sampler import *\n\nimport collections\nfrom collections import defaultdict\n\nimport timm\nfrom timm.models.efficientnet import *\n\nimport torch.cuda.amp as amp","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:45.771357Z","iopub.execute_input":"2021-08-10T10:09:45.771757Z","iopub.status.idle":"2021-08-10T10:09:45.781419Z","shell.execute_reply.started":"2021-08-10T10:09:45.771710Z","shell.execute_reply":"2021-08-10T10:09:45.780408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Unpack file","metadata":{}},{"cell_type":"code","source":"_data_dir = \"/kaggle/input/covid19512/\"","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:45.782790Z","iopub.execute_input":"2021-08-10T10:09:45.783207Z","iopub.status.idle":"2021-08-10T10:09:45.790776Z","shell.execute_reply.started":"2021-08-10T10:09:45.783150Z","shell.execute_reply":"2021-08-10T10:09:45.790043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls {_data_dir}train/000a312787f2.png","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:45.794265Z","iopub.execute_input":"2021-08-10T10:09:45.794583Z","iopub.status.idle":"2021-08-10T10:09:46.517618Z","shell.execute_reply.started":"2021-08-10T10:09:45.794559Z","shell.execute_reply":"2021-08-10T10:09:46.516634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mask_example = cv2.imread(_data_dir + \"mask/000a312787f2.png\")","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:46.521945Z","iopub.execute_input":"2021-08-10T10:09:46.522248Z","iopub.status.idle":"2021-08-10T10:09:46.536347Z","shell.execute_reply.started":"2021-08-10T10:09:46.522217Z","shell.execute_reply":"2021-08-10T10:09:46.535599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(mask_example, cmap=\"bone\")","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:46.539174Z","iopub.execute_input":"2021-08-10T10:09:46.539512Z","iopub.status.idle":"2021-08-10T10:09:46.692169Z","shell.execute_reply.started":"2021-08-10T10:09:46.539488Z","shell.execute_reply":"2021-08-10T10:09:46.691184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_example = cv2.imread(_data_dir + \"train/000a312787f2.png\")","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:46.693676Z","iopub.execute_input":"2021-08-10T10:09:46.694026Z","iopub.status.idle":"2021-08-10T10:09:46.705023Z","shell.execute_reply.started":"2021-08-10T10:09:46.693988Z","shell.execute_reply":"2021-08-10T10:09:46.704168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(train_example, cmap=\"bone\")","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:46.706403Z","iopub.execute_input":"2021-08-10T10:09:46.707008Z","iopub.status.idle":"2021-08-10T10:09:46.868793Z","shell.execute_reply.started":"2021-08-10T10:09:46.706969Z","shell.execute_reply":"2021-08-10T10:09:46.867773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare matedata ","metadata":{}},{"cell_type":"code","source":"study_df = pd.read_csv(\"/kaggle/input/siim-covid19-detection/train_study_level.csv\")\nimage_df = pd.read_csv(\"/kaggle/input/siim-covid19-detection/train_image_level.csv\")\nmeta_df = pd.read_csv(_data_dir + \"meta.csv\")\nfold_df = pd.read_csv(_data_dir + \"updated_iamge_level.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:46.870165Z","iopub.execute_input":"2021-08-10T10:09:46.870529Z","iopub.status.idle":"2021-08-10T10:09:46.943274Z","shell.execute_reply.started":"2021-08-10T10:09:46.870493Z","shell.execute_reply":"2021-08-10T10:09:46.942462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:46.944515Z","iopub.execute_input":"2021-08-10T10:09:46.945023Z","iopub.status.idle":"2021-08-10T10:09:46.955817Z","shell.execute_reply.started":"2021-08-10T10:09:46.944985Z","shell.execute_reply":"2021-08-10T10:09:46.954967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_duplicateList_path = _data_dir + 'dublicate.txt'","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:46.957196Z","iopub.execute_input":"2021-08-10T10:09:46.957745Z","iopub.status.idle":"2021-08-10T10:09:46.964987Z","shell.execute_reply.started":"2021-08-10T10:09:46.957708Z","shell.execute_reply":"2021-08-10T10:09:46.964141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_duplicateList = []\nwith open(_duplicateList_path, newline='') as csvfile:\n    spamreader = csv.reader(csvfile, delimiter=' ', quotechar='|')\n    for row in spamreader:\n        _duplicateList += row\n\n_duplicateList[:5]","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:46.966257Z","iopub.execute_input":"2021-08-10T10:09:46.966818Z","iopub.status.idle":"2021-08-10T10:09:46.977708Z","shell.execute_reply.started":"2021-08-10T10:09:46.966781Z","shell.execute_reply":"2021-08-10T10:09:46.976696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"code","source":"image_size = 512\ndata_dir = _data_dir\nstudy_name_to_label = {\n    'Negative for Pneumonia'  :0,\n    'Typical Appearance'      :1,\n    'Indeterminate Appearance':2,\n    'Atypical Appearance'     :3,\n}\nstudy_label_to_name = { v:k for k,v in study_name_to_label.items()}\nnum_study_label = len(study_name_to_label)","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:46.979504Z","iopub.execute_input":"2021-08-10T10:09:46.979841Z","iopub.status.idle":"2021-08-10T10:09:46.984740Z","shell.execute_reply.started":"2021-08-10T10:09:46.979807Z","shell.execute_reply":"2021-08-10T10:09:46.983690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_fold(mode='train-0'):\n    if 'train' in mode:\n        df = study_df.copy()\n        df.loc[:, 'id'] = df.id.str.replace('_study', '')\n        df = df.rename(columns={'id': 'study_id'})\n        \n        fold = fold_df.copy()\n        fold.loc[:, 'id'] = fold.id.str.replace('_image', '')\n        fold = fold.rename(columns={'id': 'image'})\n\n        #---\n        df = df.merge(fold, left_on='study_id', right_on='StudyInstanceUID')\n        \n        df = df[~df['image'].isin(_duplicateList)]\n\n        #---\n        fold = int(mode[-1])\n        df_train = df[df.fold != fold].reset_index(drop=True)\n        df_valid = df[df.fold == fold].reset_index(drop=True)\n        return df_train, df_valid","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:46.986406Z","iopub.execute_input":"2021-08-10T10:09:46.987071Z","iopub.status.idle":"2021-08-10T10:09:46.995933Z","shell.execute_reply.started":"2021-08-10T10:09:46.987034Z","shell.execute_reply":"2021-08-10T10:09:46.995097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset ","metadata":{}},{"cell_type":"code","source":"class SiimDataset(Dataset):\n    def __init__(self, df, augment=None):\n        super().__init__()\n        self.df = df\n        self.augment = augment\n        self.length = len(df)\n\n    def __str__(self):\n        string  = ''\n        string += '\\tlen = %d\\n'%len(self)\n        string += '\\tdf  = %s\\n'%str(self.df.shape)\n\n        string += '\\tlabel distribution\\n'\n        for i in range(num_study_label):\n            n = self.df[study_label_to_name[i]].sum()\n            string += '\\t\\t %d %26s: %5d (%0.4f)\\n'%(i, study_label_to_name[i], n, n/len(self.df) )\n        return string\n\n\n    def __len__(self):\n        return self.length\n\n    def __getitem__(self, index):\n        d = self.df.iloc[index]\n\n        #image_file = data_dir + '/%s_640/%s/%s/%s.png' % (d.set, d.study, d.series, d.image)\n        image_file = data_dir + '/train/%s.png' % (d.image)\n        image = cv2.imread(image_file,cv2.IMREAD_GRAYSCALE)\n        onehot = d[study_name_to_label.keys()].values\n\n        mask_file = data_dir + '/mask/%s.png' % (d.image)\n        mask = cv2.imread(mask_file,cv2.IMREAD_GRAYSCALE)\n        \n        if str(mask) == \"None\":\n            mask = np.zeros((image_size, image_size))\n\n        r = {\n            'index' : index,\n            'd' : d,\n            'image' : image,\n            'mask' : mask,\n            'onehot' : onehot,\n        }\n        if self.augment is not None: r = self.augment(r)\n        return r\n\n\ndef null_collate(batch):\n    collate = defaultdict(list)\n\n    for r in batch:\n        for k, v in r.items():\n            collate[k].append(v)\n\n    # ---\n    batch_size = len(batch)\n    onehot = np.ascontiguousarray(np.stack(collate['onehot'])).astype(np.float32)\n    collate['onehot'] = torch.from_numpy(onehot)\n\n    image = np.stack(collate['image'])\n    image = image.reshape(batch_size, 1, image_size,image_size).repeat(3,1)\n    image = np.ascontiguousarray(image)\n    image = image.astype(np.float32) / 255\n    collate['image'] = torch.from_numpy(image)\n\n    mask = np.stack(collate['mask'])\n    mask = mask.reshape(batch_size, 1, image_size,image_size)\n    mask = np.ascontiguousarray(mask)\n    mask = mask.astype(np.float32) / 255\n    collate['mask'] = torch.from_numpy(mask)\n\n    return collate","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:46.997295Z","iopub.execute_input":"2021-08-10T10:09:46.997712Z","iopub.status.idle":"2021-08-10T10:09:47.014364Z","shell.execute_reply.started":"2021-08-10T10:09:46.997677Z","shell.execute_reply":"2021-08-10T10:09:47.013440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# what is ascontiguousarray\n#x = np.arange(6).reshape(2,3)\n\"\"\"\nx = np.array([[1,1,0],[2,0,1], [1,2,3,]])\ny = np.ascontiguousarray(x, dtype=np.float32)\nprint(x.flags['C_CONTIGUOUS'])\nprint(x)\nprint(y)\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:47.015778Z","iopub.execute_input":"2021-08-10T10:09:47.016273Z","iopub.status.idle":"2021-08-10T10:09:47.030231Z","shell.execute_reply.started":"2021-08-10T10:09:47.016230Z","shell.execute_reply":"2021-08-10T10:09:47.029273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\ndf_train, df_valid = make_fold(mode='train-1')\n\ndataset = SiimDataset(df_valid) #null_augment\nprint(dataset)\n\n\nfor i in range(10):\n    i = np.random.choice(len(dataset))\n    r = dataset[i]\n\n    print('index ' , i)\n    print(r['d'])\n    print(r['onehot'])\n    print('')\n    plt.figure()\n    plt.imshow(r['image'])\n    plt.figure()\n    plt.imshow(r['mask'], cmap=\"bone\")\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:47.032030Z","iopub.execute_input":"2021-08-10T10:09:47.032512Z","iopub.status.idle":"2021-08-10T10:09:47.043731Z","shell.execute_reply.started":"2021-08-10T10:09:47.032471Z","shell.execute_reply":"2021-08-10T10:09:47.042757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nloader = DataLoader(\n    dataset,\n    sampler = RandomSampler(dataset),\n    batch_size  = 8,\n    drop_last   = True,\n    num_workers = 0,\n    pin_memory  = True,\n    collate_fn  = null_collate,\n)\nfor t,batch in enumerate(loader):\n    if t>5: break\n\n    print(t, '-----------')\n    print('index : ', batch['index'])\n    print('image : ')\n    print('\\t', batch['image'].shape, batch['image'].is_contiguous())\n    print('mask : ')\n    print('\\t', batch['mask'].shape, batch['mask'].is_contiguous())\n    print('onehot : ')\n    print('\\t', batch['onehot'])\n    print('\\t', batch['onehot'].shape, batch['onehot'].is_contiguous())\n    print('')\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:47.044980Z","iopub.execute_input":"2021-08-10T10:09:47.047194Z","iopub.status.idle":"2021-08-10T10:09:47.054032Z","shell.execute_reply.started":"2021-08-10T10:09:47.047149Z","shell.execute_reply":"2021-08-10T10:09:47.052867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model ","metadata":{}},{"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self):\n        super(Net, self).__init__()\n\n        #e = efficientnet_b3a(pretrained=True, drop_rate=0.3, drop_path_rate=0.2)\n        #e = efficientnetv2_rw_m(pretrained=True, drop_rate=0.5, drop_path_rate=0.2)\n        #e = tf_efficientnetv2_m_in21ft1k(pretrained=True, drop_rate=0.3, drop_path_rate=0.2)\n        e = tf_efficientnetv2_m(pretrained=True, drop_path_rate=0.4)\n        \n        self.b0 = nn.Sequential(\n            e.conv_stem,\n            e.bn1,\n            e.act1,\n        )\n        self.b1 = e.blocks[0]\n        self.b2 = e.blocks[1]\n        self.b3 = e.blocks[2]\n        self.b4 = e.blocks[3]\n        self.b5 = e.blocks[4]\n        self.b6 = e.blocks[5]\n        self.b7 = e.blocks[6]\n        self.b8 = nn.Sequential(\n            e.conv_head, #384, 1536\n            e.bn2,\n            e.act2,\n        )\n\n        self.logit = nn.Linear(1280,num_study_label)\n        self.mask = nn.Sequential(\n            nn.Conv2d(176, 128, kernel_size=3, padding=1),\n            nn.BatchNorm2d(128),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(128, 128, kernel_size=3, padding=1),\n            nn.BatchNorm2d(128),\n            nn.ReLU(inplace=True),\n\n            #nn.Conv2d(128, 128, kernel_size=3, padding=1),\n            #nn.BatchNorm2d(128),\n            #nn.ReLU(inplace=True),\n\n            nn.Conv2d(128, 1, kernel_size=1, padding=0),\n        )\n\n\n    # @torch.cuda.amp.autocast()\n    def forward(self, image):\n        batch_size = len(image)\n        x = 2*image-1     # ; print('input ',   x.shape)\n        \"\"\"\n        rw_m\n        torch.Size([2, 32, 256, 256])\n        torch.Size([2, 32, 256, 256])\n        torch.Size([2, 56, 128, 128])\n        torch.Size([2, 80, 64, 64])\n        torch.Size([2, 152, 32, 32])\n        torch.Size([2, 192, 32, 32])\n        torch.Size([2, 328, 16, 16])\n        torch.Size([2, 2152, 16, 16])\n        \"\"\"\n        x = self.b0(x) ; #print (x.shape)  # torch.Size([2, 32, 256, 256])\n        x = self.b1(x) ; #print (x.shape)  # torch.Size([2, 32, 256, 256])\n        x = self.b2(x) ; #print (x.shape)  # torch.Size([2, 56, 128, 128])\n        x = self.b3(x) ; #print (x.shape)  # torch.Size([2, 80, 64, 64])\n        x = self.b4(x) ; #print (x.shape)  # torch.Size([2, 152, 32, 32])\n        x = self.b5(x) ; #print (x.shape)  # torch.Size([2, 192, 32, 32])\n\n        #------------\n        mask = self.mask(x)\n        #-------------\n\n        x = self.b6(x) ; #print (x.shape)  # torch.Size([2, 328, 16, 16])\n        \n        x = self.b7(x) ; #print (x.shape)  # torch.Size([2, 512, 16, 16])\n        \n        x = self.b8(x) ; #print (x.shape)  # torch.Size([2, 2152, 16, 16])\n        x = F.adaptive_avg_pool2d(x,1).reshape(batch_size,-1)\n        x = F.dropout(x, 0.5, training=self.training)\n\n        logit = self.logit(x)\n        return logit, mask","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:47.059133Z","iopub.execute_input":"2021-08-10T10:09:47.059465Z","iopub.status.idle":"2021-08-10T10:09:47.073622Z","shell.execute_reply.started":"2021-08-10T10:09:47.059436Z","shell.execute_reply":"2021-08-10T10:09:47.072128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nbatch_size = 2\nC, H, W = 3, 512, 512\n#C, H, W = 3, 640, 640\nimage = torch.randn(batch_size, C, H, W).cuda()\nmask  = torch.randn(batch_size, num_study_label, H, W).cuda()\n\nnet = Net().cuda()\nlogit, mask = net(image)\n\nprint(image.shape)\nprint(logit.shape)\nprint(mask.shape)\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:47.076631Z","iopub.execute_input":"2021-08-10T10:09:47.077148Z","iopub.status.idle":"2021-08-10T10:09:47.087441Z","shell.execute_reply.started":"2021-08-10T10:09:47.077101Z","shell.execute_reply":"2021-08-10T10:09:47.086468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"class Logger(object):\n    def __init__(self):\n        self.terminal = sys.stdout  #stdout\n        self.file = None\n\n    def open(self, file, mode=None):\n        if mode is None: mode ='w'\n        self.file = open(file, mode)\n\n    def write(self, message, is_terminal=1, is_file=1 ):\n        if '\\r' in message: is_file=0\n\n        if is_terminal == 1:\n            self.terminal.write(message)\n            self.terminal.flush()\n            #time.sleep(1)\n\n        if is_file == 1:\n            self.file.write(message)\n            self.file.flush()\n\n    def flush(self):\n        # this flush method is needed for python 3 compatibility.\n        # this handles the flush command by doing nothing.\n        # you might want to specify some extra behavior here.\n        pass","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:47.089010Z","iopub.execute_input":"2021-08-10T10:09:47.089443Z","iopub.status.idle":"2021-08-10T10:09:47.099130Z","shell.execute_reply.started":"2021-08-10T10:09:47.089400Z","shell.execute_reply":"2021-08-10T10:09:47.097745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_to_str(t, mode='min'):\n    if mode=='min':\n        t  = int(t)/60\n        hr = t//60\n        min = t%60\n        return '%2d hr %02d min'%(hr,min)\n\n    elif mode=='sec':\n        t   = int(t)\n        min = t//60\n        sec = t%60\n        return '%2d min %02d sec'%(min,sec)\n\n    else:\n        raise NotImplementedError","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:47.100711Z","iopub.execute_input":"2021-08-10T10:09:47.101115Z","iopub.status.idle":"2021-08-10T10:09:47.108904Z","shell.execute_reply.started":"2021-08-10T10:09:47.101074Z","shell.execute_reply":"2021-08-10T10:09:47.107421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_learning_rate(optimizer):\n    lr=[]\n    for param_group in optimizer.param_groups:\n        lr +=[ param_group['lr'] ]\n\n    assert(len(lr)==1) #we support only one param_group\n    lr = lr[0]\n\n    return lr","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:47.110647Z","iopub.execute_input":"2021-08-10T10:09:47.111077Z","iopub.status.idle":"2021-08-10T10:09:47.117898Z","shell.execute_reply.started":"2021-08-10T10:09:47.111020Z","shell.execute_reply":"2021-08-10T10:09:47.116850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lovasz loss","metadata":{}},{"cell_type":"code","source":"from torch.autograd import Variable","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:47.119702Z","iopub.execute_input":"2021-08-10T10:09:47.120147Z","iopub.status.idle":"2021-08-10T10:09:47.126885Z","shell.execute_reply.started":"2021-08-10T10:09:47.120104Z","shell.execute_reply":"2021-08-10T10:09:47.126057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lovasz_grad(gt_sorted):\n    \"\"\"\n    Computes gradient of the Lovasz extension w.r.t sorted errors\n    See Alg. 1 in paper\n    \"\"\"\n    p = len(gt_sorted)\n    gts = gt_sorted.sum()\n    intersection = gts - gt_sorted.float().cumsum(0)\n    union = gts + (1 - gt_sorted).float().cumsum(0)\n    jaccard = 1. - intersection / union\n    if p > 1: # cover 1-pixel case\n        jaccard[1:p] = jaccard[1:p] - jaccard[0:-1]\n    return jaccard\n\n# --------------------------- BINARY LOSSES ---------------------------\n\n\ndef lovasz_hinge(logits, labels, per_image=True, ignore=None):\n    \"\"\"\n    Binary Lovasz hinge loss\n      logits: [B, H, W] Variable, logits at each pixel (between -\\infty and +\\infty)\n      labels: [B, H, W] Tensor, binary ground truth masks (0 or 1)\n      per_image: compute the loss per image instead of per batch\n      ignore: void class id\n    \"\"\"\n    if per_image:\n        loss = mean(lovasz_hinge_flat(*flatten_binary_scores(log.unsqueeze(0), lab.unsqueeze(0), ignore))\n                          for log, lab in zip(logits, labels))\n    else:\n        loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))\n    return loss\n\n\ndef lovasz_hinge_flat(logits, labels):\n    \"\"\"\n    Binary Lovasz hinge loss\n      logits: [P] Variable, logits at each prediction (between -\\infty and +\\infty)\n      labels: [P] Tensor, binary ground truth labels (0 or 1)\n      ignore: label to ignore\n    \"\"\"\n    if len(labels) == 0:\n        # only void pixels, the gradients should be 0\n        return logits.sum() * 0.\n    signs = 2. * labels.float() - 1.\n    errors = (1. - logits * Variable(signs))\n    errors_sorted, perm = torch.sort(errors, dim=0, descending=True)\n    perm = perm.data\n    gt_sorted = labels[perm]\n    grad = lovasz_grad(gt_sorted)\n    loss = torch.dot(F.relu(errors_sorted), Variable(grad))\n    return loss\n\n\ndef flatten_binary_scores(scores, labels, ignore=None):\n    \"\"\"\n    Flattens predictions in the batch (binary case)\n    Remove labels equal to 'ignore'\n    \"\"\"\n    scores = scores.view(-1)\n    labels = labels.view(-1)\n    if ignore is None:\n        return scores, labels\n    valid = (labels != ignore)\n    vscores = scores[valid]\n    vlabels = labels[valid]\n    return vscores, vlabels\n\n# --------------------------- multi LOSSES ---------------------------\n\ndef lovasz_softmax(probas, labels, classes='present', per_image=False, ignore=None):\n    \"\"\"\n    Multi-class Lovasz-Softmax loss\n      probas: [B, C, H, W] Variable, class probabilities at each prediction (between 0 and 1).\n              Interpreted as binary (sigmoid) output with outputs of size [B, H, W].\n      labels: [B, H, W] Tensor, ground truth labels (between 0 and C - 1)\n      classes: 'all' for all, 'present' for classes present in labels, or a list of classes to average.\n      per_image: compute the loss per image instead of per batch\n      ignore: void class labels\n    \"\"\"\n    if per_image:\n        loss = mean(lovasz_softmax_flat(*flatten_probas(prob.unsqueeze(0), lab.unsqueeze(0), ignore), classes=classes)\n                          for prob, lab in zip(probas, labels))\n    else:\n        loss = lovasz_softmax_flat(*flatten_probas(probas, labels, ignore), classes=classes)\n    return loss\n\n\ndef lovasz_softmax_flat(probas, labels, classes='present'):\n    \"\"\"\n    Multi-class Lovasz-Softmax loss\n      probas: [P, C] Variable, class probabilities at each prediction (between 0 and 1)\n      labels: [P] Tensor, ground truth labels (between 0 and C - 1)\n      classes: 'all' for all, 'present' for classes present in labels, or a list of classes to average.\n    \"\"\"\n    if probas.numel() == 0:\n        # only void pixels, the gradients should be 0\n        return probas * 0.\n    C = probas.size(1)\n    losses = []\n    class_to_sum = list(range(C)) if classes in ['all', 'present'] else classes\n    for c in class_to_sum:\n        fg = (labels == c).float() # foreground for class c\n        if (classes is 'present' and fg.sum() == 0):\n            continue\n        if C == 1:\n            if len(classes) > 1:\n                raise ValueError('Sigmoid output possible only with 1 class')\n            class_pred = probas[:, 0]\n        else:\n            class_pred = probas[:, c]\n        errors = (Variable(fg) - class_pred).abs()\n        errors_sorted, perm = torch.sort(errors, 0, descending=True)\n        perm = perm.data\n        fg_sorted = fg[perm]\n        losses.append(torch.dot(errors_sorted, Variable(lovasz_grad(fg_sorted))))\n    return mean(losses)\n\n\ndef flatten_probas(probas, labels, ignore=None):\n    \"\"\"\n    Flattens predictions in the batch\n    \"\"\"\n    if probas.dim() == 3:\n        # assumes output of a sigmoid layer\n        B, H, W = probas.size()\n        probas = probas.view(B, 1, H, W)\n    B, C, H, W = probas.size()\n    probas = probas.permute(0, 2, 3, 1).contiguous().view(-1, C)  # B * H * W, C = P, C\n    labels = labels.view(-1)\n    if ignore is None:\n        return probas, labels\n    valid = (labels != ignore)\n    vprobas = probas[valid.nonzero().squeeze()]\n    vlabels = labels[valid]\n    return vprobas, vlabels\n\ndef isnan(x):\n    return x != x\n    \n    \ndef mean(l, ignore_nan=False, empty=0):\n    \"\"\"\n    nanmean compatible with generators.\n    \"\"\"\n    l = iter(l)\n    if ignore_nan:\n        l = ifilterfalse(isnan, l)\n    try:\n        n = 1\n        acc = next(l)\n    except StopIteration:\n        if empty == 'raise':\n            raise ValueError('Empty mean')\n        return empty\n    for n, v in enumerate(l, 2):\n        acc += v\n    if n == 1:\n        return acc\n    return acc / n","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:47.128570Z","iopub.execute_input":"2021-08-10T10:09:47.129034Z","iopub.status.idle":"2021-08-10T10:09:47.154037Z","shell.execute_reply.started":"2021-08-10T10:09:47.128993Z","shell.execute_reply":"2021-08-10T10:09:47.153046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"class AmpNet(Net):\n    @torch.cuda.amp.autocast()\n    def forward(self,*args):\n        return super(AmpNet, self).forward(*args)\n\nis_mixed_precision = True  #True #False","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:47.155509Z","iopub.execute_input":"2021-08-10T10:09:47.155905Z","iopub.status.idle":"2021-08-10T10:09:47.165042Z","shell.execute_reply.started":"2021-08-10T10:09:47.155866Z","shell.execute_reply":"2021-08-10T10:09:47.164205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Metric ","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import average_precision_score","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:47.167119Z","iopub.execute_input":"2021-08-10T10:09:47.167748Z","iopub.status.idle":"2021-08-10T10:09:47.173527Z","shell.execute_reply.started":"2021-08-10T10:09:47.167701Z","shell.execute_reply":"2021-08-10T10:09:47.172758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def np_loss_cross_entropy(probability, truth):\n    batch_size = len(probability)\n    truth = truth.reshape(-1)\n    p = probability[np.arange(batch_size),truth]\n    loss = -np.log(np.clip(p,1e-6,1))\n    loss = loss.mean()\n    return loss\n\n\ndef np_metric_map_curve_by_class(probability, truth):\n    num_sample, num_label = probability.shape\n    score = []\n    for i in range(num_label):\n        s = average_precision_score(truth==i, probability[:,i])\n        score.append(s)\n    score = np.array(score)\n    return score","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:47.175896Z","iopub.execute_input":"2021-08-10T10:09:47.176754Z","iopub.status.idle":"2021-08-10T10:09:47.184144Z","shell.execute_reply.started":"2021-08-10T10:09:47.176719Z","shell.execute_reply":"2021-08-10T10:09:47.183308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Augment","metadata":{}},{"cell_type":"code","source":"# #--- flip ---\ndef do_random_hflip(image, mask):\n    if np.random.rand()>0.5:\n        image = cv2.flip(image,1)\n        mask = cv2.flip(mask,1)\n    return image, mask\n\n\n# #--- geometric ---\ndef do_random_rotate(image, mask, mag=15 ):\n    angle = np.random.uniform(-1, 1)*mag\n\n    height, width = image.shape[:2]\n    cx, cy = width // 2, height // 2\n\n    transform = cv2.getRotationMatrix2D((cx, cy), -angle, 1.0)\n    image = cv2.warpAffine(image, transform, (width, height), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n    mask = cv2.warpAffine(mask, transform, (width, height), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n\n    return image, mask\n\n\ndef do_random_scale( image, mask, mag=0.1 ):\n    s = 1 + np.random.uniform(-1, 1)*mag\n    height, width = image.shape[:2]\n    w,h = int(s*width), int(s*height)\n    if (h,w)==image.shape[:2]:\n        return image, mask\n\n    dst = np.array([\n        [0,0],[width,height], [width,0], #[0,height],\n    ]).astype(np.float32)\n\n    if s>1:\n        dx = np.random.choice(w-width)\n        dy = np.random.choice(h-height)\n        src = np.array([\n            [-dx,-dy],[-dx+w,-dy+h], [-dx+w,-dy],#[-dx,-dy+h],#\n        ]).astype(np.float32)\n    if s<1:\n        dx = np.random.choice(width-w)\n        dy = np.random.choice(height-h)\n        src = np.array([\n            [dx,dy], [dx+w,dy+h], [dx+w,dy],#\n        ]).astype(np.float32)\n\n    transform = cv2.getAffineTransform(src, dst)\n    image = cv2.warpAffine( image, transform, (width, height), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n    mask = cv2.warpAffine( mask, transform, (width, height), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n    return image, mask\n\n\ndef do_random_stretch_y( image, mask, mag=0.25 ):\n    s = 1 + np.random.uniform(-1, 1)*mag\n    height, width = image.shape[:2]\n    h = int(s*height)\n    w = width\n    if h==height:\n        return image, mask\n\n    dst = np.array([\n        [0,0],[width,height], [width,0], #[0,height],\n    ]).astype(np.float32)\n\n\n    if s>1:\n        dx = 0#np.random.choice(w-width)\n        dy = np.random.choice(h-height)\n        src = np.array([\n            [-dx,-dy],[-dx+w,-dy+h], [-dx+w,-dy],#[-dx,-dy+h],#\n        ]).astype(np.float32)\n    if s<1:\n        dx = 0#np.random.choice(width-w)\n        dy = np.random.choice(height-h)\n        src = np.array([\n            [dx,dy], [dx+w,dy+h], [dx+w,dy],#\n        ]).astype(np.float32)\n\n    transform = cv2.getAffineTransform(src, dst)\n    image = cv2.warpAffine( image, transform, (width, height), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n    mask = cv2.warpAffine( mask, transform, (width, height), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n    return image, mask\n\n\n\ndef do_random_stretch_x( image, mask, mag=0.25 ):\n    s = 1 + np.random.uniform(-1, 1)*mag\n    height, width = image.shape[:2]\n    h = height\n    w = int(s*width)\n    if w==width:\n        return image, mask\n\n    dst = np.array([\n        [0,0],[width,height], [width,0], #[0,height],\n    ]).astype(np.float32)\n\n    if s>1:\n        dx = np.random.choice(w-width)\n        dy = 0#np.random.choice(h-height)\n        src = np.array([\n            [-dx,-dy],[-dx+w,-dy+h], [-dx+w,-dy],#[-dx,-dy+h],#\n        ]).astype(np.float32)\n    if s<1:\n        dx = np.random.choice(width-w)\n        dy = 0#np.random.choice(height-h)\n        src = np.array([\n            [dx,dy], [dx+w,dy+h], [dx+w,dy],#\n        ]).astype(np.float32)\n\n    transform = cv2.getAffineTransform(src, dst)\n    image = cv2.warpAffine( image, transform, (width, height), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n    mask = cv2.warpAffine( mask, transform, (width, height), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n    return image, mask\n\n\ndef do_random_shift( image, mask, mag=32 ):\n    b = mag\n    height, width = image.shape[:2]\n\n    image = cv2.copyMakeBorder(image, b,b,b,b, borderType=cv2.BORDER_CONSTANT, value=0)\n    mask  = cv2.copyMakeBorder(mask, b,b,b,b, borderType=cv2.BORDER_CONSTANT, value=0)\n    x = np.random.randint(0,2*b)\n    y = np.random.randint(0,2*b)\n    image = image[y:y+height,x:x+width]\n    mask = mask[y:y+height,x:x+width]\n\n    return image, mask\n\n###########################################################################################3\n\n\n\n# #--- noise ---\ndef do_random_blurout(image, size=0.20, num_cut=3):\n    height, width = image.shape[:2]\n    size = int(size*(height+width)/2)\n    for t in range(num_cut):\n        x = np.random.randint(0,width- size)\n        y = np.random.randint(0,height-size)\n        x0 = x\n        x1 = x+size\n        y0 = y\n        y1 = y+size\n        image[y0:y1,x0:x1]=image[y0:y1,x0:x1].mean()\n\n    return image\n\ndef do_random_guassian_blur(image, mag=[0.1, 2.0]):\n    sigma = np.random.uniform(mag[0],mag[1])\n    image = cv2.GaussianBlur(image, (23, 23), sigma)\n    return image\n\ndef do_random_noise(image, mag=0.08):\n    height, width = image.shape[:2]\n\n    image = image.astype(np.float32)/255\n    noise = np.random.uniform(-1,1,size=(height,width))*mag\n    image = image+noise\n\n    image = np.clip(image,0,1)\n    image = (image*255).astype(np.uint8)\n    return image\n\n\n\n# # --- intensity ---\ndef do_random_intensity_shift_contast(image, mag=[0.3,0.2]):\n    image = (image).astype(np.float32)/255\n    alpha0 = 1 + np.random.uniform(-1,1)*mag[0]\n    alpha1 = np.random.uniform(-1,1)*mag[1]\n    image = (image+alpha1)\n    image = np.clip(image,0,1)\n    image = image**alpha0\n    image = np.clip(image,0,1)\n    image = (image*255).astype(np.uint8)\n    return image\n\n#https://answers.opencv.org/question/12024/use-of-clahe/)\ndef do_random_clahe(image, mag=[[2,4],[6,12]]):\n    l = np.random.uniform(*mag[0])\n    g = np.random.randint(*mag[1])\n    clahe = cv2.createCLAHE(clipLimit=l, tileGridSize=(g, g))\n\n    image = clahe.apply(image)\n    return image\n\n# https://github.com/facebookresearch/CovidPrognosis/blob/master/covidprognosis/data/transforms.py\ndef do_histogram_norm(image, mag=[[2,4],[6,12]]):\n    num_bin = 255\n\n    histogram, bin = np.histogram( image.flatten(), num_bin, density=True)\n    cdf = histogram.cumsum()  # cumulative distribution function\n    cdf = 255 * cdf / cdf[-1]  # normalize\n\n    # use linear interpolation of cdf to find new pixel values\n    equalized = np.interp(image.flatten(), bin[:-1], cdf)\n    image = equalized.reshape(image.shape)\n    return image","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:47.187136Z","iopub.execute_input":"2021-08-10T10:09:47.187427Z","iopub.status.idle":"2021-08-10T10:09:47.230159Z","shell.execute_reply.started":"2021-08-10T10:09:47.187393Z","shell.execute_reply":"2021-08-10T10:09:47.229264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def do_valid(net, valid_loader):\n\n    valid_probability = []\n    valid_truth = []\n    valid_num = 0\n\n    net.eval()\n    start_timer = timer()\n    for t, batch in enumerate(valid_loader):\n        batch_size = len(batch['index'])\n        image = batch['image'].cuda()\n        onehot = batch['onehot']\n        label = onehot.argmax(-1)\n\n        with torch.no_grad():\n            #with amp.autocast():\n                logit, mask = data_parallel(net,image)\n                probability = F.softmax(logit,-1)\n\n        valid_num += batch_size\n        valid_probability.append(probability.data.cpu().numpy())\n        valid_truth.append(label.data.cpu().numpy())\n        print('\\r %8d / %d  %s'%(valid_num, len(valid_loader.dataset),time_to_str(timer() - start_timer,'sec')),end='',flush=True)\n\n    assert(valid_num == len(valid_loader.dataset))\n    #print('')\n    #----------------------\n    truth = np.concatenate(valid_truth)\n    probability = np.concatenate(valid_probability)\n    predict = probability.argsort(-1)[::-1]\n\n    loss = np_loss_cross_entropy(probability,truth)\n    topk = (predict==truth.reshape(-1,1))\n    acc  = topk[:, 0]\n    topk = topk.mean(0).cumsum()\n    acc = [acc[truth==i].mean() for i in range(num_study_label)]\n\n    #---\n    map  = np_metric_map_curve_by_class(probability, truth)*(4/6)\n\n    return [loss, map.mean(), topk[0], topk[1]]","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:47.231743Z","iopub.execute_input":"2021-08-10T10:09:47.232110Z","iopub.status.idle":"2021-08-10T10:09:47.245279Z","shell.execute_reply.started":"2021-08-10T10:09:47.232071Z","shell.execute_reply":"2021-08-10T10:09:47.244284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install albumentations -U\nimport albumentations as alb","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:47.246829Z","iopub.execute_input":"2021-08-10T10:09:47.247350Z","iopub.status.idle":"2021-08-10T10:09:53.077427Z","shell.execute_reply.started":"2021-08-10T10:09:47.247292Z","shell.execute_reply":"2021-08-10T10:09:53.076464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"size = 512\ntransform = alb.Compose([\n        #位置\n        alb.OneOf([\n            alb.HorizontalFlip(p=0.5),\n            alb.VerticalFlip(p=0.5),\n        ], p=0.5),\n\n        alb.RandomRotate90(p=.5),\n\n        #真实杂音\n        alb.OneOf([\n            #alb.RandomShadow(p=0.1),  # 灰阶无法使用\n            alb.CoarseDropout(p=0.2), \n            # alb.MaskDropout(p=0.02),\n        ], p=0.15),\n\n        #模糊\n        alb.OneOf([\n            alb.MedianBlur(blur_limit=(3, 7),p=0.5),\n            alb.GlassBlur(sigma=0.9, max_delta=200, iterations=1, always_apply=False, mode='fast',p=0.05),\n            alb.GaussianBlur(p=0.5),\n        ], p=0.02),\n\n        #形状\n        alb.OneOf([\n            alb.OpticalDistortion(p=0.3),\n            alb.GridDistortion(p=0.3),\n            alb.IAAPiecewiseAffine(p=0.1),\n            alb.ElasticTransform (alpha=1, sigma=50,alpha_affine=40,\n                              interpolation=1, border_mode=4, #value=None, mask_value=None, \n                              always_apply=False, approximate=False, p=0.3)\n        ], p=0.3),\n\n          #颜色\n        alb.OneOf([\n            # alb.HueSaturationValue(10,25,10, p= 0.4),  # 灰阶不兼容\n            alb.CLAHE(clip_limit=4, p=0.1),\n            alb.OneOf([\n                #normal\n                alb.RandomBrightnessContrast(brightness_limit=[-0.4,0.4], \n                                         contrast_limit=[-0.4,0.4], brightness_by_max=False, \n                                         always_apply=False, p=0.8),\n                #lighting\n                alb.RandomBrightnessContrast(brightness_limit=[0.4,0.6], \n                                         contrast_limit=[0.4,0.6], \n                                         brightness_by_max=False, \n                                         always_apply=False, p=0.05),\n                alb.RandomBrightnessContrast(brightness_limit=[0.4,0.6], \n                                         contrast_limit=[-0.6,-0.4], brightness_by_max=False, \n                                         always_apply=False, p=0.05),\n                #darking\n                alb.RandomBrightnessContrast(brightness_limit=[-0.6,-0.4], \n                                         contrast_limit=[0.4,0.6], brightness_by_max=False, \n                                         always_apply=False, p=0.05),\n                alb.RandomBrightnessContrast(brightness_limit=[-0.6,-0.4], \n                                         contrast_limit=[-0.5,-0.3], brightness_by_max=False, \n                                         always_apply=False, p=0.05),\n            ],p=0.6)             \n        ], p=0.4),\n\n        #整体失真化\n        alb.OneOf([\n            alb.GaussNoise(p= 0.1),\n            alb.RandomGamma (gamma_limit=(70, 110), #eps=None, \n                             always_apply=False, p=0.3),           \n        ], p=0.1),\n\n        #偏移\n        alb.OneOf([\n            alb.ShiftScaleRotate(shift_limit=[-0.1,0.1], scale_limit=[-0.15,0.15], rotate_limit=15, p=0.98, \n                         border_mode=cv2.BORDER_REFLECT),\n            alb.ShiftScaleRotate(shift_limit=[0.1,0.3], scale_limit=[-0.15,0.15], rotate_limit=15, p=0.005, \n                         border_mode=0),\n            alb.ShiftScaleRotate(shift_limit=[-0.3,0.1], scale_limit=[-0.15,0.15], rotate_limit=15, p=0.005, \n                         border_mode=0),\n            alb.ShiftScaleRotate(shift_limit=0.1, scale_limit=[0.5,0.6], rotate_limit=30, p=0.01, \n                         border_mode=cv2.BORDER_REFLECT),\n        ], p=0.9),\n              \n        alb.Cutout(num_holes=10, \n                    max_h_size=int(.1 * size), max_w_size=int(.1 * size), \n                    p=.25),\n    ])\n#random.seed(42)","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:53.079379Z","iopub.execute_input":"2021-08-10T10:09:53.079746Z","iopub.status.idle":"2021-08-10T10:09:53.099028Z","shell.execute_reply.started":"2021-08-10T10:09:53.079705Z","shell.execute_reply":"2021-08-10T10:09:53.097667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_augment(r):\n    image = r['image']\n    mask = r['mask']\n\n    aug = transform(image=image, mask=mask)\n\n    r['image'] = aug[\"image\"]\n    r['mask'] = aug[\"mask\"]\n    return r","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:53.100280Z","iopub.execute_input":"2021-08-10T10:09:53.100682Z","iopub.status.idle":"2021-08-10T10:09:53.112532Z","shell.execute_reply.started":"2021-08-10T10:09:53.100644Z","shell.execute_reply":"2021-08-10T10:09:53.111789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\ndef train_augment(r):\n    image = r['image']\n    mask = r['mask']\n    # if image[:2].shape != (image_size, image_size):\n    #     image = cv2.resize(image, dsize=(image_size, image_size), interpolation=cv2.INTER_AREA)\n\n    if 1:\n        for fn in np.random.choice([\n            lambda image, mask : do_random_scale(image, mask, mag=0.20),\n            lambda image, mask : do_random_stretch_y(image, mask, mag=0.20),\n            lambda image, mask : do_random_stretch_x(image, mask, mag=0.20),\n            lambda image, mask : do_random_shift(image, mask, mag=int(0.20*image_size)),\n            lambda image, mask : (image, mask)\n        ],1):\n            image, mask = fn(image, mask)\n\n        for fn in np.random.choice([\n            lambda image, mask : do_random_rotate(image, mask, mag=15),\n            lambda image, mask : do_random_hflip(image, mask),\n            lambda image, mask : (image, mask)\n        ],1):\n            image, mask = fn(image, mask)\n\n        # ------------------------\n        for fn in np.random.choice([\n            lambda image : do_random_intensity_shift_contast(image, mag=[0.5,0.5]),\n            lambda image : do_random_noise(image, mag=0.05),\n            lambda image : do_random_guassian_blur(image),\n            lambda image : do_random_blurout(image, size=0.25, num_cut=2),\n            #lambda image : do_random_clahe(image),\n            #lambda image : do_histogram_norm(image),\n            lambda image : image,\n        ],1):\n            image = fn(image)\n\n    r['image'] = image\n    r['mask'] = mask\n    return r\n\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:53.114054Z","iopub.execute_input":"2021-08-10T10:09:53.114476Z","iopub.status.idle":"2021-08-10T10:09:53.124709Z","shell.execute_reply.started":"2021-08-10T10:09:53.114442Z","shell.execute_reply":"2021-08-10T10:09:53.123788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = {\n    \"image_size\":image_size,\n    \"start_lr\":1e-3,\n    #fold = [0, 1, 2, 3, 4]\n    \"foldlist\":[0],\n    \"batchsize\":16,\n    \"num_iteration\":18000,\n    \"iter_log\":200,\n    \"iter_valid\":200\n    \n}","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:53.125922Z","iopub.execute_input":"2021-08-10T10:09:53.126284Z","iopub.status.idle":"2021-08-10T10:09:53.134237Z","shell.execute_reply.started":"2021-08-10T10:09:53.126254Z","shell.execute_reply":"2021-08-10T10:09:53.133429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IDENTIFIER   = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')\ndef run_train():\n    \n    #wandb.watch(Net, criterion, log=\"all\", log_freq=10)\n    \n    #for fold in [0,1,2,3,4]:\n    for fold in config[\"foldlist\"]:\n        \n        #wandb.init(project=\"covid19-siim-classify\", config=config)\n        \n        out_dir = \"./\"\n        initial_checkpoint = None\n        #out_dir + '/checkpoint/00007700_model.pth' \n            \n\n        #start_lr   = 0.0001#1\n        start_lr = config[\"start_lr\"]\n        batch_size = config[\"batchsize\"] #14 #22\n\n        #num_iteration = 12000\n        num_iteration = config[\"num_iteration\"]\n        iter_log    = config[\"iter_log\"]\n        iter_valid  = config[\"iter_valid\"]\n        iter_save   = list(range(0, num_iteration+1, 200))\n        \n        accumulation_steps = 2 \n\n        ## setup  ----------------------------------------\n        for f in ['checkpoint', 'train', 'valid', 'backup']: os.makedirs(out_dir + '/' + f, exist_ok=True)\n        # backup_project_as_zip(PROJECT_PATH, out_dir +'/backup/code.train.%s.zip'%IDENTIFIER)\n\n        log = Logger()\n        log.open(out_dir + '/log.train.txt', mode='a')\n        log.write('\\n--- [START %s] %s\\n\\n' % (IDENTIFIER, '-' * 64))\n        #log.write('\\t%s\\n' % COMMON_STRING)\n        #log.write('\\t__file__ = %s\\n' % __file__)\n        log.write('\\tout_dir  = %s\\n' % out_dir)\n        log.write('\\n')\n\n        ## dataset ------------------------------------\n        df_train, df_valid = make_fold('train-%d'%fold)\n        \n        train_dataset = SiimDataset(df_train, train_augment)\n        #train_dataset = SiimDataset(df_train, )\n        valid_dataset = SiimDataset(df_valid, )\n\n        train_loader = DataLoader(\n            train_dataset,\n            sampler = RandomSampler(train_dataset),\n            batch_size = batch_size,\n            drop_last   = True,\n            num_workers = 4,\n            pin_memory  = True,\n            worker_init_fn=lambda id: np.random.seed(torch.initial_seed() // 2 ** 32 + id),\n            collate_fn  = null_collate,\n        )\n        valid_loader  = DataLoader(\n            valid_dataset,\n            sampler = SequentialSampler(valid_dataset),\n            batch_size  = 16,\n            drop_last   = False,\n            num_workers = 4,\n            pin_memory  = True,\n            collate_fn  = null_collate,\n        )\n\n        log.write('train_dataset : \\n%s\\n'%(train_dataset))\n        log.write('valid_dataset : \\n%s\\n'%(valid_dataset))\n        log.write('\\n')\n\n\n        ## net ----------------------------------------\n        log.write('** net setting **\\n')\n        if is_mixed_precision:\n            scaler = amp.GradScaler()\n            net = AmpNet().cuda()\n        else:\n            net = Net().cuda()\n\n\n        if initial_checkpoint is not None:\n            f = torch.load(initial_checkpoint, map_location=lambda storage, loc: storage)\n            start_iteration = f['iteration']\n            start_epoch = f['epoch']\n            state_dict  = f['state_dict']\n            net.load_state_dict(state_dict,strict=True)  #True\n        else:\n            start_iteration = 0\n            start_epoch = 0\n\n\n        log.write('net=%s\\n'%(type(net)))\n        log.write('\\tinitial_checkpoint = %s\\n' % initial_checkpoint)\n        log.write('\\n')\n\n        # -----------------------------------------------\n        if 0: ##freeze\n            for p in net.block0.backbone.parameters(): p.requires_grad = False\n\n\n        #optimizer = Lookahead(RAdam(filter(lambda p: p.requires_grad, net.parameters()),lr=start_lr), alpha=0.5, k=5)\n        #optimizer = RAdam(filter(lambda p: p.requires_grad, net.parameters()),lr=start_lr)\n        #optimizer = MADGRAD( filter(lambda p: p.requires_grad, net.parameters()), lr=start_lr, momentum= 0.9, weight_decay= 0, eps= 1e-06)\n        #optimizer = Adam(net.parameters(), lr=start_lr)\n        \n        optimizer = MADGRAD( filter(lambda p: p.requires_grad, net.parameters()), lr=start_lr, momentum= 0.9, weight_decay= 0, eps= 1e-06)\n        \n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=5)\n        \n        # num_iteration = 8000\n        # iter_log    = 100\n        # iter_valid  = 100\n        # iter_save   = list(range(0, num_iteration, 100))#1*1000\n\n        log.write('optimizer\\n  %s\\n'%(optimizer))\n        log.write('\\n')\n\n\n        ## start training here! ##############################################\n        log.write('** start training here! **\\n')\n        log.write('   fold = %d\\n'%(fold))\n        log.write('   is_mixed_precision = %s \\n'%str(is_mixed_precision))\n        log.write('   batch_size = %d\\n'%(batch_size))\n        #log.write('   experiment = %s\\n' % str(__file__.split('/')[-2:]))\n        log.write('                      |----- VALID ---|---- TRAIN/BATCH --------------\\n')\n        log.write('rate     iter   epoch | loss    map  topk_0 topk_1  | loss0  loss1 loss2 | time          \\n')\n        log.write('----------------------------------------------------------------------\\n')\n                  #0.00000   0.00* 0.00  | 0.000  0.000  0.000  0.000  | 0.000  0.000  0.000 |  0 hr 00 min\n\n        def message(mode='print'):\n            if mode==('print'):\n                asterisk = ' '\n                loss = batch_loss\n            if mode==('log'):\n                asterisk = '*' if iteration in iter_save else ' '\n                loss = train_loss\n\n            text = \\\n                '%0.5f  %5.3f%s %4.2f  | '%(rate, iteration/10000, asterisk, epoch,) +\\\n                '%4.3f  %4.3f  %4.3f  %4.3f  | '%(*valid_loss,) +\\\n                '%4.3f  %4.3f  %4.3f  | '%(*loss,) +\\\n                '%s' % (time_to_str(timer() - start_timer,'min'))\n\n            return text\n\n        #----\n        valid_loss = np.zeros(4,np.float32)\n        train_loss = np.zeros(3,np.float32)\n        batch_loss = np.zeros_like(train_loss)\n        sum_train_loss = np.zeros_like(train_loss)\n        sum_train = 0\n        loss0 = torch.FloatTensor([0]).cuda().sum()\n        loss1 = torch.FloatTensor([0]).cuda().sum()\n        loss2 = torch.FloatTensor([0]).cuda().sum()\n\n\n        start_timer = timer()\n        iteration = start_iteration\n        epoch = start_epoch\n        rate = 0\n        while  iteration < num_iteration:\n\n            for t, batch in enumerate(train_loader):\n\n                if iteration in iter_save:\n                    if iteration != start_iteration:\n                        \"\"\"\n                        torch.save({\n                            'state_dict': net.state_dict(),\n                            'iteration': iteration,\n                            'epoch': epoch,\n                        }, out_dir + '/checkpoint/%08d_model.pth' % (iteration))\n                        \"\"\"\n                        \n                        \n                        #wandb.save(out_dir + '/checkpoint/%08d_model.pth' % (iteration))\n                        \n                        pass\n\n                if (iteration % iter_valid == 0):\n                    #if iteration!=start_iteration:\n                        valid_loss = do_valid(net, valid_loader)  #\n                        scheduler.step(valid_loss[0])\n                        pass\n\n                if (iteration % iter_log == 0):\n                    print('\\r', end='', flush=True)\n                    log.write(message(mode='log') + '\\n')\n                    \"\"\"\n                    wandb.log({\n                        \"fold\": fold,\n                        \"epoch\": epoch, \n                        \"rate\": rate,\n                        \"iterate\": iteration/10000,\n                        \"valid_loss\": valid_loss[0], \n                        \"valid_map\": valid_loss[1],\n                        \"valid_topk_0\": valid_loss[2], \n                        \"valid_topk_1\": valid_loss[3],\n                        \"train_ce_loss\": loss0, \n                        \"train_bce_loss\": loss1\n                    }, \n                        step=iteration)\n\n                    \"\"\"\n                   \n\n                # learning rate schduler ------------\n                rate = get_learning_rate(optimizer)\n\n                # one iteration update  -------------\n                batch_size = len(batch['index'])\n                image = batch['image'].cuda()\n                truth_mask = batch['mask'].cuda()\n                truth_mask = F.interpolate(truth_mask, size=(32,32), mode='bilinear', align_corners=False)\n                #truth_mask = F.interpolate(truth_mask, size=(16,16), mode='bilinear', align_corners=False)\n                onehot = batch['onehot'].cuda()\n                label = onehot.argmax(-1)\n\n                #----\n                net.train()\n                optimizer.zero_grad()\n\n                if is_mixed_precision:\n                    with amp.autocast():\n                        logit, mask = data_parallel(net, image)\n                        loss0 = F.cross_entropy(logit, label)\n                        #loss1 = F.binary_cross_entropy_with_logits(mask, truth_mask)\n                        \n                        loss1 = F.binary_cross_entropy_with_logits(mask, truth_mask) + lovasz_softmax(torch.sigmoid(mask), truth_mask, classes=[1], ignore=255)\n                        \n                        loss2 = (loss1 + loss0) / accumulation_steps\n                        \n                    scaler.scale(loss2).backward()\n                    #scaler.scale(loss1).backward()\n                    #scaler.scale(loss0+loss1).backward()\n                    #scaler.unscale_(optimizer)\n                    #torch.nn.utils.clip_grad_norm_(net.parameters(), 2)\n                    #scaler.step(optimizer)\n                    #scaler.update()\n\n                    if (iteration+1) % accumulation_steps == 0:             # Wait for several backward steps\n                        scaler.unscale_(optimizer)\n                        scaler.step(optimizer)                           # Now we can do an optimizer step\n                        scaler.update()\n                        optimizer.zero_grad()  \n\n                else :\n                    assert(False)\n                    print('fp32')\n                    logit, mask = data_parallel(net, image)\n                    loss0 = F.cross_entropy(logit, label)\n                    loss1 = F.binary_cross_entropy_with_logits(mask, truth_mask)\n\n                    (loss0+loss1).backward()\n                    optimizer.step()\n\n\n                # print statistics  --------\n                epoch += 1 / len(train_loader)\n                iteration += 1\n\n                batch_loss = np.array([loss0.item(), loss1.item(), loss2.item()])\n                sum_train_loss += batch_loss\n                sum_train += 1\n                if iteration % 100 == 0:\n                    train_loss = sum_train_loss / (sum_train + 1e-12)\n                    sum_train_loss[...] = 0\n                    sum_train = 0\n\n                print('\\r', end='', flush=True)\n                print(message(mode='print'), end='', flush=True)\n\n\n        log.write('\\n')\n        #wandb.finish()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:53.135853Z","iopub.execute_input":"2021-08-10T10:09:53.136256Z","iopub.status.idle":"2021-08-10T10:09:53.171966Z","shell.execute_reply.started":"2021-08-10T10:09:53.136220Z","shell.execute_reply":"2021-08-10T10:09:53.170729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run_train()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:09:53.173294Z","iopub.execute_input":"2021-08-10T10:09:53.173703Z","iopub.status.idle":"2021-08-10T10:12:07.826553Z","shell.execute_reply.started":"2021-08-10T10:09:53.173654Z","shell.execute_reply":"2021-08-10T10:12:07.824664Z"},"trusted":true},"execution_count":null,"outputs":[]}]}