{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:25.847715Z","iopub.execute_input":"2021-08-02T04:44:25.848128Z","iopub.status.idle":"2021-08-02T04:44:34.572723Z","shell.execute_reply.started":"2021-08-02T04:44:25.848032Z","shell.execute_reply":"2021-08-02T04:44:34.571817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport random\nimport os\nimport numpy as np\nfrom collections import defaultdict\nimport cv2\nimport pandas as pd\nimport matplotlib.pyplot as plt\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nSEED = 1234\nseed_everything(SEED) ","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:34.574473Z","iopub.execute_input":"2021-08-02T04:44:34.574838Z","iopub.status.idle":"2021-08-02T04:44:35.885097Z","shell.execute_reply.started":"2021-08-02T04:44:34.574797Z","shell.execute_reply":"2021-08-02T04:44:35.884267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    imgsz = 512\n    masksz = 512","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:35.886869Z","iopub.execute_input":"2021-08-02T04:44:35.887198Z","iopub.status.idle":"2021-08-02T04:44:35.89326Z","shell.execute_reply.started":"2021-08-02T04:44:35.887163Z","shell.execute_reply":"2021-08-02T04:44:35.890719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_list_from_file():\n    with open('../input/datasiimheng/duplicate.txt', 'r') as f:\n        gg = f.read().splitlines()\n    f.close()\n    return gg","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:35.894973Z","iopub.execute_input":"2021-08-02T04:44:35.895339Z","iopub.status.idle":"2021-08-02T04:44:35.901299Z","shell.execute_reply.started":"2021-08-02T04:44:35.895299Z","shell.execute_reply":"2021-08-02T04:44:35.900195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_fold(mode='train', fold=0):\n    data_dir = '/kaggle/input/datasiimheng'\n    if 'train' in mode:\n        df_study = pd.read_csv('/kaggle/input/siim-covid19-detection/train_study_level.csv')\n        df_fold  = pd.read_csv(data_dir+'/df_fold_rand830.csv')\n        df_meta  = pd.read_csv(data_dir+'/df_meta.csv')\n\n        df_study.loc[:, 'id'] = df_study.id.str.replace('_study', '')\n        df_study = df_study.rename(columns={'id': 'study_id'})\n\n        #---\n        df = df_study.copy()\n        df = df.merge(df_fold, on='study_id')\n        df = df.merge(df_meta, left_on='study_id', right_on='study')\n\n        duplicate = read_list_from_file()\n        df = df[~df['image'].isin(duplicate)]\n\n        #---\n        df_train = df[df.fold != fold].reset_index(drop=True)\n        df_valid = df[df.fold == fold].reset_index(drop=True)\n        return df_train, df_valid\n\n    if 'test' in mode:\n        df_meta  = pd.read_csv(data_dir+'/df_meta.csv')\n        df_valid = df_meta[df_meta['set']=='test'].copy()\n\n        for l in study_name_to_label.keys():\n            df_valid.loc[:,l]=0\n        df_valid = df_valid.reset_index(drop=True)\n        return df_valid\n\ndef null_augment(r):\n    image = r['image']\n    # if image[:2].shape != (image_size, image_size):\n    #     r['image'] = cv2.resize(image, dsize=(image_size, image_size), interpolation=cv2.INTER_AREA)\n    return r\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:35.902755Z","iopub.execute_input":"2021-08-02T04:44:35.903129Z","iopub.status.idle":"2021-08-02T04:44:35.914435Z","shell.execute_reply.started":"2021-08-02T04:44:35.903092Z","shell.execute_reply":"2021-08-02T04:44:35.913397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train, df_valid = make_fold('train', fold=0)\ndf_train","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:35.91587Z","iopub.execute_input":"2021-08-02T04:44:35.91622Z","iopub.status.idle":"2021-08-02T04:44:36.054432Z","shell.execute_reply.started":"2021-08-02T04:44:35.916184Z","shell.execute_reply":"2021-08-02T04:44:36.053594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_name_to_predict_string = {\n    'Negative for Pneumonia'  :'negative',\n    'Typical Appearance'      :'typical',\n    'Indeterminate Appearance':'indeterminate',\n    'Atypical Appearance'     :'atypical',\n}\n\nstudy_name_to_label = {\n    'Negative for Pneumonia'  :0,\n    'Typical Appearance'      :1,\n    'Indeterminate Appearance':2,\n    'Atypical Appearance'     :3,\n}\nstudy_label_to_name = { v:k for k,v in study_name_to_label.items()}\nnum_study_label = len(study_name_to_label)","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:36.055603Z","iopub.execute_input":"2021-08-02T04:44:36.055967Z","iopub.status.idle":"2021-08-02T04:44:36.063277Z","shell.execute_reply.started":"2021-08-02T04:44:36.05593Z","shell.execute_reply":"2021-08-02T04:44:36.062435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_label_to_name[2]","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:36.067279Z","iopub.execute_input":"2021-08-02T04:44:36.067567Z","iopub.status.idle":"2021-08-02T04:44:36.075878Z","shell.execute_reply.started":"2021-08-02T04:44:36.067542Z","shell.execute_reply":"2021-08-02T04:44:36.075075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader, RandomSampler\nclass SiimDataset(Dataset):\n    def __init__(self, df, augment=null_augment):\n        super().__init__()\n        self.df = df\n        self.augment = augment\n        self.length = len(df)\n\n    def __str__(self):\n        string  = ''\n        string += '\\tlen = %d\\n'%len(self)\n        string += '\\tdf  = %s\\n'%str(self.df.shape)\n\n        string += '\\tlabel distribution\\n'\n        for i in range(num_study_label):\n            n = self.df[study_label_to_name[i]].sum()\n            string += '\\t\\t %d %26s: %5d (%0.4f)\\n'%(i, study_label_to_name[i], n, n/len(self.df) )\n        return string\n\n\n    def __len__(self):\n        return self.length\n\n    def __getitem__(self, index):\n        d = self.df.iloc[index]\n        roi_path = '/kaggle/input/covid19-detection-890pxpng-study/ROI Mask'\n        train_path = '/kaggle/input/covid19-detection-890pxpng-study/train'\n        test_path = '/kaggle/input/covid19-detection-890pxpng-study/test'\n        #image_file = data_dir + '/%s_640/%s/%s/%s.png' % (d.set, d.study, d.series, d.image)\n        \n\n        if d.set == 'train':\n            image_file = f'{train_path}/{d.image}.png'\n            image = cv2.imread(image_file,cv2.IMREAD_GRAYSCALE)\n#             image = cv2.resize(image, (CFG.imgsz, CFG.imgsz))\n            onehot = d[study_name_to_label.keys()].values\n            mask_file = f'{roi_path}/{d.image}.png'\n            try:\n                mask = cv2.imread(mask_file,cv2.IMREAD_GRAYSCALE)\n                mask[mask > 0] = 255\n            except:\n                mask = np.zeros_like(image)\n        else:\n            image_file = f'{test_path}/{d.image}.png'\n            image = cv2.imread(image_file,cv2.IMREAD_GRAYSCALE)\n            \n            mask = np.zeros_like(image)\n        image = cv2.resize(image, (CFG.imgsz, CFG.imgsz))\n        mask = cv2.resize(mask, (CFG.masksz, CFG.masksz))\n        r = {\n            'index' : index,\n            'd' : d,\n            'image' : image,\n            'mask' : mask,\n            'onehot' : onehot,\n        }\n        if self.augment is not None: r = self.augment(r)\n        return r\n\n\ndef null_collate(batch):\n    collate = defaultdict(list)\n\n    for r in batch:\n        for k, v in r.items():\n            collate[k].append(v)\n\n    # ---\n    image_size= CFG.imgsz\n    batch_size = len(batch)\n    onehot = np.ascontiguousarray(np.stack(collate['onehot'])).astype(np.float32)\n    collate['onehot'] = torch.from_numpy(onehot)\n\n    image = np.stack(collate['image'])\n    image = image.reshape(batch_size, 1, image_size,image_size).repeat(3,1)\n    image = np.ascontiguousarray(image)\n    image = image.astype(np.float32) / 255\n    collate['image'] = torch.from_numpy(image)\n\n\n    mask = np.stack(collate['mask'])\n    mask = mask.reshape(batch_size, 1, image_size,image_size)\n    mask = np.ascontiguousarray(mask)\n    mask = mask.astype(np.float32) / 255\n    collate['mask'] = torch.from_numpy(mask)\n\n    return collate","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:36.079236Z","iopub.execute_input":"2021-08-02T04:44:36.07951Z","iopub.status.idle":"2021-08-02T04:44:36.097267Z","shell.execute_reply.started":"2021-08-02T04:44:36.079486Z","shell.execute_reply":"2021-08-02T04:44:36.096395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:36.099173Z","iopub.execute_input":"2021-08-02T04:44:36.09946Z","iopub.status.idle":"2021-08-02T04:44:36.125621Z","shell.execute_reply.started":"2021-08-02T04:44:36.099438Z","shell.execute_reply":"2021-08-02T04:44:36.124843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# image = cv2.imread('../input/covid19-detection-890pxpng-study/train/0023f02ae886.png')\n# try:\n#     mask = cv2.imread('../input/covid19-detection-890pxpng-study/ROI Mask/0023f02ae886.png')\n#     mask[mask > 0] = \n# except:\n#     mask = np.zeros_like(image)\n# plt.imshow(mask)","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:36.127543Z","iopub.execute_input":"2021-08-02T04:44:36.127843Z","iopub.status.idle":"2021-08-02T04:44:36.13497Z","shell.execute_reply.started":"2021-08-02T04:44:36.127819Z","shell.execute_reply":"2021-08-02T04:44:36.134164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train, df_valid = make_fold(mode='train', fold=0)\n    #df_valid = make_fold(mode='test')\n\ndataset = SiimDataset(df_valid) #null_augment\n\n    ","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:36.136785Z","iopub.execute_input":"2021-08-02T04:44:36.137055Z","iopub.status.idle":"2021-08-02T04:44:36.190188Z","shell.execute_reply.started":"2021-08-02T04:44:36.137028Z","shell.execute_reply":"2021-08-02T04:44:36.189463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"re = dataset.__getitem__(3)\nplt.imshow(re['mask'])","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:36.192381Z","iopub.execute_input":"2021-08-02T04:44:36.192714Z","iopub.status.idle":"2021-08-02T04:44:36.461055Z","shell.execute_reply.started":"2021-08-02T04:44:36.192682Z","shell.execute_reply":"2021-08-02T04:44:36.460291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# #--- flip ---\ndef do_random_hflip(image, mask):\n    if np.random.rand()>0.5:\n        image = cv2.flip(image,1)\n        mask = cv2.flip(mask,1)\n    return image, mask\n\n\n# #--- geometric ---\ndef do_random_rotate(image, mask, mag=15 ):\n    angle = np.random.uniform(-1, 1)*mag\n\n    height, width = image.shape[:2]\n    cx, cy = width // 2, height // 2\n\n    transform = cv2.getRotationMatrix2D((cx, cy), -angle, 1.0)\n    image = cv2.warpAffine(image, transform, (width, height), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n    mask = cv2.warpAffine(mask, transform, (width, height), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n\n    return image, mask\n\n\ndef do_random_scale( image, mask, mag=0.1 ):\n    s = 1 + np.random.uniform(-1, 1)*mag\n    height, width = image.shape[:2]\n    w,h = int(s*width), int(s*height)\n    if (h,w)==image.shape[:2]:\n        return image, mask\n\n    dst = np.array([\n        [0,0],[width,height], [width,0], #[0,height],\n    ]).astype(np.float32)\n\n    if s>1:\n        dx = np.random.choice(w-width)\n        dy = np.random.choice(h-height)\n        src = np.array([\n            [-dx,-dy],[-dx+w,-dy+h], [-dx+w,-dy],#[-dx,-dy+h],#\n        ]).astype(np.float32)\n    if s<1:\n        dx = np.random.choice(width-w)\n        dy = np.random.choice(height-h)\n        src = np.array([\n            [dx,dy], [dx+w,dy+h], [dx+w,dy],#\n        ]).astype(np.float32)\n\n    transform = cv2.getAffineTransform(src, dst)\n    image = cv2.warpAffine( image, transform, (width, height), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n    mask = cv2.warpAffine( mask, transform, (width, height), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n    return image, mask\n\n\ndef do_random_stretch_y( image, mask, mag=0.25 ):\n    s = 1 + np.random.uniform(-1, 1)*mag\n    height, width = image.shape[:2]\n    h = int(s*height)\n    w = width\n    if h==height:\n        return image, mask\n\n    dst = np.array([\n        [0,0],[width,height], [width,0], #[0,height],\n    ]).astype(np.float32)\n\n\n    if s>1:\n        dx = 0#np.random.choice(w-width)\n        dy = np.random.choice(h-height)\n        src = np.array([\n            [-dx,-dy],[-dx+w,-dy+h], [-dx+w,-dy],#[-dx,-dy+h],#\n        ]).astype(np.float32)\n    if s<1:\n        dx = 0#np.random.choice(width-w)\n        dy = np.random.choice(height-h)\n        src = np.array([\n            [dx,dy], [dx+w,dy+h], [dx+w,dy],#\n        ]).astype(np.float32)\n\n    transform = cv2.getAffineTransform(src, dst)\n    image = cv2.warpAffine( image, transform, (width, height), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n    mask = cv2.warpAffine( mask, transform, (width, height), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n    return image, mask\n\n\n\ndef do_random_stretch_x( image, mask, mag=0.25 ):\n    s = 1 + np.random.uniform(-1, 1)*mag\n    height, width = image.shape[:2]\n    h = height\n    w = int(s*width)\n    if w==width:\n        return image, mask\n\n    dst = np.array([\n        [0,0],[width,height], [width,0], #[0,height],\n    ]).astype(np.float32)\n\n    if s>1:\n        dx = np.random.choice(w-width)\n        dy = 0#np.random.choice(h-height)\n        src = np.array([\n            [-dx,-dy],[-dx+w,-dy+h], [-dx+w,-dy],#[-dx,-dy+h],#\n        ]).astype(np.float32)\n    if s<1:\n        dx = np.random.choice(width-w)\n        dy = 0#np.random.choice(height-h)\n        src = np.array([\n            [dx,dy], [dx+w,dy+h], [dx+w,dy],#\n        ]).astype(np.float32)\n\n    transform = cv2.getAffineTransform(src, dst)\n    image = cv2.warpAffine( image, transform, (width, height), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n    mask = cv2.warpAffine( mask, transform, (width, height), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n    return image, mask\n\n\ndef do_random_shift( image, mask, mag=32 ):\n    b = mag\n    height, width = image.shape[:2]\n\n    image = cv2.copyMakeBorder(image, b,b,b,b, borderType=cv2.BORDER_CONSTANT, value=0)\n    mask  = cv2.copyMakeBorder(mask, b,b,b,b, borderType=cv2.BORDER_CONSTANT, value=0)\n    x = np.random.randint(0,2*b)\n    y = np.random.randint(0,2*b)\n    image = image[y:y+height,x:x+width]\n    mask = mask[y:y+height,x:x+width]\n\n    return image, mask\n\n###########################################################################################3\n\n\n\n# #--- noise ---\ndef do_random_blurout(image, size=0.20, num_cut=3):\n    height, width = image.shape[:2]\n    size = int(size*(height+width)/2)\n    for t in range(num_cut):\n        x = np.random.randint(0,width- size)\n        y = np.random.randint(0,height-size)\n        x0 = x\n        x1 = x+size\n        y0 = y\n        y1 = y+size\n        image[y0:y1,x0:x1]=image[y0:y1,x0:x1].mean()\n\n    return image\n\ndef do_random_guassian_blur(image, mag=[0.1, 2.0]):\n    sigma = np.random.uniform(mag[0],mag[1])\n    image = cv2.GaussianBlur(image, (23, 23), sigma)\n    return image\n\ndef do_random_noise(image, mag=0.08):\n    height, width = image.shape[:2]\n\n    image = image.astype(np.float32)/255\n    noise = np.random.uniform(-1,1,size=(height,width))*mag\n    image = image+noise\n\n    image = np.clip(image,0,1)\n    image = (image*255).astype(np.uint8)\n    return image\n\n\n\n# # --- intensity ---\ndef do_random_intensity_shift_contast(image, mag=[0.3,0.2]):\n    image = (image).astype(np.float32)/255\n    alpha0 = 1 + random.uniform(-1,1)*mag[0]\n    alpha1 = random.uniform(-1,1)*mag[1]\n    image = (image+alpha1)\n    image = np.clip(image,0,1)\n    image = image**alpha0\n    image = np.clip(image,0,1)\n    image = (image*255).astype(np.uint8)\n    return image\n\n#https://answers.opencv.org/question/12024/use-of-clahe/)\ndef do_random_clahe(image, mag=[[2,4],[6,12]]):\n    l = np.random.uniform(*mag[0])\n    g = np.random.randint(*mag[1])\n    clahe = cv2.createCLAHE(clipLimit=l, tileGridSize=(g, g))\n\n    image = clahe.apply(image)\n    return image\ndef do_histogram_norm(image, mag=[[2,4],[6,12]]):\n    num_bin = 255\n\n    histogram, bin = np.histogram( image.flatten(), num_bin, density=True)\n    cdf = histogram.cumsum()  # cumulative distribution function\n    cdf = 255 * cdf / cdf[-1]  # normalize\n\n    # use linear interpolation of cdf to find new pixel values\n    equalized = np.interp(image.flatten(), bin[:-1], cdf)\n    image = equalized.reshape(image.shape)\n    return image","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:36.4624Z","iopub.execute_input":"2021-08-02T04:44:36.462756Z","iopub.status.idle":"2021-08-02T04:44:36.502918Z","shell.execute_reply.started":"2021-08-02T04:44:36.462719Z","shell.execute_reply":"2021-08-02T04:44:36.502084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# e = tf_efficientnet_b5_ns(pretrained=True, drop_rate=0.3, drop_path_rate=0.2)\n# e.conv_head","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:36.504308Z","iopub.execute_input":"2021-08-02T04:44:36.504865Z","iopub.status.idle":"2021-08-02T04:44:36.517142Z","shell.execute_reply.started":"2021-08-02T04:44:36.504828Z","shell.execute_reply":"2021-08-02T04:44:36.516369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import timm\nimport torch\nimport torch.nn as nn\nimport torchvision\nimport torch.nn.functional as F\nfrom timm.models.efficientnet import *\n\nclass Net(nn.Module):\n    def __init__(self):\n        super(Net, self).__init__()\n\n        e = tf_efficientnet_b5_ns(pretrained=True, drop_rate=0.3, drop_path_rate=0.2)\n        \n        self.b0 = nn.Sequential(\n            e.conv_stem,\n            e.bn1,\n            e.act1,\n        )\n        self.b1 = e.blocks[0]\n        self.b2 = e.blocks[1]\n        self.b3 = e.blocks[2]\n        self.b4 = e.blocks[3]\n        self.b5 = e.blocks[4]\n        self.b6 = e.blocks[5]\n        self.b7 = e.blocks[6]\n        self.b8 = nn.Sequential(\n            e.conv_head, #384, 1536\n            e.bn2,\n            e.act2\n        )\n\n        self.logit = nn.Sequential(\n            nn.Linear(2048, 1024),\n            nn.ReLU(inplace=True),\n            nn.Linear(1024, num_study_label)\n        )\n        self.mask = nn.Sequential(\n            nn.Conv2d(304, 128, kernel_size=3, padding=1),\n            nn.BatchNorm2d(128),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(128, 128, kernel_size=3, padding=1),\n            nn.BatchNorm2d(128),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(128, 1, kernel_size=1, padding=0),\n        )\n\n\n    # @torch.cuda.amp.autocast()\n    def forward(self, image):\n        batch_size = len(image)\n             # ; print('input ',   x.shape)\n        x =  image.float()\n        x = self.b0(x) #; print (x.shape)  # torch.Size([2, 40, 256, 256])\n        x = self.b1(x) #; print (x.shape)  # torch.Size([2, 24, 256, 256])\n        x = self.b2(x) #; print (x.shape)  # torch.Size([2, 32, 128, 128])\n        x = self.b3(x) #; print (x.shape)  # torch.Size([2, 48, 64, 64])\n        x = self.b4(x) #; print (x.shape)  # torch.Size([2, 96, 32, 32])\n        x = self.b5(x) #; print (x.shape)  # torch.Size([2, 136, 32, 32])\n        #------------\n       \n        #-------------\n        x = self.b6(x) #; print (x.shape)  # torch.Size([2, 232, 16, 16])\n        mask = self.mask(x)\n        x = self.b7(x) #; print (x.shape)  # torch.Size([2, 384, 16, 16])\n        x = self.b8(x) #; print (x.shape)  # torch.Size([2, 1536, 16, 16])\n#         x = torch.sigmoid(x)\n#         x = F.dropout(x, 0.4)\n        x = F.adaptive_avg_pool2d(x,1).reshape(batch_size,-1)\n        #x = F.dropout(x, 0.5, training=self.training)\n        logit = self.logit(x)\n        return logit, mask\n","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:36.520188Z","iopub.execute_input":"2021-08-02T04:44:36.520451Z","iopub.status.idle":"2021-08-02T04:44:37.428455Z","shell.execute_reply.started":"2021-08-02T04:44:36.520426Z","shell.execute_reply":"2021-08-02T04:44:37.427526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net = Net()\ndef run_check_net():\n    batch_size = 2\n    C, H, W = 3, 512, 512\n    #C, H, W = 3, 640, 640\n    image = torch.randn(batch_size, C, H, W).cuda()\n    mask  = torch.randn(batch_size, num_study_label, H, W).cuda()\n\n    net = Net().cuda()\n    logit, mask = net(image)\n\n    print(image.shape)\n    print(logit.shape)\n    print(mask.shape)\nrun_check_net()","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:37.430703Z","iopub.execute_input":"2021-08-02T04:44:37.431182Z","iopub.status.idle":"2021-08-02T04:44:46.584114Z","shell.execute_reply.started":"2021-08-02T04:44:37.431141Z","shell.execute_reply":"2021-08-02T04:44:46.583122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.cuda.amp as amp\n\nclass AmpNet(Net):\n    @torch.cuda.amp.autocast()\n    def forward(self,*args):\n        return super(AmpNet, self).forward(*args)\n\nis_mixed_precision = True  #True #False","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:46.585518Z","iopub.execute_input":"2021-08-02T04:44:46.585877Z","iopub.status.idle":"2021-08-02T04:44:46.591383Z","shell.execute_reply.started":"2021-08-02T04:44:46.58584Z","shell.execute_reply":"2021-08-02T04:44:46.590118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_augment(r, image_size=512):\n    \n    image = r['image']\n    mask = r['mask']\n    # if image[:2].shape != (image_size, image_size):\n    #     image = cv2.resize(image, dsize=(image_size, image_size), interpolation=cv2.INTER_AREA)\n\n    if 1:\n        for fn in np.random.choice([\n            lambda image, mask : do_random_scale(image, mask, mag=0.20),\n            lambda image, mask : do_random_stretch_y(image, mask, mag=0.20),\n#             lambda image, mask : do_random_stretch_x(image, mask, mag=0.20),\n            lambda image, mask : do_random_shift(image, mask, mag=int(0.20*image_size)),\n            lambda image, mask : (image, mask)\n        ],1):\n            image, mask = fn(image, mask)\n\n        for fn in np.random.choice([\n            lambda image, mask : do_random_rotate(image, mask, mag=15),\n            lambda image, mask : do_random_hflip(image, mask),\n            lambda image, mask : (image, mask)\n        ],1):\n            image, mask = fn(image, mask)\n\n        # ------------------------\n        for fn in np.random.choice([\n            lambda image : do_random_intensity_shift_contast(image, mag=[0.5,0.5]),\n#             lambda image : do_random_noise(image, mag=0.05),\n            lambda image : do_random_guassian_blur(image),\n            lambda image : do_random_blurout(image, size=0.25, num_cut=2),\n            #lambda image : do_random_clahe(image),\n            lambda image : do_histogram_norm(image),\n            lambda image : image,\n        ],1):\n            image = fn(image)\n\n    r['image'] = image\n    r['mask'] = mask\n    return r","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:46.592971Z","iopub.execute_input":"2021-08-02T04:44:46.59337Z","iopub.status.idle":"2021-08-02T04:44:46.606063Z","shell.execute_reply.started":"2021-08-02T04:44:46.593282Z","shell.execute_reply":"2021-08-02T04:44:46.605224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def np_metric_map_curve_by_class(probability, truth):\n    num_sample, num_label = probability.shape\n    score = []\n    for i in range(num_label):\n        s = average_precision_score(truth==i, probability[:,i])\n        score.append(s)\n    score = np.array(score)\n    return score","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:46.60885Z","iopub.execute_input":"2021-08-02T04:44:46.609116Z","iopub.status.idle":"2021-08-02T04:44:46.616911Z","shell.execute_reply.started":"2021-08-02T04:44:46.609083Z","shell.execute_reply":"2021-08-02T04:44:46.616074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for t, batch in enumerate(valid_loader):\n#     batch_size = len(batch['index'])\n#     image = batch['image'].cuda()\n#     valid_probability = []\n#     valid_truth = []\n#     onehot = batch['onehot']\n#     label = onehot.argmax(-1)\n\n#     with torch.no_grad():\n#         logit, mask = data_parallel(net,image)\n# #         probability = F.softmax(logit,-1)\n# #         valid_probability.append(probability.data.cpu().numpy())\n# #         valid_truth.append(label.data.cpu().numpy())\n# #     truth = np.concatenate(valid_truth)\n# #     probability = np.concatenate(valid_probability)\n# #     predict = probability.argsort(-1)[::-1]\n# #     batch_size = len(probability)\n# #     truth = truth.reshape(-1)\n    \n# #     p = probability[np.arange(batch_size),truth]\n#     print(mask)\n#     if t == 0:\n#         break","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:46.618072Z","iopub.execute_input":"2021-08-02T04:44:46.61843Z","iopub.status.idle":"2021-08-02T04:44:46.625332Z","shell.execute_reply.started":"2021-08-02T04:44:46.618394Z","shell.execute_reply":"2021-08-02T04:44:46.62453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def do_valid(net, valid_loader):\n\n    valid_probability = []\n    valid_truth = []\n    valid_num = 0\n\n    net.eval()\n    start_timer = timer()\n    for t, batch in enumerate(valid_loader):\n        batch_size = len(batch['index'])\n        image = batch['image'].cuda()\n        onehot = batch['onehot']\n        label = onehot.argmax(-1)\n\n        with torch.no_grad():\n            #with amp.autocast():\n                logit, mask = data_parallel(net,image)\n                probability = F.softmax(logit,-1)\n\n        valid_num += batch_size\n        valid_probability.append(probability.data.cpu().numpy())\n        valid_truth.append(label.data.cpu().numpy())\n        print('\\r %8d / %d  %s'%(valid_num, len(valid_loader.dataset),time_to_str(timer() - start_timer,'sec')),end='',flush=True)\n\n    assert(valid_num == len(valid_loader.dataset))\n    #print('')\n    #----------------------\n    truth = np.concatenate(valid_truth)\n    probability = np.concatenate(valid_probability)\n    predict = probability.argsort(-1)[:, ::-1]\n\n    loss = np_loss_cross_entropy(probability,truth)\n    topk = (predict==truth.reshape(-1,1))\n    acc  = topk[:, 0]\n    topk = topk.mean(0).cumsum()\n    acc = [acc[truth==i].mean() for i in range(num_study_label)]\n\n    #---\n    map  = np_metric_map_curve_by_class(probability, truth)*(4/6)\n\n    return [loss, map.mean(), topk[0], topk[1]]","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:46.626707Z","iopub.execute_input":"2021-08-02T04:44:46.627133Z","iopub.status.idle":"2021-08-02T04:44:46.639511Z","shell.execute_reply.started":"2021-08-02T04:44:46.627097Z","shell.execute_reply":"2021-08-02T04:44:46.638338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install madgrad","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:46.644028Z","iopub.execute_input":"2021-08-02T04:44:46.644309Z","iopub.status.idle":"2021-08-02T04:44:53.336631Z","shell.execute_reply.started":"2021-08-02T04:44:46.644285Z","shell.execute_reply":"2021-08-02T04:44:53.335716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import SequentialSampler\nfrom madgrad import MADGRAD\nimport collections\nfrom collections import defaultdict\nimport copy\nimport numbers\nimport inspect\nimport shutil\nfrom timeit import default_timer as timer\nimport itertools\nfrom collections import OrderedDict\nfrom multiprocessing import Pool\nimport multiprocessing as mp\n\n#from pprintpp import pprint, pformat\nimport json\nimport zipfile\nfrom shutil import copyfile\n\nimport csv\nimport pandas as pd\nimport pickle\nimport glob\nimport sys\nfrom distutils.dir_util import copy_tree\nimport time\n\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.mplot3d import Axes3D\n#torch lib\nimport torch\nfrom torch.utils.data.dataset import Dataset\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data.sampler import *\n\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.nn.parallel.data_parallel import data_parallel\n\nfrom torch.nn.utils.rnn import *\nfrom contextlib import contextmanager\n","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:53.338529Z","iopub.execute_input":"2021-08-02T04:44:53.338836Z","iopub.status.idle":"2021-08-02T04:44:53.353593Z","shell.execute_reply.started":"2021-08-02T04:44:53.338807Z","shell.execute_reply":"2021-08-02T04:44:53.352808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Logger(object):\n    def __init__(self):\n        self.terminal = sys.stdout  #stdout\n        self.file = None\n\n    def open(self, file, mode=None):\n        if mode is None: mode ='w'\n        self.file = open(file, mode)\n\n    def write(self, message, is_terminal=1, is_file=1 ):\n        if '\\r' in message: is_file=0\n\n        if is_terminal == 1:\n            self.terminal.write(message)\n            self.terminal.flush()\n            #time.sleep(1)\n\n        if is_file == 1:\n            self.file.write(message)\n            self.file.flush()\n\n    def flush(self):\n        # this flush method is needed for python 3 compatibility.\n        # this handles the flush command by doing nothing.\n        # you might want to specify some extra behavior here.\n        pass\n","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:53.355181Z","iopub.execute_input":"2021-08-02T04:44:53.355626Z","iopub.status.idle":"2021-08-02T04:44:53.365836Z","shell.execute_reply.started":"2021-08-02T04:44:53.355585Z","shell.execute_reply":"2021-08-02T04:44:53.365022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_to_str(t, mode='min'):\n    if mode=='min':\n        t  = int(t)/60\n        hr = t//60\n        min = t%60\n        return '%2d hr %02d min'%(hr,min)\n\n    elif mode=='sec':\n        t   = int(t)\n        min = t//60\n        sec = t%60\n        return '%2d min %02d sec'%(min,sec)\n\n    else:\n        raise NotImplementedError","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:53.367249Z","iopub.execute_input":"2021-08-02T04:44:53.367668Z","iopub.status.idle":"2021-08-02T04:44:53.377132Z","shell.execute_reply.started":"2021-08-02T04:44:53.36761Z","shell.execute_reply":"2021-08-02T04:44:53.376148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def np_loss_cross_entropy(probability, truth):\n    batch_size = len(probability)\n    truth = truth.reshape(-1)\n    p = probability[np.arange(batch_size),truth]\n    loss = -np.log(np.clip(p,1e-6,1))\n    loss = loss.mean()\n    return loss","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:53.378699Z","iopub.execute_input":"2021-08-02T04:44:53.379183Z","iopub.status.idle":"2021-08-02T04:44:53.390435Z","shell.execute_reply.started":"2021-08-02T04:44:53.379143Z","shell.execute_reply":"2021-08-02T04:44:53.389568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_lovasz_gradient(truth): #sorted\n    truth_sum    = truth.sum()\n    intersection = truth_sum - truth.cumsum(0)\n    union        = truth_sum + (1 - truth).cumsum(0)\n    jaccard      = 1. - intersection / union\n    T = len(truth)\n    jaccard[1:T] = jaccard[1:T] - jaccard[0:T-1]\n\n    gradient = jaccard\n    return gradient\n\n\n\n\ndef lovasz_hinge_one(logit , truth):\n    truth = truth.float()\n    sign  = 2. * truth - 1.\n    hinge = (1. - logit * sign)\n    hinge, permutation = torch.sort(hinge, dim=0, descending=True)\n    hinge = F.relu(hinge)\n\n    truth = truth[permutation.data]\n    gradient = compute_lovasz_gradient(truth)\n\n    loss = torch.dot(hinge, gradient)\n    return loss\n\n\n\ndef lovasz_loss(logit, truth, mode='hinge'):\n\n    if mode=='hinge':\n        lovasz_one = lovasz_hinge_one\n    elif mode=='soft_hinge':\n        lovasz_one = lovasz_soft_hinge_one\n    elif mode=='logistic':\n        lovasz_one = lovasz_logistic_one\n    elif mode=='exp':\n        lovasz_one = lovasz_exp_one\n    else:\n        raise NotImplementedError\n\n    batch_size = len(truth)\n    loss = torch.zeros(batch_size).cuda()\n    for b in range(batch_size):\n        l, t = logit[b].view(-1), truth[b].view(-1)\n        loss[b] = lovasz_one(l, t)\n\n\n    return loss\n\n\n#####################################################################\n\n\n#https://www.groundai.com/project/a-new-smooth-approximation-to-the-zero-one-loss-with-a-probabilistic-interpretation/\ndef lovasz_logistic_one(logit , truth):\n    lamda = 8\n    logit = torch.clamp(logit,-4,4)\n\n    truth = truth.float()\n    sign  = 2. * truth - 1.\n    logistic = torch.log(1+torch.exp(lamda*(1- logit*sign)))/lamda\n    logistic, permutation = torch.sort(logistic, dim=0, descending=True)\n\n    truth = truth[permutation.data]\n    gradient = compute_lovasz_gradient(truth)\n\n    loss = torch.dot(logistic, gradient)\n    return loss\n\n\n\n\ndef lovasz_exp_one(logit , truth):\n\n    truth = truth.float()\n    sign  = 2. * truth - 1.\n    exp   = torch.exp(-logit*sign)\n    exp, permutation = torch.sort(exp, dim=0, descending=True)\n\n    truth = truth[permutation.data]\n    gradient = compute_lovasz_gradient(truth)\n\n    loss = torch.dot(exp, gradient)\n    return loss\n\n\n\n# https://github.com/pytorch/pytorch/blob/master/torch/legacy/nn/SoftPlus.py\ndef lovasz_soft_hinge_one(logit , truth):\n\n    truth = truth.float()\n    sign  = 2. * truth - 1.\n    hinge = (1. - logit * sign)\n    hinge, permutation = torch.sort(hinge, dim=0, descending=True)\n    hinge = nn.Softplus()(hinge)\n\n    truth = truth[permutation.data]\n    gradient = compute_lovasz_gradient(truth)\n\n    loss = torch.dot(hinge, gradient)\n    return loss\ndef symmetric_lovasz(logits, labels):\n    return (lovasz_soft_hinge_one(logits, labels) + lovasz_soft_hinge_one(-logits, 1 - labels)) / 2","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:53.392058Z","iopub.execute_input":"2021-08-02T04:44:53.392551Z","iopub.status.idle":"2021-08-02T04:44:53.41291Z","shell.execute_reply.started":"2021-08-02T04:44:53.392512Z","shell.execute_reply":"2021-08-02T04:44:53.412156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score, roc_curve, average_precision_score\nfrom scipy.optimize import brentq\nfrom scipy.interpolate import interp1d\n","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:53.414259Z","iopub.execute_input":"2021-08-02T04:44:53.414605Z","iopub.status.idle":"2021-08-02T04:44:54.176485Z","shell.execute_reply.started":"2021-08-02T04:44:53.41457Z","shell.execute_reply":"2021-08-02T04:44:54.175666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import shutil\n# shutil.rmtree('./fold1')","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:54.177826Z","iopub.execute_input":"2021-08-02T04:44:54.178181Z","iopub.status.idle":"2021-08-02T04:44:54.181931Z","shell.execute_reply.started":"2021-08-02T04:44:54.178147Z","shell.execute_reply":"2021-08-02T04:44:54.180935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def adjust_learning_rate(optimizer, lr):\n    for param_group in optimizer.param_groups:\n        param_group['lr'] = lr\n\ndef get_learning_rate(optimizer):\n    lr=[]\n    for param_group in optimizer.param_groups:\n       lr +=[ param_group['lr'] ]\n\n    assert(len(lr)==1) #we support only one param_group\n    lr = lr[0]\n\n    return lr","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:54.183136Z","iopub.execute_input":"2021-08-02T04:44:54.183543Z","iopub.status.idle":"2021-08-02T04:44:54.193046Z","shell.execute_reply.started":"2021-08-02T04:44:54.183506Z","shell.execute_reply":"2021-08-02T04:44:54.192223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# os.listdir('./fold3/checkpoint')","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:54.194312Z","iopub.execute_input":"2021-08-02T04:44:54.194844Z","iopub.status.idle":"2021-08-02T04:44:54.202515Z","shell.execute_reply.started":"2021-08-02T04:44:54.194805Z","shell.execute_reply":"2021-08-02T04:44:54.201418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Logger(objectject):\n    def __init__(self):\n        self.terminal = sys.stdout  #stdout\n        self.file = None\n\n    def open(self, file, mode=None):\n        if mode is None: mode ='w'\n        self.file = open(file, mode)\n\n    def write(self, message, is_terminal=1, is_file=1 ):\n        if '\\r' in message: is_file=0\n\n        if is_terminal == 1:\n            self.terminal.write(message)\n            self.terminal.flush()\n            #time.sleep(1)\n\n        if is_file == 1:\n            self.file.write(message)\n            self.file.flush()\n\n    def flush(self):\n        # this flush method is needed for python 3 compatibility.\n        # this handles the flush command by doing nothing.\n        # you might want to specify some extra behavior here.\n        pass\nlog = Logger()\nlog.write('bb')","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:48:51.533271Z","iopub.execute_input":"2021-08-02T04:48:51.533673Z","iopub.status.idle":"2021-08-02T04:48:51.567146Z","shell.execute_reply.started":"2021-08-02T04:48:51.533608Z","shell.execute_reply":"2021-08-02T04:48:51.565583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold in [2]:\n    os.makedirs(f'/kaggle/working/fold{fold + 1}', exist_ok=True)\n    out_dir = f'/kaggle/working/fold{fold + 1}'\n    initial_checkpoint = None\n\n    start_lr   = 0.0001#1\n    batch_size = 8 #14 #22\n\n    num_iteration = 21600\n    iter_log    = 200\n    iter_valid  = 200\n    iter_save   = list(range(0, num_iteration+1, 200))\n\n    ## setup  ----------------------------------------\n    for f in ['checkpoint', 'train', 'valid', 'backup']: os.makedirs(out_dir + '/' + f, exist_ok=True)\n    # backup_project_as_zip(PROJECT_PATH, out_dir +'/backup/code.train.%s.zip'%IDENTIFIER)\n\n    log = Logger()\n    try:\n        log.open(out_dir + '/log.train.txt', mode='a')\n        log.write('\\n--- [START %s] %s\\n\\n' % ('continue', '-' * 64))\n        \n    except:\n        initial_checkpoint = None\n\n    ## dataset ------------------------------------\n    df_train, df_valid = make_fold('train', fold)\n    train_dataset = SiimDataset(df_train, train_augment)\n    valid_dataset = SiimDataset(df_valid, )\n\n    train_loader = DataLoader(\n        train_dataset,\n        sampler = RandomSampler(train_dataset),\n        batch_size = batch_size,\n        drop_last   = True,\n        num_workers = 4,\n        pin_memory  = True,\n        worker_init_fn=lambda id: np.random.seed(torch.initial_seed() // 2 ** 32 + id),\n        collate_fn  = null_collate,\n    )\n    valid_loader  = DataLoader(\n        valid_dataset,\n        sampler = SequentialSampler(valid_dataset),\n        batch_size  = 16,\n        drop_last   = False,\n        num_workers = 4,\n        pin_memory  = True,\n        collate_fn  = null_collate,\n    )\n\n    log.write('train_dataset : \\n%s\\n'%(train_dataset))\n    log.write('valid_dataset : \\n%s\\n'%(valid_dataset))\n    log.write('\\n')\n\n\n    ## net ----------------------------------------\n    log.write('** net setting **\\n')\n    if is_mixed_precision:\n        scaler = amp.GradScaler()\n        net = AmpNet().cuda()\n    else:\n        net = Net().cuda()\n\n    #check initital_checkpoit if have weight you can train with epoch 10 20 etc\n    if initial_checkpoint is not None:\n        f = torch.load(initial_checkpoint, map_location=lambda storage, loc: storage)\n        start_iteration = f['iteration']\n        start_epoch = f['epoch']\n        state_dict  = f['state_dict']\n        net.load_state_dict(state_dict,strict=True)  #True\n    else:\n        start_iteration = 0\n        start_epoch = 0\n\n\n    log.write('net=%s\\n'%(type(net)))\n#         log.write('\\tinitial_checkpoint = %s\\n' % initial_checkpoint)\n    log.write('\\n')\n\n    # -----------------------------------------------\n    if 0: ##freeze\n        for p in net.block0.backbone.parameters(): \n            p.requires_grad = False\n\n    #optimizer = Lookahead(RAdam(filter(lambda p: p.requires_grad, net.parameters()),lr=start_lr), alpha=0.5, k=5)\n    #optimizer = RAdam(filter(lambda p: p.requires_grad, net.parameters()),lr=start_lr)\n    optimizer = MADGRAD( filter(lambda p: p.requires_grad, net.parameters()), lr=start_lr, momentum= 0.9, weight_decay= 0, eps= 1e-06)\n\n\n    # num_iteration = 8000\n    # iter_log    = 100\n    # iter_valid  = 100\n    # iter_save   = list(range(0, num_iteration, 100))#1*1000\n\n    log.write('optimizer\\n  %s\\n'%(optimizer))\n    log.write('\\n')\n\n\n    ## start training here! ##############################################\n    log.write('** start training here! **\\n')\n    log.write('   fold = %d\\n'%(fold + 1))\n    log.write('   is_mixed_precision = %s \\n'%str(is_mixed_precision))\n    log.write('   batch_size = %d\\n'%(batch_size))\n#     log.write('   experiment = %s\\n' % str(__file__.split('/')[-2:]))\n    log.write('                      |----- VALID ---|---- TRAIN/BATCH --------------\\n')\n    log.write('rate     iter   epoch | loss    map   | loss0  loss1  | time          \\n')\n    log.write('----------------------------------------------------------------------\\n')\n              #0.00000   0.00* 0.00  | 0.000  0.000  | 0.000  0.000  |  0 hr 00 min\n\n    def message(mode='print'):\n        if mode==('print'):\n            asterisk = ' '\n            loss = batch_loss\n        if mode==('log'):\n            asterisk = '*' if iteration in iter_save else ' '\n            loss = train_loss\n\n        text = \\\n            '%0.5f  %5.3f%s %4.2f  | '%(rate, iteration/10000, asterisk, epoch,) +\\\n            '%4.3f  %4.3f  %4.3f  %4.3f  | '%(*valid_loss,) +\\\n            '%4.3f  %4.3f  %4.3f  | '%(*loss,) +\\\n            '%s' % (time_to_str(timer() - start_timer,'min'))\n\n        return text\n\n    #----\n    valid_loss = np.zeros(4,np.float32)\n    train_loss = np.zeros(3,np.float32)\n    batch_loss = np.zeros_like(train_loss)\n    sum_train_loss = np.zeros_like(train_loss)\n    sum_train = 0\n    loss0 = torch.FloatTensor([0]).cuda().sum()\n    loss1 = torch.FloatTensor([0]).cuda().sum()\n    loss2 = torch.FloatTensor([0]).cuda().sum()\n\n\n    start_timer = timer()\n    iteration = start_iteration\n    epoch = start_epoch\n    rate = 0\n    while  iteration < num_iteration:\n\n        for t, batch in enumerate(train_loader):\n\n            if iteration in iter_save:\n                if iteration != start_iteration:\n                    torch.save({\n                        'state_dict': net.state_dict(),\n                        'iteration': iteration,\n                        'epoch': epoch,\n                    }, out_dir + '/checkpoint/%08d_model.pth' % (iteration))\n                    pass\n\n            if (iteration % iter_valid == 0):\n                #if iteration!=start_iteration:\n                    valid_loss = do_valid(net, valid_loader)  #\n                    pass\n\n            if (iteration % iter_log == 0):\n                print('\\r', end='', flush=True)\n                log.write(message(mode='log') + '\\n')\n\n\n            # learning rate schduler ------------\n            rate = get_learning_rate(optimizer)\n\n            # one iteration update  -------------\n            batch_size = len(batch['index'])\n            image = batch['image'].cuda()\n            truth_mask = batch['mask'].cuda()\n            truth_mask = F.interpolate(truth_mask, size=(16,16), mode='bilinear', align_corners=False)\n            onehot = batch['onehot'].cuda()\n            label = onehot.argmax(-1)\n\n            #----\n            net.train()\n            optimizer.zero_grad()\n\n            if is_mixed_precision:\n                with amp.autocast():\n                    logit, mask = data_parallel(net, image)\n                    loss0 = F.cross_entropy(logit, label)\n                    loss1 = 0.35*F.binary_cross_entropy_with_logits(mask, truth_mask) + 0.65 * symmetric_lovasz(mask.reshape(-1), truth_mask.reshape(-1))\n\n                #scaler.scale(loss0).backward()\n                #scaler.scale(loss1).backward()\n                scaler.scale(loss0+loss1).backward()\n                scaler.unscale_(optimizer)\n                #torch.nn.utils.clip_grad_norm_(net.parameters(), 2)\n                scaler.step(optimizer)\n                scaler.update()\n\n\n            else :\n                assert(False)\n                print('fp32')\n                logit, mask = data_parallel(net, image)\n                loss0 = F.cross_entropy(logit, label)\n                loss1 = 0.25 * F.binary_cross_entropy_with_logits(mask, truth_mask) + 0.75 * lovasz_soft_hinge_one(mask.reshape(-1), truth_math.reshape(-1))\n\n                (loss0+loss1).backward()\n                optimizer.step()\n\n\n            # print statistics  --------\n            epoch += 1 / len(train_loader)\n            iteration += 1\n\n            batch_loss = np.array([loss0.item(), loss1.item(), loss2.item()])\n            sum_train_loss += batch_loss\n            sum_train += 1\n            if iteration % 100 == 0:\n                train_loss = sum_train_loss / (sum_train + 1e-12)\n                sum_train_loss[...] = 0\n                sum_train = 0\n\n            print('\\r', end='', flush=True)\n            print(message(mode='print'), end='', flush=True)\n\n\n    log.write('\\n')","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:45:43.27247Z","iopub.execute_input":"2021-08-02T04:45:43.272808Z","iopub.status.idle":"2021-08-02T04:45:47.335597Z","shell.execute_reply.started":"2021-08-02T04:45:43.272777Z","shell.execute_reply":"2021-08-02T04:45:47.331886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train, df_valid = make_fold('train', 0)\ndf_train","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:54.375876Z","iopub.status.idle":"2021-08-02T04:44:54.376444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.copy('./fold4/checkpoint/00005200_model.pth', '/kaggle/working/b5_ns_fold4_0.372.pth')","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:54.377866Z","iopub.status.idle":"2021-08-02T04:44:54.378422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.copy('./fold4/checkpoint/00001000_model.pth', '/kaggle/working/effb3-aux-fold4.pth')","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:54.379703Z","iopub.status.idle":"2021-08-02T04:44:54.380257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.copy('./fold3/checkpoint/00002400_model.pth', '/kaggle/working/effb3-aux-fold3.pth')","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:54.381624Z","iopub.status.idle":"2021-08-02T04:44:54.382192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.copy('./fold2/checkpoint/00001000_model.pth', '/kaggle/working/effb3-aux-fold2.pth')","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:54.383486Z","iopub.status.idle":"2021-08-02T04:44:54.384053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.copy('./fold1/checkpoint/00001800_model.pth', '/kaggle/working/effb3-aux-fold1.pth')","metadata":{"execution":{"iopub.status.busy":"2021-08-02T04:44:54.385313Z","iopub.status.idle":"2021-08-02T04:44:54.385882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}