{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# pytorch-baseline for classify data(study -label)","metadata":{}},{"cell_type":"markdown","source":"# 0 导入程序包","metadata":{}},{"cell_type":"code","source":"package_paths = [\n    '../input/pytorch-image-models/pytorch-image-models-master', #'../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\n    '../input/image-fmix/FMix-master'\n]\nimport sys; \n\nfor pth in package_paths:\n    sys.path.append(pth)\n    \nfrom fmix import sample_mask, make_low_freq_image, binarise_mask","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:27.513518Z","iopub.execute_input":"2021-06-07T03:07:27.513933Z","iopub.status.idle":"2021-06-07T03:07:28.136668Z","shell.execute_reply.started":"2021-06-07T03:07:27.513826Z","shell.execute_reply":"2021-06-07T03:07:28.135833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nfrom skimage import exposure\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.nn.modules.loss import _WeightedLoss\nimport torch.nn.functional as F\n\nimport timm\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\n#from efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:28.138125Z","iopub.execute_input":"2021-06-07T03:07:28.138457Z","iopub.status.idle":"2021-06-07T03:07:31.043306Z","shell.execute_reply.started":"2021-06-07T03:07:28.138423Z","shell.execute_reply":"2021-06-07T03:07:31.042448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:31.045186Z","iopub.execute_input":"2021-06-07T03:07:31.045542Z","iopub.status.idle":"2021-06-07T03:07:31.051898Z","shell.execute_reply.started":"2021-06-07T03:07:31.045505Z","shell.execute_reply":"2021-06-07T03:07:31.051179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1 数据预处理","metadata":{}},{"cell_type":"code","source":"# 将训练csv读入\nCOMPETITION_NAME = \"siimcovid19-512-img-png-600-study-png\"\nload_dir = f\"/kaggle/input/{COMPETITION_NAME}/\"\ndf = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:31.053379Z","iopub.execute_input":"2021-06-07T03:07:31.053716Z","iopub.status.idle":"2021-06-07T03:07:31.092685Z","shell.execute_reply.started":"2021-06-07T03:07:31.053682Z","shell.execute_reply":"2021-06-07T03:07:31.091775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 为操作方便修改表头 inplace参数决定是否修改原df\ndf.rename(columns={'Negative for Pneumonia':'0','Typical Appearance':'1',\"Indeterminate Appearance\":'2',\n                   \"Atypical Appearance\":\"3\"}, inplace=True)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:31.093944Z","iopub.execute_input":"2021-06-07T03:07:31.094289Z","iopub.status.idle":"2021-06-07T03:07:31.104715Z","shell.execute_reply.started":"2021-06-07T03:07:31.094255Z","shell.execute_reply":"2021-06-07T03:07:31.103833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 解码one-hot\nlabels = []\ndef get_label(row):\n    for c in df.columns:\n        if row[c]==1:\n            labels.append(int(c))\ndf.apply(get_label, axis=1)\nprint(\"label modified\")","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:31.106183Z","iopub.execute_input":"2021-06-07T03:07:31.106549Z","iopub.status.idle":"2021-06-07T03:07:31.295400Z","shell.execute_reply.started":"2021-06-07T03:07:31.106516Z","shell.execute_reply":"2021-06-07T03:07:31.294392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 合并两份DataFrame,注意axis = 1参数\nlabels = {'label':labels}\nstudy_label = pd.DataFrame(labels)\ntrain_study = pd.concat([df, study_label], axis = 1)\n#print(train_study)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:31.296783Z","iopub.execute_input":"2021-06-07T03:07:31.297129Z","iopub.status.idle":"2021-06-07T03:07:31.305205Z","shell.execute_reply.started":"2021-06-07T03:07:31.297092Z","shell.execute_reply":"2021-06-07T03:07:31.304318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_study ['0'];del train_study ['1'];del train_study ['2'];del train_study ['3']\ntrain_study","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:31.306579Z","iopub.execute_input":"2021-06-07T03:07:31.306921Z","iopub.status.idle":"2021-06-07T03:07:31.328381Z","shell.execute_reply.started":"2021-06-07T03:07:31.306881Z","shell.execute_reply":"2021-06-07T03:07:31.327670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study.label.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:31.331715Z","iopub.execute_input":"2021-06-07T03:07:31.331950Z","iopub.status.idle":"2021-06-07T03:07:31.341547Z","shell.execute_reply.started":"2021-06-07T03:07:31.331928Z","shell.execute_reply":"2021-06-07T03:07:31.340657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 随机seed种子很重要\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:31.343299Z","iopub.execute_input":"2021-06-07T03:07:31.343626Z","iopub.status.idle":"2021-06-07T03:07:31.349058Z","shell.execute_reply.started":"2021-06-07T03:07:31.343592Z","shell.execute_reply":"2021-06-07T03:07:31.348045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 从公开分享的notebook中学习到：灰度值均衡化可能会增加准确率，所以这里尝试一下\ndef preprocess_image(img):\n    #img[:,:,0] = exposure.equalize_hist(img[:,:,0])\n    #img[:,:,1] = exposure.equalize_hist(img[:,:,1])\n    #img[:,:,2] = exposure.equalize_hist(img[:,:,2])\n    eimg = exposure.equalize_hist(img)\n    return eimg\n# 目前实现的效果并没有太大提升，但感觉训练过程更稳定了一些","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:31.350467Z","iopub.execute_input":"2021-06-07T03:07:31.351168Z","iopub.status.idle":"2021-06-07T03:07:31.358676Z","shell.execute_reply.started":"2021-06-07T03:07:31.351133Z","shell.execute_reply":"2021-06-07T03:07:31.357888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 打开一张图看看长什么样\ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    return im_rgb\n    \n    '''\n    #打开这两条程序可以看到处理前后两者的对比\n    #processimg = np.concatenate((im_rgb/255, equ_img), axis=1)\n    #return processimg\n    '''\n    #equ_img = preprocess_image(im_rgb)  #打开或关闭通道灰度均值化\n    #return equ_img.astype(np.float32)\n\nyuan = get_img('../input/siimcovid19-512-img-png-600-study-png/study/00086460a852_study.png')\nplt.imshow(yuan)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:31.359801Z","iopub.execute_input":"2021-06-07T03:07:31.360152Z","iopub.status.idle":"2021-06-07T03:07:31.571290Z","shell.execute_reply.started":"2021-06-07T03:07:31.360119Z","shell.execute_reply":"2021-06-07T03:07:31.570365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 调试\n'''\npath = '../input/siimcovid19-512-img-png-600-study-png/study/00086460a852_study.png'\nimg = cv2.imread(path)\nprint(img)\n'''","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:31.572723Z","iopub.execute_input":"2021-06-07T03:07:31.573067Z","iopub.status.idle":"2021-06-07T03:07:31.578249Z","shell.execute_reply.started":"2021-06-07T03:07:31.573031Z","shell.execute_reply":"2021-06-07T03:07:31.577483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2 构建网络及其基本参数设置","metadata":{}},{"cell_type":"code","source":"# 基本超参数设置\nCFG = {\n    'fold_num': 5,\n    'seed': 2021,\n    #'model_arch': 'tf_efficientnet_b7',\n    'model_arch': 'tf_efficientnet_b4_ns',\n    'img_size': 512,\n    'epochs': 60,\n    'train_bs': 16,\n    'valid_bs': 32,\n    'T_0': 10,\n    'lr': 1e-6,\n    'min_lr': 1e-6,\n    'weight_decay':1e-6,\n    'num_workers': 4,\n    'accum_iter': 2, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0'\n}","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:31.579550Z","iopub.execute_input":"2021-06-07T03:07:31.580096Z","iopub.status.idle":"2021-06-07T03:07:31.587095Z","shell.execute_reply.started":"2021-06-07T03:07:31.580063Z","shell.execute_reply":"2021-06-07T03:07:31.586074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rand_bbox(size, lam):\n    W = size[0]\n    H = size[1]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    return bbx1, bby1, bbx2, bby2\n\n\nclass CassavaDataset(Dataset):\n    def __init__(self, train_study, data_root, \n                 transforms=None, \n                 output_label=True, \n                 one_hot_label=False,\n                 do_fmix=False, \n                 fmix_params={\n                     'alpha': 1., \n                     'decay_power': 3., \n                     'shape': (CFG['img_size'], CFG['img_size']),\n                     'max_soft': True, \n                     'reformulate': False\n                 },\n                 do_cutmix=False,\n                 cutmix_params={\n                     'alpha': 1,\n                 }\n                ):\n        \n        super().__init__()\n        self.df = train_study.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.do_fmix = do_fmix\n        self.fmix_params = fmix_params\n        self.do_cutmix = do_cutmix\n        self.cutmix_params = cutmix_params\n        \n        self.output_label = output_label\n        self.one_hot_label = one_hot_label\n        \n        if output_label == True:\n            self.labels = self.df['label'].values\n            #print(self.labels)\n            \n            if one_hot_label is True:\n                self.labels = np.eye(self.df['label'].max()+1)[self.labels]\n                #print(self.labels)\n            \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.labels[index]\n        #print('查查路径正确吗',\"{}/{}\".format(self.data_root, self.df.loc[index]['id']))\n        img  = get_img(\"{}/{}\".format(self.data_root, self.df.loc[index]['id'])+'.png')\n        #print(img)\n\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        \n        if self.do_fmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n            with torch.no_grad():\n                #lam, mask = sample_mask(**self.fmix_params)\n                \n                lam = np.clip(np.random.beta(self.fmix_params['alpha'], self.fmix_params['alpha']),0.6,0.7)\n                \n                # Make mask, get mean / std\n                mask = make_low_freq_image(self.fmix_params['decay_power'], self.fmix_params['shape'])\n                mask = binarise_mask(mask, lam, self.fmix_params['shape'], self.fmix_params['max_soft'])\n    \n                fmix_ix = np.random.choice(self.df.index, size=1)[0]\n                fmix_img  = get_img(\"{}/{}\".format(self.data_root, self.df.iloc[fmix_ix]['id'])+'.png')\n\n                if self.transforms:\n                    fmix_img = self.transforms(image=fmix_img)['image']\n\n                mask_torch = torch.from_numpy(mask)\n                \n                # mix image\n                img = mask_torch*img+(1.-mask_torch)*fmix_img\n\n                #print(mask.shape)\n\n                #assert self.output_label==True and self.one_hot_label==True\n\n                # mix target\n                rate = mask.sum()/CFG['img_size']/CFG['img_size']\n                target = rate*target + (1.-rate)*self.labels[fmix_ix]\n                #print(target, mask, img)\n                #assert False\n        \n        if self.do_cutmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n            #print(img.sum(), img.shape)\n            with torch.no_grad():\n                cmix_ix = np.random.choice(self.df.index, size=1)[0]\n                cmix_img  = get_img(\"{}/{}\".format(self.data_root, self.df.iloc[cmix_ix]['id'])+'.png')\n                if self.transforms:\n                    cmix_img = self.transforms(image=cmix_img)['image']\n                    \n                lam = np.clip(np.random.beta(self.cutmix_params['alpha'], self.cutmix_params['alpha']),0.3,0.4)\n                bbx1, bby1, bbx2, bby2 = rand_bbox((CFG['img_size'], CFG['img_size']), lam)\n\n                img[:, bbx1:bbx2, bby1:bby2] = cmix_img[:, bbx1:bbx2, bby1:bby2]\n\n                rate = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (CFG['img_size'] * CFG['img_size']))\n                target = rate*target + (1.-rate)*self.labels[cmix_ix]\n                \n            #print('-', img.sum())\n            #print(target)\n            #assert False\n                            \n        # do label smoothing\n        #print(type(img), type(target))\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:31.588448Z","iopub.execute_input":"2021-06-07T03:07:31.588861Z","iopub.status.idle":"2021-06-07T03:07:31.612961Z","shell.execute_reply.started":"2021-06-07T03:07:31.588827Z","shell.execute_reply":"2021-06-07T03:07:31.612273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \n        \ndef get_valid_transforms():\n    return Compose([\n            CenterCrop(CFG['img_size'], CFG['img_size'], p=1.),\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:31.614929Z","iopub.execute_input":"2021-06-07T03:07:31.615467Z","iopub.status.idle":"2021-06-07T03:07:32.292240Z","shell.execute_reply.started":"2021-06-07T03:07:31.615434Z","shell.execute_reply":"2021-06-07T03:07:32.291387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n        '''\n        self.model.classifier = nn.Sequential(\n            nn.Dropout(0.3),\n            #nn.Linear(n_features, hidden_size,bias=True), nn.ELU(),\n            nn.Linear(n_features, n_class, bias=True)\n        )\n        '''\n    def forward(self, x):\n        x = self.model(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:32.293572Z","iopub.execute_input":"2021-06-07T03:07:32.293910Z","iopub.status.idle":"2021-06-07T03:07:32.300598Z","shell.execute_reply.started":"2021-06-07T03:07:32.293874Z","shell.execute_reply":"2021-06-07T03:07:32.299003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#绘制loss，方便查看\ndef PlotLossFun(epoch, Cur_List, IFTRAIN='TRAIN'):\n    x = range(len(Cur_List))\n    y = Cur_List\n    \n    IFTRAIN = IFTRAIN\n    EPOCHNUM = str(epoch)\n    \n    plt.plot(x,y)\n    plt.xlabel(f'EpochNum:{EPOCHNUM}')\n    plt.ylabel(f'{IFTRAIN}cur')\n    \n    #if epoch == CFG['epochs']-1:\n    plt.show()\n    plt.savefig(f\"{IFTRAIN}_epoch{EPOCHNUM}_cur.jpg\")","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:32.302255Z","iopub.execute_input":"2021-06-07T03:07:32.302589Z","iopub.status.idle":"2021-06-07T03:07:32.313027Z","shell.execute_reply.started":"2021-06-07T03:07:32.302556Z","shell.execute_reply":"2021-06-07T03:07:32.312130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_dataloader(df, trn_idx, val_idx, data_root='../input/siimcovid19-512-img-png-600-study-png/study'):\n    \n    from catalyst.data.sampler import BalanceClassSampler\n    \n    train_ = df.loc[trn_idx,:].reset_index(drop=True)\n    valid_ = df.loc[val_idx,:].reset_index(drop=True)\n    #print(train_, valid_)\n        \n    train_ds = CassavaDataset(train_, data_root, transforms=get_train_transforms(), output_label=True, one_hot_label=False, do_fmix=False, do_cutmix=False)\n    valid_ds = CassavaDataset(valid_, data_root, transforms=get_valid_transforms(), output_label=True)\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=CFG['train_bs'],\n        pin_memory=False,\n        drop_last=False,\n        shuffle=True,        \n        num_workers=CFG['num_workers'],\n        #sampler=BalanceClassSampler(labels=train_['label'].values, mode=\"downsampling\")\n    )\n    val_loader = torch.utils.data.DataLoader(\n        valid_ds, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=False,\n    )\n    return train_loader, val_loader\n\ndef train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, scheduler=None, schd_batch_update=False):   \n       \n    model.train()\n    Temp_Loss_List = []\n\n    t = time.time()\n    running_loss = None\n\n    pbar = tqdm(enumerate(train_loader), total=len(train_loader))\n    #print(enumerate(train_loader),'len(train_loader)',train_loader)\n    \n    for step, (imgs, image_labels) in pbar:\n        \n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n\n        #print(image_labels.shape, exam_label.shape)\n        with autocast():\n            image_preds = model(imgs)   #output = model(input)\n\n            loss = loss_fn(image_preds, image_labels)\n            \n            scaler.scale(loss).backward()\n\n            if running_loss is None:\n                running_loss = loss.item()\n            else:\n                running_loss = running_loss * .99 + loss.item() * .01\n\n            if ((step + 1) %  CFG['accum_iter'] == 0) or ((step + 1) == len(train_loader)):\n                # may unscale_ here if desired (e.g., to allow clipping unscaled gradients)\n\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad() \n                \n                if scheduler is not None and schd_batch_update:\n                    scheduler.step()\n\n            if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(train_loader)):\n                description = f'epoch {epoch} loss: {running_loss:.4f}'\n                Temp_Loss_List.append(running_loss)\n                \n                pbar.set_description(description)\n    \n    # 绘制loss曲线\n    Train_Loss_List.append(np.sum(Temp_Loss_List) / len(Temp_Loss_List))\n    if epoch == CFG['epochs']-1:\n        PlotLossFun(epoch, Train_Loss_List,IFTRAIN='TRAIN')            \n    if scheduler is not None and not schd_batch_update:\n        scheduler.step()\n    return Train_Loss_List\n        \ndef valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False):\n    model.eval()\n\n    t = time.time()\n    loss_sum = 0\n    sample_num = 0\n    image_preds_all = []\n    image_targets_all = []\n    \n    pbar = tqdm(enumerate(val_loader), total=len(val_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n        \n        image_preds = model(imgs)   #output = model(input)\n        #print(image_preds.shape, exam_pred.shape)\n        image_preds_all += [torch.argmax(image_preds, 1).detach().cpu().numpy()]\n        image_targets_all += [image_labels.detach().cpu().numpy()]\n        \n        loss = loss_fn(image_preds, image_labels)\n        \n        loss_sum += loss.item()*image_labels.shape[0]\n        sample_num += image_labels.shape[0]  \n\n        if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(val_loader)):\n            description = f'epoch {epoch} loss: {loss_sum/sample_num:.4f}'\n            pbar.set_description(description)\n    \n    image_preds_all = np.concatenate(image_preds_all)\n    image_targets_all = np.concatenate(image_targets_all)\n    print('validation multi-class accuracy = {:.4f}'.format((image_preds_all==image_targets_all).mean()))\n    \n    Val_Acc_List.append((image_preds_all==image_targets_all).mean())\n    if epoch == CFG['epochs']-1:\n        PlotLossFun(epoch, Val_Acc_List, IFTRAIN='VAL') \n    \n    if scheduler is not None:\n        if schd_loss_update:\n            scheduler.step(loss_sum/sample_num)\n        else:\n            scheduler.step()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:32.314446Z","iopub.execute_input":"2021-06-07T03:07:32.314907Z","iopub.status.idle":"2021-06-07T03:07:32.337069Z","shell.execute_reply.started":"2021-06-07T03:07:32.314755Z","shell.execute_reply":"2021-06-07T03:07:32.336253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reference: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/173733\nclass MyCrossEntropyLoss(_WeightedLoss):\n    def __init__(self, weight=None, reduction='mean'):\n        super().__init__(weight=weight, reduction=reduction)\n        self.weight = weight\n        self.reduction = reduction\n\n    def forward(self, inputs, targets):\n        lsm = F.log_softmax(inputs, -1)\n\n        if self.weight is not None:\n            lsm = lsm * self.weight.unsqueeze(0)\n\n        loss = -(targets * lsm).sum(-1)\n\n        if  self.reduction == 'sum':\n            loss = loss.sum()\n        elif  self.reduction == 'mean':\n            loss = loss.mean()\n\n        return loss","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:32.338277Z","iopub.execute_input":"2021-06-07T03:07:32.338666Z","iopub.status.idle":"2021-06-07T03:07:32.353163Z","shell.execute_reply.started":"2021-06-07T03:07:32.338630Z","shell.execute_reply":"2021-06-07T03:07:32.352333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3 主循环，开始训练网络","metadata":{}},{"cell_type":"code","source":"if __name__ == '__main__':\n    \n    # for training only, need nightly build pytorch\n    seed_everything(CFG['seed'])\n    \n    folds = StratifiedKFold(n_splits=CFG['fold_num'], shuffle=True, random_state=CFG['seed']).split(np.arange(train_study.shape[0]), train_study.label.values)\n    \n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        \n        # we'll train fold 0 first\n        if fold > 0:\n            break \n\n        print('Training with {} started'.format(fold))\n\n        print(len(trn_idx), len(val_idx))\n        train_loader, val_loader = prepare_dataloader(train_study, trn_idx, val_idx, \n                                                      data_root='../input/siimcovid19-512-img-png-600-study-png/study')\n        device = torch.device(CFG['device'])\n        model = CassvaImgClassifier(CFG['model_arch'], train_study.label.nunique(), pretrained=True).to(device)\n        scaler = GradScaler()   \n        optimizer = torch.optim.Adam(model.parameters(), lr=CFG['lr'], weight_decay=CFG['weight_decay'])\n        #scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=CFG['epochs']-1)\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG['T_0'], T_mult=1, eta_min=CFG['min_lr'], last_epoch=-1)\n        #scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=25, \n        #                                                max_lr=CFG['lr'], epochs=CFG['epochs'], steps_per_epoch=len(train_loader))\n        \n        loss_tr = nn.CrossEntropyLoss().to(device) #MyCrossEntropyLoss().to(device)\n        loss_fn = nn.CrossEntropyLoss().to(device)\n        \n        #建立空数组，用来绘制曲线\n        global Train_Loss_List, Val_Acc_List\n        Train_Loss_List = []\n        Val_Acc_List = []\n            \n        for epoch in range(CFG['epochs']):\n\n            train_one_epoch(epoch, model, loss_tr, optimizer, train_loader, device, scheduler=scheduler, schd_batch_update=False)\n            \n            with torch.no_grad():\n                valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False)\n            \n            # 保存模型\n            if epoch > CFG['epochs']-10:\n                torch.save(model.state_dict(),'{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch))\n            \n        #torch.save(model.cnn_model.state_dict(),'{}/cnn_model_fold_{}_{}'.format(CFG['model_path'], fold, CFG['tag']))\n        del model, optimizer, train_loader, val_loader, scaler, scheduler\n        torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T03:07:32.354974Z","iopub.execute_input":"2021-06-07T03:07:32.355209Z","iopub.status.idle":"2021-06-07T03:18:05.349197Z","shell.execute_reply.started":"2021-06-07T03:07:32.355187Z","shell.execute_reply":"2021-06-07T03:18:05.347048Z"},"trusted":true},"execution_count":null,"outputs":[]}]}