{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# fmixというライブりをimportできるようにする\nfrom os.path import exists\nif not exists('fmix.zip'):\n    !wget -O fmix.zip https://github.com/ecs-vlc/fmix/archive/master.zip\n    !unzip -qq fmix.zip\n    !mv FMix-master/* ./\n    !rm -r FMix-master","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"package_paths = [\n    '../input/pytorch-image-models/pytorch-image-models-master', #'../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\n    '../input/image-fmix/FMix-master'\n]\nimport sys; \n\nfor pth in package_paths:\n    sys.path.append(pth)\n    \n# sample_mask:引数のアルファでパラメトリック化されたベータ分布から平均ラムダをサンプリングし，低周波画像を作成し，このラムダに基づいて2値化します．\n# make_low_freq_image:\n# binarise_mask:\nfrom fmix import sample_mask, make_low_freq_image, binarise_mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from glob import glob   # 引数に指定されたパターンにマッチするファイルパス名を取得する\n# GroupKFold->グループの概念を追加したKFold, StartifiedKFold->目的変数の偏りがないように、データを分割します\n# 参考：https://qiita.com/Mukomiz/items/1156ea693dc4f077ca79#group-kfold\n# グループのメリット：複数人の画像データ（1人の画像が表情を変えて複数ある前提）を入力に男性・女性を判断したい場合、学習用データ\n# とテストデータに同じ人が存在してしまうと、その画像の精度がよくなってしまうことがある。\n# そのため、同じ人の画像をグループ化して学習用とテスト用のどちらかだけで使うようにする。\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io # 画像処理ライブラリ\nimport torch           # deeplearningライブラリ\nfrom torch import nn   # ニューラルネットワーク機能\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport torchvision      # 画像の前処理や学習済みモデルなどを提供する\nfrom torchvision import transforms   # 画像の編集ライブラリ   参考：https://pystyle.info/pytorch-list-of-transforms/\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm   # プログレスバーを表示する\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.nn.modules.loss import _WeightedLoss\nimport torch.nn.functional as F\n\nimport timm\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\n#from efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG = {\n    'fold_num': 5,\n    'seed': 719,\n    'model_arch': 'tf_efficientnet_b4_ns',\n    'img_size': 512,\n    'epochs': 10,   # 一つの訓練データを何回繰り返して学習させるか\n    'train_bs': 16,\n    'valid_bs': 32,\n    'T_0': 10,\n    'lr': 1e-4,\n    'min_lr': 1e-6,\n    'weight_decay':1e-6,\n    'num_workers': 4,\n    'accum_iter': 2, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.label.value_counts()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"各フォールドで層別検証分割を行うことで、各フォールドの訓練集合と検証集合がターゲット分布の訓練集合全体のように見えるようにすることができます。"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsubmission.head()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper Functions\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)                # 乱数生成器を初期化\n    os.environ['PYTHONHASHSEED'] = str(seed) # 乱数生成器の固定化\n    np.random.seed(seed)                     # 発生する乱数をあらかじめ固定する\n    torch.manual_seed(seed)                  # 発生する乱数をあらかじめ固定する\n    torch.cuda.manual_seed(seed)             # 発生する乱数をあらかじめ固定する\n    # 参考：https://nuka137.hatenablog.com/entry/2020/09/01/080038\n    torch.backends.cudnn.deterministic = True # 乱数の固定化\n    torch.backends.cudnn.benchmark = True\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    #print(im_rgb)\n    return im_rgb\n\nimg = get_img('../input/cassava-leaf-disease-classification/train_images/1000015157.jpg')\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ランダムなバウンディングボックスを取得?\ndef rand_bbox(size, lam):   # bbox->バウンディングボックス\n    W = size[0]\n    H = size[1]\n    cut_rat = np.sqrt(1. - lam)   # sqrt->√\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)  # clip->NumPy配列ndarrayの要素の値を任意の範囲内に収める\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    return bbx1, bby1, bbx2, bby2\n\n\nclass CassavaDataset(Dataset):\n    def __init__(self, df, data_root, \n                 transforms=None, \n                 output_label=True, \n                 one_hot_label=False,\n                 do_fmix=False, \n                 fmix_params={\n                     'alpha': 1., \n                     'decay_power': 3., \n                     'shape': (CFG['img_size'], CFG['img_size']),\n                     'max_soft': True, \n                     'reformulate': False\n                 },\n                 do_cutmix=False,\n                 cutmix_params={\n                     'alpha': 1,\n                 }\n                ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()  # reset_index0->インデックスを0から振りなおす drop->元のインデックスの削除\n        self.transforms = transforms\n        self.data_root = data_root\n        self.do_fmix = do_fmix\n        self.fmix_params = fmix_params\n        self.do_cutmix = do_cutmix\n        self.cutmix_params = cutmix_params\n        \n        self.output_label = output_label\n        self.one_hot_label = one_hot_label\n        \n        if output_label == True:\n            self.labels = self.df['label'].values\n            #print(self.labels)\n            \n            if one_hot_label is True:\n                self.labels = np.eye(self.df['label'].max()+1)[self.labels]   # eye->単位行列を作成する\n                #print(self.labels)\n            \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.labels[index]\n          \n        img  = get_img(\"{}/{}\".format(self.data_root, self.df.loc[index]['image_id']))\n\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        \n        if self.do_fmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n            with torch.no_grad():\n                #lam, mask = sample_mask(**self.fmix_params)\n                \n                lam = np.clip(np.random.beta(self.fmix_params['alpha'], self.fmix_params['alpha']),0.6,0.7)\n                \n                # Make mask, get mean / std\n                mask = make_low_freq_image(self.fmix_params['decay_power'], self.fmix_params['shape'])\n                mask = binarise_mask(mask, lam, self.fmix_params['shape'], self.fmix_params['max_soft'])\n    \n                fmix_ix = np.random.choice(self.df.index, size=1)[0]\n                fmix_img  = get_img(\"{}/{}\".format(self.data_root, self.df.iloc[fmix_ix]['image_id']))\n\n                if self.transforms:\n                    fmix_img = self.transforms(image=fmix_img)['image']\n\n                mask_torch = torch.from_numpy(mask)\n                \n                # mix image\n                img = mask_torch*img+(1.-mask_torch)*fmix_img\n\n                #print(mask.shape)\n\n                #assert self.output_label==True and self.one_hot_label==True\n\n                # mix target\n                rate = mask.sum()/CFG['img_size']/CFG['img_size']\n                target = rate*target + (1.-rate)*self.labels[fmix_ix]\n                #print(target, mask, img)\n                #assert False\n        \n        if self.do_cutmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n            #print(img.sum(), img.shape)\n            with torch.no_grad():\n                cmix_ix = np.random.choice(self.df.index, size=1)[0]\n                cmix_img  = get_img(\"{}/{}\".format(self.data_root, self.df.iloc[cmix_ix]['image_id']))\n                if self.transforms:\n                    cmix_img = self.transforms(image=cmix_img)['image']\n                    \n                lam = np.clip(np.random.beta(self.cutmix_params['alpha'], self.cutmix_params['alpha']),0.3,0.4)\n                bbx1, bby1, bbx2, bby2 = rand_bbox((CFG['img_size'], CFG['img_size']), lam)\n\n                img[:, bbx1:bbx2, bby1:bby2] = cmix_img[:, bbx1:bbx2, bby1:bby2]\n\n                rate = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (CFG['img_size'] * CFG['img_size']))\n                target = rate*target + (1.-rate)*self.labels[cmix_ix]\n                \n            #print('-', img.sum())\n            #print(target)\n            #assert False\n                            \n        # do label smoothing\n        #print(type(img), type(target))\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define Train\\Validation Image Augmentations(訓練検証画像の拡張を定義する)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# HorizontalFlip->左右反転, VerticalFlip->上下反転, IAAPerspective->4点透視変換??, ShiftScaleRotate->平行移動、拡大縮小、回転, \n# CLAHE->画像全体の画素値をみて平滑化するのではなく、いくつかのタイル（上の例では 8 × 8 ）に分割した上で、その中で平滑化を行なう。\n# RandomRotate90->ランダムに90°単位で回転させる, Transpose->行と列を入れ替えて転置, Blur->画像をぼかす, OpticalDistortion->光学的な歪みを再現する\n# GridDistortion->Grid歪みを再現する, HueSaturationValue->ランダムに色相、彩度、輝度を変える\n# IAAAdditiveGaussianNoise->入力画像にガウスノイズを追加, GaussNoise->ガウスノイズを付加, MotionBlur->ランダムなカーネルサイズでモーションブラー(ボックスフィルタ)をかける\n# IAAPiecewiseAffine->入力上に点の規則的なグリッドを配置し，アフィン変換によってこれらの点の近傍をランダムに移動させます\n# RandomResizedCrop->ランダムにリサイズ及び切り抜きを行う\n# 参考URL:https://qiita.com/kurilab/items/b69e1be8d0224ae139ad\n# https://pystyle.info/pytorch-list-of-transforms/\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),  # 行と列を入れ替えて転置\n            HorizontalFlip(p=0.5),  # 左右反転\n            VerticalFlip(p=0.5),    # 上下反転\n            ShiftScaleRotate(p=0.5), # 平行移動、拡大縮小、回転\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5), # ランダムに色相、彩度、輝度を変える\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5), # ランダムに明るさとコントラストを変える\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0), # 画素値を255で除算→チャンネルごとの平均値を減算→チャンネルごとの標準偏差で除算\n\n            CoarseDropout(p=0.5), # 矩形領域の粗いDropout\n            Cutout(p=0.5),        # 矩形領域の粗いDropout\n            ToTensorV2(p=1.0),    # torch.Tensorに変換\n        ], p=1.)\n  \n        \ndef get_valid_transforms():\n    return Compose([\n            CenterCrop(CFG['img_size'], CFG['img_size'], p=1.),\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        # tf_efficientnet_b4_nsというモデルを作成\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        # ???\n        n_features = self.model.classifier.in_features\n        # ニューラルネットワークの1層を作成。n_features=各入力サンプルのサイズ, n_class=各出力サンプルのサイズ\n        self.model.classifier = nn.Linear(n_features, n_class)\n        '''\n        self.model.classifier = nn.Sequential(\n            nn.Dropout(0.3),\n            #nn.Linear(n_features, hidden_size,bias=True), nn.ELU(),\n            nn.Linear(n_features, n_class, bias=True)\n        )\n        '''\n    def forward(self, x):\n        x = self.model(x)\n        return x\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training APIs"},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_dataloader(df, trn_idx, val_idx, data_root='../input/cassava-leaf-disease-classification/train_images/'):\n    \n    from catalyst.data.sampler import BalanceClassSampler\n    print(data_root)\n    \n    # tran_idx, val_idxの逆順のリスト作成\n    train_ = df.loc[trn_idx,:].reset_index(drop=True)\n    valid_ = df.loc[val_idx,:].reset_index(drop=True)\n    \n    # データセット作成\n    train_ds = CassavaDataset(train_, data_root, transforms=get_train_transforms(), output_label=True, one_hot_label=False, do_fmix=False, do_cutmix=False)\n    valid_ds = CassavaDataset(valid_, data_root, transforms=get_valid_transforms(), output_label=True)\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=CFG['train_bs'],\n        pin_memory=False,               # Trueの時、テンソルを CUDA ピン留めメモリにコピーする。\n        drop_last=False,                # Trueの時、データセットのサイズがバッチサイズで割り切れない場合、最後の不完全なバッチを削除する。Falseの場合、最後のデータが小さくなる\n        shuffle=True,                   # Trueの時、エポックごとにデータが再シャッフルする。\n        num_workers=CFG['num_workers'],   # 処理を行うCPU数？\n        #sampler=BalanceClassSampler(labels=train_['label'].values, mode=\"downsampling\")\n    )\n    val_loader = torch.utils.data.DataLoader(\n        valid_ds, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=False,\n    )\n    return train_loader, val_loader\n\ndef train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, scheduler=None, schd_batch_update=False):\n    model.train()\n\n    t = time.time()  # 現在時刻\n    running_loss = None\n\n    # train_loaderの中身がよくわからない。\n    pbar = tqdm(enumerate(train_loader), total=len(train_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n\n        #print(image_labels.shape, exam_label.shape)\n        # 演算処理をampの対象にキャスト??\n        # amp->いくつかの演算で torch.float32 (float) のデータ型を使用し、その他の演算では torch.float16 (ハーフ) を使用する、混合精度の便利なメソッド\n        with autocast():\n            image_preds = model(imgs)   #output = model(input)\n            #print(image_preds.shape, exam_pred.shape)\n\n            loss = loss_fn(image_preds, image_labels)\n            \n            # スケーラーの勾配を作る\n            scaler.scale(loss).backward()\n\n            if running_loss is None:\n                running_loss = loss.item()\n            else:\n                running_loss = running_loss * .99 + loss.item() * .01\n\n            if ((step + 1) %  CFG['accum_iter'] == 0) or ((step + 1) == len(train_loader)):\n                # may unscale_ here if desired (e.g., to allow clipping unscaled gradients)\n\n                # 最初にオプティマイザの代入されたパラメータのグラデーションをアンスケールします??\n                scaler.step(optimizer)\n                # 次のイテレーションのスケールを更新します。??\n                scaler.update()\n                # 勾配をゼロクリアする\n                optimizer.zero_grad() \n                \n                if scheduler is not None and schd_batch_update:\n                    scheduler.step()\n\n            if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(train_loader)):\n                description = f'epoch {epoch} loss: {running_loss:.4f}'\n                # 説明文の設定\n                pbar.set_description(description)\n                \n    if scheduler is not None and not schd_batch_update:\n        scheduler.step()\n        \ndef valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False):\n    # https://qiita.com/tatsuya11bbs/items/86141fe3ca35bdae7338\n    # eval()はdropoutやbatch normの on/offの切替\n    model.eval()\n\n    t = time.time()\n    loss_sum = 0\n    sample_num = 0\n    image_preds_all = []\n    image_targets_all = []\n    \n    pbar = tqdm(enumerate(val_loader), total=len(val_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n        \n        image_preds = model(imgs)   #output = model(input)\n        #print(image_preds.shape, exam_pred.shape)\n        image_preds_all += [torch.argmax(image_preds, 1).detach().cpu().numpy()]\n        image_targets_all += [image_labels.detach().cpu().numpy()]\n        \n        loss = loss_fn(image_preds, image_labels)\n        \n        loss_sum += loss.item()*image_labels.shape[0]\n        sample_num += image_labels.shape[0]  \n\n        if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(val_loader)):\n            description = f'epoch {epoch} loss: {loss_sum/sample_num:.4f}'\n            pbar.set_description(description)\n    \n    # concatenate->結合\n    image_preds_all = np.concatenate(image_preds_all)\n    image_targets_all = np.concatenate(image_targets_all)\n    print('validation multi-class accuracy = {:.4f}'.format((image_preds_all==image_targets_all).mean()))\n    \n    if scheduler is not None:\n        if schd_loss_update:\n            scheduler.step(loss_sum/sample_num)\n        else:\n            scheduler.step()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# reference: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/173733\nclass MyCrossEntropyLoss(_WeightedLoss):\n    def __init__(self, weight=None, reduction='mean'):\n        super().__init__(weight=weight, reduction=reduction)\n        self.weight = weight\n        self.reduction = reduction\n\n    def forward(self, inputs, targets):\n        lsm = F.log_softmax(inputs, -1)\n\n        if self.weight is not None:\n            lsm = lsm * self.weight.unsqueeze(0)\n\n        loss = -(targets * lsm).sum(-1)\n\n        if  self.reduction == 'sum':\n            loss = loss.sum()\n        elif  self.reduction == 'mean':\n            loss = loss.mean()\n\n        return loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Main Loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n     # for training only, need nightly build pytorch\n\n    seed_everything(CFG['seed'])\n    \n    # n_splits->分割数(5), shuffle->分割前に各クラスのサンプルをシャフルする。 np.arange(train.shape[0])->trainデータの画像の数だけ0からの配列を作成(0,1,2…) \n    # train.label.values->ラベルごとの数\n    folds = StratifiedKFold(n_splits=CFG['fold_num'], shuffle=True, random_state=CFG['seed']).split(np.arange(train.shape[0]), train.label.values)\n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        # we'll train fold 0 first\n        if fold > 0:\n            break \n\n        print('Training with {} started'.format(fold))\n\n        print(\"fold={} trn_idx={} val_idx={}\".format(fold, trn_idx, val_idx))\n        print(len(trn_idx), len(val_idx))\n        train_loader, val_loader = prepare_dataloader(train, trn_idx, val_idx, data_root='../input/cassava-leaf-disease-classification/train_images/')\n\n        # pytorchで使用するGPU(cuda)を指定します。\n        device = torch.device(CFG['device'])\n        \n        model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique(), pretrained=True).to(device)\n        # スケーラーの定義\n        # 勾配のアンダーフローを最小化することで、float16 の勾配を持つネットワークの収束性を向上させます。？？？\n        scaler = GradScaler()   \n        # モデルの最適化  lr=学習率  weight_decay=重みの減少率？？ \n        # 参考：https://rightcode.co.jp/blog/information-technology/torch-optim-optimizer-compare-and-verify-update-process-and-performance-of-optimization-methods\n        optimizer = torch.optim.Adam(model.parameters(), lr=CFG['lr'], weight_decay=CFG['weight_decay'])\n        #scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=CFG['epochs']-1)\n        # COS曲線を描くスケジューラー？？\n        # スケジューラーとは、学習率のスケジューリングのこと。1回目は0.1、2回目は0.2・・・　https://wonderfuru.com/scheduler/\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG['T_0'], T_mult=1, eta_min=CFG['min_lr'], last_epoch=-1)\n        #scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=25, \n        #                                                max_lr=CFG['lr'], epochs=CFG['epochs'], steps_per_epoch=len(train_loader))\n        # 損失関数（交差エントロピー誤差）\n        loss_tr = nn.CrossEntropyLoss().to(device) #MyCrossEntropyLoss().to(device)\n        loss_fn = nn.CrossEntropyLoss().to(device)\n        \n        for epoch in range(CFG['epochs']):\n            train_one_epoch(epoch, model, loss_tr, optimizer, train_loader, device, scheduler=scheduler, schd_batch_update=False)\n\n            with torch.no_grad():\n                valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False)\n\n            torch.save(model.state_dict(),'{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch))\n            \n        #torch.save(model.cnn_model.state_dict(),'{}/cnn_model_fold_{}_{}'.format(CFG['model_path'], fold, CFG['tag']))\n        del model, optimizer, train_loader, val_loader, scaler, scheduler\n        torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}