{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport json\nfrom PIL import Image\nimport os\n\nimport torch\nimport torch.nn as nn\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master/')\nsys.path.append('../input/adamp-optimizer/AdamP-master')\nimport timm\nfrom adamp import AdamP\n!conda install -c conda-forge nvidia-apex --yes\nfrom apex import amp","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 파일들 불러오기"},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_DIR='../input/cassava-leaf-disease-classification/train_images/'\nTEST_DIR='../input/cassava-leaf-disease-classification/test_images/'\nlabels=json.load(open(\"../input/cassava-leaf-disease-classification/label_num_to_disease_map.json\"))\n\ntrain=pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.head()\n\nDEVICE=torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n#print(DEVICE)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 학습 파라미터 설정"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG:\n    BATCH=32\n    EPOCHS=20\n    LR=0.0001\n    IMG_SIZE=256\n    model_name='tf_efficientnet_b3_ns'\n    target_size=5\n    weight_decay=1e-6","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 학습 이미지와 라벨 구분"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_Train , Y_Train = train['image_id'].values, train['label'].values\nX_Test=[name for name in (os.listdir(TEST_DIR))]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 데이터 불러오는 클래스 선언"},{"metadata":{"trusted":true},"cell_type":"code","source":"class GetData(Dataset):\n    def __init__(self, Dir, FNames, Labels, Transform):\n        self.dir=Dir\n        self.fnames=FNames\n        self.transform=Transform\n        self.labels=Labels\n        \n    def __len__(self):\n        return len(self.fnames)\n    \n    def __getitem__(self, index):\n        x=Image.open(os.path.join(self.dir, self.fnames[index]))\n        \n        if \"train\" in self.dir:\n            return self.transform(x), self.labels[index]\n        \n        elif \"test\" in self.dir:\n            return self.transform(x), self.fnames[index]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Transform 정의 (Augmentation? 그럼 test에선 왜하지? TTA?)"},{"metadata":{"trusted":true},"cell_type":"code","source":"Transform = transforms.Compose(\n    [transforms.ToTensor(),\n    transforms.Resize((CFG.IMG_SIZE, CFG.IMG_SIZE)),\n    transforms.RandomRotation(90),\n    transforms.RandomHorizontalFlip(p=0.5),\n    transforms.Normalize((0.485,0.456,0.406), (0.229,0.224,0.225))])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainset= GetData(TRAIN_DIR, X_Train, Y_Train, Transform)\ntrainloader=DataLoader(trainset,batch_size=CFG.BATCH, shuffle=True, num_workers=4)\n\n# 테스트 데이터는 왜 라벨 안 넣어줌?? -> 애초에 테스트 라벨링 파일 자체가 없음\ntestset=GetData(TEST_DIR, X_Test, None, Transform)\ntestloader=DataLoader(testset, batch_size=1, shuffle=False, num_workers=4)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 모델 선언"},{"metadata":{"trusted":true},"cell_type":"code","source":"class EfficientNet(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(CFG.model_name,pretrained=pretrained)\n        #n_features = self.model.fc.in_features\n        #self.model.fc = nn.Linear(n_features, CFG.target_size)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n        # ResNext에서는 self.model.fc 였던 것이 \n        # EfficientNet에서는 self.model.classifier로 바뀜\n        # 왜 그런지 확인할 것\n        \n    def forward(self,x):\n        x=self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SymmeetricCrossEntropy"},{"metadata":{"trusted":true},"cell_type":"code","source":"class SymmetricCrossEntropy(nn.Module):\n    \n    def __init__(self, alpha=0.1, beta=1.0, num_classes=5):\n        super(SymmetricCrossEntropy, self).__init__()\n        self.alpha = alpha\n        self.beta = beta\n        self.num_classes = num_classes\n        \n    def forward(self, logits, targets, reduction='mean'):\n        onehot_targets = torch.eye(self.num_classes)[targets].cuda()\n        ce_loss = F.cross_entropy(logits, targets, reduction=reduction)\n        rce_loss = (-onehot_targets*logits.softmax(1).clamp(1e-7, 1.0).log()).sum(1)\n        \n        if reduction == 'mean':\n            rce_loss = rce_loss.mean()\n        elif reduction == 'sum':\n            rce_loss = rce_loss.sum()\n        return self.alpha * ce_loss + self.beta * rce_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model=CustomResNext(model_name=CFG.model_name, pretrained=False)\nmodel = EfficientNet(CFG.model_name, train.label.nunique(), pretrained=True).to(DEVICE)\n# 모델을 GPU로 올려줘야함\nmodel.to(DEVICE)\n#criterion=nn.CrossEntropyLoss()\ncriterion = SymmetricCrossEntropy().to(DEVICE)\n#optimizer=torch.optim.Adam(model.parameters(),lr=CFG.LR)\noptimizer = AdamP(model.parameters(), lr=CFG.LR, weight_decay=CFG.weight_decay)\n\n#Apex의 MixedPrecision => 학습 속도 빨라짐\nmodel, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Training Start!\\n')\nfor epoch in range(CFG.EPOCHS):\n    train_loss=0.0\n    \n    model=model.train()\n    \n    for i, (images,labels) in enumerate(trainloader):\n        images=images.to(DEVICE)\n        labels=labels.to(DEVICE)\n        \n        logits=model(images)\n        loss=criterion(logits, labels)\n        optimizer.zero_grad()\n        \n        # Apex의 MixedPrecision 사용 \n        with amp.scale_loss(loss, optimizer) as scaled_loss: \n            scaled_loss.backward()\n\n        #oss.backward()\n        optimizer.step()\n        \n        # 찾아볼 것\n        train_loss+=loss.detach().item()\n    #찾아볼 것 \n    model.eval()\n    print('Epoch: %d | Loss: %.4f'%(epoch, train_loss / i))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"apex의 AMP를 사용했을 때 \nGradient overflow.  Skipping step, loss scaler 0 reducing loss scale to~~~ 에 대한 답변 (깃헙 이슈)\n\nOccasionally seeing a message like “overflow detected, skipping step, reducing loss scale” is normal behavior with dynamic loss scaling, and it usually happens in the first few iterations because Amp begins by trying a high loss scale.\n\nSeeing nan loss values (ie, the loss scalar resulting from the forward pass is nan or inf) is NOT normal, and indicates something has gone wrong. As Piotr says, if this is the case, a minimal repro would be helpful."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}