{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ../input/efficientnet/EfficientNet-PyTorch -f ./ --no-index","metadata":{"execution":{"iopub.status.busy":"2021-05-26T00:21:22.371704Z","iopub.execute_input":"2021-05-26T00:21:22.372192Z","iopub.status.idle":"2021-05-26T00:21:31.373098Z","shell.execute_reply.started":"2021-05-26T00:21:22.372107Z","shell.execute_reply":"2021-05-26T00:21:31.372166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nfrom tqdm import tqdm\nimport copy \nimport pandas as pd\nfrom torchvision import transforms, models\nimport torchvision\nfrom torch import optim\nfrom torch.optim import lr_scheduler\nimport torch\nimport os\n\nfrom efficientnet_pytorch import EfficientNet","metadata":{"execution":{"iopub.status.busy":"2021-05-26T00:21:33.443092Z","iopub.execute_input":"2021-05-26T00:21:33.443425Z","iopub.status.idle":"2021-05-26T00:21:34.818409Z","shell.execute_reply.started":"2021-05-26T00:21:33.44339Z","shell.execute_reply":"2021-05-26T00:21:34.817574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_train = transforms.Compose([\n#         transforms.Resize(256),\n#         transforms.CenterCrop(224),\n        transforms.RandomResizedCrop(224),\n        transforms.RandomAffine(degrees=10, translate=(0.2, 0.2), scale=(0.8, 1.2), shear=15),\n        transforms.RandomHorizontalFlip(),\n        transforms.ColorJitter(.2, .2, .2, .2),\n        transforms.ToTensor(),\n        transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n    ])\ntransform_valid = transforms.Compose([\n        transforms.Resize(256),\n        transforms.CenterCrop(224),\n        transforms.ToTensor(),\n        transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n    ])","metadata":{"execution":{"iopub.status.busy":"2021-05-26T00:21:38.476137Z","iopub.execute_input":"2021-05-26T00:21:38.476457Z","iopub.status.idle":"2021-05-26T00:21:38.484251Z","shell.execute_reply.started":"2021-05-26T00:21:38.476425Z","shell.execute_reply":"2021-05-26T00:21:38.483349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nwith open('../input/plant-pathology-2021-fgvc8/train.csv', 'r') as f:\n    csv = f.readlines()[1:]\n    for _ in range(5):\n        random.shuffle(csv)\n    cnt = int(len(csv)*0.9)\n    train_csv = csv[:cnt]\n    valid_csv = csv[cnt:]","metadata":{"execution":{"iopub.status.busy":"2021-05-26T00:21:38.986502Z","iopub.execute_input":"2021-05-26T00:21:38.986822Z","iopub.status.idle":"2021-05-26T00:21:39.089438Z","shell.execute_reply.started":"2021-05-26T00:21:38.986794Z","shell.execute_reply":"2021-05-26T00:21:39.08852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label = {i.split(',')[1].strip() for i in train_csv}\nlabel = sorted(label)\nlabel2idx = {label:idx for idx, label in enumerate(label)}\nlabel2idx","metadata":{"execution":{"iopub.status.busy":"2021-05-26T00:21:40.03687Z","iopub.execute_input":"2021-05-26T00:21:40.037231Z","iopub.status.idle":"2021-05-26T00:21:40.056049Z","shell.execute_reply.started":"2021-05-26T00:21:40.037198Z","shell.execute_reply":"2021-05-26T00:21:40.054856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class torchvision_Dataset(torch.utils.data.Dataset): \n    def __init__(self, data_root, csv, label, transforms=None):\n        self.data = csv\n        self.image_path = data_root\n        self.label = label\n        self.transform = transforms\n        \n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx): \n        image_name, label_name = self.data[idx].split(',')\n        img = Image.open(os.path.join(self.image_path, image_name))\n        if self.transform:\n            x = self.transform(img)\n        \n        return x, self.label[label_name]","metadata":{"execution":{"iopub.status.busy":"2021-05-26T00:21:52.967366Z","iopub.execute_input":"2021-05-26T00:21:52.967689Z","iopub.status.idle":"2021-05-26T00:21:52.97469Z","shell.execute_reply.started":"2021-05-26T00:21:52.967661Z","shell.execute_reply":"2021-05-26T00:21:52.973459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = torchvision_Dataset('train_images', train_csv, label2idx, transform_train)\nvalid_dataset = torchvision_Dataset('train_images', valid_csv, label2idx, transform_valid)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T00:21:53.690334Z","iopub.execute_input":"2021-05-26T00:21:53.690656Z","iopub.status.idle":"2021-05-26T00:21:53.694651Z","shell.execute_reply.started":"2021-05-26T00:21:53.690627Z","shell.execute_reply":"2021-05-26T00:21:53.693558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloaders = torch.utils.data.DataLoader(train_dataset, batch_size=16, shuffle=True, num_workers=1)\nvalid_dataloaders = torch.utils.data.DataLoader(valid_dataset, batch_size=16, shuffle=False, num_workers=1)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T00:21:54.214648Z","iopub.execute_input":"2021-05-26T00:21:54.215006Z","iopub.status.idle":"2021-05-26T00:21:54.220332Z","shell.execute_reply.started":"2021-05-26T00:21:54.214967Z","shell.execute_reply":"2021-05-26T00:21:54.218961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T00:21:54.802571Z","iopub.execute_input":"2021-05-26T00:21:54.802916Z","iopub.status.idle":"2021-05-26T00:21:54.807564Z","shell.execute_reply.started":"2021-05-26T00:21:54.802882Z","shell.execute_reply":"2021-05-26T00:21:54.806621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_ft = models.resnet34(pretrained=False)\nmodel_ft = EfficientNet.from_name('efficientnet-b4', num_classes=12)\nmodel_ft.to(device)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T00:21:55.361298Z","iopub.execute_input":"2021-05-26T00:21:55.361612Z","iopub.status.idle":"2021-05-26T00:21:59.941087Z","shell.execute_reply.started":"2021-05-26T00:21:55.361584Z","shell.execute_reply":"2021-05-26T00:21:59.940256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FocalLoss(torch.nn.Module):\n    \"\"\"\n    The focal loss for fighting against class-imbalance\n    \"\"\"\n    def __init__(self, alpha=1, gamma=2):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.epsilon = 1e-12  # prevent training from Nan-loss error\n\n    def forward(self, logits, target):\n        \"\"\"\n        logits & target should be tensors with shape [batch_size, num_classes]\n        \"\"\"\n        probs = torch.sigmoid(logits)\n        one_subtract_probs = 1.0 - probs\n        # add epsilon\n        probs_new = probs + self.epsilon\n        one_subtract_probs_new = one_subtract_probs + self.epsilon\n        # calculate focal loss\n        log_pt = target * torch.log(probs_new) + (1.0 - target) * torch.log(one_subtract_probs_new)\n        pt = torch.exp(log_pt)\n        focal_loss = -1.0 * (self.alpha * (1 - pt) ** self.gamma) * log_pt\n        return torch.mean(focal_loss)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = FocalLoss()\noptimizer_ft = optim.Adam(model_ft.parameters(), lr=1e-3)\nexp_lr_scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer_ft, 40, eta_min=1e-6, verbose=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T00:22:07.950394Z","iopub.execute_input":"2021-05-26T00:22:07.950732Z","iopub.status.idle":"2021-05-26T00:22:07.963921Z","shell.execute_reply.started":"2021-05-26T00:22:07.950701Z","shell.execute_reply":"2021-05-26T00:22:07.963095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model, criterion, optimizer, scheduler, num_epochs=25):\n    # since = time.time()\n    \n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n    \n    for epoch in range(num_epochs):\n        running_loss = 0.0\n        train_corrects = 0\n        train_data_cnt = 0\n        train_progress_bar = tqdm(train_dataloaders) \n        for inputs, labels in train_progress_bar:\n            model.train()\n            \n            inputs = inputs.to(device)\n            labels = labels.to(device)\n            \n            optimizer.zero_grad()\n            \n            outputs = model(inputs)\n            _, preds = torch.max(outputs, 1)\n            loss = criterion(outputs, labels)\n            \n            loss.backward()\n            optimizer.step()\n            \n            running_loss += loss.item() * inputs.size(0)\n            train_corrects += torch.sum(preds == labels.data)\n            train_data_cnt += inputs.size(0)\n            train_progress_bar.set_description(f\" Epoch[{epoch+1}/{num_epochs}] train : runing_Loss {running_loss / train_data_cnt:.5f}, train_acc {train_corrects / train_data_cnt:.5f}\")\n        scheduler.step()\n \n        valid_corrects = 0\n        valid_data_cnt = 0\n        valid_progress_bar = tqdm(valid_dataloaders)\n        for inputs, labels in valid_progress_bar:    \n            model.eval()\n           \n            inputs = inputs.to(device)\n            labels = labels.to(device)\n           \n            with torch.no_grad():\n                outputs = model(inputs)\n                _, preds = torch.max(outputs, 1)\n                \n            valid_corrects += torch.sum(preds == labels.data)\n            valid_data_cnt += inputs.size(0)\n            valid_progress_bar.set_description(f\" Epoch[{epoch+1}/{num_epochs}] valid : valid_acc {valid_corrects / valid_data_cnt}\")\n            \n        epoch_acc = valid_corrects / valid_dataset.__len__()\n        if epoch_acc > best_acc:\n            best_acc = epoch_acc\n            best_epoch = epoch\n            best_model_wts = copy.deepcopy(model.state_dict())\n            torch.save(model_ft.state_dict(), f\"outputs/{best_epoch}.pth\")\n            print(f\"best epoch : {best_epoch}\")\n    best_model_wts = copy.deepcopy(model.state_dict())        \n    return best_model_wts","metadata":{"execution":{"iopub.status.busy":"2021-05-26T00:22:10.438618Z","iopub.execute_input":"2021-05-26T00:22:10.438963Z","iopub.status.idle":"2021-05-26T00:22:10.45022Z","shell.execute_reply.started":"2021-05-26T00:22:10.438928Z","shell.execute_reply":"2021-05-26T00:22:10.44934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,\n#                       num_epochs=30)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T00:22:11.02428Z","iopub.execute_input":"2021-05-26T00:22:11.024641Z","iopub.status.idle":"2021-05-26T00:22:11.028019Z","shell.execute_reply.started":"2021-05-26T00:22:11.024612Z","shell.execute_reply":"2021-05-26T00:22:11.027116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_ft.load_state_dict(torch.load('../input/efficient-adam-focalloss/39.pth'))\nmodel_ft.to(device)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T00:22:12.149284Z","iopub.execute_input":"2021-05-26T00:22:12.149607Z","iopub.status.idle":"2021-05-26T00:22:14.126768Z","shell.execute_reply.started":"2021-05-26T00:22:12.149579Z","shell.execute_reply":"2021-05-26T00:22:14.125994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx2label = {label2idx[label]:label for label in label2idx}","metadata":{"execution":{"iopub.status.busy":"2021-05-26T00:22:15.586737Z","iopub.execute_input":"2021-05-26T00:22:15.587091Z","iopub.status.idle":"2021-05-26T00:22:15.591575Z","shell.execute_reply.started":"2021-05-26T00:22:15.58706Z","shell.execute_reply":"2021-05-26T00:22:15.590328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob\nimport csv\nimg_paths = glob(\"../input/plant-pathology-2021-fgvc8/test_images/*\")\n\nsubmit =[] \n\nfor img_path in img_paths:\n    model_ft.eval()\n    img = Image.open(img_path)\n    img = transform_valid(img)\n    img = img.unsqueeze(0)\n    with torch.no_grad():\n        pred = model_ft(img.cuda())\n    _, top_one = torch.max(pred, 1)\n    img_path = img_path.split('/')[-1]\n    \n    submit.append([img_path,idx2label[int(top_one)]])\n    \nsubmission = pd.DataFrame(submit, columns=[\"image\", \"labels\"])\nsubmission.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T00:22:18.075397Z","iopub.execute_input":"2021-05-26T00:22:18.075723Z","iopub.status.idle":"2021-05-26T00:22:19.994381Z","shell.execute_reply.started":"2021-05-26T00:22:18.075693Z","shell.execute_reply":"2021-05-26T00:22:19.993562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}