{"cells":[{"metadata":{},"cell_type":"markdown","source":"[reference of preprocess](https://www.pluralsight.com/guides/image-classification-with-pytorch)"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --no-deps ../input/pretrined-models/timm-0.3.3-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nimport timm\nfrom PIL import Image, ImageDraw, ImageChops\nimport matplotlib.pyplot as plt\nfrom torchvision.utils import make_grid\nfrom tqdm import tqdm\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = \"../input/cassava-leaf-disease-classification\"\nos.listdir(path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(path + \"/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\n\npath_json = '../input/cassava-leaf-disease-classification/label_num_to_disease_map.json'\n\nwith open(path_json, mode = 'r') as f:\n    label_to_name = json.load(f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_to_name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dataframeの写真のIDをpathに変更する\ndf[\"path\"] = df[\"image_id\"].map(lambda x: path + \"/train_images/\" + x)\ndf = df.drop(columns=[\"image_id\"])\ndf = df.sample(frac=1).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import model_selection\n\ntrain_df, valid_df = model_selection.train_test_split(\n    df, test_size=0.2, random_state=42, stratify=df.label.values\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_label_id(name):\n    id = 0\n    if name == 'cbb':\n        id = 0\n    elif name == 'cbsd':\n        id = 1\n    elif name == 'cgm':\n        id = 2\n    elif name == 'healthy':\n        id = 4\n    return id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"paths, labels = [], []\nimg_labels = os.listdir('../input/cassava-2019-compe-data/kaggle_upload/train/')\nprint(img_labels)\nfor label in img_labels:\n    if label != \"cmd\":\n        img_ids = os.listdir(\"../input/cassava-2019-compe-data/kaggle_upload/train/\"+label+\"/\")\n        for img_id in img_ids:\n            paths.append(\"../input/cassava-2019-compe-data/kaggle_upload/train/\"+label+\"/\"+img_id)\n            labels.append(get_label_id(label))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"extra_df = pd.DataFrame({'label':labels, 'path':paths})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"extra_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.concat([train_df, extra_df])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.label.value_counts().plot(kind=\"bar\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_df.label.value_counts().plot(kind=\"bar\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.reset_index().drop(columns=[\"index\"])\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_df = valid_df.reset_index().drop(columns=[\"index\"])\nvalid_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p_df = pd.read_csv('../input/pretrined-models/extra_label.csv', index_col = 0).reset_index().drop(columns = ['index'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert(p):\n    tmp = p.replace('[',\"\").replace(']','').split(',')\n    tmp = [float(i) for i in tmp]\n    return tmp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p_df['p_label'] = p_df['p_label'].map(convert)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_p(df):\n    paths = df['path']\n    list_df = pd.DataFrame(columns = ['label', 'path', 'p_label'])\n    for path in tqdm(paths):\n        tmp = p_df[p_df['path'] == path]\n        # print(tmp['p_label'])\n        list_df = list_df.append(tmp, ignore_index = True)\n    return list_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_df = add_p(train_df)\n#valid_df = add_p(valid_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"im = Image.open(train_df[\"path\"][0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"im","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\n\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.utils.data.dataset import Subset\n\nfrom sklearn.model_selection import KFold\n\nimport matplotlib.image as img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# データ用の関数の定義"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, dataframe, transform=None, p_label = False):\n        super().__init__()\n        self.df = dataframe\n        self.transform = transform\n        self.p = p_label\n\n    def __len__(self):\n        return len(self.df[\"path\"])\n\n    def __getitem__(self, index):\n        # pathと正解ラベルの入手\n        path = self.df[\"path\"][index]\n        if self.p:\n            label = self.df['p_label'][index]\n            label = np.array(label)\n        else:\n            label = self.df[\"label\"][index]\n        # 画像の読み込み\n        with open(path, \"rb\") as f:\n            image = Image.open(f)\n            image = image.convert(\"RGB\")\n        # transformがあるときには、画像に適用する\n        if self.transform is not None:\n            image = self.transform(image)\n\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class make_mask_image:\n    def __init__(self, p, mask_size=50):\n        self.p = p\n        self.mask_size = mask_size\n\n    def __call__(self, image):\n        start_width, start_height = [], []\n        if random.random() < self.p:\n            draw = ImageDraw.Draw(image)\n            width, height = image.size\n            for i in range(10):\n                start_width.append(random.randrange(0, width - self.mask_size))\n                start_height.append(random.randrange(0, height - self.mask_size))\n            for x, y in zip(start_width, start_height):\n                draw.rectangle(\n                    (x, y, x + self.mask_size, y + self.mask_size),\n                    fill=(0, 0, 0),\n                    outline=(0, 0, 0),\n                )\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 学習データと評価データに対する画像の前処理の定義\nimage_size = 384\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\ntrain_transform = transforms.Compose(\n    [  # 大きさの変更\n        transforms.RandomHorizontalFlip(p=0.5),\n        transforms.RandomVerticalFlip(p=0.5),\n        transforms.RandomResizedCrop(image_size),\n        make_mask_image(p=0.5, mask_size=40),\n        # tensor型に変更\n        transforms.ToTensor(),\n        # 正規化する\n        transforms.Normalize(mean=mean, std=std),\n    ]\n)\n\nvalid_transform = transforms.Compose(\n    [\n        transforms.Resize((image_size, image_size)),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=mean, std=std),\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = CassavaDataset(train_df, train_transform, False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset.__getitem__(3)[1]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# show labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Unnormalize(object):\n    def __init__(self, mean, std):\n        self.mean = mean\n        self.std = std\n        \n    def __call__(self, tensor):\n        for t, m, s in zip(tensor, self.mean, self.std):\n            t.mul_(s).add_(m)\n        return tensor","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unnorm = Unnormalize(mean, std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_img(img, unnorm = None, label = None):\n    if unnorm != None:\n        img = unnorm(img)\n        \n    plt.imshow(img.permute(1, 2, 0))\n    \n    if label != None:\n        plt.title(label_to_name[str(label)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_batch(batch, unnorm = None):\n    imgs, labels = batch\n    \n    if unnorm:\n        unnorm_imgs = []\n        for img in imgs:\n            unnorm_imgs.append(unnorm(img))\n        imgs = unnorm_imgs\n        \n    ig, ax = plt.subplots(figsize=(16, 8))\n    ax.set_xticks([]); ax.set_yticks([])\n    ax.imshow(make_grid(imgs, nrow=8).permute(1, 2, 0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TaylorSoftmax(nn.Module):\n\n    def __init__(self, dim=1, n=2):\n        super(TaylorSoftmax, self).__init__()\n        assert n % 2 == 0\n        self.dim = dim\n        self.n = n\n\n    def forward(self, x):\n        \n        fn = torch.ones_like(x)\n        denor = 1.\n        for i in range(1, self.n+1):\n            denor *= i\n            fn = fn + x.pow(i) / denor\n        out = fn / fn.sum(dim=self.dim, keepdims=True)\n        return out\n\nclass LabelSmoothingLoss(nn.Module):\n\n    def __init__(self, classes, smoothing=0.0, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.cls = classes \n        self.dim = dim \n    def forward(self, pred, target): \n        \"\"\"Taylor Softmax and log are already applied on the logits\"\"\"\n        #pred = pred.log_softmax(dim=self.dim) \n        with torch.no_grad(): \n            true_dist = torch.zeros_like(pred) \n            true_dist.fill_(self.smoothing / (self.cls - 1)) \n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))\n    \n\nclass TaylorCrossEntropyLoss(nn.Module):\n\n    def __init__(self, n=2, ignore_index=-1, reduction='mean', smoothing=0.2):\n        super(TaylorCrossEntropyLoss, self).__init__()\n        assert n % 2 == 0\n        self.taylor_softmax = TaylorSoftmax(dim=1, n=n)\n        self.reduction = reduction\n        self.ignore_index = ignore_index\n        self.lab_smooth = LabelSmoothingLoss(num_classes, smoothing=smoothing)\n\n    def forward(self, logits, labels):\n\n        log_probs = self.taylor_softmax(logits).log()\n        #loss = F.nll_loss(log_probs, labels, reduction=self.reduction,\n        #        ignore_index=self.ignore_index)\n        loss = self.lab_smooth(log_probs, labels)\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tensor, label = dataset[3]\ndisplay_img(tensor, unnorm, label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loader = DataLoader(dataset, 16, shuffle = True)\ndisplay_batch(next(iter(loader)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epoch = 5\nbatch_size = 16\nnum_classes = 5\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resNet = timm.create_model(\"resnet50\", pretrained=False)\nresNet.fc = nn.Linear(resNet.fc.in_features, num_classes)\nresNet = resNet.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ef_model = timm.create_model(\"tf_efficientnet_b4_ns\", pretrained=False)\nef_model.classifier = nn.Linear(ef_model.classifier.in_features, num_classes)\nef_model = ef_model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ef_optimizer = torch.optim.AdamW(ef_model.parameters(), lr=1e-4, weight_decay=0.0001)\nef_scheduler = torch.optim.lr_scheduler.StepLR(ef_optimizer, step_size=2, gamma=0.1)\n\nresNet_optimizer = torch.optim.AdamW(resNet.parameters(), lr=1e-4, weight_decay=0.0001)\nresNet_scheduler = torch.optim.lr_scheduler.StepLR(resNet_optimizer, step_size=2, gamma=0.1)\ncriterion=TaylorCrossEntropyLoss()\n# criterion = nn.MSELoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ef_model.load_state_dict(torch.load(\"../input/models/ef_model.pth\", map_location = device))\nresNet.load_state_dict(torch.load(\"../input/models/res_model.pth\", map_location = device))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def collate_fn(batch):\n    images, targets = list(zip(*batch))\n    images = torch.stack(images)\n    targets = torch.Tensor(targets)\n    return images, targets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calc_correction(model, df):\n    model.eval()\n    path = df[\"path\"]\n    label = df[\"label\"]\n    count = 0\n    pred_list = [0, 0, 0, 0, 0]\n    model = model.to(device)\n    with torch.no_grad():\n        for i in tqdm(range(len(path))):\n            image_path = path[i]\n            image_label = label[i]\n            image = Image.open(image_path)\n            image = valid_transform(image)\n            image = image.unsqueeze(0).to(device)\n            pred = model(image)\n            pred = pred.argmax(1).item()\n            pred_list[pred] += 1\n            if pred == image_label:\n                count += 1\n    percent = count / len(path)\n    return percent, pred_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# lossの表示をする\nfrom matplotlib import pyplot as plt\ndef plot_losses(epoch, title, train_losses, valid_losses):\n    y = list(range(len(train_losses)))\n    train_loss = plt.plot(y, train_losses)\n    valid_loss = plt.plot(y, valid_losses)\n    plt.title(title)\n    plt.ylabel(\"loss\")\n    plt.legend(\n        (train_loss[0], valid_loss[0]), (\"train loss\", \"valid loss\"),\n    )\n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\ndef train_model(model, dataset, batch_size, optimizer, criterion, scheduler,  epoch, model_title):\n    best_model = None\n    best_loss = float(\"inf\")\n    train_losses, valid_losses = [], []\n    kf = KFold(n_splits = 5)\n    \n    for fold, (train_index, valid_index) in enumerate(kf.split(dataset)):\n        print(\"fold: \", fold)\n        train_dataset = Subset(dataset, train_index)\n        train_loader = DataLoader(train_dataset, batch_size, shuffle = True, num_workers = 4)#, collate_fn = collate_fn)\n        valid_dataset = Subset(dataset, valid_index)\n        valid_loader= DataLoader(valid_dataset, batch_size, shuffle = False)#, collate_fn = collate_fn)\n        \n        for epoch in range(1, epoch + 1):\n            epoch_start_time = time.time()\n            acc = []\n            train_loss = 0\n            valid_loss = 0\n\n            model.train()\n            for data, target in train_loader:\n                data = data.to(device)\n                target = target.to(device)\n                optimizer.zero_grad()\n                output = model(data)\n                loss = criterion(output, target)\n                loss.backward()\n                optimizer.step()\n                train_loss += loss.item()*len(data)\n            train_loss = train_loss/len(train_loader.sampler)\n            train_losses.append(train_loss)\n\n            model.eval()\n            for data, target in valid_loader:\n                data = data.to(device)\n                target = target.to(device)\n\n                with torch.no_grad():\n                    output = model(data)\n                    # pred = (output.argmax(1) == target)\n                    # acc.append(sum(pred)/len(pred))\n\n                    loss = criterion(output, target)\n\n                    valid_loss += loss.item()*len(data)\n                    \n            if valid_loss < best_loss:\n                best_loss = valid_loss\n                best_model = model\n                    \n            scheduler.step()\n\n            # collection = sum(acc)/len(acc)\n            valid_loss = valid_loss/len(valid_loader.sampler)\n            valid_losses.append(valid_loss)\n            print('Time: {:.3f}\\t Epoch: {} \\tTraining Loss: {:.3f} \\tValidation Loss: {:.3f}'# \\t Acc: {:.2f}'\n                  .format(time.time() - epoch_start_time, epoch, train_loss, valid_loss))#, collection))\n            num_collection = []\n    torch.save(model.state_dict(), model_title)\n    \n    return model, train_losses, valid_losses","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_models(resNet, ef_model):\n    model_title = \"./ef_model.pth\"\n    ef_model, train_losses, valid_losses = train_model(ef_model, dataset, batch_size, ef_optimizer, criterion, ef_scheduler, epoch, model_title)\n    print(calc_correction(ef_model, valid_df))\n    title = \"ef losses\"\n    plot_losses(epoch, title, train_losses, valid_losses)\n    \n    model_title = \"./res_model.pth\"\n    resNet, train_losses, valid_losses = train_model(resNet, dataset, batch_size, resNet_optimizer, criterion,resNet_scheduler, epoch, model_title)\n    print(calc_correction(resNet, valid_df))\n    title = \"resNet losses\"\n    plot_losses(epoch, title, train_losses, valid_losses)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_models(resNet, ef_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassaveClassifier(nn.Module):\n    def __init__(self, model, ef_model):\n        super().__init__()\n        self.model = model\n        self.ef_model = ef_model\n    \n    def forward(self, x):\n        x1 = self.model(x)\n        x2 = self.ef_model(x)\n        return (0.5 * x1 + 0.5 * x2)\n\n    def test(self, x, rate):\n        x1 = self.model(x)\n        x2 = self.ef_model(x)\n        p = rate * x1 + (1 - rate) * x2\n        return p","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier = CassaveClassifier(resNet, ef_model)\nclassifier = classifier.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def test_rate():\n    for rate in range(1, 10):\n        classifier.eval()\n        path = valid_df[\"path\"]\n        label = valid_df[\"label\"]\n        count = 0\n        pred_list = [0, 0, 0, 0, 0]\n        for i in tqdm(range(len(path))):\n            image_path = path[i]\n            image_label = label[i]\n            image = Image.open(image_path)\n            image = valid_transform(image)\n            image = image.unsqueeze(0).to(device)\n            pred = classifier.test(image, rate/10).argmax(1).item()\n            pred_list[pred] += 1\n            if pred == image_label:\n                count += 1\n        percent = count / len(path)\n        print(\"rate: \", rate/10)\n        print(\"percent: \", percent)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test_rate()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"calc_correction(classifier, valid_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = \"../input/cassava-leaf-disease-classification/test_images/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_path = []\nimage_id = []\nfor i in os.listdir(path):\n    image_id.append(str(i))\n    image_path.append(path + str(i))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = []\nfor path in image_path:\n    image = Image.open(path)\n    image = valid_transform(image)\n    image = image.unsqueeze(0).to(device)\n    predict = classifier(image).argmax(1).item()\n    pred.append(predict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.DataFrame({\"image_id\": image_id, \"label\": pred})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}