{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"pip install ttach","metadata":{"execution":{"iopub.status.busy":"2021-11-06T14:34:18.990615Z","iopub.execute_input":"2021-11-06T14:34:18.990948Z","iopub.status.idle":"2021-11-06T14:34:28.755263Z","shell.execute_reply.started":"2021-11-06T14:34:18.990889Z","shell.execute_reply":"2021-11-06T14:34:28.754203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install albumentations","metadata":{"execution":{"iopub.status.busy":"2021-11-06T14:34:28.761168Z","iopub.execute_input":"2021-11-06T14:34:28.761503Z","iopub.status.idle":"2021-11-06T14:34:39.178069Z","shell.execute_reply.started":"2021-11-06T14:34:28.761436Z","shell.execute_reply":"2021-11-06T14:34:39.177066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install 'git+https://github.com/katsura-jp/pytorch-cosine-annealing-with-warmup'","metadata":{"execution":{"iopub.status.busy":"2021-11-06T14:34:39.179859Z","iopub.execute_input":"2021-11-06T14:34:39.180166Z","iopub.status.idle":"2021-11-06T14:34:51.202707Z","shell.execute_reply.started":"2021-11-06T14:34:39.180132Z","shell.execute_reply":"2021-11-06T14:34:51.201451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport os\nimport torch.optim as optim\nimport sys\nimport glob\nimport random\nimport csv\nimport cv2\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings('ignore')\nfrom contextlib import contextmanager\nfrom cosine_annealing_warmup import CosineAnnealingWarmupRestarts\n\nfrom PIL import Image\nimport torch\nimport torch.nn as nn\nfrom torchvision import transforms\nimport torchvision.models as models\nfrom torch.utils.data import DataLoader,Dataset\nfrom sklearn.metrics import accuracy_score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-06T14:34:51.207146Z","iopub.execute_input":"2021-11-06T14:34:51.20742Z","iopub.status.idle":"2021-11-06T14:34:52.294768Z","shell.execute_reply.started":"2021-11-06T14:34:51.207382Z","shell.execute_reply":"2021-11-06T14:34:52.293634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg={}\ncfg['batch_size'] = 16\ncfg['lr'] = 1e-4\ncfg['min_lr'] = 1e-7\ncfg['max_lr'] = 1e-3\ncfg['image_size'] = 512\ncfg['div'] = 1e+2\ncfg['final_div'] = 1\ncfg['pct_start'] = 0.111\ncfg['datasplit'] = 0.7\ncfg['epoch'] = 20\ncfg['device'] = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ncfg['seed'] = 121117102\ncfg['T0'] = 10\ncfg['num_samples'] = 3000","metadata":{"execution":{"iopub.status.busy":"2021-11-06T14:34:52.297192Z","iopub.execute_input":"2021-11-06T14:34:52.297617Z","iopub.status.idle":"2021-11-06T14:34:52.34434Z","shell.execute_reply.started":"2021-11-06T14:34:52.297569Z","shell.execute_reply":"2021-11-06T14:34:52.342922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_path = '../input/cassava-leaf-disease-classification'\ntrain_image_path = os.path.join(base_path,'train_images')\ntest_image_path = os.path.join(base_path,'test_images')\n\nsample_sub = os.path.join(base_path,'sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-11-06T14:34:52.346756Z","iopub.execute_input":"2021-11-06T14:34:52.347683Z","iopub.status.idle":"2021-11-06T14:34:52.356823Z","shell.execute_reply.started":"2021-11-06T14:34:52.347631Z","shell.execute_reply":"2021-11-06T14:34:52.355541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed = cfg['seed']\n\ndef setSeed(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    pd.core.common.random_state(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nsetSeed(seed)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T14:34:52.358623Z","iopub.execute_input":"2021-11-06T14:34:52.359226Z","iopub.status.idle":"2021-11-06T14:34:52.374699Z","shell.execute_reply.started":"2021-11-06T14:34:52.359177Z","shell.execute_reply":"2021-11-06T14:34:52.373539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom collections import Counter\n\nalldata_df = pd.read_csv(os.path.join(base_path,'train.csv'))\n\ndisplay(alldata_df.info())\n\nCLASSES = set(alldata_df['label'].values)\nNUMCLASS=len(CLASSES)\ndisplay(CLASSES)\n\nc = Counter(alldata_df['label'].values)\ndisplay(c)\nplt.pie(c.values(),labels=c.keys(),autopct='%1.1f%%')\nplt.axis('equal')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-06T14:34:52.376014Z","iopub.execute_input":"2021-11-06T14:34:52.376258Z","iopub.status.idle":"2021-11-06T14:34:52.575883Z","shell.execute_reply.started":"2021-11-06T14:34:52.376227Z","shell.execute_reply":"2021-11-06T14:34:52.574652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getImagePath(imageid,BASEPATH = train_image_path):\n    return os.path.join(BASEPATH,imageid + '.jpg')\n\n\ndef getImage(imagepath):\n    img = cv2.imread(imagepath)\n    imgRGB = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n    return imgRGB\n\n\ndef showExample(df,rd=True):\n    fig,axis = plt.subplots(NUMCLASS,2,figsize=(10,5*NUMCLASS))\n    for idx,cls in enumerate(CLASSES):\n        if rd:\n            sample = df[df['label'] == cls].sample(1).reset_index(drop=True)\n        else:\n            sample = df[df['label'] == cls].sample(1,random_state=seed).reset_index(drop=True)\n        \n        imgid = sample.iloc[0]['image_id'].split('.')[0]\n        img = getImage(getImagePath(imgid)).astype(np.uint8)\n        \n        axis[idx,0].set_title(str(imgid+'_'+str(cls)))\n        axis[idx,0].axis='off'\n        axis[idx,0].imshow(img)\n        axis[idx,1].set_title(str(imgid+'_'+str(cls)))\n        axis[idx,1].axis='off'\n        axis[idx,1].imshow(img)\n\nshowExample(alldata_df)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T14:34:52.578107Z","iopub.execute_input":"2021-11-06T14:34:52.578549Z","iopub.status.idle":"2021-11-06T14:34:55.550603Z","shell.execute_reply.started":"2021-11-06T14:34:52.578476Z","shell.execute_reply":"2021-11-06T14:34:55.549273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def splitDataset(df,p=0.8):\n    df=df.sample(frac=1,random_state=seed).reset_index(drop=True)\n    train_df = df.iloc[:int(len(df)*p)]\n    valid_df = df.iloc[int(len(df)*p):]\n    return train_df , valid_df","metadata":{"execution":{"iopub.status.busy":"2021-11-06T14:34:55.551799Z","iopub.execute_input":"2021-11-06T14:34:55.552078Z","iopub.status.idle":"2021-11-06T14:34:55.560911Z","shell.execute_reply.started":"2021-11-06T14:34:55.552041Z","shell.execute_reply":"2021-11-06T14:34:55.559641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CLDCDataset(Dataset):\n    def __init__(self,df,transform=None,testmodel=False):\n\n        self.df = df\n        self.transform = transform\n        self.testmodel = testmodel\n\n        \n    def __getitem__(self,idx):\n\n        imgid = self.df.iloc[idx]['image_id'].split('.')[0]\n        img = getImage(getImagePath(imgid,test_image_path if self.testmodel else train_image_path))\n        \n        if self.transform:\n            img = self.transform(image=img)['image']\n            \n        if not self.testmodel:\n            labels = self.df.iloc[idx]['label']\n            return imgid,img,labels\n        \n        return imgid,img\n\n\n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T14:34:55.562632Z","iopub.execute_input":"2021-11-06T14:34:55.563184Z","iopub.status.idle":"2021-11-06T14:34:55.576533Z","shell.execute_reply.started":"2021-11-06T14:34:55.563141Z","shell.execute_reply":"2021-11-06T14:34:55.575216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.__version__","metadata":{"execution":{"iopub.status.busy":"2021-11-06T14:34:55.578447Z","iopub.execute_input":"2021-11-06T14:34:55.579256Z","iopub.status.idle":"2021-11-06T14:34:55.58968Z","shell.execute_reply.started":"2021-11-06T14:34:55.579209Z","shell.execute_reply":"2021-11-06T14:34:55.58844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations\nfrom albumentations import (\n    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, \n    IAAAdditiveGaussianNoise, Transpose, HueSaturationValue, \n    )\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms(Config):\n    return albumentations.Compose([\n        albumentations.RandomCrop(cfg['image_size'], cfg['image_size']),\n        albumentations.Resize(cfg['image_size'], cfg['image_size']),\n        albumentations.HorizontalFlip(p=0.5),\n        albumentations.VerticalFlip(0.5),\n        albumentations.Transpose(p=0.5),\n        albumentations.Rotate(limit=(-90, 90), p=0.5),\n        albumentations.OneOf([\n            albumentations.ShiftScaleRotate(),\n            albumentations.ElasticTransform(alpha=3)\n        ], p=0.5),\n        albumentations.OneOf([\n            albumentations.OpticalDistortion(distort_limit=1.0),\n            albumentations.GridDistortion(num_steps=5, distort_limit=1.0)\n        ], p=0.5),\n        albumentations.OneOf([\n            albumentations.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2),\n            albumentations.RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1)),\n            albumentations.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.2),\n            albumentations.FancyPCA(),\n            albumentations.CLAHE(clip_limit=4.0)\n        ], p=0.5),\n        albumentations.OneOf([\n            albumentations.IAAAffine(),\n            albumentations.IAAPerspective(),\n            albumentations.IAAPiecewiseAffine(),\n            albumentations.IAASuperpixels()\n        ], p=0.5),\n        albumentations.Cutout(max_h_size=int(cfg['image_size'] * 0.375), max_w_size=int(cfg['image_size'] * 0.375), num_holes=1, p=0.5),\n        albumentations.Normalize(mean=[0.485,0.456,0.406], std=[0.229,0.224,0.225], max_pixel_value=255, p=1.0),\n        ToTensorV2(p=1.0)\n    ])\ndef get_test_transforms(Config):\n    return albumentations.Compose([\n        albumentations.CenterCrop(p=1.0, height = cfg['image_size'], width = cfg['image_size']),\n        albumentations.Normalize(mean=[0.485,0.456,0.406], std=[0.229,0.224,0.225], max_pixel_value=255, p=1.0),\n        ToTensorV2(p=1.0)\n    ])","metadata":{"execution":{"iopub.status.busy":"2021-11-06T14:34:55.595091Z","iopub.execute_input":"2021-11-06T14:34:55.595864Z","iopub.status.idle":"2021-11-06T14:34:56.025284Z","shell.execute_reply.started":"2021-11-06T14:34:55.595817Z","shell.execute_reply":"2021-11-06T14:34:56.024236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df , valid_df = splitDataset(alldata_df,cfg['datasplit'])\nweight = []\nfor index,row in train_df.iterrows():\n    if row['label'] == 0:\n        weight.append(12)\n    elif row['label'] == 1:\n        weight.append(6)\n    elif row['label'] == 2:\n        weight.append(6)\n    elif row['label'] == 3:\n        weight.append(1)\n    elif row['label'] == 4:\n        weight.append(5)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T14:34:56.027617Z","iopub.execute_input":"2021-11-06T14:34:56.027861Z","iopub.status.idle":"2021-11-06T14:34:57.566159Z","shell.execute_reply.started":"2021-11-06T14:34:56.027826Z","shell.execute_reply":"2021-11-06T14:34:57.564839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainTransforms = get_train_transforms(cfg)\nvalidTransforms = get_test_transforms(cfg)\n\n#train_sampler = torch.utils.data.WeightedRandomSampler(weight, num_samples = 25000, replacement = True, generator = None)\n\ntrainDataset = CLDCDataset(train_df,transform=trainTransforms)\nvalidDataset = CLDCDataset(valid_df,transform=validTransforms)\n\ntrainDataLoader = DataLoader(trainDataset,batch_size=cfg['batch_size'],num_workers=4,drop_last=True,shuffle = True)\nvalidDataLoader = DataLoader(validDataset,batch_size=cfg['batch_size'],shuffle=False,num_workers=4)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T14:34:57.56886Z","iopub.execute_input":"2021-11-06T14:34:57.569675Z","iopub.status.idle":"2021-11-06T14:34:57.578399Z","shell.execute_reply.started":"2021-11-06T14:34:57.569624Z","shell.execute_reply":"2021-11-06T14:34:57.577153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resnext50 = torch.hub.load('pytorch/vision:v0.10.0', 'resnext50_32x4d', pretrained = True)\n\nfc_features = resnext50.fc.in_features\nresnext50.fc = nn.Linear(fc_features,5)\n\nprint(resnext50)\n\nmodel = resnext50\n\nmodel = model.to(cfg['device'])\ncertification = torch.nn.CrossEntropyLoss()\noptimizer = torch.optim.AdamW(model.parameters(), lr=cfg['lr'])","metadata":{"execution":{"iopub.status.busy":"2021-11-06T14:34:57.58005Z","iopub.execute_input":"2021-11-06T14:34:57.580813Z","iopub.status.idle":"2021-11-06T14:35:00.068547Z","shell.execute_reply.started":"2021-11-06T14:34:57.580767Z","shell.execute_reply":"2021-11-06T14:35:00.067519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pip install efficientnet_pytorch","metadata":{"execution":{"iopub.status.busy":"2021-11-06T14:35:00.070073Z","iopub.execute_input":"2021-11-06T14:35:00.071973Z","iopub.status.idle":"2021-11-06T14:35:00.077413Z","shell.execute_reply.started":"2021-11-06T14:35:00.071922Z","shell.execute_reply":"2021-11-06T14:35:00.075999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from efficientnet_pytorch import EfficientNet\n#EfficientNet = EfficientNet.from_pretrained('efficientnet-b4')\n#fc_features = EfficientNet._fc.in_features\n#EfficientNet._fc = nn.Linear(fc_features,5)\n#print(EfficientNet)\n\n#model = EfficientNet\n\n#model = model.to(cfg['device'])\n#certification = torch.nn.CrossEntropyLoss()\n#optimizer = torch.optim.AdamW(model.parameters(), lr=cfg['lr'])","metadata":{"execution":{"iopub.status.busy":"2021-11-06T14:35:00.079035Z","iopub.execute_input":"2021-11-06T14:35:00.07991Z","iopub.status.idle":"2021-11-06T14:35:00.090134Z","shell.execute_reply.started":"2021-11-06T14:35:00.079864Z","shell.execute_reply":"2021-11-06T14:35:00.089096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_data(model, dataloader, certification, optimizer):\n    model.train()\n    totalloss=0\n    totalacc=0\n    with tqdm(dataloader,unit='batch',desc='Train') as tqdm_loader:\n        for idx, (imgid, img,label) in enumerate(tqdm_loader):\n            img = img.to(device=cfg['device'])\n            label = label.to(device=cfg['device'])\n            label = torch.tensor(label, dtype=torch.long) \n\n            pred = model(img).to(device=cfg['device'])\n\n            loss = certification(pred,label)\n            \n            pred = pred.cpu().detach().argmax(dim=1)\n\n            optimizer.zero_grad()\n            loss.backward()           \n            optimizer.step()\n            \n            nowloss = loss.detach().item()\n            totalloss +=nowloss\n            \n            acc = accuracy_score(pred,label.cpu())\n            totalacc +=acc\n            \n            tqdm_loader.set_postfix(loss=nowloss,avgloss=totalloss/(idx+1),avgACC=totalacc/(idx+1) )\n\n@torch.no_grad()\ndef eval_data(model, dataloader, certification):\n    model.eval()\n    totalloss=0\n    bestloss=10\n    bestacc = 0\n    totalacc=0\n    with torch.no_grad():\n        with tqdm(dataloader,unit='batch',desc='Valid') as tqdm_loader:\n            for idx, (imgid, img,label) in enumerate(tqdm_loader):\n\n                img = img.to(device=cfg['device'])\n                label = label.to(device=cfg['device'])\n                label = torch.tensor(label, dtype=torch.long) \n\n                pred = model(img)\n                \n                loss = certification(pred,label)\n                \n                pred = pred.cpu().detach().argmax(dim=1)\n                \n                nowloss = loss.detach().item()\n                totalloss +=nowloss\n                \n                acc = accuracy_score(pred,label.cpu())\n                totalacc +=acc\n\n                tqdm_loader.set_postfix(loss=nowloss,avgloss=totalloss/(idx+1),avgACC=totalacc/(idx+1) )\n                \n            avgloss = totalloss/len(tqdm_loader)\n            avgacc = totalacc/len(tqdm_loader)\n            if avgacc > bestacc:\n                bestacc = avgacc\n                bestloss = avgloss\n                torch.save(model,'../best_model.pkl')\n            elif avgacc == bestacc:\n                if avgloss <= bestloss:\n                    bestloss = avgloss\n                    torch.save(model,'../best_model.pkl')","metadata":{"execution":{"iopub.status.busy":"2021-11-06T15:11:47.272921Z","iopub.execute_input":"2021-11-06T15:11:47.273461Z","iopub.status.idle":"2021-11-06T15:11:47.305857Z","shell.execute_reply.started":"2021-11-06T15:11:47.273348Z","shell.execute_reply":"2021-11-06T15:11:47.303853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=cfg['T0'], T_mult=1, eta_min=cfg['min_lr'], last_epoch=-1)\n\nfor epoch in range(cfg['epoch']):\n    print('\\nEpoch {}'.format(epoch))\n    train_data(model,trainDataLoader,certification,optimizer)\n    eval_data(model,validDataLoader,certification)\n    scheduler.step()","metadata":{"execution":{"iopub.status.busy":"2021-11-06T15:11:48.190359Z","iopub.execute_input":"2021-11-06T15:11:48.190795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test(model,dataloader):\n    model.eval()\n    pred_list =[]\n    pred_id_list =[]\n    with torch.no_grad():\n        with tqdm(dataloader,unit='batch',desc='Test') as tqdm_loader:\n            for idx, (imgid, img) in enumerate(tqdm_loader):\n\n                img = img.to(device=cfg['device'])\n                pred = model(img).detach().cpu().argmax(dim=1)\n\n                pred_list.append(pred)\n                pred_id_list.append(imgid)\n    pred_list = np.concatenate(pred_list,axis=0)\n    pred_id_list =  np.concatenate(pred_id_list,axis=0)\n    return pred_list,pred_id_list\n\ntest_transforms = get_test_transforms(cfg)\ntest_df = pd.read_csv(sample_sub)\ntest_dataset = CLDCDataset(test_df, transform = test_transforms,testmodel=True)\ntest_dataloader = DataLoader(test_dataset, batch_size = cfg['batch_size'], shuffle=False, num_workers=4)\n\nmodel.load_state_dict(torch.load('../best_model.pkl'))\npred_list,pred_id_list = test(model,test_dataloader)\n\npred_id_list = [imgid +'.jpg' for imgid in pred_id_list]\n \nsubmit_df  = pd.DataFrame(columns=test_df.columns)\nsubmit_df['image_id'] = pred_id_list\nsubmit_df['label']=pred_list\nsubmit_df.to_csv(os.path.join('./', 'submission.csv'),index=False)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T23:33:12.349379Z","iopub.execute_input":"2021-11-06T23:33:12.349926Z","iopub.status.idle":"2021-11-06T23:33:12.433574Z","shell.execute_reply.started":"2021-11-06T23:33:12.349843Z","shell.execute_reply":"2021-11-06T23:33:12.432663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_state_dict(torch.load('../best_model.pkl'))\ntorch.save(model,'./bestmodel.pkl')","metadata":{"execution":{"iopub.status.busy":"2021-11-06T15:06:57.370701Z","iopub.status.idle":"2021-11-06T15:06:57.37175Z","shell.execute_reply.started":"2021-11-06T15:06:57.371386Z","shell.execute_reply":"2021-11-06T15:06:57.371418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}