{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nplt.style.use(\"ggplot\")\n\nimport torch\nimport torch.nn as nn\nfrom torchvision import datasets, models, transforms\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch import nn, optim\nfrom torch.optim import lr_scheduler\nfrom torch.autograd import Variable\nfrom torchvision import datasets, models, transforms  \nfrom torch.utils.data.sampler import SubsetRandomSampler  \nfrom torch.utils.data import Dataset, DataLoader\n\nimport pandas as pd\nimport torch.nn.functional as F\nimport os\n\nfrom tqdm import tqdm\nimport tqdm.notebook as tq\n\nimport time\nimport json\nimport copy\nimport gc\nimport os\nimport time\nimport random\nfrom datetime import datetime\n\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\nfrom sklearn import model_selection, metrics\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    \"\"\"\n    Seeds basic parameters for reproductibility of results\n    \n    Arguments:\n        seed {int} -- Number of the seed\n    \"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\n\nseed_everything(1001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_PATH = \"../input/cassava-leaf-disease-classification\"\nTRAIN_PATH = \"../input/cassava-leaf-disease-classification/train_images/\"\nTEST_PATH = \"../input/cassava-leaf-disease-classification/test_images/\"\nMODEL_PATH = (\n    \"../input/vit-base-models-pretrained-pytorch/jx_vit_base_p16_224-80ecf9dd.pth\"\n)\n\n# model specific global variables\nIMG_SIZE = 400","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(os.path.join(DATA_PATH, \"train.csv\"))\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.label.value_counts().plot(kind=\"bar\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.label.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df, valid_df = model_selection.train_test_split(\n    df, test_size=0.2, random_state=42, stratify=df.label.values\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(torch.utils.data.Dataset):\n    \"\"\"\n    Helper Class to create the pytorch dataset\n    \"\"\"\n\n    def __init__(self, df, data_path=DATA_PATH, mode=\"train\", transforms=None):\n        super().__init__()\n        self.df_data = df.values\n        self.data_path = data_path\n        self.transforms = transforms\n        self.mode = mode\n        self.data_dir = \"train_images\" if mode == \"train\" else \"test_images\"\n\n    def __len__(self):\n        return len(self.df_data)\n\n    def __getitem__(self, index):\n        img_name, label = self.df_data[index]\n        img_path = os.path.join(self.data_path, self.data_dir, img_name)\n        img = Image.open(img_path).convert(\"RGB\")\n\n        if self.transforms is not None:\n            image = self.transforms(img)\n\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#oversampling using wighted class\nimport numpy as np\nfrom torch.utils.data.sampler import WeightedRandomSampler\n\ncounts = np.bincount(train_df['label'])\nlabels_weights = 1. / counts\nweights = labels_weights[train_df['label']]\nsampler = WeightedRandomSampler(torch.DoubleTensor(weights), len(weights))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"counts2 = np.bincount(valid_df['label'])\nlabels_weights2 = 1. / counts2\nweights2 = labels_weights2[valid_df['label']]\nsampler2 = WeightedRandomSampler(torch.DoubleTensor(weights2), len(weights2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_train = transforms.Compose(\n    [\n          transforms.Resize((IMG_SIZE, IMG_SIZE)),\n     transforms.RandomRotation(45),\n        transforms.RandomHorizontalFlip(),\n        transforms.RandomVerticalFlip(),\n        transforms.RandomResizedCrop(IMG_SIZE),\n        transforms.RandomAffine(degrees=40, scale=(.9, 1.1), shear=0),\n        transforms.RandomPerspective(distortion_scale=0.2),\n        transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5),\n        transforms.GaussianBlur(3),\n        transforms.ToTensor(),\n        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n    ]\n)\n\n\ntransforms_valid = transforms.Compose(\n    [\n        transforms.Resize((IMG_SIZE, IMG_SIZE)),\n        transforms.ToTensor(),\n        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n    ]\n)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 32","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = CassavaDataset(train_df, transforms=transforms_train)\nvalid_dataset = CassavaDataset(valid_df, transforms=transforms_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(valid_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loader =  DataLoader( dataset=train_dataset,\n        batch_size=BATCH_SIZE,\n      \n        drop_last=True,\n        num_workers=8,shuffle=False  ,sampler = sampler)\nvalid_loader =    DataLoader(  dataset=valid_dataset,\n        batch_size=BATCH_SIZE,\n        \n        drop_last=True,\n        num_workers=8,shuffle=False ,sampler = sampler2)\ndataloaders = {'train':train_loader, 'valid':valid_loader}\ndataset_sizes = {'train':len(train_dataset), 'valid':len(valid_dataset) }\n\n\n\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = torch.hub.load('pytorch/vision:v0.6.0', 'resnext50_32x4d', pretrained=True)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfor param in model.layer1.parameters():\n            param.requires_grad = False\nfor param in model.layer2.parameters():\n            param.requires_grad = True\nfor param in model.layer3.parameters():\n            param.requires_grad = True\nfor param in model.layer4.parameters():\n            param.requires_grad = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fc  =  nn.Sequential(nn.Linear(2048, 1024),\n                     nn.Dropout(0.2),\n                      nn.ReLU(),\nnn.Linear(1024, 512),\n                     nn.Dropout(0.2),\n                      nn.ReLU(),\n                         \n nn.Linear(512, 5)\n\n        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\n\noptimizer = torch.optim.Adam(model.parameters(), lr=0.0001)\n\nsched = lr_scheduler.ReduceLROnPlateau(optimizer,mode='min', factor=0.5, patience=2 , verbose=True)\neps=14","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, criteria, optimizer, scheduler,    \n                                      num_epochs=25, device='cuda'):\n    model.cuda()\n    since = time.time()\n\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n    best_loss = np.Inf\n    for epoch in range(num_epochs):\n        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n        print('-' * 10)\n\n        # Each epoch has a training and validation phase\n        for phase in ['train', 'valid']:\n            if phase == 'train':\n                #scheduler.step()\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            running_corrects = 0\n \n            # Iterate over data.\n            for img , labels in dataloaders[phase]:\n                inputs = img.to(device)\n                #labels = labels.view(labels.shape[0],1)\n                labels = labels.to(device)\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    outputs = model(inputs)\n                    #preds = torch.round(torch.sigmoid(outputs))\n                    _,preds = torch.max(outputs,1)\n                    loss = criterion(outputs, labels)\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                # statistics\n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n\n            epoch_loss = running_loss / dataset_sizes[phase]\n            epoch_acc = running_corrects.double() / dataset_sizes[phase]\n            if phase == 'valid':\n                scheduler.step(epoch_loss)\n\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n                phase, epoch_loss, epoch_acc))\n\n            # deep copy the model\n            if phase == 'valid' and epoch_loss < best_loss:\n                best_acc = epoch_acc\n                best_loss = epoch_loss\n                best_model_wts = copy.deepcopy(model.state_dict())\n                torch.save(model.state_dict(), 'resnext-bestwight-phase1-ep-'+str(epoch)+'.pth')\n\n        print()\n\n    time_elapsed = time.time() - since\n    print('Training complete in {:.0f}m {:.0f}s'.format(\n        time_elapsed // 60, time_elapsed % 60))\n    print('Best val Acc: {:4f}'.format(best_acc))\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gpu = torch.cuda.is_available()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if gpu else \"cpu\")   # use CPU or GPU\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_ft = train_model(model, criterion, optimizer, sched, eps, device)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.save(model.state_dict(), 'resnext-bestwight-oversamplining-76.pth')\n                                 \n                                                     ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save\nPATH = \"resnext-facebook-fin848.pt\"\ntorch.save(model_ft, PATH)\n                                                ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"               \n                           ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}