{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n!pip install efficientnet_pytorch\n!pip install torchsummary\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nfrom efficientnet_pytorch import EfficientNet\nimport torchvision\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nimport torchvision.models as models\nfrom torchsummary import summary\nimport torch.optim as optim\nimport copy\nfrom tqdm.autonotebook import tqdm\nfrom torch.optim.lr_scheduler import _LRScheduler\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\nfrom torch.utils.data import DataLoader\nimport os\nfrom torch.utils.data.sampler import SubsetRandomSampler\nimport cv2\nimport random\nfrom torch.autograd import Variable\nimport sys\nprint(os.listdir(\"../input\"))\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data preparation"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv = pd.read_csv('../input/train_labels.csv')\ntrain_csv.head(10)\nclasses = train_csv['label'].unique()\nencoder = {0:'negative',1:'positive'}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Outlier detection"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Outlier removal\nimport cv2\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport random\nfrom sklearn.utils import shuffle\nfrom tqdm import tqdm_notebook\n\ndata = pd.read_csv('/kaggle/input/train_labels.csv')\ntrain_path = '/kaggle/input/train/'\ntest_path = '/kaggle/input/test/'\n# quick look at the label stats\nprint(data['label'].value_counts())\n\n\n\n##Function to read images using openCv\ndef readImage(path):\n    # OpenCV reads the image in bgr format by default\n    bgr_img = cv2.imread(path)\n    # We flip it to rgb for visualization purposes\n    b,g,r = cv2.split(bgr_img)\n    rgb_img = cv2.merge([r,g,b])\n    return rgb_img\n\n\n\n## time to plot\n\n\n### for plotting negative and positive labled data.. just for getting visualisation into data \n### not at all required\n# random sampling\nshuffled_data = shuffle(data)\n\nfig, ax = plt.subplots(2,5, figsize=(20,8))\nfig.suptitle('Histopathologic scans of lymph node sections',fontsize=20)\n# Negatives\nfor i, idx in enumerate(shuffled_data[shuffled_data['label'] == 0]['id'][:5]):\n    path = os.path.join(train_path, idx)\n    ax[0,i].imshow(readImage(path + '.tif'))\n    # Create a Rectangle patch\n    box = patches.Rectangle((32,32),32,32,linewidth=4,edgecolor='b',facecolor='none', linestyle=':', capstyle='round')\n    ax[0,i].add_patch(box)\nax[0,0].set_ylabel('Negative samples', size='large')\n# Positives\nfor i, idx in enumerate(shuffled_data[shuffled_data['label'] == 1]['id'][:5]):\n    path = os.path.join(train_path, idx)\n    ax[1,i].imshow(readImage(path + '.tif'))\n    # Create a Rectangle patch\n    box = patches.Rectangle((32,32),32,32,linewidth=4,edgecolor='r',facecolor='none', linestyle=':', capstyle='round')\n    ax[1,i].add_patch(box)\nax[1,0].set_ylabel('Tumor tissue samples', size='large')\n\n\n\n\n\n\n\n## data augmentation part\n\n\nimport random\nORIGINAL_SIZE = 96      # original size of the images - do not change\n\n# AUGMENTATION VARIABLES\nCROP_SIZE = 90          # final size after crop\nRANDOM_ROTATION = 3    # range (0-180), 180 allows all rotation variations, 0=no change\nRANDOM_SHIFT = 2        # center crop shift in x and y axes, 0=no change. This cannot be more than (ORIGINAL_SIZE - CROP_SIZE)//2 \nRANDOM_BRIGHTNESS = 7  # range (0-100), 0=no change\nRANDOM_CONTRAST = 5    # range (0-100), 0=no change\nRANDOM_90_DEG_TURN = 1  # 0 or 1= random turn to left or right\n\ndef readCroppedImage(path, augmentations = True):\n    # augmentations parameter is included for counting statistics from images, where we don't want augmentations\n    \n    # OpenCV reads the image in bgr format by default\n    bgr_img = cv2.imread(path)\n    # We flip it to rgb for visualization purposes\n    b,g,r = cv2.split(bgr_img)\n    rgb_img = cv2.merge([r,g,b])\n    \n    if(not augmentations):\n        return rgb_img / 255\n    \n    #random rotation\n    rotation = random.randint(-RANDOM_ROTATION,RANDOM_ROTATION)\n    if(RANDOM_90_DEG_TURN == 1):\n        rotation += random.randint(-1,1) * 90\n    M = cv2.getRotationMatrix2D((48,48),rotation,1)   # the center point is the rotation anchor\n    rgb_img = cv2.warpAffine(rgb_img,M,(96,96))\n    \n    #random x,y-shift\n    x = random.randint(-RANDOM_SHIFT, RANDOM_SHIFT)\n    y = random.randint(-RANDOM_SHIFT, RANDOM_SHIFT)\n    \n    # crop to center and normalize to 0-1 range\n    start_crop = (ORIGINAL_SIZE - CROP_SIZE) // 2\n    end_crop = start_crop + CROP_SIZE\n    rgb_img = rgb_img[(start_crop + x):(end_crop + x), (start_crop + y):(end_crop + y)] / 255\n    \n    # Random flip\n    flip_hor = bool(random.getrandbits(1))\n    flip_ver = bool(random.getrandbits(1))\n    if(flip_hor):\n        rgb_img = rgb_img[:, ::-1]\n    if(flip_ver):\n        rgb_img = rgb_img[::-1, :]\n        \n    # Random brightness\n    br = random.randint(-RANDOM_BRIGHTNESS, RANDOM_BRIGHTNESS) / 100.\n    rgb_img = rgb_img + br\n    \n    # Random contrast\n    cr = 1.0 + random.randint(-RANDOM_CONTRAST, RANDOM_CONTRAST) / 100.\n    rgb_img = rgb_img * cr\n    \n    # clip values to 0-1 range\n    rgb_img = np.clip(rgb_img, 0, 1.0)\n    \n    return rgb_img\n\n\n\n\n\n\n\n### Plotting the augumented data for same image just for fun xD\n\n\nfig, ax = plt.subplots(2,5, figsize=(20,8))\nfig.suptitle('Cropped histopathologic scans of lymph node sections',fontsize=20)\n# Negatives\nfor i, idx in enumerate(shuffled_data[shuffled_data['label'] == 0]['id'][:5]):\n    path = os.path.join(train_path, idx)\n    ax[0,i].imshow(readCroppedImage(path + '.tif'))\nax[0,0].set_ylabel('Negative samples', size='large')\n# Positives\nfor i, idx in enumerate(shuffled_data[shuffled_data['label'] == 1]['id'][:5]):\n    path = os.path.join(train_path, idx)\n    ax[1,i].imshow(readCroppedImage(path + '.tif'))\nax[1,0].set_ylabel('Tumor tissue samples', size='large')\n\n\n\n\n\n### Script for finding the images that are almost black or almost white\n\n\n## here comes the main part\n\n\n# As we count the statistics, we can check if there are any completely black or white images\ndark_th = 10 / 255      # If no pixel reaches this threshold, image is considered too dark\nbright_th = 245 / 255   # If no pixel is under this threshold, image is considerd too bright\ntoo_dark_idx = []\ntoo_bright_idx = []\n\nx_tot = np.zeros(3)\nx2_tot = np.zeros(3)\ncounted_ones = 0\nfor i, idx in tqdm_notebook(enumerate(shuffled_data['id']), 'computing statistics...(220025 it total)'):\n    path = os.path.join(train_path, idx)\n    imagearray = readCroppedImage(path + '.tif', augmentations = False).reshape(-1,3)\n    # is this too dark\n    if(imagearray.max() < dark_th):\n        too_dark_idx.append(idx)\n        continue # do not include in statistics\n    # is this too bright\n    if(imagearray.min() > bright_th):\n        too_bright_idx.append(idx)\n        continue # do not include in statistics\n    x_tot += imagearray.mean(axis=0)\n    x2_tot += (imagearray**2).mean(axis=0)\n    counted_ones += 1\n    \nchannel_avr = x_tot/counted_ones\nchannel_std = np.sqrt(x2_tot/counted_ones - channel_avr**2)\nchannel_avr,channel_std\n\nprint('There was {0} extremely dark image'.format(len(too_dark_idx)))\nprint('and {0} extremely bright images'.format(len(too_bright_idx)))\nprint('Dark one:')\nprint(too_dark_idx)\nprint('Bright ones:')\nprint(too_bright_idx)\n\n\n\n\n\n## this part is for displaying the outliers in the dataset( any 6)\nfig, ax = plt.subplots(2,6, figsize=(25,9))\nfig.suptitle('Almost completely black or white images',fontsize=20)\n# Too dark\ni = 0\nfor idx in np.asarray(too_dark_idx)[:min(6, len(too_dark_idx))]:\n    lbl = shuffled_data[shuffled_data['id'] == idx]['label'].values[0]\n    path = os.path.join(train_path, idx)\n    ax[0,i].imshow(readCroppedImage(path + '.tif', augmentations = False))\n    ax[0,i].set_title(idx + '\\n label=' + str(lbl), fontsize = 8)\n    i += 1\nax[0,0].set_ylabel('Extremely dark images', size='large')\nfor j in range(min(6, len(too_dark_idx)), 6):\n    ax[0,j].axis('off') # hide axes if there are less than 6\n# Too bright\ni = 0\nfor idx in np.asarray(too_bright_idx)[:min(6, len(too_bright_idx))]:\n    lbl = shuffled_data[shuffled_data['id'] == idx]['label'].values[0]\n    path = os.path.join(train_path, idx)\n    ax[1,i].imshow(readCroppedImage(path + '.tif', augmentations = False))\n    ax[1,i].set_title(idx + '\\n label=' + str(lbl), fontsize = 8)\n    i += 1\nax[1,0].set_ylabel('Extremely bright images', size='large')\nfor j in range(min(6, len(too_bright_idx)), 6):\n    ax[1,j].axis('off') # hide axes if there are less than 6","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Removing too bright and dark images from the dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in too_dark_idx:\n    train_csv = train_csv[train_csv.id != i]\nfor i in too_bright_idx:\n    train_csv = train_csv[train_csv.id != i]\ntrain_csv = train_csv.reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_df,val_df = train_test_split(train_csv,test_size = 0.1)\nval_df = val_df.reset_index()\nval_df = val_df.drop(['index'],axis = 1)\ntrain_df = train_df.reset_index()\ntrain_df = train_df.drop(['index'],axis = 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset class"},{"metadata":{"trusted":true},"cell_type":"code","source":"class cancer_dataset(Dataset):\n    def __init__(self,image_dir,train_csv,transform = None):\n        self.img_dir = image_dir\n        self.transform = transform\n        self.id = train_csv.id\n        self.classes =  train_csv.label\n    def __len__(self):\n        return len(self.id)\n    def __getitem__(self,idx):\n        img_name = os.path.join(self.img_dir, self.id[idx]+'.tif')\n        image = cv2.imread(img_name)\n        if self.transform:\n            image = self.transform(image)\n        label = self.classes[idx]\n        return image,label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Function for creating dataloader dictionary based on the train,test,val split"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#data loader\n\ndef data_loader(train_data,encoder,test_data = None,valid_data = None , valid_size = None,test_size = None , batch_size = 32,inv_normalize = None):\n    #class_plot(train_data,encoder,inv_normalize)\n    if(test_data == None and valid_size == None and valid_data == None and test_size == None):\n        train_loader =  DataLoader(train_data, batch_size = batch_size , shuffle = True)\n        dataloaders = {'train':train_loader}\n        return dataloaders\n    if(test_data == None and valid_size == None and valid_data != None and test_size == None):\n        train_loader =  DataLoader(train_data, batch_size = batch_size , shuffle = True)\n        valid_loader = DataLoader(valid_data,batch_size = batch_size,shuffle = True)\n        dataloaders = {'train':train_loader,'val':valid_loader}\n        return dataloaders\n\n    if(test_data !=None and valid_size==None and valid_data == None):\n        test_loader = DataLoader(test_data, batch_size= batch_size,shuffle = True)\n        train_loader =  DataLoader(train_data, batch_size = batch_size , shuffle = True)\n\n        dataloaders = {'train':train_loader,'test':test_loader}\n\n    if(test_data == None and valid_size!=None and valid_data == None):\n        if(test_size==None):\n            data_len = len(train_data)\n            indices = list(range(data_len))\n            np.random.shuffle(indices)\n            split1 = int(np.floor(valid_size * data_len))\n            valid_idx , train_idx = indices[:split1], indices[split1:]\n            valid_sampler = SubsetRandomSampler(valid_idx)\n            train_sampler = SubsetRandomSampler(train_idx)\n            valid_loader = DataLoader(train_data, batch_size= batch_size, sampler=valid_sampler)\n            train_loader =  DataLoader(train_data, batch_size = batch_size , sampler=valid_sampler)\n            dataloaders = {'train':train_loader,'val':valid_loader}\n            return dataloaders\n        if(test_size !=None):\n            data_len = len(train_data)\n            indices = list(range(data_len))\n            np.random.shuffle(indices)\n            split1 = int(np.floor(valid_size * data_len))\n            split2 = int(np.floor(test_size * data_len))\n            valid_idx , test_idx,train_idx = indices[:split1], indices[split1:split1+split2],indices[split1+split2:]\n            valid_sampler = SubsetRandomSampler(valid_idx)\n            test_sampler = SubsetRandomSampler(test_idx)\n            train_sampler = SubsetRandomSampler(train_idx)\n            valid_loader = DataLoader(test_data, batch_size= batch_size, sampler=valid_sampler)\n            test_loader = DataLoader(test_data, batch_size= batch_size, sampler=test_sampler)\n            train_loader =  DataLoader(train_data, batch_size = batch_size , sampler=valid_sampler)\n            dataloaders = {'train':train_loader,'val':valid_loader,'test':test_loader}\n            return dataloaders\n    if(test_data != None and valid_size!=None):\n        data_len = len(test_data)\n        indices = list(range(data_len))\n        np.random.shuffle(indices)\n        split1 = int(np.floor(valid_size * data_len))\n        valid_idx , test_idx = indices[:split1], indices[split1:]\n        valid_sampler = SubsetRandomSampler(valid_idx)\n        test_sampler = SubsetRandomSampler(test_idx)\n        valid_loader = DataLoader(test_data, batch_size= batch_size, sampler=valid_sampler)\n        test_loader = DataLoader(test_data, batch_size= batch_size, sampler=test_sampler)\n        train_loader =  DataLoader(train_data, batch_size = batch_size , shuffle = True)\n\n        dataloaders = {'train':train_loader,'val':valid_loader,'test':test_loader}\n        return dataloaders\n    if(test_data!=None and valid_data !=None):\n        valid_loader = DataLoader(valid_data, batch_size= batch_size,shuffle  = True)\n        test_loader = DataLoader(test_data, batch_size= batch_size,shuffle = True)\n        train_loader =  DataLoader(train_data, batch_size = batch_size , shuffle = True)\n\n        dataloaders = {'train':train_loader,'val':valid_loader,'test':test_loader}\n        return dataloaders\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Code to calculate mean and standard deviation of custom dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"def normalization_parameter(dataloader):\n    mean = 0.\n    std = 0.\n    nb_samples = len(dataloader.dataset)\n    for data,_ in tqdm(dataloader):\n        batch_samples = data.size(0)\n        data = data.view(batch_samples, data.size(1), -1)\n        mean += data.mean(2).sum(0)\n        std += data.std(2).sum(0)\n    mean /= nb_samples\n    std /= nb_samples\n    return mean,std","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"im_size = 96\nbatch_size = 64\ntrain_transforms = transforms.Compose([\n                                        transforms.ToPILImage(),\n                                        transforms.Resize((im_size,im_size)),\n                                        transforms.ToTensor()])\n\ntrain_data = cancer_dataset('../input/train',train_csv,transform = train_transforms)\ntrain_loader =  DataLoader(train_data, batch_size = batch_size , shuffle = True)\nmean,std = normalization_parameter(train_loader)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Training and test data augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transforms = transforms.Compose([\n                                        transforms.ToPILImage(),\n                                        transforms.Resize((im_size,im_size)),\n                                        transforms.RandomHorizontalFlip(),\n                                        transforms.transforms.RandomRotation(10),\n                                        transforms.RandomVerticalFlip(p=0.5),\n                                        transforms.ToTensor(),\n                                        transforms.Normalize(mean,std)])\ntest_transforms = transforms.Compose([\n                                        transforms.ToPILImage(),\n                                        transforms.Resize((im_size,im_size)),\n                                        transforms.ToTensor(),\n                                        transforms.Normalize(mean,std)])\n\n#inverse normalization for image plot\n\ninv_normalize =  transforms.Normalize(\n    mean=-1*np.divide(mean,std),\n    std=1/std\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = cancer_dataset('../input/train',train_df,transform = train_transforms)\nval_data = cancer_dataset('../input/train',val_df,transform = test_transforms)\ndataloaders =  data_loader(train_data,encoder = encoder,valid_data = val_data, batch_size = batch_size)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model with freezed pretrain layer"},{"metadata":{"trusted":true},"cell_type":"code","source":"class classifie(nn.Module):\n    def __init__(self):\n        super(classifie, self).__init__()\n        model = models.densenet201(pretrained = True)\n        model = model.features\n        for child in model.children():\n          for layer in child.modules():\n            layer.requires_grad = False\n            if(isinstance(layer,torch.nn.modules.batchnorm.BatchNorm2d)):\n              layer.requires_grad = True\n        #model = EfficientNet.from_pretrained('efficientnet-b3')\n        #model =  nn.Sequential(*list(model.children())[:-3])\n        self.model = model\n        self.linear = nn.Linear(3840, 512)\n        self.bn = nn.BatchNorm1d(512)\n        self.dropout = nn.Dropout(0.2)\n        self.elu = nn.ELU()\n        self.out = nn.Linear(512, 2)\n        self.bn1 = nn.BatchNorm1d(3840)\n        self.dropout2 = nn.Dropout(0.2)\n    def forward(self, x):\n        out = self.model(x)\n        avg_pool = nn.functional.adaptive_avg_pool2d(out, output_size = 1)\n        max_pool = nn.functional.adaptive_max_pool2d(out, output_size = 1)\n        out = torch.cat((avg_pool,max_pool),1)\n        batch = out.shape[0]\n        out = out.view(batch, -1)\n        conc = self.linear(self.dropout2(self.bn1(out)))\n        conc = self.elu(conc)\n        conc = self.bn(conc)\n        conc = self.dropout(conc)\n        res = self.out(conc)\n        return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nclassifier = classifie().to(device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Learning rate finder as seen in fast ai.\nImplementation for pytorch copied from https://github.com/davidtvs/pytorch-lr-finder"},{"metadata":{"trusted":true},"cell_type":"code","source":"from __future__ import print_function, with_statement, division\nimport copy\nimport os\nimport torch\nfrom tqdm.autonotebook import tqdm\nfrom torch.optim.lr_scheduler import _LRScheduler\nimport matplotlib.pyplot as plt\n\n\nclass LRFinder(object):\n    \"\"\"Learning rate range test.\n    The learning rate range test increases the learning rate in a pre-training run\n    between two boundaries in a linear or exponential manner. It provides valuable\n    information on how well the network can be trained over a range of learning rates\n    and what is the optimal learning rate.\n    Arguments:\n        model (torch.nn.Module): wrapped model.\n        optimizer (torch.optim.Optimizer): wrapped optimizer where the defined learning\n            is assumed to be the lower boundary of the range test.\n        criterion (torch.nn.Module): wrapped loss function.\n        device (str or torch.device, optional): a string (\"cpu\" or \"cuda\") with an\n            optional ordinal for the device type (e.g. \"cuda:X\", where is the ordinal).\n            Alternatively, can be an object representing the device on which the\n            computation will take place. Default: None, uses the same device as `model`.\n        memory_cache (boolean): if this flag is set to True, `state_dict` of model and\n            optimizer will be cached in memory. Otherwise, they will be saved to files\n            under the `cache_dir`.\n        cache_dir (string): path for storing temporary files. If no path is specified,\n            system-wide temporary directory is used.\n            Notice that this parameter will be ignored if `memory_cache` is True.\n    Example:\n        >>> lr_finder = LRFinder(net, optimizer, criterion, device=\"cuda\")\n        >>> lr_finder.range_test(dataloader, end_lr=100, num_iter=100)\n    Cyclical Learning Rates for Training Neural Networks: https://arxiv.org/abs/1506.01186\n    fastai/lr_find: https://github.com/fastai/fastai\n    \"\"\"\n\n    def __init__(self, model, optimizer, criterion, device=None, memory_cache=True, cache_dir=None):\n        self.model = model\n        self.optimizer = optimizer\n        self.criterion = criterion\n        self.history = {\"lr\": [], \"loss\": []}\n        self.best_loss = None\n        self.memory_cache = memory_cache\n        self.cache_dir = cache_dir\n\n        # Save the original state of the model and optimizer so they can be restored if\n        # needed\n        self.model_device = next(self.model.parameters()).device\n        self.state_cacher = StateCacher(memory_cache, cache_dir=cache_dir)\n        self.state_cacher.store('model', self.model.state_dict())\n        self.state_cacher.store('optimizer', self.optimizer.state_dict())\n\n        # If device is None, use the same as the model\n        if device:\n            self.device = device\n        else:\n            self.device = self.model_device\n\n    def reset(self):\n        \"\"\"Restores the model and optimizer to their initial states.\"\"\"\n        self.model.load_state_dict(self.state_cacher.retrieve('model'))\n        self.optimizer.load_state_dict(self.state_cacher.retrieve('optimizer'))\n        self.model.to(self.model_device)\n\n    def range_test(\n        self,\n        train_loader,\n        val_loader=None,\n        end_lr=10,\n        num_iter=100,\n        step_mode=\"exp\",\n        smooth_f=0.05,\n        diverge_th=5,\n    ):\n        \"\"\"Performs the learning rate range test.\n        Arguments:\n            train_loader (torch.utils.data.DataLoader): the training set data laoder.\n            val_loader (torch.utils.data.DataLoader, optional): if `None` the range test\n                will only use the training loss. When given a data loader, the model is\n                evaluated after each iteration on that dataset and the evaluation loss\n                is used. Note that in this mode the test takes significantly longer but\n                generally produces more precise results. Default: None.\n            end_lr (float, optional): the maximum learning rate to test. Default: 10.\n            num_iter (int, optional): the number of iterations over which the test\n                occurs. Default: 100.\n            step_mode (str, optional): one of the available learning rate policies,\n                linear or exponential (\"linear\", \"exp\"). Default: \"exp\".\n            smooth_f (float, optional): the loss smoothing factor within the [0, 1[\n                interval. Disabled if set to 0, otherwise the loss is smoothed using\n                exponential smoothing. Default: 0.05.\n            diverge_th (int, optional): the test is stopped when the loss surpasses the\n                threshold:  diverge_th * best_loss. Default: 5.\n        \"\"\"\n        # Reset test results\n        self.history = {\"lr\": [], \"loss\": []}\n        self.best_loss = None\n\n        # Move the model to the proper device\n        self.model.to(self.device)\n\n        # Initialize the proper learning rate policy\n        if step_mode.lower() == \"exp\":\n            lr_schedule = ExponentialLR(self.optimizer, end_lr, num_iter)\n        elif step_mode.lower() == \"linear\":\n            lr_schedule = LinearLR(self.optimizer, end_lr, num_iter)\n        else:\n            raise ValueError(\"expected one of (exp, linear), got {}\".format(step_mode))\n\n        if smooth_f < 0 or smooth_f >= 1:\n            raise ValueError(\"smooth_f is outside the range [0, 1[\")\n\n        # Create an iterator to get data batch by batch\n        iterator = iter(train_loader)\n        for iteration in tqdm(range(num_iter)):\n            # Get a new set of inputs and labels\n            try:\n                inputs, labels = next(iterator)\n            except StopIteration:\n                iterator = iter(train_loader)\n                inputs, labels = next(iterator)\n\n            # Train on batch and retrieve loss\n            loss = self._train_batch(inputs, labels)\n            if val_loader:\n                loss = self._validate(val_loader)\n\n            # Update the learning rate\n            lr_schedule.step()\n            self.history[\"lr\"].append(lr_schedule.get_lr()[0])\n\n            # Track the best loss and smooth it if smooth_f is specified\n            if iteration == 0:\n                self.best_loss = loss\n            else:\n                if smooth_f > 0:\n                    loss = smooth_f * loss + (1 - smooth_f) * self.history[\"loss\"][-1]\n                if loss < self.best_loss:\n                    self.best_loss = loss\n\n            # Check if the loss has diverged; if it has, stop the test\n            self.history[\"loss\"].append(loss)\n            if loss > diverge_th * self.best_loss:\n                print(\"Stopping early, the loss has diverged\")\n                break\n\n        print(\"Learning rate search finished. See the graph with {finder_name}.plot()\")\n\n    def _train_batch(self, inputs, labels):\n        # Set model to training mode\n        self.model.train()\n\n        # Move data to the correct device\n        inputs = inputs.to(self.device)\n        labels = labels.to(self.device)\n\n        # Forward pass\n        self.optimizer.zero_grad()\n        outputs = self.model(inputs)\n        loss = self.criterion(outputs, labels)\n\n        # Backward pass\n        loss.backward()\n        self.optimizer.step()\n\n        return loss.item()\n\n    def _validate(self, dataloader):\n        # Set model to evaluation mode and disable gradient computation\n        running_loss = 0\n        self.model.eval()\n        with torch.no_grad():\n            for inputs, labels in dataloader:\n                # Move data to the correct device\n                inputs = inputs.to(self.device)\n                labels = labels.to(self.device)\n\n                # Forward pass and loss computation\n                outputs = self.model(inputs)\n                loss = self.criterion(outputs, labels)\n                running_loss += loss.item() * inputs.size(0)\n\n        return running_loss / len(dataloader.dataset)\n\n    def plot(self, skip_start=10, skip_end=5, log_lr=True):\n        \"\"\"Plots the learning rate range test.\n        Arguments:\n            skip_start (int, optional): number of batches to trim from the start.\n                Default: 10.\n            skip_end (int, optional): number of batches to trim from the start.\n                Default: 5.\n            log_lr (bool, optional): True to plot the learning rate in a logarithmic\n                scale; otherwise, plotted in a linear scale. Default: True.\n        \"\"\"\n\n        if skip_start < 0:\n            raise ValueError(\"skip_start cannot be negative\")\n        if skip_end < 0:\n            raise ValueError(\"skip_end cannot be negative\")\n\n        # Get the data to plot from the history dictionary. Also, handle skip_end=0\n        # properly so the behaviour is the expected\n        lrs = self.history[\"lr\"]\n        losses = self.history[\"loss\"]\n        if skip_end == 0:\n            lrs = lrs[skip_start:]\n            losses = losses[skip_start:]\n        else:\n            lrs = lrs[skip_start:-skip_end]\n            losses = losses[skip_start:-skip_end]\n\n        # Plot loss as a function of the learning rate\n        plt.plot(lrs, losses)\n        if log_lr:\n            plt.xscale(\"log\")\n        plt.xlabel(\"Learning rate\")\n        plt.ylabel(\"Loss\")\n        plt.show()\n\n\nclass LinearLR(_LRScheduler):\n    \"\"\"Linearly increases the learning rate between two boundaries over a number of\n    iterations.\n    Arguments:\n        optimizer (torch.optim.Optimizer): wrapped optimizer.\n        end_lr (float, optional): the initial learning rate which is the lower\n            boundary of the test. Default: 10.\n        num_iter (int, optional): the number of iterations over which the test\n            occurs. Default: 100.\n        last_epoch (int): the index of last epoch. Default: -1.\n    \"\"\"\n\n    def __init__(self, optimizer, end_lr, num_iter, last_epoch=-1):\n        self.end_lr = end_lr\n        self.num_iter = num_iter\n        super(LinearLR, self).__init__(optimizer, last_epoch)\n\n    def get_lr(self):\n        curr_iter = self.last_epoch + 1\n        r = curr_iter / self.num_iter\n        return [base_lr + r * (self.end_lr - base_lr) for base_lr in self.base_lrs]\n\n\nclass ExponentialLR(_LRScheduler):\n    \"\"\"Exponentially increases the learning rate between two boundaries over a number of\n    iterations.\n    Arguments:\n        optimizer (torch.optim.Optimizer): wrapped optimizer.\n        end_lr (float, optional): the initial learning rate which is the lower\n            boundary of the test. Default: 10.\n        num_iter (int, optional): the number of iterations over which the test\n            occurs. Default: 100.\n        last_epoch (int): the index of last epoch. Default: -1.\n    \"\"\"\n\n    def __init__(self, optimizer, end_lr, num_iter, last_epoch=-1):\n        self.end_lr = end_lr\n        self.num_iter = num_iter\n        super(ExponentialLR, self).__init__(optimizer, last_epoch)\n\n    def get_lr(self):\n        curr_iter = self.last_epoch + 1\n        r = curr_iter / self.num_iter\n        return [base_lr * (self.end_lr / base_lr) ** r for base_lr in self.base_lrs]\n\n\nclass StateCacher(object):\n    def __init__(self, in_memory, cache_dir=None):\n        self.in_memory = in_memory\n        self.cache_dir = cache_dir\n\n        if self.cache_dir is None:\n            import tempfile\n            self.cache_dir = tempfile.gettempdir()\n        else:\n            if not os.path.isdir(self.cache_dir):\n                raise ValueError('Given `cache_dir` is not a valid directory.')\n\n        self.cached = {}\n\n    def store(self, key, state_dict):\n        if self.in_memory:\n            self.cached.update({key: copy.deepcopy(state_dict)})\n        else:\n            fn = os.path.join(self.cache_dir, 'state_{}_{}.pt'.format(key, id(self)))\n            self.cached.update({key: fn})\n            torch.save(state_dict, fn)\n\n    def retrieve(self, key):\n        if key not in self.cached:\n            raise KeyError('Target {} was not cached.'.format(key))\n\n        if self.in_memory:\n            return self.cached.get(key)\n        else:\n            fn = self.cached.get(key)\n            if not os.path.exists(fn):\n                raise RuntimeError('Failed to load state in {}. File does not exist anymore.'.format(fn))\n            state_dict = torch.load(fn, map_location=lambda storage, location: storage)\n            return state_dict\n\n    def __del__(self):\n        \"\"\"Check whether there are unused cached files existing in `cache_dir` before\n        this instance being destroyed.\"\"\"\n        if self.in_memory:\n            return\n\n        for k in self.cached:\n            if os.path.exists(self.cached[k]):\n                os.remove(self.cached[k])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def lr_finder(model,train_loader):\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    criterion = nn.CrossEntropyLoss()\n    optimizer_ft = optim.Adam(model.parameters(), lr=0.0000001)\n    lr_finder = LRFinder(model, optimizer_ft, criterion, device=device)\n    lr_finder.range_test(train_loader, end_lr=1, num_iter=1000)\n    lr_finder.reset()\n    lr_finder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loader = dataloaders['train']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_finder(classifier,train_loader)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Cyclical learning rate implementation similar to fast ai fit_one_cycle in pytorch.\nImplementation copied from - https://github.com/nachiket273/One_Cycle_Policy"},{"metadata":{"trusted":true},"cell_type":"code","source":"class OneCycle(object):\n    \"\"\"\n    In paper (https://arxiv.org/pdf/1803.09820.pdf), author suggests to do one cycle during \n    whole run with 2 steps of equal length. During first step, increase the learning rate \n    from lower learning rate to higher learning rate. And in second step, decrease it from \n    higher to lower learning rate. This is Cyclic learning rate policy. Author suggests one \n    addition to this. - During last few hundred/thousand iterations of cycle reduce the \n    learning rate to 1/100th or 1/1000th of the lower learning rate.\n    Also, Author suggests that reducing momentum when learning rate is increasing. So, we make \n    one cycle of momentum also with learning rate - Decrease momentum when learning rate is \n    increasing and increase momentum when learning rate is decreasing.\n    Args:\n        nb              Total number of iterations including all epochs\n        max_lr          The optimum learning rate. This learning rate will be used as highest \n                        learning rate. The learning rate will fluctuate between max_lr to\n                        max_lr/div and then (max_lr/div)/div.\n        momentum_vals   The maximum and minimum momentum values between which momentum will\n                        fluctuate during cycle.\n                        Default values are (0.95, 0.85)\n        prcnt           The percentage of cycle length for which we annihilate learning rate\n                        way below the lower learnig rate.\n                        The default value is 10\n        div             The division factor used to get lower boundary of learning rate. This\n                        will be used with max_lr value to decide lower learning rate boundary.\n                        This value is also used to decide how much we annihilate the learning \n                        rate below lower learning rate.\n                        The default value is 10.\n    \"\"\"\n    def __init__(self, nb, max_lr, momentum_vals=(0.95, 0.85), prcnt= 10 , div=10):\n        self.nb = nb\n        self.div = div\n        self.step_len =  int(self.nb * (1- prcnt/100)/2)\n        self.high_lr = max_lr\n        self.low_mom = momentum_vals[1]\n        self.high_mom = momentum_vals[0]\n        self.prcnt = prcnt\n        self.iteration = 0\n        self.lrs = []\n        self.moms = []\n        \n    def calc(self):\n        self.iteration += 1\n        lr = self.calc_lr()\n        mom = self.calc_mom()\n        return (lr, mom)\n        \n    def calc_lr(self):\n        if self.iteration==self.nb:\n            self.iteration = 0\n            self.lrs.append(self.high_lr/self.div)\n            return self.high_lr/self.div\n        if self.iteration > 2 * self.step_len:\n            ratio = (self.iteration - 2 * self.step_len) / (self.nb - 2 * self.step_len)\n            lr = self.high_lr * ( 1 - 0.99 * ratio)/self.div\n        elif self.iteration > self.step_len:\n            ratio = 1- (self.iteration -self.step_len)/self.step_len\n            lr = self.high_lr * (1 + ratio * (self.div - 1)) / self.div\n        else :\n            ratio = self.iteration/self.step_len\n            lr = self.high_lr * (1 + ratio * (self.div - 1)) / self.div\n        self.lrs.append(lr)\n        return lr\n    \n    def calc_mom(self):\n        if self.iteration==self.nb:\n            self.iteration = 0\n            self.moms.append(self.high_mom)\n            return self.high_mom\n        if self.iteration > 2 * self.step_len:\n            mom = self.high_mom\n        elif self.iteration > self.step_len:\n            ratio = (self.iteration -self.step_len)/self.step_len\n            mom = self.low_mom + ratio * (self.high_mom - self.low_mom)\n        else :\n            ratio = self.iteration/self.step_len\n            mom = self.high_mom - ratio * (self.high_mom - self.low_mom)\n        self.moms.append(mom)\n        return mom","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_lr(optimizer, lr):\n    for g in optimizer.param_groups:\n        g['lr'] = lr\ndef update_mom(optimizer, mom):\n    for g in optimizer.param_groups:\n        g['momentum'] = mom","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Training step with one cycle policy"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model,dataloaders,device,num_epochs,lr,batch_size,patience):\n    phase1 = dataloaders.keys()\n    losses = list()\n    criterion = nn.CrossEntropyLoss()\n    acc = list()\n    flag = 0\n    optimizer = optim.SGD(model.parameters(), lr=lr,momentum = 0.9)\n    for epoch in range(num_epochs):\n        print('Epoch:',epoch)\n        for phase in phase1:\n            epoch_metrics = {\"loss\": [], \"acc\": []}\n            if phase == ' train':\n                model.train()\n            else:\n                model.eval()\n            for  batch_idx, (data, target) in enumerate(dataloaders[phase]):\n                data, target = Variable(data), Variable(target)\n                data = data.type(torch.FloatTensor).to(device)\n                target = target.type(torch.LongTensor).to(device)\n                optimizer.zero_grad()\n                output = model(data)\n                loss = criterion(output, target)\n                acc = 100 * (output.detach().argmax(1) == target).cpu().numpy().mean()\n                epoch_metrics[\"loss\"].append(loss.item())\n                epoch_metrics[\"acc\"].append(acc)\n                lr,mom = onecyc.calc()\n                update_lr(optimizer, lr)\n                update_mom(optimizer, mom)\n                \n                if(phase =='train'):\n                    loss.backward()\n                    optimizer.step()\n                sys.stdout.write(\n                \"\\r[Epoch %d/%d] [Batch %d/%d] [Loss: %f (%f), Acc: %.2f%% (%.2f%%)]\"\n                % (\n                    epoch,\n                    num_epochs,\n                    batch_idx,\n                    len(dataloaders[phase]),\n                    loss.item(),\n                    np.mean(epoch_metrics[\"loss\"]),\n                    acc,\n                    np.mean(epoch_metrics[\"acc\"]),\n                    )\n                )\n               \n            epoch_acc = np.mean(epoch_metrics[\"acc\"])\n            epoch_loss = np.mean(epoch_metrics[\"loss\"])\n        print('')  \n        print('{} Accuracy: {}'.format(phase,epoch_acc.item()))\n    return losses,acc\n\ndef train_model(model,dataloaders,encoder,lr_scheduler = None,inv_normalize = None,num_epochs=10,lr=0.0001,batch_size=8,patience = None,classes = None):\n    dataloader_train = {}\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    losses = list()\n    accuracy = list()\n    key = dataloaders.keys()\n    perform_test = False\n    for phase in key:\n        if(phase == 'test'):\n            perform_test = True\n        else:\n            dataloader_train.update([(phase,dataloaders[phase])])\n    losses,accuracy = train(model,dataloader_train,device,num_epochs,lr,batch_size,patience)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_epochs = 5\nlr = 0.003\nonecyc = OneCycle(len(train_loader)*n_epochs,lr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_model(classifier,dataloaders,encoder,inv_normalize = None,num_epochs=n_epochs,lr = lr,batch_size = batch_size,patience = None,classes = classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Unfreeze pretrained layers of model\nfor param in classifier.parameters():\n    param.requires_grad = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = 0.001\ntrain_model(classifier,dataloaders,encoder,inv_normalize,num_epochs=n_epochs,lr = lr,batch_size = batch_size,patience = None,classes = classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class cancer_dataset_test(Dataset):\n    def __init__(self,image_dir,transform = None):\n        self.img_dir = image_dir\n        self.transform = transform\n        self.id = os.listdir(image_dir)\n    def __len__(self):\n        return len(self.id)\n    def __getitem__(self,idx):\n        img_name = os.path.join(self.img_dir, self.id[idx])\n        image = cv2.imread(img_name)\n        if self.transform:\n            image = self.transform(image)\n        return self.id[idx],image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = cancer_dataset_test('/kaggle/input/test',test_transforms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_loader = DataLoader(test_data, batch_size =128, shuffle = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def test(model,dataloader,device,batch_size):\n    running_corrects = 0\n    running_loss=0\n    pred = []\n    id = list()\n    sm = nn.Softmax(dim = 1)\n    criterion = nn.CrossEntropyLoss()\n    for batch_idx, (id_1,data) in enumerate(dataloader):\n        data = Variable(data)\n        data = data.type(torch.FloatTensor).to(device)\n        model.eval()\n        output = model(data)\n        #output = sm(output)\n        _, preds = torch.max(output, 1)\n        preds = preds.cpu().numpy()\n        preds = np.reshape(preds,(len(preds),1))\n        \n        for i in range(len(preds)):\n            pred.append(preds[i])\n            id.append(id_1[i])\n    return id,pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"id,pred = test(classifier,test_loader,'cuda',128)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a = list()\nfor i in range(len(pred)):\n    a.append(pred[i][0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"b = b[:-4]\nb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"id1 = list()\nfor i in id:\n    i = i[:-4]\n    id1.append(i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a = np.asarray(a)\na = np.reshape(a,(-1,1))\nb = np.asarray(id1)\nb = np.reshape(b,(-1,1))\nsub = np.concatenate((b,a),axis = 1)\nsub_df = pd.DataFrame(sub)\nsub_df.columns = ['id','has_cactus']\nsub_df.head(10)\nsub_df.to_csv(\"/kaggle/working/submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.head(10)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}