{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import Subset\nfrom torch.utils.data import DataLoader\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam\nfrom torch.nn import CosineSimilarity\nfrom torchvision.transforms import ToTensor\nfrom torchvision import transforms\nimport torchvision.models as models\nfrom pathlib import Path\nimport PIL.Image\nimport random\nimport math\nfrom tqdm import tqdm_notebook as tqdm\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom torch.optim.lr_scheduler import ExponentialLR","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('../input')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_df = pd.read_csv(path/'train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"va_ratio = 0.05","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_df, va_df = train_test_split(tr_df, \n#                                 train_size=0.05,\n                                test_size=va_ratio, \n                                random_state=42, \n                                stratify=tr_df['diagnosis'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# From https://www.kaggle.com/taindow/pre-processing-train-and-test-images\n\nimport os\nimport glob\nimport cv2\nimport numpy as np\nimport PIL\n\ndef crop_image_from_gray(img,tol=7):\n    \"\"\"\n    Crop out black borders\n    https://www.kaggle.com/ratthachat/aptos-updated-preprocessing-ben-s-cropping\n    \"\"\"  \n    \n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n        mask = gray_img>tol        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0):\n            return img\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n            img = np.stack([img1,img2,img3],axis=-1)\n        return img\n\n\ndef circle_crop(img):   \n    \"\"\"\n    Create circular crop around image centre    \n    \"\"\"    \n    \n    img = cv2.imread(img)\n    img = crop_image_from_gray(img)    \n    \n    height, width, depth = img.shape    \n    \n    x = int(width/2)\n    y = int(height/2)\n    r = np.amin((x,y))\n    \n    circle_img = np.zeros((height, width), np.uint8)\n    cv2.circle(circle_img, (x,y), int(r), 1, thickness=-1)\n    img = cv2.bitwise_and(img, img, mask=circle_img)\n    img = crop_image_from_gray(img)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    return PIL.Image.fromarray(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class APTOSDataset(Dataset):\n    def __init__(self, path, df, labels=None):\n        self.len = len(df)\n        self.path = path\n        self.labels = labels\n        self.images = list(map(lambda f: path/(f + '.png'), df['id_code']))\n        self.trfms = transforms.Compose([\n            transforms.Resize(240),\n            transforms.CenterCrop(240),\n            #     transforms.Grayscale(num_output_channels=3),\n            transforms.ToTensor(),\n            ])\n        \n    def __getitem__(self, index):\n        if self.labels is not None:\n            label = self.labels[index]\n        else:\n            label = 0\n        path = self.images[index]\n        return self.trfms(circle_crop(str(path))), label\n    \n    def __len__(self):\n        return self.len","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_ds = APTOSDataset(path/'train_images', tr_df, tr_df['diagnosis'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"va_ds = APTOSDataset(path/'train_images', va_df, va_df['diagnosis'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(tr_ds), len(va_ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bs = 16\nnw = 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_dl = DataLoader(tr_ds, batch_size=bs, num_workers=nw, drop_last=True, pin_memory=True, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"va_dl = DataLoader(va_ds, batch_size=bs, num_workers=nw, drop_last=True, pin_memory=True, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ArcMarginProduct(nn.Module):\n    def __init__(self, s=32.0, m=0.50, easy_margin=True):\n        super(ArcMarginProduct, self).__init__()\n        self.s = s\n        self.m = m\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n\n        # make the function cos(theta+m) monotonic decreasing while theta in [0°,180°]\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n\n    def forward(self, cosine, label):\n        # cos(theta + m)\n#         cosine = torch.clamp(cosine, -1.0, 1.0)\n#         print(\"cos:\", cosine)#, cosine >= 1., cosine <= -1.)\n#         assert(not (cosine.clone().detach().cpu().numpy() >= 1.).any())\n#         assert(not (cosine.clone().detach().cpu().numpy() <= -1.).any())\n#         sine_2 = F.relu(1.0 - torch.pow(cosine, 2))\n#         print(sine_2)#, sine_2 > 1.0)\n#         assert(not (sine_2.detach().cpu().numpy() > 1.0).any())\n#         sine = torch.sqrt(sine_2)\n#         sine = torch.clamp(sine, -1.0, 1.0)\n#         print(\"sin:\", sine)#, sine >= 1., sine <= -1.)\n#         assert(not (sine.clone().detach().cpu().numpy() >= 1.).any())\n#         assert(not (sine.clone().detach().cpu().numpy() <= -1.).any())\n#         phi = cosine * self.cos_m - sine * self.sin_m\n        phi = cosine - self.m\n\n#         if self.easy_margin:\n#             phi = torch.where(cosine > 0, phi, cosine)\n#         else:\n#             phi = torch.where((cosine - self.th) > 0, phi, cosine - self.mm)\n\n        #one_hot = torch.zeros(cosine.size(), device='cuda' if torch.cuda.is_available() else 'cpu')\n        one_hot = torch.zeros_like(cosine)\n        one_hot.scatter_(1, label.view(-1, 1), 1)\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output = output * self.s\n        \n        output = F.cross_entropy(output, label)\n#         print(\"loss:\", output)\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CosineEmbedding(nn.Module):\n    def __init__(self, embedding_dim, num_classes):\n        super(CosineEmbedding, self).__init__()\n        self.weight = nn.Parameter(torch.Tensor(num_classes, embedding_dim))\n        nn.init.xavier_uniform_(self.weight)\n        \n    def forward(self, inputs):\n#         print(\"emb weight:\", self.weight)\n#         print(\"inp:\", inputs)\n        return F.linear(F.normalize(inputs), F.normalize(self.weight))\n#         return F.linear(inputs, self.weight)\n\n    def normalize(self):\n#         return\n        with torch.no_grad():\n            torch.div(self.weight, self.weight.norm(p=None, dim=1, keepdim=True), out=self.weight)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = models.resnet18(pretrained=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_dim = 2\nnum_classes = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fc = nn.Sequential(\n    nn.Linear(in_features=512, out_features=512, bias=True),\n    nn.ReLU(),\n    nn.BatchNorm1d(num_features=512),\n    nn.Dropout(),\n#     nn.Linear(in_features=512, out_features=num_classes, bias=True))\n    nn.Linear(in_features=512, out_features=embedding_dim, bias=False))\n# model.fc = nn.Linear(in_features=512, out_features=embedding_dim, bias=False)\nmodel.cuda();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding = CosineEmbedding(embedding_dim, num_classes)\nembedding.cuda();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for param in model.parameters():\n    param.requires_grad = False\nfor param in model.fc.parameters():\n    param.requires_grad = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = ArcMarginProduct(s=16.0, m=0.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = 1e-2\noptimizer = Adam(model.fc.parameters(), lr=lr)\noptimizer.add_param_group({'params': embedding.parameters(), 'lr': lr})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scheduler = ExponentialLR(optimizer=optimizer, gamma=0.95)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_epochs = 10\nmodel_dir = Path('/kaggle/working/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_metrics = []\nva_metrics = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for epoch in tqdm(range(num_epochs)):\n# for epoch in range(num_epochs):\n    print('Epoch: {:02d}\\n'.format(epoch))\n    \n    iterations = 0\n    running_loss = 0.0\n    running_acc = 0.0\n    running_ce = 0.0\n    model.train()\n    for inputs, targets in tqdm(tr_dl):\n#     for inputs, targets in tr_dl:\n        inputs = inputs.cuda()\n        targets = targets.cuda()\n        optimizer.zero_grad()\n        outputs = model(inputs)\n        outputs = embedding(outputs)\n        loss = criterion(outputs, targets)\n        loss.backward()\n        embedding.normalize()\n        optimizer.step()\n        \n        loss = loss.item()\n        running_loss += loss\n        _, preds = torch.max(outputs, 1)\n        acc = torch.sum(preds == targets.data, dtype=torch.float32) / inputs.shape[0]\n        running_acc += acc\n        ce = F.cross_entropy(outputs, targets).mean().item()\n        running_ce += ce\n        tr_metrics.append((loss, acc, ce))\n        iterations += 1\n    \n    tr_loss = running_loss / iterations\n    tr_acc = running_acc / iterations\n    tr_ce = running_ce / iterations\n    print('Train: Loss: {:.6f} Acc: {:.6f} CE: {:.6f}'.format(tr_loss, tr_acc, tr_ce))\n    \n    iterations = 0\n    running_loss = 0.0\n    running_ce = 0.0\n    running_acc = 0.0\n    model.eval()\n    with torch.no_grad():\n        for inputs, targets in tqdm(va_dl):\n#         for inputs, targets in va_dl:\n            inputs = inputs.cuda()\n            targets = targets.cuda()\n            outputs = model(inputs)\n            outputs = embedding(outputs)\n            \n            loss = criterion(outputs, targets)\n            loss = loss.item()\n            running_loss += loss\n            _, preds = torch.max(outputs, 1)\n#             print(preds, targets.data)\n            acc = torch.sum(preds == targets.data, dtype=torch.float32) / inputs.shape[0]\n#             print(acc)\n            running_acc += acc\n            running_ce += F.cross_entropy(outputs, targets).mean().item()\n            iterations += 1\n            \n    va_loss = running_loss / iterations\n    va_acc = running_acc / iterations\n    va_ce = running_ce / iterations\n    va_metrics.append((va_loss, va_acc, va_ce))\n    print('Val: Loss: {:.6f} Acc: {:.6f} CE: {:.6f}'.format(va_loss, va_acc, va_ce))\n#     print(embedding.weight)\n    scheduler.step()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(list(map(lambda t:t[0], tr_metrics)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(list(map(lambda t:t[1], tr_metrics)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(list(map(lambda t:t[2], tr_metrics)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(list(map(lambda t:t[0], va_metrics)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(list(map(lambda t:t[1], va_metrics)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(list(map(lambda t:t[2], va_metrics)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}