{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport random\nimport pandas as pd\nfrom sklearn.model_selection import KFold\nimport matplotlib.pyplot as plt\nimport gc\nimport tqdm\n\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom fastai.vision.all import *\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\n\nclass SemisupervisedKLDiv(nn.KLDivLoss):\n#   Labels with less total votes are less confident\n#   Smoothing those labels with semisupervised votes\n#   will reduce his loss so gradient will be more\n#   affected by samples with higher total votes\n    def __init__(\n            self,\n            min_v=1# The minium votes to be accepted\n        ):\n        super().__init__(reduction=\"batchmean\")\n        self.min_v = torch.tensor(min_v,dtype=torch.float).to(device)\n\n    def set_min_v(self,new_min_v):\n        self.min_v = new_min_v\n\n    def forward(\n            self,\n            y,# Raw predictions\n            t # Raw targets\n        ):\n    #   How many votes do we have?\n        v = t.sum(-1,keepdim=True)\n    #   Where votes < min_v add model votes\n        mask = v[:,0] < self.min_v\n        t[mask] += (self.min_v - v[mask])*torch.softmax(y[mask].detach(),-1)\n        v[mask] = self.min_v\n        t /= v\n        y = nn.functional.log_softmax(y,  dim=1)\n        loss = super().forward(y, t)\n        return loss","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CosineAnnealingVotes\ndef nt(nmin,nmax,tcur,tmax):\n    return nmax - .5*(nmax-nmin)*(1+np.cos(tcur*np.pi/tmax))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nstart_min_v = 1# No smothing\nend_min_v = 4# 3 model votes for single voted samples, 2 model votes for double voted samples and 1 vote for triple voted samples\nEPOCHS = 30\n# By this implementation min_v no need to be integer\nplt.plot(nt(start_min_v,end_min_v,np.arange(EPOCHS),EPOCHS))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# callback to update min_v during training\ndef cb(self):\n    learn.loss_func.set_min_v(nt(start_min_v,end_min_v,learn.epoch,EPOCHS))\n#   print(nt(start_min_v,end_min_v,learn.epoch,EPOCHS))\nmin_v_cb = Callback(before_epoch=cb)","metadata":{},"execution_count":null,"outputs":[]}]}