{"cells":[{"metadata":{},"cell_type":"markdown","source":"Versions:\n* v6: first scoring kernel using resnet50 (LB 0.576)"},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport os\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom PIL import Image\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.metrics import confusion_matrix\n\nimport albumentations\nfrom albumentations import torch as AT\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nclass_desc = {\n    0:\"0 - No DR\",\n    1:\"1 - Mild\",\n    2:\"2 - Moderate\",\n    3:\"3 - Severe\",\n    4:\"4 - Proliferative DR\"\n}","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ntest = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\nsubmission = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\nprint(test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train['diagnosis'].value_counts())\ntrain['diagnosis'].value_counts().plot(kind='bar',title='Class Counts');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Disbalanced Training Set"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(25,16))\n#display 10 images from each class\nfor class_id in sorted(train['diagnosis'].unique()):\n    for i, (idx,row) in enumerate(train.loc[train['diagnosis'] == class_id].sample(10).iterrows()):\n#         print(f\"class_id {class_id} i {i} idx {idx} row {row['id_code']}\")\n        ax = fig.add_subplot(5,10,class_id * 10 + i + 1,xticks=[],yticks=[])\n        im = Image.open(f\"../input/aptos2019-blindness-detection/train_images/{row['id_code']}.png\")\n        plt.imshow(im)\n        ax.set_title(f'Label: {class_id}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There seems to be no significant difference.\nLets see zoomed versions"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"for class_id in sorted(train.diagnosis.unique()):\n    for i, (index,rows) in enumerate(train.loc[train.diagnosis == class_id].sample(3).iterrows()):\n        plt.figure(figsize=(15,15))\n        im = Image.open(f\"../input/aptos2019-blindness-detection/train_images/{rows['id_code']}.png\")\n        plt.xticks([]);plt.xticks([]);\n        plt.title(class_desc[class_id])\n        plt.imshow(im)\n        plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder, OneHotEncoder\ndef prepare_labels_2darray(y):\n    '''\n    Input : labels =>   idx  class\n                        0    2\n                        1    4\n                        2    1\n    Output : 2d array => array([[0., 0., 1., 0., 0.],\n                                [0., 0., 0., 0., 1.],\n                                [0., 1., 0., 0., 0.],...])\n    '''\n    y = np.array(y)\n    # y = array([2, 4, 1, ..., 2, 0, 2])\n    le = LabelEncoder()\n    int_enc = le.fit_transform(y)\n    # int_enc = array([2, 4, 1, ..., 2, 0, 2])\n    # LE is not required here, since label values start from 0 and ends with 4 for 5 classes\n    # useful if class values are arbitrary eg. 2,4,6 -> transformed = 0,1,2\n    int_enc = int_enc.reshape(len(int_enc),1)\n    ohe = OneHotEncoder(sparse=False)\n    ohe_enc = ohe.fit_transform(int_enc)\n    # default is sparse=True, if that's the case,\n    #ohe_enc = (0, 2)\t1.0\n    #          (1, 4)\t1.0 , etc\n    # if sparse=False,\n    #ohe_enc = array([[0., 0., 1., 0., 0.],\n    #                [0., 0., 0., 0., 1.],...])\n    y = ohe_enc\n    return y, le\n\ny,le = prepare_labels_2darray(train[\"diagnosis\"]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import torchvision\nimport torchvision.transforms as transforms\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class GlassDataset():\n    def __init__(self, df, datatype=\"train\", transform=transforms.Compose([transforms.CenterCrop(32),transforms.ToTensor()]),y=None):\n        self.df = df\n        self.datatype = datatype\n        self.image_files_list = [f'../input/aptos2019-blindness-detection/{self.datatype}_images/{i}.png' for i in df['id_code'].values]\n        if self.datatype == 'train':\n            self.labels = y\n        else:\n            self.labels = np.zeros((df.shape[0],5))\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.image_files_list)\n    \n    def __getitem__(self,idx):\n        img_name = self.image_files_list[idx]\n        img = cv2.imread(img_name)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        image = self.transform(image=img)\n        image = image['image']\n        \n        #img_name_short = self.image_files_list[idx].split('.')[0]\n        \n        label = self.labels[idx]\n        if self.datatype == 'test':\n            return image, label, img_name\n        else:\n            return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"data_transformations = albumentations.Compose([\n    albumentations.Resize(224,224),\n    albumentations.HorizontalFlip(),\n    albumentations.RandomBrightnessContrast(),\n    albumentations.ShiftScaleRotate(rotate_limit=15,scale_limit=0.10),\n    albumentations.JpegCompression(80),\n    albumentations.HueSaturationValue(),\n    albumentations.Normalize(),\n    AT.ToTensor()\n])\ndata_transformations_test = albumentations.Compose([\n    albumentations.Resize(224,224),\n    albumentations.Normalize(),\n    AT.ToTensor()\n])\ndataset = GlassDataset(df=train, datatype='train', transform=data_transformations, y=y)\ntest_set = GlassDataset(df=test, datatype='test', transform=data_transformations_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrn, validation = train_test_split(train.diagnosis, stratify=train.diagnosis, test_size=0.1)\n\"\"\"\ntrn - type = pandas.series, content = train.diagnosis values with arbitrarily ordered idx's\n\nprint(trn[:3]) ##similar for validation\n           id_code  diagnosis\n2009  8d4ff745a409          0\n494   233d948e2544          0\n50    03e25101e8e8          1\n\nprint(trn.index[:3])\nInt64Index([2009, 494, 50], dtype='int64')\n\nprint(trn.values[:3])\n[0 0 1]\n\"\"\";\nfrom torch.utils.data.sampler import SubsetRandomSampler\ntrain_sampler = SubsetRandomSampler(list(trn.index))\nvalidation_sampler = SubsetRandomSampler(list(validation.index))\n\"\"\"\nprint(trn.index[:3])\nInt64Index([527, 879, 1959], dtype='int64')\n\nprint(list(trn.index[:3]))\n[527, 879, 1959]\n\ntype(train_sampler)\ntorch.utils.data.sampler.SubsetRandomSampler\n\"\"\";","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import torch\nbatch_size = 64\nnum_workers = 0\n\ntrain_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers)\nvalid_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=validation_sampler, num_workers=num_workers)\ntest_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"model_conv = torchvision.models.resnet50()\nmodel_conv.load_state_dict(torch.load('../input/pytorch-pretrained-image-models/resnet50.pth'))\nnum_features = model_conv.fc.in_features # 2048\nfrom torch import nn\nmodel_conv.fc = nn.Linear(num_features,5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"torch.cuda.current_device() #0\ntorch.cuda.is_available() #True\ntorch.cuda.get_device_name(0) #Tesla P100-PCIE-16GB","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"model_conv.cuda()\ncriterion = nn.BCEWithLogitsLoss()\nfrom torch import optim\noptimizer = optim.SGD(model_conv.fc.parameters(), lr=0.01, momentum=0.99)\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\n# scheduler = StepLR(optimizer, step_size=3, gamma=0.1)\nscheduler = ReduceLROnPlateau(optimizer, factor=0.5, patience=2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model Training"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"valid_loss_min = np.Inf; #inf, type-float\npatience = 5\np = 0\n\n\nstop = False\n\n#number of epochs to train the model\nn_epoch = 20\nfor epoch in range(1, n_epoch+1):\n    print(time.ctime(), 'Epoch:',epoch)\n    \n    train_loss = []\n    train_auc = []\n    \n    for batch_idx, (data,target) in enumerate(train_loader):\n        #print(batch_idx, \"data\",data.shape, \"target\", target.shape) #data torch.Size([64, 3, 224, 224]) target torch.Size([64, 5])\n        \n        data, target = data.cuda(), target.cuda()\n        optimizer.zero_grad()\n        output = model_conv(data)\n        loss = criterion(output, target.float())\n        train_loss.append(loss.item())\n        \n        a = target.data.cpu().numpy()\n        b = output[:,-1].detach().cpu().numpy()\n        loss.backward()\n        optimizer.step()\n    model_conv.eval()\n    val_loss = []\n    val_auc = []\n    for batch_i, (data,target) in enumerate(valid_loader):\n        data,target = data.cuda(), target.cuda()\n        output = model_conv(data)\n        loss = criterion(output, target.float())\n        val_loss.append(loss.item())\n        a = target.data.cpu().numpy()\n        b = output[:,-1].detach().cpu().numpy()\n    print(f\"train_loss = {np.mean(train_loss):.4f} validation_loss = {np.mean(val_loss):.4f}\")\n    \n    valid_loss = np.mean(val_loss)\n    scheduler.step(valid_loss)\n    if valid_loss <= valid_loss_min:\n        print(f'Validation loss decreased ({valid_loss_min:.6f} --> {valid_loss:.6f})')\n        torch.save(model_conv.state_dict(),'model.pth')\n        valid_loss_min = valid_loss\n        p = 0\n        \n    # check if validation loss didn't improve\n    if valid_loss > valid_loss_min:\n        p += 1\n        print(f\"{p} epochs of increasing val loss\")\n        if p > patience:\n            print(\"Stopping Training\")\n            stop = True\n            break\n    \n    if stop:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"sub = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\n\nmodel_conv.eval()\nfor (data, target, name) in test_loader:\n    data = data.cuda()\n    output = model_conv(data)\n    output = output.cpu().detach().numpy()\n    for i, (e, n) in enumerate(list(zip(output, name))):\n        sub.loc[sub['id_code'] == n.split('/')[-1].split('.')[0], 'diagnosis'] = le.inverse_transform([np.argmax(e)])\n        \nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['diagnosis'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Understanding the evaluation metric (Quadratic Weighted Kappa):\nFrom [Wiki](https://en.wikipedia.org/wiki/Cohen%27s_kappa):\n1. **Cohen's kappa coefficient (κ) is a statistic which measures inter-rater agreement for qualitative (categorical) items. **\n2. It is generally thought to be a more robust measure than simple percent agreement calculation, as κ takes into account the possibility of the agreement occurring by chance.\n3. Note that Cohen's kappa measures agreement between two raters only.\n\n## Calculation :\n**Cohen's kappa measures the agreement between two raters who each classify N items into C mutually exclusive categories**.\n\n[Kappa Wiki Example](https://en.wikipedia.org/wiki/Cohen%27s_kappa#Example)\n\nThe definition of ${\\textstyle \\kappa }$ is:\n\n**${\\displaystyle \\kappa \\equiv {\\frac {p_{o}-p_{e}}{1-p_{e}}}=1-{\\frac {1-p_{o}}{1-p_{e}}},\\!}$** <br>\n* where po is the relative observed agreement among raters (identical to accuracy), and \n* pe is the hypothetical probability of chance agreement, using the observed data to calculate the probabilities of each observer randomly seeing each category."},{"metadata":{},"cell_type":"markdown","source":"## Interpreting the Quadratic Weighted Kappa Metric\n* A weighted Kappa is a metric which is used to calculate the amount of similarity between predictions and actuals. c\n* A perfect score of 1.0 is granted when both the predictions and actuals are the same. \n* If there is no agreement among the raters other than what would be expected by chance (as given by pe), ${\\textstyle \\kappa =0}$. \n* Whereas, the least possible score is -1 which is given when the predictions are furthest away from actuals. In our case, consider all actuals were 0's and all predictions were 4's.This would lead to a QWKP score of -1.\n* The aim is to get as close to 1 as possible. Generally a score of 0.6+ is considered to be a really good score.  <br><br>\nNote:  \nQuadratic Kappa Metric is the same as cohen kappa metric in Sci-kit learn <br>\n```sklearn.metrics.cohen_kappa_score``` when weights are set to 'Quadratic'. <br><br>"},{"metadata":{},"cell_type":"markdown","source":"## Weighted-Kappa\n\n* The weighted kappa allows disagreements to be weighted differently and is especially useful when codes are ordered.  \n* Three matrices are involved:\n    1. Matrix of observed scores O (N×N(N is number of categories) histogram matrix O, where each element eij of O corresponds to the number of observations that received a category i by A and a category j by B. In our case, N=5 so O is 5×5 matrix. Each element eij will represent count of images that recieved category i by A(say human) and category j by B(our models). So greater the number in diagonal, greater good.)\n    2. Matrix of expected scores E (N×N histogram matrix of expected ratings E, which is calculated as the outer product between each rater's histogram vector of ratings. E is normalized so that E and O have the same sum. Now, each cell in O is multiplied by corresponding cell in W and sum the results across all the cells. Call this Po. Same is done for E. Call this Pe.\n    3. Matrix of weights W calculated based on the difference between actual and predicted rating scores. (see [Weights-Matrix-Construction](#Weights-Matrix-Construction)). Weight matrix cells located on the diagonal (upper-left to bottom-right) represent agreement and thus contain zeros. Off-diagonal cells contain weights indicating the seriousness of that disagreement.\n\n* From these three matrices, the quadratic weighted kappa is calculated.  \n    * The equation for weighted κ is: ${\\displaystyle \\kappa =1-{\\frac {\\sum _{i=1}^{k}\\sum _{j=1}^{k}w_{ij}O_{ij}}{\\sum _{i=1}^{k}\\sum _{j=1}^{k}w_{ij}E_{ij}}}}$ <br>\n    where k=number of codes and ${\\displaystyle w_{ij}}$, ${\\displaystyle O_{ij}}$, and ${\\displaystyle E_{ij}}$ are elements in the weight, observed, and expected matrices, respectively.\n    \n[Weighted Kappa Example](http://vassarstats.net/kappaexp.html)\n\n## Weights-Matrix-Construction\n* When no weight matrix is involved, its called unweighted kappa. This means that there is no progression between categories. They are nominal.\n* But when categories are ordinal, i.e., they have some kind of progression relationship, for example: sad, ok, happy, very happy or No DR, Mild, Moderate, Severe, Proliferative DR, then weighted kappa is used.\n* Concept of distance is used to to calculate each element of W. Distance between category 2 and 0 is 2, between 3, 0 is 3, between 4 and 1 is 3 and so on.\n* Linear weight is calculated as:<br>\nweight$=1-\\frac{ | \\text { distance } |}{\\text { Maximum Possible Distance }}$\n \n* Quadratic weight is calculated as:<br>\nweight$=1-\\frac{ | \\text { distance }\\left.\\right|^{2}}{\\left(\\text { Maximum Possible Distance) }^{2}\\right.}$  \n* In this competition, maximum possible distance will be 4.\n\n## So, what this means for us?\n* 1st thing 1st, any random guess will be penalized.  \n* If you use ensemble of model by averaging their prediction, you will get floating values between 0 and 4, which you will then have to convert to integer using round, clip or some other method. But keep in mind that you will be penalized more if distance is more. So if your average is 1.6, and true label is 3, then if you predict to 1, distance would be 2, hence less weight:  \n$\\Large1-\\frac{2^{2}}{4^{2}}=0.75$  \n \n* But if you predict 2, then distance will be 1 and thus, comparitively more weight:  \n$\\Large1-\\frac{1^{2}}{4^{2}}=0.9375$  \n* This means we need to come with threshold for every category. Here's where, [OptimizedRounder class for Quadratic Weighted Kappa (QWK)](https://www.kaggle.com/abhishek/optimizer-for-quadratic-weighted-kappa) by Abhishek, comes into play.\n\n## Each Step Explained\n* Step-1: Under Step-1, we shall be calculating a confusion_matrix between the Predicted and Actual values. Here is a great resource to know more about confusion_matrix. \n* Step-2: Under Step-2, each element is weighted. Predictions that are further away from actuals are marked harshly than predictions that are closer to actuals. We will have a less score if our prediction is 5 and actual is 3 as compared to a prediction of 4 in the same case.\n* Step-3: We create two vectors, one for preds and one for actuals, which tells us how many values of each rating exist in both vectors. \n* Step-4:E is the Expected Matrix which is the outer product of the two vectors calculated in step-3.\n* Step-5: Normalise both matrices to have same sum. Since, it is easiest to get sum to be '1', we will simply divide each matrix by it's sum to normalise the data. \n* Step-6: Calculated numerator and denominator of Weighted Kappa and return the Weighted Kappa metric as 1-(num/den)"},{"metadata":{},"cell_type":"markdown","source":"### Step 1 : Confusion Matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"actuals = np.array([4, 4, 3, 4, 4, 4, 1, 1, 2, 1])\npreds   = np.array([0, 2, 1, 0, 0, 0, 1, 1, 2, 1])\nO = confusion_matrix(actuals, preds); O","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Step 2 : Weighted Matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"w = np.zeros((5,5)); w","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N=5\nfor i in range(len(w)):\n    for j in range(len(w)):\n        w[i][j] = float(((i-j)**2)/(N-1)**2) #as per formula, for this competition, N=5\nw","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Note that all values lying on the diagonal are penalised the least with a penalty of 0, whereas predictions and actuals furthest away from each other are penalised the most."},{"metadata":{},"cell_type":"markdown","source":"### Step 3 : Histogram"},{"metadata":{"trusted":true},"cell_type":"code","source":"act_hist=np.zeros([N])\nfor item in actuals: \n    act_hist[item]+=1\n    \npred_hist=np.zeros([N])\nfor item in preds: \n    pred_hist[item]+=1\n\nprint(f'Actuals value counts:   {act_hist} \\nPrediction value counts:{pred_hist}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Therefore, we have 3 values with adoption rating 1, 1 value with adoption rating 2, 1 value with adoption rating 1 an 5 values with adoption rating of 5 in the actuals."},{"metadata":{},"cell_type":"markdown","source":"### Step-4: Expected Value (Outer product of histograms)"},{"metadata":{"trusted":true},"cell_type":"code","source":"E = np.outer(act_hist, pred_hist); E","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Step-5: Normalise E and O matrix\nE and O are normalized such that E and O have the same sum."},{"metadata":{"trusted":true},"cell_type":"code","source":"print(E.sum());E = E/E.sum(); print(E.sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(O.sum()); O = O/O.sum(); print(O.sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"E","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"O","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Step-6: Calculate Weighted Kappa"},{"metadata":{"trusted":true},"cell_type":"code","source":"num=0\nden=0\nfor i in range(len(w)):\n    for j in range(len(w)):\n        num+=w[i][j]*O[i][j]\n        den+=w[i][j]*E[i][j]\n \nweighted_kappa = (1 - (num/den)); weighted_kappa","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def quadratic_kappa(actuals, preds, N=5):\n    \"\"\"This function calculates the Quadratic Kappa Metric used for Evaluation in the PetFinder competition\n    at Kaggle. It returns the Quadratic Weighted Kappa metric score between the actual and the predicted values \n    of adoption rating.\"\"\"\n    O = confusion_matrix(actuals, preds)\n    w = np.zeros((N,N))\n    for i in range(len(w)): \n        for j in range(len(w)):\n            w[i][j] = float(((i-j)**2)/(N-1)**2)\n    \n    act_hist=np.zeros([N])\n    for item in actuals: \n        act_hist[item]+=1\n    \n    pred_hist=np.zeros([N])\n    for item in preds: \n        pred_hist[item]+=1\n                         \n    E = np.outer(act_hist, pred_hist);\n    E = E/E.sum();\n    O = O/O.sum();\n    num=0\n    den=0\n    for i in range(len(w)):\n        for j in range(len(w)):\n            num+=w[i][j]*O[i][j]\n            den+=w[i][j]*E[i][j]\n    return (1 - (num/den))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### What if both actuals and predictions match 100%?"},{"metadata":{"trusted":true},"cell_type":"code","source":"actuals = np.array([4, 4, 3, 4, 4, 4, 1, 1, 2, 0])\npreds   = np.array([4, 4, 3, 4, 4, 4, 1, 1, 2, 0])\nquadratic_kappa(actuals, preds)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"References for Kappa Explanation:  \nhttps://www.kaggle.com/aroraaman/quadratic-kappa-metric-explained-in-5-simple-steps  \nhttps://www.kaggle.com/ashwan1/understanding-kappa-using-dummy-classifier"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}