{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Import Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import os\nimport math\nimport random\nimport time\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom glob import glob\nfrom tqdm import tqdm\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n\nimport optuna\n\nfrom pandas_profiling import ProfileReport","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configuration"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"CFG = {'seed': 1337}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper Functions"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def seed_everything(seed: int = 1337):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"seed_everything(CFG['seed'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Test Data"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-merged/merged.csv')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=CFG['seed'])\n\ntrain['fold'] = -1\nfor fold, (_, val_idx) in enumerate(skf.split(np.arange(train.shape[0]), train['label'].values)):\n    train.at[val_idx, 'fold'] = fold","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"train = train.set_index('image_id')\ntrain.head(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load predictions"},{"metadata":{},"cell_type":"markdown","source":"## Align image_id"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def align_image_ids(train: pd.DataFrame, df: pd.DataFrame):\n    df = df.set_index('image_id')\n\n    y = pd.DataFrame()\n    y_image_ids, y_prob0, y_prob1, y_prob2, y_prob3, y_prob4 = [], [], [], [], [], []\n\n    for image_id in train.index:\n        row = df.loc[image_id][[f'logits{i}' for i in range(5)]]\n\n        p0, p1, p2, p3, p4 = row.values\n\n        y_image_ids.append(image_id)\n        y_prob0.append(p0)\n        y_prob1.append(p1)\n        y_prob2.append(p2)\n        y_prob3.append(p3)\n        y_prob4.append(p4)\n\n    y['image_id'] = y_image_ids\n    y['logits0'] = y_prob0\n    y['logits1'] = y_prob1\n    y['logits2'] = y_prob2\n    y['logits3'] = y_prob3\n    y['logits4'] = y_prob4\n\n    return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def get_predictions(preds):\n    return np.array([preds['logits0'], preds['logits1'], preds['logits2'], preds['logits3'], preds['logits4']]).T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calc_p(preds, a0, a1, a2, a3, a4, a5, a6):\n# def calc_p(preds, a0, a1, a2, a3, a4, a5):\n# def calc_p(preds, a0, a1, a2, a3, a4):\n# def calc_p(preds, a0, a1, a2, a3):\n    # logits = a0 * preds[0] + a1 * preds[1] + a2 * preds[2] + a3 * preds[3]\n    # logits = a0 * preds[0] + a1 * preds[1] + a2 * preds[2] + a3 * preds[3] + a4 * preds[4]\n    # logits = a0 * preds[0] + a1 * preds[1] + a2 * preds[2] + a3 * preds[3] + a4 * preds[4] + a5 * preds[5]\n    logits = a0 * preds[0] + a1 * preds[1] + a2 * preds[2] + a3 * preds[3] + a4 * preds[4] + a5 * preds[5] + a6 * preds[6]\n    return logits.argmax(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"cv_preds = sorted(glob(os.path.join('../input/leaf-disease-validation', '*.csv')))[:-1]\ncv_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"cv_dfs = [align_image_ids(train, pd.read_csv(path)) for path in cv_preds]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"preds = [\n    get_predictions(cv_dfs[0]),\n    get_predictions(cv_dfs[1]),\n    get_predictions(cv_dfs[2]),\n    get_predictions(cv_dfs[3]),\n    get_predictions(cv_dfs[4]),\n    get_predictions(cv_dfs[5]),\n    get_predictions(cv_dfs[6]),  # corrected one\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"labels = np.asarray([train.loc[image_id]['label'] for image_id in cv_dfs[0]['image_id'].values])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# # cutting 2019 data\n# preds = [pred[:-4940, ...] for pred in preds]\n# labels = labels[:-4940, ...]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# weights = [0.64438387, 0.06787352, 0.21374317, 0.92894338, 0.30073056, 0.25681572]\n# logits = weights[0] * preds[0] + weights[1] * preds[1] + weights[2] * preds[2] + weights[3] * preds[3] + weights[4] * preds[4] + weights[5] * preds[5]\n\n# train['pseudo'] = logits.argmax(1)\n# train.to_csv('pseudo_label_train.csv')\n# train","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Tuning Ensemble Weights"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"from heapq import heappush, heappop, heappushpop\n\nCAPACITY_INCREMENT = 1000\n\n\nclass _Simplex:\n\tdef __init__(self, pointIndices, testCoords, contentFractions, objectiveScore, opportunityCost, contentFraction, difference):\n\t\tself.pointIndices = pointIndices\n\t\tself.testCoords = testCoords\n\t\tself.contentFractions = contentFractions\n\t\tself.contentFraction = contentFraction\n\t\tself.__objectiveScore = objectiveScore\n\t\tself.__opportunityCost = opportunityCost\n\t\tself.update(difference)\n\n\tdef update(self, difference):\n\t\tself.acquisitionValue = -(self.__objectiveScore + (self.__opportunityCost * difference))\n\t\tself.difference = difference\n\n\tdef __eq__(self, other):\n\t\treturn self.acquisitionValue == other.acquisitionValue\n\n\tdef __lt__(self, other):\n\t\treturn self.acquisitionValue < other.acquisitionValue\n\n    \nclass SimpleTuner:\n\tdef __init__(self, cornerPoints, objectiveFunction, exploration_preference=0.15):\n\t\tself.__cornerPoints = cornerPoints\n\t\tself.__numberOfVertices = len(cornerPoints)\n\t\tself.queue = []\n\t\tself.capacity = self.__numberOfVertices + CAPACITY_INCREMENT\n\t\tself.testPoints = np.empty((self.capacity, self.__numberOfVertices))\n\t\tself.objective = objectiveFunction\n\t\tself.iterations = 0\n\t\tself.maxValue = None\n\t\tself.minValue = None\n\t\tself.bestCoords = []\n\t\tself.opportunityCostFactor = exploration_preference #/ self.__numberOfVertices\n\n\tdef optimize(self, maxSteps=10):\n\t\tfor step in tqdm(range(maxSteps)):\n\t\t\tif len(self.queue) > 0:\n\t\t\t\ttargetSimplex = self.__getNextSimplex()\n\t\t\t\tnewPointIndex = self.__testCoords(targetSimplex.testCoords)\n\t\t\t\tfor i in range(self.__numberOfVertices):\n\t\t\t\t\ttempIndex = targetSimplex.pointIndices[i]\n\t\t\t\t\ttargetSimplex.pointIndices[i] = newPointIndex\n\t\t\t\t\tnewContentFraction = targetSimplex.contentFraction * targetSimplex.contentFractions[i]\n\t\t\t\t\tnewSimplex = self.__makeSimplex(targetSimplex.pointIndices, newContentFraction)\n\t\t\t\t\theappush(self.queue, newSimplex)\n\t\t\t\t\ttargetSimplex.pointIndices[i] = tempIndex\n\t\t\telse:\n\t\t\t\ttestPoint = self.__cornerPoints[self.iterations]\n\t\t\t\ttestPoint.append(0)\n\t\t\t\ttestPoint = np.array(testPoint, dtype=np.float64)\n\t\t\t\tself.__testCoords(testPoint)\n\t\t\t\tif self.iterations == (self.__numberOfVertices - 1):\n\t\t\t\t\tinitialSimplex = self.__makeSimplex(np.arange(self.__numberOfVertices, dtype=np.intp), 1)\n\t\t\t\t\theappush(self.queue, initialSimplex)\n\t\t\tself.iterations += 1\n\n\tdef get_best(self):\n\t\treturn (self.maxValue, self.bestCoords[0:-1])\n\n\tdef __getNextSimplex(self):\n\t\ttargetSimplex = heappop(self.queue)\n\t\tcurrentDifference = self.maxValue - self.minValue\n\t\twhile currentDifference > targetSimplex.difference:\n\t\t\ttargetSimplex.update(currentDifference)\n\t\t\t# if greater than because heapq is in ascending order\n\t\t\tif targetSimplex.acquisitionValue > self.queue[0].acquisitionValue:\n\t\t\t\ttargetSimplex = heappushpop(self.queue, targetSimplex)\n\t\treturn targetSimplex\n\t\t\n\tdef __testCoords(self, testCoords):\n\t\tobjectiveValue = self.objective(testCoords[0:-1])\n\t\tif self.maxValue == None or objectiveValue > self.maxValue: \n\t\t\tself.maxValue = objectiveValue\n\t\t\tself.bestCoords = testCoords\n\t\t\tif self.minValue == None: self.minValue = objectiveValue\n\t\telif objectiveValue < self.minValue:\n\t\t\tself.minValue = objectiveValue\n\t\ttestCoords[-1] = objectiveValue\n\t\tif self.capacity == self.iterations:\n\t\t\tself.capacity += CAPACITY_INCREMENT\n\t\t\tself.testPoints.resize((self.capacity, self.__numberOfVertices))\n\t\tnewPointIndex = self.iterations\n\t\tself.testPoints[newPointIndex] = testCoords\n\t\treturn newPointIndex\n\n\n\tdef __makeSimplex(self, pointIndices, contentFraction):\n\t\tvertexMatrix = self.testPoints[pointIndices]\n\t\tcoordMatrix = vertexMatrix[:, 0:-1]\n\t\tbarycenterLocation = np.sum(vertexMatrix, axis=0) / self.__numberOfVertices\n\n\t\tdifferences = coordMatrix - barycenterLocation[0:-1]\n\t\tdistances = np.sqrt(np.sum(differences * differences, axis=1))\n\t\ttotalDistance = np.sum(distances)\n\t\tbarycentricTestCoords = distances / totalDistance\n\n\t\teuclideanTestCoords = vertexMatrix.T.dot(barycentricTestCoords)\n\t\t\n\t\tvertexValues = vertexMatrix[:,-1]\n\n\t\ttestpointDifferences = coordMatrix - euclideanTestCoords[0:-1]\n\t\ttestPointDistances = np.sqrt(np.sum(testpointDifferences * testpointDifferences, axis=1))\n\n\t\tinverseDistances = 1 / testPointDistances\n\t\tinverseSum = np.sum(inverseDistances)\n\t\tinterpolatedValue = inverseDistances.dot(vertexValues) / inverseSum\n\n\t\tcurrentDifference = self.maxValue - self.minValue\n\t\topportunityCost = self.opportunityCostFactor * math.log(contentFraction, self.__numberOfVertices)\n\n\t\treturn _Simplex(pointIndices.copy(), euclideanTestCoords, barycentricTestCoords, interpolatedValue, opportunityCost, contentFraction, currentDifference)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def f(weights):\n    norm_weights = weights / np.sum(weights)\n    valid_preds = np.average(preds, axis=0, weights=norm_weights)\n    return [np.argmax(pred) for pred in valid_preds]\n\n\ndef acc_function(weights):\n    y_preds = f(weights)\n    n_eq = [result == ref for result, ref in zip(y_preds, labels)]\n    return np.sum(n_eq) / len(y_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# optimization_domain_vertices = [\n#     [0, 0, 0, 0, 0, 0], \n#     [0, 0, 0, 0, 0, 1], \n#     [0, 0, 0, 0, 1, 0], \n#     [0, 0, 0, 1, 0, 0], \n#     [0, 0, 1, 0, 0, 0], \n#     [0, 1, 0, 0, 0, 0],\n#     [1, 0, 0, 0, 0, 0],\n# ]\noptimization_domain_vertices = [[0, 0, 0, 0, 0], [0, 0, 0, 0, 1], [0, 0, 0, 1, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0]]\n# optimization_domain_vertices = [[0, 0, 0, 0], [0, 0, 0, 1], [0, 0, 1, 0], [0, 1, 0, 0], [1, 0, 0, 0]]\n# optimization_domain_vertices = [[0, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0]]\n\nnumber_of_iterations = 5000\nexploration = 0.01 # optional, default 0.15","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"tuner = SimpleTuner(optimization_domain_vertices, acc_function, exploration_preference=exploration)\ntuner.optimize(number_of_iterations)\n\nbest_objective_value, best_weights = tuner.get_best()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print(f'Best objective value = {best_objective_value:.6f}')\nprint(f'Optimum weights = {best_weights}')\nprint(f'Ensembled Accuracy (same as best objective value) = {acc_function(best_weights):.6f}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Tuning with Optuna"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"r_min, r_max = .0, 1.\n\n# You can increase iteration number.\niteration = 5000\n\noptuna.logging.disable_default_handler()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def objective(trial):\n    a = trial.suggest_uniform('a', r_min, r_max)\n    # a = trial.suggest_uniform('a', .1, .3)\n    b = trial.suggest_uniform('b', r_min, r_max)\n    # b = trial.suggest_uniform('b', .1, .3)\n    c = trial.suggest_uniform('c', r_min, r_max)\n    # c = trial.suggest_uniform('c', .1, .3)\n    d = trial.suggest_uniform('d', r_min, r_max)\n    # d = trial.suggest_uniform('d', .1, .3)\n    e = trial.suggest_uniform('e', r_min, r_max)\n    # e = trial.suggest_uniform('e', .1, .3)\n    f = trial.suggest_uniform('f', r_min, r_max)\n    # f = trial.suggest_uniform('e', .25, .35)\n    g = trial.suggest_uniform('g', r_min, r_max)\n    # g = trial.suggest_uniform('e', .25, .35)\n\n    # score = accuracy_score(calc_p(preds, a, b, c, d), labels)\n    # score = accuracy_score(calc_p(preds, a, b, c, d, e), labels)\n    # score = accuracy_score(calc_p(preds, a, b, c, d, e, f), labels)\n    score = accuracy_score(calc_p(preds, a, b, c, d, e, f, g), labels)\n\n    # print(f'a:{a:.6f}, b:{b:.6f}, c:{c:.6f}, d:{d:.6f}, score:{score:.6f}')\n    # print(f'a:{a:.6f}, b:{b:.6f}, c:{c:.6f}, d:{d:.6f}, e:{e:.6f}, score:{score:.6f}')\n    # print(f'a:{a:.6f}, b:{b:.6f}, c:{c:.6f}, d:{d:.6f}, e:{e:.6f}, f:{f:.6f}, score:{score:.6f}')\n    print(f'a:{a:.6f}, b:{b:.6f}, c:{c:.6f}, d:{d:.6f}, e:{e:.6f}, f:{f:.6f}, g:{g:.6f}, score:{score:.6f}')\n    return score","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"%%time\nSEED: int = 1337\n\nstudy = optuna.create_study(direction='maximize', sampler=optuna.samplers.TPESampler(seed=SEED))\nstudy.optimize(objective, n_trials=iteration, n_jobs=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'cv score : {study.best_value:.6f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study.best_params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"plt.plot([trial.params['a'] for trial in study.trials], label='a')\nplt.plot([trial.params['b'] for trial in study.trials], label='b')\nplt.plot([trial.params['c'] for trial in study.trials], label='c')\nplt.plot([trial.params['d'] for trial in study.trials], label='d')\nplt.plot([trial.params['e'] for trial in study.trials], label='e')\nplt.plot([trial.params['f'] for trial in study.trials], label='f')\nplt.plot([trial.params['g'] for trial in study.trials], label='g')\nplt.legend()\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"plt.plot([trial.value for trial in study.trials])\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from optuna.visualization import plot_optimization_history\n\nplot_optimization_history(study)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# from optuna.visualization import plot_param_importances\n\n# plot_param_importances(study)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# from optuna.visualization import plot_contour\n\n# plot_contour(study)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Metrics"},{"metadata":{"trusted":true},"cell_type":"code","source":"params = study.best_params\n\n# weights = [params['a'], params['b'], params['c'], params['d']]\n# weights = [params['a'], params['b'], params['c'], params['d'], params['e']]\n# weights = [params['a'], params['b'], params['c'], params['d'], params['e'], params['f']]\nweights = [params['a'], params['b'], params['c'], params['d'], params['e'], params['f'], params['g']]\n\nweights = [round(weight, 8) for weight in weights]\nweights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# weights = best_weights\n\n# weights = [0.28008428, 0.08930099, 0.19287446, 0.13415098, 0.2855688]\n# logits = weights[0] * preds[0] + weights[1] * preds[1] + weights[2] * preds[2] + weights[3] * preds[3] + weights[4] * preds[4]\n\n# weights = [0.34618164, 0.19092364, 0.38515934, 0.91232422, 0.00026995, 0.70023081]\n# weights = [0.64438387, 0.06787352, 0.21374317, 0.92894338, 0.30073056, 0.25681572]\n# logits = weights[0] * preds[0] + weights[1] * preds[1] + weights[2] * preds[2] + weights[3] * preds[3] + weights[4] * preds[4] + weights[5] * preds[5]\n\nlogits = weights[0] * preds[0] + weights[1] * preds[1] + weights[2] * preds[2] + weights[3] * preds[3] + weights[4] * preds[4] + weights[5] * preds[5] + weights[6] * preds[6]\n\n# weights = [0.32264375, 0.19517635, 0.10858799, 0.33353971]\n# weights = [0.22155271, 0.1881944, 0.38943474, 0.1644162]\n# logits = weights[0] * preds[0] + weights[1] * preds[1] + weights[2] * preds[2] + weights[3] * preds[3]\n\n# weights = [0.33547759, 0.30181755, 0.28914882]\n# logits = weights[0] * preds[0] + weights[1] * preds[1] + weights[2] * preds[2]\n\n# weights = [0.15093364, 0.15979473, 0.38676311, 0.26914147]\n# logits = weights[0] * preds[0] + weights[1] * preds[1] + weights[2] * preds[2] + weights[3] * preds[3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# logits = np.mean(preds, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"valid_acc = np.sum(labels == logits.argmax(1)) / len(logits) * 100.\nprint(f'[*] valid top-1 acc : {valid_acc:.4f}')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"cm = confusion_matrix(labels, logits.argmax(1))\ncm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"for i, val in enumerate(cm):\n    print(f'[+] Class {i} | top-1 acc : {val[i] / sum(val) * 100.:.4f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"for i, val in enumerate(cm[:-1]):\n    print(f'[+] Class {i} | possibility to mistake for healthy : {val[4] / val[i] * 100.:.4f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"print(classification_report(labels, logits.argmax(1), digits=6))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Public vs Private"},{"metadata":{"trusted":true},"cell_type":"code","source":"n_iters: int = 100000\nratio: float = 0.31\n\nn_samples: int = labels.shape[0]\nn_pub_samples: int = int(ratio * n_samples)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p = (logits.argmax(1) == labels)\np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\npub, priv = [], []\nindexes = np.arange(n_samples)\nfor i in tqdm(range(n_iters)):\n    np.random.shuffle(indexes)\n    pub_idx = indexes[:n_pub_samples]\n    priv_idx = indexes[n_pub_samples:]\n\n    pub_score = np.sum(p[pub_idx]) / pub_idx.shape[0]\n    priv_score = np.sum(p[priv_idx]) / priv_idx.shape[0]\n\n    pub.append(pub_score)\n    priv.append(priv_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.mean(priv), np.median(priv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(pub, hist=False, rug=True, color='blue', label='public')\nsns.distplot(priv, hist=False, rug=True, color='red', label='private')\n\nplt.title('public vs private')\nplt.xlabel('score')\nplt.ylabel('n_samples')\nplt.legend(prop={'size': 12}, title='group')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EOF"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}