{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# What would the leaderboard look like if the private test labels were inadvertently shuffled?","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.metrics import roc_auc_score\n\nnp.random.seed(0)\n\nleaderboard = pd.read_csv('../input/final-lb-rsnamiccai-brain-tumor-classification/rsna-private.csv')\nteam_n = leaderboard.shape[0]\nentries = np.array(leaderboard['Entries'], dtype=np.int32)\n\n# Size of the public test set\npublic_n = 87\n# Size of the complete test set\ntotal_n = 400","metadata":{"execution":{"iopub.status.busy":"2021-10-21T17:04:47.882905Z","iopub.execute_input":"2021-10-21T17:04:47.883190Z","iopub.status.idle":"2021-10-21T17:04:47.917351Z","shell.execute_reply.started":"2021-10-21T17:04:47.883161Z","shell.execute_reply":"2021-10-21T17:04:47.916676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = np.random.random(total_n).round()\nbest_scores = np.empty(team_n, dtype=np.float32)\n\nfor team in range(team_n):\n    public_scores = np.empty(entries[team], dtype=np.float32)\n    private_scores = np.empty_like(public_scores)\n    for entry in range(entries[team]):\n        preds = np.random.random(total_n)\n        public_auc = roc_auc_score(labels[:public_n], preds[:public_n])\n        private_auc = roc_auc_score(labels[public_n:], preds[public_n:])\n        public_scores[entry] = public_auc\n        private_scores[entry] = private_auc\n\n    # Get the private score for the entry that scored the best on public LB\n    idx = np.argmax(public_scores)\n    best_scores[team] = round(private_scores[idx], 5)","metadata":{"execution":{"iopub.status.busy":"2021-10-21T17:04:47.918702Z","iopub.execute_input":"2021-10-21T17:04:47.919458Z","iopub.status.idle":"2021-10-21T17:05:44.917152Z","shell.execute_reply.started":"2021-10-21T17:04:47.919418Z","shell.execute_reply":"2021-10-21T17:05:44.916136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the top scores in the private LB\nsorted_scores = -np.sort(-best_scores)\nsimulated_top_ten = sorted_scores[:10]\nactual_top_ten = leaderboard['Score'].values[:10]\nprint(f'Simulated top 10: {simulated_top_ten}')\nprint(f'Actual top 10   : {actual_top_ten}')","metadata":{"execution":{"iopub.status.busy":"2021-10-21T17:05:44.918671Z","iopub.execute_input":"2021-10-21T17:05:44.918906Z","iopub.status.idle":"2021-10-21T17:05:44.926024Z","shell.execute_reply.started":"2021-10-21T17:05:44.918878Z","shell.execute_reply":"2021-10-21T17:05:44.925050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](https://emojipedia-us.s3.dualstack.us-west-1.amazonaws.com/thumbs/120/apple/285/thinking-face_1f914.png)","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\ndata = pd.DataFrame({'actual':leaderboard['Score'].values, 'simulated':best_scores})\nsns.histplot(data, kde=True)","metadata":{"execution":{"iopub.status.busy":"2021-10-21T17:06:02.402846Z","iopub.execute_input":"2021-10-21T17:06:02.403654Z","iopub.status.idle":"2021-10-21T17:06:02.940367Z","shell.execute_reply.started":"2021-10-21T17:06:02.403587Z","shell.execute_reply":"2021-10-21T17:06:02.939367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](https://emojipedia-us.s3.dualstack.us-west-1.amazonaws.com/thumbs/120/apple/285/relieved-face_1f60c.png)","metadata":{}},{"cell_type":"markdown","source":"## While the top scores on the actual leaderboard seem to line up with that of the simulated leaderboard, the score distributions are different.","metadata":{}}]}