{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is a fork of osciiart's notebook: https://www.kaggle.com/osciiart/public-lb-simulation\n\nI continued the analysis to simulate LB score after multiple random-value submits.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport os, glob, pickle, time, gc, copy, sys\nimport warnings\nfrom tqdm import tqdm\nfrom sklearn import metrics\n\ntqdm.pandas()\nwarnings.filterwarnings('ignore')\npd.set_option('display.max_columns', 100)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-24T12:40:27.687177Z","iopub.execute_input":"2021-09-24T12:40:27.687855Z","iopub.status.idle":"2021-09-24T12:40:28.789063Z","shell.execute_reply.started":"2021-09-24T12:40:27.687749Z","shell.execute_reply":"2021-09-24T12:40:28.787931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data loading","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv\")\nprint('len(df_train): {}'.format(len(df_train)))\nprint(\"df_train['MGMT_value'].mean(): {:.6f}\".format(df_train['MGMT_value'].mean()))\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-24T12:40:28.790885Z","iopub.execute_input":"2021-09-24T12:40:28.791307Z","iopub.status.idle":"2021-09-24T12:40:28.831009Z","shell.execute_reply.started":"2021-09-24T12:40:28.79126Z","shell.execute_reply":"2021-09-24T12:40:28.829952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv\")\nprint('len(df_test): {}'.format(len(df_test)))\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-24T12:40:28.83247Z","iopub.execute_input":"2021-09-24T12:40:28.832774Z","iopub.status.idle":"2021-09-24T12:40:28.849802Z","shell.execute_reply.started":"2021-09-24T12:40:28.832743Z","shell.execute_reply":"2021-09-24T12:40:28.848828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The public test data has only 87 samples. So that, the public LB score must be unstable. I check it below.","metadata":{}},{"cell_type":"markdown","source":"# Public LB simulation","metadata":{}},{"cell_type":"code","source":"# assume that positive rate of the test data is same with the train data.\nnum_positive = int(len(df_test)*df_train['MGMT_value'].mean())\nprint(\"num_positive: {}\".format(num_positive))","metadata":{"execution":{"iopub.status.busy":"2021-09-24T12:41:32.577522Z","iopub.execute_input":"2021-09-24T12:41:32.577901Z","iopub.status.idle":"2021-09-24T12:41:32.584346Z","shell.execute_reply.started":"2021-09-24T12:41:32.57787Z","shell.execute_reply":"2021-09-24T12:41:32.583425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make true labels\ny_true = np.zeros(len(df_test))\ny_true[:num_positive] = 1\nprint(\"y_true.mean(): {:.6f}\".format(y_true.mean()))","metadata":{"execution":{"iopub.status.busy":"2021-09-24T12:41:33.573151Z","iopub.execute_input":"2021-09-24T12:41:33.573765Z","iopub.status.idle":"2021-09-24T12:41:33.580992Z","shell.execute_reply.started":"2021-09-24T12:41:33.573724Z","shell.execute_reply":"2021-09-24T12:41:33.57929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make random prediction\ny_pred = np.random.rand(len(df_test))\ny_pred","metadata":{"execution":{"iopub.status.busy":"2021-09-24T12:41:35.260455Z","iopub.execute_input":"2021-09-24T12:41:35.2609Z","iopub.status.idle":"2021-09-24T12:41:35.272839Z","shell.execute_reply.started":"2021-09-24T12:41:35.260861Z","shell.execute_reply":"2021-09-24T12:41:35.271605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calculate the score\nfrom sklearn import metrics\nscore = metrics.roc_auc_score(y_true, y_pred)\nprint(\"score: {:.6f}\".format(score))","metadata":{"execution":{"iopub.status.busy":"2021-09-24T12:41:38.820371Z","iopub.execute_input":"2021-09-24T12:41:38.820756Z","iopub.status.idle":"2021-09-24T12:41:38.831792Z","shell.execute_reply.started":"2021-09-24T12:41:38.820724Z","shell.execute_reply":"2021-09-24T12:41:38.830483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.auto import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-09-24T12:42:42.252502Z","iopub.execute_input":"2021-09-24T12:42:42.253185Z","iopub.status.idle":"2021-09-24T12:42:42.343532Z","shell.execute_reply.started":"2021-09-24T12:42:42.253124Z","shell.execute_reply":"2021-09-24T12:42:42.342592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# try random prediction 1000 times.\nscores = []\nfor i in tqdm(range(100000)):\n    np.random.seed(i)\n    y_pred = np.random.rand(len(df_test))\n    score = metrics.roc_auc_score(y_true, y_pred)\n    scores.append(score)\ndf_score = pd.DataFrame(scores, columns=['score'])\ndf_score = df_score.sort_values('score').reset_index(drop=True)\nplt.hist(df_score['score'], bins=10)\nplt.show()\ndf_score.tail()","metadata":{"execution":{"iopub.status.busy":"2021-09-24T12:42:51.79873Z","iopub.execute_input":"2021-09-24T12:42:51.799398Z","iopub.status.idle":"2021-09-24T12:44:38.233922Z","shell.execute_reply.started":"2021-09-24T12:42:51.799356Z","shell.execute_reply":"2021-09-24T12:44:38.23301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The simulations above shows that a score of about 0.65 is almost by chance. Don't believe the public LB score too much. ","metadata":{}},{"cell_type":"markdown","source":"## One submission\n\nWith one all-random submit, the median LB score is 0.5 AUC. ","metadata":{}},{"cell_type":"code","source":"print(f'(1 random sub) percentile-50 : {np.percentile(scores, 50)}')\nprint(f'(1 random sub) percentile-90 : {np.percentile(scores, 90)}')\nprint(f'(1 random sub) percentile-95 : {np.percentile(scores, 95)}')\nprint(f'(1 random sub) percentile-99 : {np.percentile(scores, 99)}')\nprint(f'(1 random sub) percentile-99.9 : {np.percentile(scores, 99.9)}')\nprint(f'(1 random sub) percentile-99.99 : {np.percentile(scores, 99.99)}')","metadata":{"execution":{"iopub.status.busy":"2021-09-24T12:56:36.856764Z","iopub.execute_input":"2021-09-24T12:56:36.857161Z","iopub.status.idle":"2021-09-24T12:56:36.980157Z","shell.execute_reply.started":"2021-09-24T12:56:36.857131Z","shell.execute_reply":"2021-09-24T12:56:36.978848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Simulate LB score after multiple all-random submits\n\nLB shows the maximum score, so if you submit all-random multiple times, you are more likely to get a high scoring.","metadata":{}},{"cell_type":"code","source":"import random\n\nscores_10 = []\nscores_20 = []\nscores_30 = []\nscores_50 = []\nscores_100 = []\n\n# pick k random scores randomly from the 1M pool\nfor i in tqdm(range(10000)):\n    random.seed(i)\n    \n    scores_10.append(np.max(random.choices(scores, k=10)))\n    scores_20.append(np.max(random.choices(scores, k=20)))\n    scores_30.append(np.max(random.choices(scores, k=30)))\n    scores_50.append(np.max(random.choices(scores, k=50)))\n    scores_100.append(np.max(random.choices(scores, k=100)))","metadata":{"execution":{"iopub.status.busy":"2021-09-24T12:50:33.988821Z","iopub.execute_input":"2021-09-24T12:50:33.989236Z","iopub.status.idle":"2021-09-24T12:50:36.556901Z","shell.execute_reply.started":"2021-09-24T12:50:33.989203Z","shell.execute_reply":"2021-09-24T12:50:36.556173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'(10 random subs) percentile-50 : {np.percentile(scores_10, 50)}')\nprint(f'(10 random subs) percentile-90 : {np.percentile(scores_10, 90)}')\nprint(f'(10 random subs) percentile-95 : {np.percentile(scores_10, 95)}')\nprint(f'(10 random subs) percentile-99 : {np.percentile(scores_10, 99)}')\nprint(f'(10 random subs) percentile-99.9 : {np.percentile(scores_10, 99.9)}')\nprint(f'(10 random subs) percentile-99.99 : {np.percentile(scores_10, 99.99)}')","metadata":{"execution":{"iopub.status.busy":"2021-09-24T12:56:41.676542Z","iopub.execute_input":"2021-09-24T12:56:41.676924Z","iopub.status.idle":"2021-09-24T12:56:41.700646Z","shell.execute_reply.started":"2021-09-24T12:56:41.67689Z","shell.execute_reply":"2021-09-24T12:56:41.699141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'(20 random subs) percentile-50 : {np.percentile(scores_20, 50)}')\nprint(f'(20 random subs) percentile-90 : {np.percentile(scores_20, 90)}')\nprint(f'(20 random subs) percentile-95 : {np.percentile(scores_20, 95)}')\nprint(f'(20 random subs) percentile-99 : {np.percentile(scores_20, 99)}')\nprint(f'(20 random subs) percentile-99.9 : {np.percentile(scores_20, 99.9)}')\nprint(f'(20 random subs) percentile-99.99 : {np.percentile(scores_20, 99.99)}')","metadata":{"execution":{"iopub.status.busy":"2021-09-24T12:57:04.390004Z","iopub.execute_input":"2021-09-24T12:57:04.390601Z","iopub.status.idle":"2021-09-24T12:57:04.414682Z","shell.execute_reply.started":"2021-09-24T12:57:04.390565Z","shell.execute_reply":"2021-09-24T12:57:04.41377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'(50 random subs) percentile50 : {np.percentile(scores_50, 50)}')\nprint(f'(50 random subs) percentile90 : {np.percentile(scores_50, 90)}')\nprint(f'(50 random subs) percentile95 : {np.percentile(scores_50, 95)}')\nprint(f'(50 random subs) percentile99 : {np.percentile(scores_50, 99)}')\nprint(f'(50 random subs) percentile99.9 : {np.percentile(scores_50, 99.9)}')\nprint(f'(50 random subs) percentile99.99 : {np.percentile(scores_50, 99.99)}')","metadata":{"execution":{"iopub.status.busy":"2021-09-24T12:53:14.446967Z","iopub.execute_input":"2021-09-24T12:53:14.447385Z","iopub.status.idle":"2021-09-24T12:53:14.472558Z","shell.execute_reply.started":"2021-09-24T12:53:14.447352Z","shell.execute_reply":"2021-09-24T12:53:14.471474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'(100 random subs) percentile50 : {np.percentile(scores_100, 50)}')\nprint(f'(100 random subs) percentile90 : {np.percentile(scores_100, 90)}')\nprint(f'(100 random subs) percentile95 : {np.percentile(scores_100, 95)}')\nprint(f'(100 random subs) percentile99 : {np.percentile(scores_100, 99)}')\nprint(f'(100 random subs) percentile99.9 : {np.percentile(scores_100, 99.9)}')\nprint(f'(100 random subs) percentile99.99 : {np.percentile(scores_100, 99.99)}')","metadata":{"execution":{"iopub.status.busy":"2021-09-24T12:53:04.408747Z","iopub.execute_input":"2021-09-24T12:53:04.409184Z","iopub.status.idle":"2021-09-24T12:53:04.435709Z","shell.execute_reply.started":"2021-09-24T12:53:04.409148Z","shell.execute_reply":"2021-09-24T12:53:04.434377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}