{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is a fork of osciiart's notebook: https://www.kaggle.com/osciiart/public-lb-simulation\n\nI continued the analysis to simulate private score after **two** random-value submits.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport os, glob, pickle, time, gc, copy, sys\nimport warnings\nfrom tqdm import tqdm\nfrom sklearn import metrics\n\ntqdm.pandas()\nwarnings.filterwarnings('ignore')\npd.set_option('display.max_columns', 100)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-21T12:45:42.660448Z","iopub.execute_input":"2021-10-21T12:45:42.660988Z","iopub.status.idle":"2021-10-21T12:45:42.667665Z","shell.execute_reply.started":"2021-10-21T12:45:42.660952Z","shell.execute_reply":"2021-10-21T12:45:42.667042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data loading","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv\")\nprint('len(df_train): {}'.format(len(df_train)))\nprint(\"df_train['MGMT_value'].mean(): {:.6f}\".format(df_train['MGMT_value'].mean()))\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-21T12:45:44.139745Z","iopub.execute_input":"2021-10-21T12:45:44.140100Z","iopub.status.idle":"2021-10-21T12:45:44.161649Z","shell.execute_reply.started":"2021-10-21T12:45:44.140067Z","shell.execute_reply":"2021-10-21T12:45:44.160904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv\")\nprint('len(df_test): {}'.format(len(df_test)))\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-21T12:46:02.719977Z","iopub.execute_input":"2021-10-21T12:46:02.720490Z","iopub.status.idle":"2021-10-21T12:46:02.737557Z","shell.execute_reply.started":"2021-10-21T12:46:02.720458Z","shell.execute_reply":"2021-10-21T12:46:02.736830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The public test data has only 87 samples and the private test was approx. four times larger.\n\n> The private leaderboard is calculated with approximately 78% of the test data.\n\nWe can multiply the public test set to four times bigger to simulate the private set.","metadata":{}},{"cell_type":"code","source":"df_test = pd.concat([df_test, df_test, df_test, df_test], ignore_index = True)\nprint('len(df_test): {}'.format(len(df_test)))\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-21T12:46:03.992054Z","iopub.execute_input":"2021-10-21T12:46:03.992652Z","iopub.status.idle":"2021-10-21T12:46:04.003688Z","shell.execute_reply.started":"2021-10-21T12:46:03.992614Z","shell.execute_reply":"2021-10-21T12:46:04.002925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Private LB simulation","metadata":{}},{"cell_type":"code","source":"# assume that positive rate of the test data is same with the train data.\nnum_positive = int(len(df_test)*df_train['MGMT_value'].mean())\nprint(\"num_positive: {}\".format(num_positive))","metadata":{"execution":{"iopub.status.busy":"2021-10-21T12:46:17.060356Z","iopub.execute_input":"2021-10-21T12:46:17.060920Z","iopub.status.idle":"2021-10-21T12:46:17.066625Z","shell.execute_reply.started":"2021-10-21T12:46:17.060884Z","shell.execute_reply":"2021-10-21T12:46:17.065827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make true labels\ny_true = np.zeros(len(df_test))\ny_true[:num_positive] = 1\nprint(\"y_true.mean(): {:.6f}\".format(y_true.mean()))","metadata":{"execution":{"iopub.status.busy":"2021-10-21T12:46:20.399920Z","iopub.execute_input":"2021-10-21T12:46:20.400515Z","iopub.status.idle":"2021-10-21T12:46:20.406104Z","shell.execute_reply.started":"2021-10-21T12:46:20.400474Z","shell.execute_reply":"2021-10-21T12:46:20.405033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make random prediction\ny_pred = np.random.rand(len(df_test))\ny_pred","metadata":{"execution":{"iopub.status.busy":"2021-10-21T12:46:21.860317Z","iopub.execute_input":"2021-10-21T12:46:21.860664Z","iopub.status.idle":"2021-10-21T12:46:21.870836Z","shell.execute_reply.started":"2021-10-21T12:46:21.860633Z","shell.execute_reply":"2021-10-21T12:46:21.869696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calculate the score\nfrom sklearn import metrics\nscore = metrics.roc_auc_score(y_true, y_pred)\nprint(\"score: {:.6f}\".format(score))","metadata":{"execution":{"iopub.status.busy":"2021-10-21T12:46:31.239673Z","iopub.execute_input":"2021-10-21T12:46:31.240049Z","iopub.status.idle":"2021-10-21T12:46:31.247899Z","shell.execute_reply.started":"2021-10-21T12:46:31.240016Z","shell.execute_reply":"2021-10-21T12:46:31.246868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.auto import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-10-21T12:46:34.608284Z","iopub.execute_input":"2021-10-21T12:46:34.608745Z","iopub.status.idle":"2021-10-21T12:46:34.688542Z","shell.execute_reply.started":"2021-10-21T12:46:34.608713Z","shell.execute_reply":"2021-10-21T12:46:34.687620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# try random prediction 100000 times.\nscores = []\nfor i in tqdm(range(100000)):\n    np.random.seed(i)\n    y_pred = np.random.rand(len(df_test))\n    score = metrics.roc_auc_score(y_true, y_pred)\n    scores.append(score)\ndf_score = pd.DataFrame(scores, columns=['score'])\ndf_score = df_score.sort_values('score').reset_index(drop=True)\nplt.hist(df_score['score'], bins=10)\nplt.show()\ndf_score.tail()","metadata":{"execution":{"iopub.status.busy":"2021-10-21T12:46:39.980882Z","iopub.execute_input":"2021-10-21T12:46:39.981195Z","iopub.status.idle":"2021-10-21T12:48:24.498548Z","shell.execute_reply.started":"2021-10-21T12:46:39.981168Z","shell.execute_reply":"2021-10-21T12:48:24.497804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The simulations above shows that a score of about 0.65 is almost by chance. Don't believe the public LB score too much. ","metadata":{}},{"cell_type":"markdown","source":"## One submission\n\nWith one all-random submit, the median LB score is 0.5 AUC. ","metadata":{}},{"cell_type":"code","source":"print(f'(1 random sub) percentile-50 : {np.percentile(scores, 50)}')\nprint(f'(1 random sub) percentile-90 : {np.percentile(scores, 90)}')\nprint(f'(1 random sub) percentile-95 : {np.percentile(scores, 95)}')\nprint(f'(1 random sub) percentile-99 : {np.percentile(scores, 99)}')\nprint(f'(1 random sub) percentile-99.9 : {np.percentile(scores, 99.9)}')\nprint(f'(1 random sub) percentile-99.99 : {np.percentile(scores, 99.99)}')","metadata":{"execution":{"iopub.status.busy":"2021-10-21T12:48:43.040604Z","iopub.execute_input":"2021-10-21T12:48:43.041081Z","iopub.status.idle":"2021-10-21T12:48:43.107491Z","shell.execute_reply.started":"2021-10-21T12:48:43.041048Z","shell.execute_reply":"2021-10-21T12:48:43.106568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Simulate Private score after two allowed all-random submits\n\nPrivate shows the maximum score out of two selected submit.","metadata":{}},{"cell_type":"code","source":"import random\n\nscores_2 = []\n\n# pick k random scores randomly from the 1M pool\nfor i in tqdm(range(10000)):\n    random.seed(i)\n    \n    scores_2.append(np.max(random.choices(scores, k=2)))","metadata":{"execution":{"iopub.status.busy":"2021-10-21T12:50:03.101255Z","iopub.execute_input":"2021-10-21T12:50:03.101607Z","iopub.status.idle":"2021-10-21T12:50:03.341458Z","shell.execute_reply.started":"2021-10-21T12:50:03.101575Z","shell.execute_reply":"2021-10-21T12:50:03.340258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'(2 random subs) percentile-50 : {np.percentile(scores_2, 50)}')\nprint(f'(2 random subs) percentile-90 : {np.percentile(scores_2, 90)}')\nprint(f'(2 random subs) percentile-95 : {np.percentile(scores_2, 95)}')\nprint(f'(2 random subs) percentile-99 : {np.percentile(scores_2, 99)}')\nprint(f'(2 random subs) percentile-99.9 : {np.percentile(scores_2, 99.9)}')\nprint(f'(2 random subs) percentile-99.99 : {np.percentile(scores_2, 99.99)}')","metadata":{"execution":{"iopub.status.busy":"2021-10-21T12:50:26.102129Z","iopub.execute_input":"2021-10-21T12:50:26.102631Z","iopub.status.idle":"2021-10-21T12:50:26.119038Z","shell.execute_reply.started":"2021-10-21T12:50:26.102585Z","shell.execute_reply":"2021-10-21T12:50:26.117912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit all random\n\nFor fun :)","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv')\n\n# Fill with random values\nmgmt_values = sub.MGMT_value.values\nmgmt_values = [np.random.random() for _ in mgmt_values]\nsub['MGMT_value'] = mgmt_values\n\n# save\nsub.to_csv('submission.csv',index=False)\nsub.head()","metadata":{},"execution_count":null,"outputs":[]}]}