{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport os, glob, pickle, time, gc, copy, sys\nimport warnings\nfrom tqdm import tqdm\nfrom sklearn import metrics\n\ntqdm.pandas()\nwarnings.filterwarnings('ignore')\npd.set_option('display.max_columns', 100)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-27T15:38:25.631031Z","iopub.execute_input":"2021-07-27T15:38:25.631436Z","iopub.status.idle":"2021-07-27T15:38:25.640209Z","shell.execute_reply.started":"2021-07-27T15:38:25.631403Z","shell.execute_reply":"2021-07-27T15:38:25.639098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data loading","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv\")\nprint('len(df_train): {}'.format(len(df_train)))\nprint(\"df_train['MGMT_value'].mean(): {:.6f}\".format(df_train['MGMT_value'].mean()))\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-27T15:30:59.084114Z","iopub.execute_input":"2021-07-27T15:30:59.084462Z","iopub.status.idle":"2021-07-27T15:30:59.104343Z","shell.execute_reply.started":"2021-07-27T15:30:59.084433Z","shell.execute_reply":"2021-07-27T15:30:59.103201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv\")\nprint('len(df_test): {}'.format(len(df_test)))\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-27T15:31:40.441086Z","iopub.execute_input":"2021-07-27T15:31:40.441493Z","iopub.status.idle":"2021-07-27T15:31:40.461974Z","shell.execute_reply.started":"2021-07-27T15:31:40.441460Z","shell.execute_reply":"2021-07-27T15:31:40.460732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The public test data has only 87 samples. So that, the public LB score must be unstable. I check it below.","metadata":{}},{"cell_type":"markdown","source":"# Public LB simulation","metadata":{}},{"cell_type":"code","source":"# assume that positive rate of the test data is same with the train data.\nnum_positive = int(len(df_test)*df_train['MGMT_value'].mean())\nprint(\"num_positive: {}\".format(num_positive))","metadata":{"execution":{"iopub.status.busy":"2021-07-27T15:34:29.185589Z","iopub.execute_input":"2021-07-27T15:34:29.185983Z","iopub.status.idle":"2021-07-27T15:34:29.193205Z","shell.execute_reply.started":"2021-07-27T15:34:29.185950Z","shell.execute_reply":"2021-07-27T15:34:29.192116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make true labels\ny_true = np.zeros(len(df_test))\ny_true[:num_positive] = 1\nprint(\"y_true.mean(): {:.6f}\".format(y_true.mean()))","metadata":{"execution":{"iopub.status.busy":"2021-07-27T15:36:26.262321Z","iopub.execute_input":"2021-07-27T15:36:26.262800Z","iopub.status.idle":"2021-07-27T15:36:26.269540Z","shell.execute_reply.started":"2021-07-27T15:36:26.262753Z","shell.execute_reply":"2021-07-27T15:36:26.268336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make random prediction\ny_pred = np.random.rand(len(df_test))\ny_pred","metadata":{"execution":{"iopub.status.busy":"2021-07-27T15:36:56.836556Z","iopub.execute_input":"2021-07-27T15:36:56.836904Z","iopub.status.idle":"2021-07-27T15:36:56.846157Z","shell.execute_reply.started":"2021-07-27T15:36:56.836875Z","shell.execute_reply":"2021-07-27T15:36:56.844952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calculate the score\nfrom sklearn import metrics\nscore = metrics.roc_auc_score(y_true, y_pred)\nprint(\"score: {:.6f}\".format(score))","metadata":{"execution":{"iopub.status.busy":"2021-07-27T15:38:17.017110Z","iopub.execute_input":"2021-07-27T15:38:17.017682Z","iopub.status.idle":"2021-07-27T15:38:17.025794Z","shell.execute_reply.started":"2021-07-27T15:38:17.017647Z","shell.execute_reply":"2021-07-27T15:38:17.024898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# try random prediction 1000 times.\nscores = []\nfor i in range(100):\n    np.random.seed(i)\n    y_pred = np.random.rand(len(df_test))\n    score = metrics.roc_auc_score(y_true, y_pred)\n    scores.append(score)\ndf_score = pd.DataFrame(scores, columns=['score'])\ndf_score = df_score.sort_values('score').reset_index(drop=True)\nplt.hist(df_score['score'], bins=10)\nplt.show()\ndf_score.tail()","metadata":{"execution":{"iopub.status.busy":"2021-07-27T15:43:23.864380Z","iopub.execute_input":"2021-07-27T15:43:23.864839Z","iopub.status.idle":"2021-07-27T15:43:24.138047Z","shell.execute_reply.started":"2021-07-27T15:43:23.864801Z","shell.execute_reply":"2021-07-27T15:43:24.136911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The simulations above shows that a score of about 0.65 is almost by chance. Don't believe the public LB score too much. ","metadata":{}}]}