{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\nimport multiprocessing\nfrom sklearn.metrics import roc_auc_score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-28T12:11:53.056045Z","iopub.execute_input":"2021-09-28T12:11:53.056401Z","iopub.status.idle":"2021-09-28T12:11:54.104305Z","shell.execute_reply.started":"2021-09-28T12:11:53.056309Z","shell.execute_reply":"2021-09-28T12:11:54.103631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare OOF\n\nCV 0.867 OOF was taken from @hidehisaarai1213 's notebook.\n\nhttps://www.kaggle.com/hidehisaarai1213/g2net-tf-on-the-fly-cqt-tpu-training","metadata":{}},{"cell_type":"code","source":"def calc_metric(df):\n    return roc_auc_score(df[\"y_true\"], df[\"y_pred\"])","metadata":{"execution":{"iopub.status.busy":"2021-09-28T12:11:54.105602Z","iopub.execute_input":"2021-09-28T12:11:54.106242Z","iopub.status.idle":"2021-09-28T12:11:54.109387Z","shell.execute_reply.started":"2021-09-28T12:11:54.106207Z","shell.execute_reply":"2021-09-28T12:11:54.108864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof = pd.read_csv(\"../input/g2net-tf-on-the-fly-cqt-tpu-training/oof/oof.csv\")\nprint(f\"CV score is {calc_metric(oof):.3f}\")","metadata":{"execution":{"iopub.status.busy":"2021-09-28T12:11:54.110345Z","iopub.execute_input":"2021-09-28T12:11:54.110899Z","iopub.status.idle":"2021-09-28T12:11:54.651352Z","shell.execute_reply.started":"2021-09-28T12:11:54.110871Z","shell.execute_reply":"2021-09-28T12:11:54.650426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Leaderboard simulation\nWe have 226000 test data, and Public leaderboard is calculated with 16%, Private is on the other 84%.\n\nWe assume that the number of each target is eqaul as same as the training datasets distribution.\n\n226000 samples are randomly extracted from 560000 oof so that the ratio of each target is the same. \n\nThis random sampling is performed 10000 times.\n\nNote that there is possibility that the same data in oof can be used for both Public and Private estimation.","metadata":{}},{"cell_type":"code","source":"num_data = 226000\nnum_public = int(0.16 * num_data)\nnum_private = int(0.84 * num_data)\nsampling_times = 10000","metadata":{"execution":{"iopub.status.busy":"2021-09-28T12:11:54.652702Z","iopub.execute_input":"2021-09-28T12:11:54.653040Z","iopub.status.idle":"2021-09-28T12:11:54.658357Z","shell.execute_reply.started":"2021-09-28T12:11:54.652999Z","shell.execute_reply":"2021-09-28T12:11:54.657348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def public_estimation(dummy):\n    df = pd.concat([oof[oof[\"y_true\"] == 1].sample(num_public//2), \n                    oof[oof[\"y_true\"] == 0].sample(num_public//2)])\n\n    return calc_metric(df)\n\n\ndef private_estimation(dummy):\n    df = pd.concat([oof[oof[\"y_true\"] == 1].sample(num_private//2), \n                    oof[oof[\"y_true\"] == 0].sample(num_private//2)])\n\n    return calc_metric(df)","metadata":{"execution":{"iopub.status.busy":"2021-09-28T12:11:54.660950Z","iopub.execute_input":"2021-09-28T12:11:54.661263Z","iopub.status.idle":"2021-09-28T12:11:54.671120Z","shell.execute_reply.started":"2021-09-28T12:11:54.661225Z","shell.execute_reply":"2021-09-28T12:11:54.670364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processes = multiprocessing.cpu_count()\nwith multiprocessing.Pool(processes=processes) as pool:\n    public_auc = pool.imap_unordered(public_estimation, range(sampling_times))\n    public_auc = tqdm(public_auc, total=sampling_times)\n    public_auc = list(public_auc)","metadata":{"execution":{"iopub.status.busy":"2021-09-28T12:11:54.672378Z","iopub.execute_input":"2021-09-28T12:11:54.672591Z","iopub.status.idle":"2021-09-28T12:14:22.194801Z","shell.execute_reply.started":"2021-09-28T12:11:54.672568Z","shell.execute_reply":"2021-09-28T12:14:22.193947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processes = multiprocessing.cpu_count()\nwith multiprocessing.Pool(processes=processes) as pool:\n    private_auc = pool.imap_unordered(private_estimation, range(sampling_times))\n    private_auc = tqdm(private_auc, total=sampling_times)\n    private_auc = list(private_auc)","metadata":{"execution":{"iopub.status.busy":"2021-09-28T12:14:22.196200Z","iopub.execute_input":"2021-09-28T12:14:22.196443Z","iopub.status.idle":"2021-09-28T12:20:30.433071Z","shell.execute_reply.started":"2021-09-28T12:14:22.196411Z","shell.execute_reply":"2021-09-28T12:20:30.432107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 8))\nsns.distplot(public_auc, color=\"red\")\nsns.distplot(private_auc, color=\"blue\")\nplt.legend(labels=[\"Public\", \"Private\"], fontsize=12)","metadata":{"execution":{"iopub.status.busy":"2021-09-28T12:21:10.669614Z","iopub.execute_input":"2021-09-28T12:21:10.670071Z","iopub.status.idle":"2021-09-28T12:21:11.308291Z","shell.execute_reply.started":"2021-09-28T12:21:10.670040Z","shell.execute_reply":"2021-09-28T12:21:11.307467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Conclusions\nIf you have stable relationship between CV and LB, it is estimated that...\n- public LB is CV ± 0.005.\n- private LB is CV ± 0.002.\n- 0.007 shake up/down is possible.\n\nIn this notebook, I proposed a very simple shake estimation.\n\nUsing multiple oof and calculating std of difference between public and private may be better way of another simulation.\n\nAny opinions are welcome!","metadata":{}}]}