{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"# Benchmark\nThis is a very simple benchmark to find out how many `new_whales` are in the test set."},{"metadata":{"trusted":true,"_uuid":"a9e402f384a05c2c83a8d3850fd7feee3e302f18"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a6b6e4dccc8d0a95fcf337aa3c01f61808494faa"},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4244148b8db1980951d53c517964508a9f3ff424"},"cell_type":"code","source":"n_all = train_df.shape[0]\nn_new_whale = train_df[train_df['Id'] == 'new_whale'].shape[0]\nprint(\"We have {}/{} ({:.2f}%) `new_whale` in the training set.\".format(n_new_whale, n_all, n_new_whale/n_all*100))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5d15624e2bebf3c978a026face6fea5ab36cbed9"},"cell_type":"markdown","source":"Let's see if we get similar distribution if we use a random fraction of the data."},{"metadata":{"trusted":true,"_uuid":"42c8c2ce1ed5d9f9be8b034aded79e233f720435"},"cell_type":"code","source":"res = []\nfor i in range(10):\n    tmp_df = train_df.sample(frac=.2)\n    n_all = tmp_df.shape[0]\n    n_new_whale = tmp_df[tmp_df['Id'] == 'new_whale'].shape[0]\n    res.append(n_new_whale/n_all)\n    print(\"We have {}/{} ({:.2f}%) `new_whale` in the random subset #{}.\".format(n_new_whale, n_all, n_new_whale/n_all*100, i))\n    \nprint(\"Average: {:.4f}, std: {:.4f}\".format(np.mean(res), np.std(res)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3aaf0395e8e5e9f577d9f1a8422bd5f1e6d44dd8"},"cell_type":"markdown","source":"*I assume a similar 'new_whale' distribution between the public (20%) and the private (80%) set.*"},{"metadata":{"trusted":true,"_uuid":"d8ea8ab563081b2db29dc02d0864c5b3dbcbf653"},"cell_type":"code","source":"test_df = pd.read_csv('../input/sample_submission.csv')\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71e196900fe2e29a5a39ade729cac0148f6f8a27"},"cell_type":"code","source":"test_df['Id'] = 'new_whale'\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5c371f4133d20868ceadf4f763242e3c52f8c175"},"cell_type":"markdown","source":"I got `0.276` LB score when I submitted the `new_whale` predictions. For every image I either got `P(1) = 0` or `1` score, see [my MAP@5 explanation kernel](https://www.kaggle.com/pestipeti/explanation-of-map5-scoring-metric) for more details."},{"metadata":{"trusted":true,"_uuid":"756725ae89f7244ba9a98263d3d9980eb108b144"},"cell_type":"code","source":"n_all = test_df.shape[0]\n# After submission (public LB score: 0.276)\nn_new_whale = 2197\nprint(\"We have {}/{} ({:.2f}%) `new_whale` in the test set.\".format(n_new_whale, n_all, n_new_whale/n_all*100))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f1f14703227109e9d96598075ed099f137d8c85"},"cell_type":"code","source":"test_df.to_csv('new_whale_benchmark.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eb2b4205fddedaec143e00925c091fa0bb9a0fd7"},"cell_type":"markdown","source":"**Thanks for reading.**"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}