{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Prediction and score dependency\nLet's understand what LogLoss.\n\nExamine the relationship between the range of values output by Prediction and the distribution of scores."},{"metadata":{},"cell_type":"markdown","source":"### Import Lib"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import argparse\nimport sys\nimport math\nimport numpy as np\nimport random\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\n\nfrom multiprocessing import Pool","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define Loss function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def logloss(true_label, predicted, eps=1e-15):\n  p = np.clip(predicted, eps, 1 - eps)\n  if true_label == 1:\n    return -math.log(p)\n  else:\n    return -math.log(1 - p)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visalize\nFor example, try to visualize the distribution of random values ​​that follow a Gaussian distribution with several half widths.\n\nThe size is fixed at 4000, which is the size of the public test dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_loss_uniform(w, size):\n    gt = (0, 1)\n    true_labels = [random.choice(gt) for _ in range(size)]\n    pred_labels = [random.uniform(0.5 - w, 0.5 + w) for _ in range(size)]\n    res = 0\n    for t, p in zip(true_labels, pred_labels):\n        res += logloss(t, p)\n    return res / size\n\ndef create_hist(w, size):\n    trial = 1000\n    with Pool(processes=4) as pool:\n        ret = pool.starmap(get_loss_uniform, [(w, size) for _ in range(trial)])\n    plt.hist(ret, bins=50)\n    plt.savefig(\"hist_Range{}_{}_uniform.png\".format(w, size))\n    plt.title(\"hist_Range{}_{}_uniform\".format(w, size))\n    plt.show()\n    plt.close(\"all\")\n    \nsize = 4000\ncreate_hist(0.5, size)\ncreate_hist(0.1, size)\ncreate_hist(0.01, size)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Maximum, minimum\nFind the maximum and minimum score obtained when the range of Prediction is limited."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_extreme_score(w, size, b=\"best\"):\n    gt = 1\n    if b!=\"best\":\n        gt = 0\n    true_labels = [gt for _ in range(size)]\n    pred_labels = [0.5 + w for _ in range(size)]\n    res = 0\n    for t, p in zip(true_labels, pred_labels):\n        res += logloss(t, p)\n    return res / size\n\nws = np.arange(0.01, 0.5, 0.01)\nbests = []\nworsts = []\nfor w in ws:\n    bests.append(get_extreme_score(w, size, b=\"best\"))\n    worsts.append(get_extreme_score(w, size, b=\"worst\"))\n# Plot both\nplt.title(\"Best and Worst\")\nplt.xlabel(\"w (range of prediction from 0.5)\")\nplt.ylabel(\"logloss\")\nplt.plot(list(ws), bests, label=\"best\")\nplt.plot(list(ws), (np.array(bests) + np.array(worsts))/2, label=\"middle\")\nplt.plot(list(ws), worsts, label=\"worst\")\nplt.legend()\nplt.show()\nplt.close(\"all\")\n\n# Plot Best only\nplt.title(\"Best only\")\nplt.xlabel(\"w (range of prediction from 0.5)\")\nplt.ylabel(\"logloss\")\nplt.plot(list(ws), bests, label=\"best\")\nplt.show()\n\n# Plot Middle only\nplt.title(\"Middle only\")\nplt.xlabel(\"w (range of prediction from 0.5)\")\nplt.ylabel(\"logloss\")\nplt.plot(list(ws), (np.array(bests) + np.array(worsts))/2, label=\"middle\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Conclusion\nLook before you leap.\n\nBut Nothing ventured, nothing gained."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}