{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Exploratory Data Analysis\n\n1. Browsing the Private Scores\n2. Regression with our team's personal private-public scores"},{"metadata":{},"cell_type":"markdown","source":"\n# Browsing the Private Scores"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom matplotlib import pyplot as plt\nfrom sklearn import datasets, linear_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### The blue verticle lines indicate all the private scores in Private Leader Board.\n### The red lines indicate the places of 0 and 1"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"f = open(\"../input/cassava-lb/Private_LB.log\",'r')\nf = list(map(float,f.read().split()))\narray = np.array(f)\n\nplt.hlines(1,-0.05,1.05,color='k')\nplt.xlim(-0.1,1.1)\nplt.ylim(0.5,1.5)\n\ny = np.ones(np.shape(array))\ny2 = np.ones((2))\nplt.plot(array,y,'|', ms=30)\nplt.plot([0,1],y2,'r|', ms=60)\nplt.axis('off')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Let's draw the Public - Private graph"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"../input/cassava-lb/Private_Public.log\", names=[\"data\"],encoding='utf-8')\ndf2 = pd.DataFrame(columns=['private', 'public'])\ndf.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### We submitted 204 times and only 187 submissions are not failed ones."},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(0,len(df),2):\n    try:\n        df2 = df2.append({'private':float(df[\"data\"][i].replace('\\u200b','')),'public':float(df[\"data\"][i+1].replace('\\u200b',''))},ignore_index=True)\n    except:\n        break\nprint(len(df2))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Let's just sample the private scores above 0.8"},{"metadata":{"trusted":true},"cell_type":"code","source":"df2 = df2[df2['private']>0.8]\nlength = len(df2)\nprint(length)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df2.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"private = df2['private'].to_numpy().reshape(length,1)\npublic = df2['public'].to_numpy().reshape(length,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"regr = linear_model.LinearRegression()\nregr.fit(public, private)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot it as in the example at http://scikit-learn.org/\nplt.scatter(public, private,  color='black')\nplt.plot(public, regr.predict(public), color='blue', linewidth=3)\nplt.title(\"Public - Private\")\nplt.xlabel(\"Public\")\nplt.ylabel(\"Private\")\nplt.show()\nprint(regr.coef_)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### What about choose the private scores above 0.87?"},{"metadata":{"trusted":true},"cell_type":"code","source":"df2 = df2[df2['private']>0.87]\nlength = len(df2)\nprint(length)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"private = df2['private'].to_numpy().reshape(length,1)\npublic = df2['public'].to_numpy().reshape(length,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"regr = linear_model.LinearRegression()\nregr.fit(public, private)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot it as in the example at http://scikit-learn.org/\nplt.scatter(public, private,  color='black')\nplt.plot(public, regr.predict(public), color='blue', linewidth=3)\nplt.title(\"Public - Private\")\nplt.xlabel(\"Public\")\nplt.ylabel(\"Private\")\nplt.show()\nprint(regr.coef_)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}