{"cells": [{"metadata": {"_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"}, "cell_type": "markdown", "source": "# Tracking the Public Leaderboard for the Riiid! Answer Correctness Prediction Competition\n* I was curious to see how the leaderboard was shaping up.\n* This will now automatically update at 9am UK time everyday.\n* Now with leaderboard animation.\n* I had hoped to plot the all teams vs the highest scoring public notebook, but that doesn't appear to be readily available.     \n* Let me know in the comments if you know how to do this and I'll add it in.\n* 7th Nov 2020: Fix a bug in the data where a team name change resulted in additional teams "}, {"metadata": {"trusted": true, "_kg_hide-input": true, "_kg_hide-output": true}, "cell_type": "code", "source": "import os\nimport zipfile\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pylab as plt\nimport plotly\nimport plotly.graph_objs as go\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\nfrom sklearn.linear_model import LinearRegression\nimport datetime\nimport colorlover as cl\n\nplt.style.use('ggplot')\ncolor_pal = [x['color'] for x in plt.rcParams['axes.prop_cycle']]", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true, "_kg_hide-input": true, "_kg_hide-output": true}, "cell_type": "code", "source": "# Format the data\ndf = pd.read_csv('../input/riiid-leaderboad/leaderboard.csv')\ndf['SubmissionDate'] = pd.to_datetime(df['SubmissionDate'])\ndf = df.set_index('SubmissionDate')\ndf.columns = [name for name in df.columns]\ndf.drop(columns=['Unnamed: 0'], inplace=True)\ndf.drop_duplicates(inplace=True)", "execution_count": null, "outputs": []}, {"metadata": {"_kg_hide-input": true, "trusted": true}, "cell_type": "code", "source": "tmp = df.fillna(0)\ntmp = tmp.max()\nteams_to_delete = list(tmp[tmp > 0.999].to_dict().keys())", "execution_count": null, "outputs": []}, {"metadata": {"_kg_hide-input": true, "trusted": true}, "cell_type": "code", "source": "print('These teams will be deleted for appearing to be using an exploit:')\nprint(\"\\n\".join(teams_to_delete))", "execution_count": null, "outputs": []}, {"metadata": {"_kg_hide-input": true, "trusted": true}, "cell_type": "code", "source": "df.drop(columns=teams_to_delete, inplace=True)", "execution_count": null, "outputs": []}, {"metadata": {"_kg_hide-input": true, "_kg_hide-output": true, "trusted": true}, "cell_type": "code", "source": "FIFTEENTH_SCORE = df.max().sort_values(ascending=False)[15]\nFIFTYTH_SCORE = df.max().sort_values(ascending=False)[50]\nTOP_SCORE = df.max().sort_values(ascending=False)[0]\nQUANTILE_25 = df.max().sort_values(ascending=False).quantile(0.25)\nQUANTILE_50 = df.max().sort_values(ascending=False).quantile(0.5)\nQUANTILE_75 = df.max().sort_values(ascending=False).quantile(0.75)", "execution_count": null, "outputs": []}, {"metadata": {}, "cell_type": "markdown", "source": "# Interactive Plot of the Top Teams"}, {"metadata": {"_kg_hide-input": true, "trusted": true}, "cell_type": "code", "source": "# Interative Plotly\nmypal = cl.scales['9']['div']['Spectral']\ncolors = cl.interp( mypal, 15 )\nannotations = []\ninit_notebook_mode(connected=True)\nTOP_TEAMS = df.max().loc[df.max() > FIFTEENTH_SCORE].index.values\ndf_filtered = df[TOP_TEAMS].ffill()\ndf_filtered = df_filtered.iloc[df_filtered.index >= df.index.min()]\nteam_ordered = df_filtered.max(axis=0).sort_values(ascending=False).index.tolist()\n\ndata = []\ni = 0\nfor col in df_filtered[team_ordered].columns:\n    data.append(go.Scatter(x = df_filtered.index, y = df_filtered[col], name=col, line=dict(color=colors[i], width=2),))\n    i += 1\n\nannotations.append(dict(xref='paper', yref='paper', x=0.0, y=1.05, xanchor='left', yanchor='bottom', \n                        text='Top Teams Public Leaderboard (Last 7 days)',\n                        font=dict(family='Arial', size=20, color='rgb(37,37,37)'), showarrow=False))\n\nlayout = go.Layout(xaxis=dict(range=[df_filtered.index.max() - pd.Timedelta(days=7), df_filtered.index.max()]),\n                   yaxis=dict(range=[FIFTEENTH_SCORE-0.0001, TOP_SCORE+0.0001]), hovermode='x', plot_bgcolor='white', annotations=annotations)\nfig = go.Figure(data=data, layout=layout)\nfig.update_layout(\n    legend=go.layout.Legend(\n        traceorder=\"normal\",\n        font=dict(family=\"sans-serif\", size=12, color=\"black\"),\n        bgcolor=\"LightSteelBlue\",\n        bordercolor=\"Black\",\n        borderwidth=2,\n    )\n)\n\nfig.update_layout(legend_orientation=\"h\")\nfig.update_layout(template=\"plotly_white\")\nfig.update_xaxes(showgrid=False)\n\niplot(fig)", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "markdown", "source": "# Scores over time (Last 7 days)\nOnly the 75% quantile plotted."}, {"metadata": {"_kg_hide-input": true, "trusted": true}, "cell_type": "code", "source": "# Scores of top teams over time\nplt.rcParams[\"font.size\"] = \"12\"\nALL_TEAMS = df.columns.values[1:]\ndf_ffill = df[ALL_TEAMS].ffill()\n\ndf_ffill.plot(figsize=(20, 10), color=color_pal[0], legend=False, alpha=0.2, \n              xlim=(df_filtered.index.max() - pd.Timedelta(days=7), df_ffill.index.max()),\n              ylim=(QUANTILE_75, TOP_SCORE+0.01), \n              title='All Teams Public Leaderboard Scores over the Last Week (75% Quantile)')\n\ndf_ffill.max(axis=1).plot(color=color_pal[1], label='1st Place Public LB', legend=True)\n\n# df_ffill['GhostSkipper'].plot(color='k', label='GhostSkipper', legend=True)\n\nplt.show()", "execution_count": null, "outputs": []}, {"metadata": {}, "cell_type": "markdown", "source": "# Scores over time (Last 7 days)\nOnly the 50% quantile plotted."}, {"metadata": {"trusted": true, "_kg_hide-input": true}, "cell_type": "code", "source": "# Scores of top teams over time\nplt.rcParams[\"font.size\"] = \"12\"\nALL_TEAMS = df.columns.values[1:]\ndf_ffill = df[ALL_TEAMS].ffill()\n\ndf_ffill.plot(figsize=(20, 10), color=color_pal[0], legend=False, alpha=0.2, \n              xlim=(df_filtered.index.max() - pd.Timedelta(days=7), df_ffill.index.max()),\n              ylim=(QUANTILE_50, TOP_SCORE+0.01), \n              title='All Teams Public Leaderboard Scores over the Last Week (50% Quantile)')\n\ndf_ffill.max(axis=1).plot(color=color_pal[1], label='1st Place Public LB', legend=True)\n\nplt.show()", "execution_count": null, "outputs": []}, {"metadata": {}, "cell_type": "markdown", "source": "# Scores over time (All time)"}, {"metadata": {"trusted": true, "_kg_hide-input": true}, "cell_type": "code", "source": "# Scores of top teams over time\nplt.rcParams[\"font.size\"] = \"12\"\nALL_TEAMS = df.columns.values[1:]\ndf_ffill = df[ALL_TEAMS].ffill()\n\ndf_ffill.plot(figsize=(20, 10), color=color_pal[0], legend=False, alpha=0.1, \n              xlim=(df_filtered.index.min(), df_ffill.index.max()),\n              ylim=(0.495, TOP_SCORE+0.01), \n              title='All Teams Public Leaderboard Scores over Time')\n\ndf_ffill.max(axis=1).plot(color=color_pal[1], label='1st Place Public LB', legend=True)\n\ndf['sample_submission.csv'] = 0.5\ndf['sample_submission.csv'].plot(color='k', label='Sample Submission', legend=True)\n\nplt.show()", "execution_count": null, "outputs": []}, {"metadata": {}, "cell_type": "markdown", "source": "# Teams By Date"}, {"metadata": {"_kg_hide-input": true, "trusted": true}, "cell_type": "code", "source": "plt.rcParams[\"font.size\"] = \"13\"\nax = df.ffill().count(axis=1).plot(figsize=(20, 8), title='Number of Teams in the Competition by Date', color=color_pal[5], lw=5)\nax.set_ylabel('Number of Teams')\nplt.show()", "execution_count": null, "outputs": []}, {"metadata": {}, "cell_type": "markdown", "source": "# Top 50 Leaderboard"}, {"metadata": {"_kg_hide-input": true, "trusted": true}, "cell_type": "code", "source": "plt.rcParams[\"font.size\"] = \"12\"\n\ncolours = ['green'] * 5\ncolours += ['gold'] * (5 + int(0.002 * df.shape[0])-1)\ncolours += ['silver'] * (int(0.05*df.shape[0]) - (5 + int(0.002 * df.shape[0])-1))\ncolours += ['peru'] * (int(0.10*df.shape[0]) - (int(0.05*df.shape[0]) + 5 + int(0.002 * df.shape[0])-1))\ncolours = colours[::-1]\n\n\n# Create Top Teams List\nTOP_TEAMS = df.max().loc[df.max() > FIFTYTH_SCORE].index.values\ndf[TOP_TEAMS].max().sort_values(ascending=True).plot(kind='barh',\n                                                     xlim=(FIFTYTH_SCORE-0.0005, TOP_SCORE+0.0005),\n                                                     title='Top 50 Public Leaderboard',\n                                                     figsize=(12, 15),\n                                                     color=colours[-len(TOP_TEAMS):])\nplt.show()", "execution_count": null, "outputs": []}, {"metadata": {}, "cell_type": "markdown", "source": "# Leaderboard Distrubtion Over Time"}, {"metadata": {"_kg_hide-input": true, "trusted": true, "_kg_hide-output": false}, "cell_type": "code", "source": "plt.rcParams[\"font.size\"] = \"7\"\nn_weeks = (datetime.date.today() - datetime.date(2020, 10, 6)).days #/ 7 # Num days of the comp\nn_weeks = int(n_weeks)\nfig, axes = plt.subplots(n_weeks, 1, figsize=(15, 25), sharex=True)\n#plt.subplots_adjust(top=8, bottom=2)\nfor x in range(n_weeks):\n    date2 = df.loc[df.index.date == datetime.date(2020, 10, 6) + datetime.timedelta(x+1)].index.min()\n    num_teams = len(df.ffill().loc[date2].dropna())\n    max_cutoff = df.ffill().loc[date2] > 0.5\n    df.ffill().loc[date2].loc[max_cutoff].plot(kind='hist',\n                               bins=50,\n                               ax=axes[x],\n                               title='{} ({} Teams)'.format(date2.date().isoformat(), num_teams), xlim=(0.5, TOP_SCORE + 0.005))\n    y_axis = axes[x].yaxis\n    y_axis.set_label_text('')\n    y_axis.label.set_visible(False)", "execution_count": null, "outputs": []}, {"metadata": {}, "cell_type": "markdown", "source": "# Animation of theTop Leaderboard Scores"}, {"metadata": {"_kg_hide-input": true, "_kg_hide-output": true, "trusted": true}, "cell_type": "code", "source": "%%capture\n%matplotlib inline\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pylab as plt\n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.ticker as ticker\nimport matplotlib.animation as animation\nfrom IPython.display import HTML\n\nimport matplotlib.colors as mcolors\n\nimport seaborn as sns", "execution_count": null, "outputs": []}, {"metadata": {"_kg_hide-input": true, "_kg_hide-output": true, "trusted": true}, "cell_type": "code", "source": "mypal = plt.rcParams['axes.prop_cycle'].by_key()['color'] # Grab the color pal\ncm = plt.get_cmap('tab20')\n\nNUM_COLORS = 20\nmypal = [mcolors.to_hex(cm(1.*i/NUM_COLORS)) for i in range(NUM_COLORS)]\n\nmy_df = df.T\n\nmin_sub_dict = {}\nfor c in df.columns:\n    min_sub_dict[c] =  df[c].dropna().index.min()\n    \n\nmy_df['colors'] = [np.random.choice(mypal) for c in range(len(my_df))]\ncolor_map = my_df['colors'].to_dict()", "execution_count": null, "outputs": []}, {"metadata": {"_kg_hide-input": true, "_kg_hide-output": true, "trusted": true}, "cell_type": "code", "source": "def draw_barchart(mydate):\n    mydate = pd.to_datetime(mydate)\n    dff = df_ffill.loc[df_ffill.index <= mydate].iloc[-1].sort_values(ascending=True).dropna().tail(25)\n\n    last_sub_date = {}\n    df2 = df.loc[df.index <= mydate]\n    for c in df2.columns:\n        last_sub_date[c] = df2[c].dropna().index.max()\n\n    ax.clear()\n    ax.barh(dff.index, dff.values, color=[color_map[x] for x in dff.index])\n    ax.set_xlim(dff.min()-0.01, dff.max()+0.0005)\n    dx = dff.values.max() / 10000\n    for i, (value, name) in enumerate(zip(dff.values, dff.index)):\n        ax.text(dff.min()-0.0099,\n                i,\n                abs(i-25),\n                size=14, weight=600, ha='left', va='center')\n        ax.text(value-dx,\n                i,\n                name,\n                size=14, weight=600, ha='right', va='bottom')\n        ax.text(value-dx,\n                i-.25,\n                f'first sub: {min_sub_dict[name]:%d-%b-%Y} / last sub {last_sub_date[name]:%d-%b-%Y}',\n                size=10,\n                color='#444444',\n                ha='right',\n                va='baseline')\n        ax.text(value+dx, i,     f'{value:,.3f}',  size=14, ha='left',  va='center')\n        \n    # ... polished styles\n    ax.text(1.0, 1.05, mydate.strftime('%d-%b-%Y'), transform=ax.transAxes, color='#777777', size=32, ha='right', weight=800)\n    ax.text(0, 1.06, 'Score', transform=ax.transAxes, size=12, color='#777777')\n    ax.xaxis.set_major_formatter(ticker.StrMethodFormatter('{x:,.3f}'))\n    ax.xaxis.set_ticks_position('top')\n    ax.tick_params(axis='x', colors='#777777', labelsize=12)\n    ax.set_yticks([])\n    ax.margins(0, 0.01)\n    ax.grid(which='major', axis='x', linestyle='-')\n    ax.set_axisbelow(True)\n    ax.text(0, 1.12, 'Top 25 Public Leaderboard Animation', transform=ax.transAxes, size=24, weight=600, ha='left')\n    plt.box(False)", "execution_count": null, "outputs": []}, {"metadata": {"_kg_hide-input": true, "_kg_hide-output": true, "trusted": true}, "cell_type": "code", "source": "fig, ax = plt.subplots(figsize=(15, 18))\ndraw_barchart('09-Oct-2020')", "execution_count": null, "outputs": []}, {"metadata": {"_kg_hide-input": true, "_kg_hide-output": true, "trusted": true}, "cell_type": "code", "source": "dates = [pd.to_datetime(x) for x in pd.Series(df.index.date).unique() if x > pd.to_datetime('08-Oct-2020')]\ndates = dates + [dates[-1] + pd.Timedelta('1 day')]\nfig, ax = plt.subplots(figsize=(15, 18))\nanimator = animation.FuncAnimation(fig,\n                                   draw_barchart,\n                                   frames=dates,\n                                   interval=750)\nani = HTML(animator.to_jshtml())", "execution_count": null, "outputs": []}, {"metadata": {"_kg_hide-input": true, "trusted": true}, "cell_type": "code", "source": "ani", "execution_count": null, "outputs": []}, {"metadata": {}, "cell_type": "markdown", "source": "This notebook is based on https://www.kaggle.com/robikscube/the-race-for-nfl-big-data-bowl-2020/ and https://www.kaggle.com/gogo827jz/moa-leaderboard-animation"}], "metadata": {"kernelspec": {"language": "python", "display_name": "Python 3", "name": "python3"}, "language_info": {"pygments_lexer": "ipython3", "nbconvert_exporter": "python", "version": "3.6.4", "file_extension": ".py", "codemirror_mode": {"name": "ipython", "version": 3}, "name": "python", "mimetype": "text/x-python"}}, "nbformat": 4, "nbformat_minor": 4}