{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Cassava"},{"metadata":{},"cell_type":"markdown","source":"<H3> This notebook performs a post-competition public vs private LB rank analysis and visualizes the shake-up <H3>\n\nNote: Updated with final LB standings!\n    "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import os, sys, subprocess\nimport numpy as np\nimport pandas as pd\nimport gc\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom IPython.core.display import HTML, Image\nfrom scipy.stats import skew, kurtosis\n\n# import chart_studio.plotly as py\nimport plotly.graph_objs as go\nimport plotly.express as px\nfrom plotly.offline import init_notebook_mode, iplot\ninit_notebook_mode(connected=True)\n\nfrom html.parser import HTMLParser\n\npd.set_option('display.max_rows', 180)\n\nprint('List of datasets:')\nprint(os.listdir(\"../input/\"))\nprint('LB files:')\nprint(os.listdir(\"../input/cassava-lb/\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helpers"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# see: https://www.kaggle.com/pednoi/visualize-the-shakeups-of-10-recent-competitions/data?select=Elo+Merchant+Category+Recommendation+_+Kaggle.html\n\nclass RankParser(HTMLParser):\n\n    def __init__(self):\n        self.entered = False\n        self.sign = ''\n        self.data = []\n        \n        super().__init__()\n    \n    def handle_starttag(self, tag, attrs):\n        if tag == 'td' and ('data-th', 'Change') in attrs:\n            self.entered = True\n            \n        if self.entered and tag == 'span':\n            if len(attrs) > 0 and len(attrs[0]) > 1 and attrs[0][1].startswith('position-change'):\n                direction = attrs[0][1][len('position-change__'):]\n                if direction == 'fallen':\n                    self.sign = '-'                    \n\n    def handle_endtag(self, tag):\n        if self.entered and tag == 'td':\n            self.entered = False\n            self.sign = ''\n\n    def handle_data(self, data):\n        if self.entered:\n            data = '0' if data == '—' else data\n            self.data.append(int(self.sign+data.strip()))\n    \n    def get_data(self):\n        return self.data\n\ndef read_html(file_path):\n    content = open(file_path, encoding='utf-8').read()    \n    parser = RankParser()\n    parser.feed(content)\n    return parser.get_data()\n\n\ndef do_read_csv(name):\n    df = pd.read_csv(name, low_memory=False)\n    print('-'*50)\n    print ('> Loaded:', name, df.shape)\n    print('-'*50)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def plot_hist(title, diff):\n    stats = \"\"\n    stats += \"count = %d\\n\" % len(diff)\n    stats += \"mean = %.2f\\n\" % np.mean(diff) # always zero because the data are zero-sum\n    stats += \"std = %.4f\\n\" % np.std(diff)\n    stats += \"skew = %.4f\\n\" % skew(diff)\n    stats += \"kurtosis = %.4f\\n\" % kurtosis(diff)\n    \n    print(\"Mean shake-up       \" ,np.mean(diff))\n    print(\"\\nMedian shake-up     \" ,np.median(diff))\n    print(\"\\nMax shake-up        \" ,np.max(diff))\n    print(\"\\nMin shake-down ;)   \" ,np.min(diff))\n    print(\"\\nStd shake-up        \" ,np.std(diff))\n    \n    fig = plt.figure(figsize=(16, 6))\n    #     sns.distplot(diff, bins=100)\n    plt.hist(diff, bins = 50, edgecolor = 'black', color = 'green')\n    plt.text(0.05, 0.5, stats, transform=plt.gca().transAxes)\n    plt.xlabel(\"Places Shake-up\")\n    plt.ylabel(\"Frequency\")\n    plt.title(title, fontsize=16)\n    plt.show()\n\n\ndef color_negative_red(val):\n    \"\"\"\n    Takes a scalar and returns a string with\n    the css property `'color: red'` for negative\n    strings, black otherwise.\n    \"\"\"\n    try:\n        color = 'red' if val < 0 else 'black'\n    except:\n        color = 'black'\n    return 'color: %s' % color\n\n\ndef plot_candle(title, diff):\n    closes = np.array(range(len(diff)))+1\n    opens = closes + np.array(diff)\n    highs = np.where(np.array(diff)<0, closes, opens)\n    lows =  np.where(np.array(diff)>=0, closes, opens)\n    \n    hovertext = ['private lb: '+str(c)+'<br>public lb: '+ str(o) +'<br>TeamName: '+str(pvt_lb.iloc[c-1]['TeamName']) +'<br>shake: '+str(abs(o-c))  for o, c in zip(opens, closes)]\n\n    trace = go.Ohlc(x=list(range(1, len(diff)+1)), open=opens, high=highs, low=lows, close=closes,\n                    increasing=dict(line=dict(color='#800000')), # '#FF6699'\n                    decreasing=dict(line=dict(color='#228B22')),          # '#66DD99'\n                    text=hovertext, \n                    hoverinfo='text')\n    \n    layout = go.Layout(\n        title = \"<b>%s</b>\" % title,\n        xaxis = dict(\n            title='Final ranks (Pvt LB)',\n            rangeslider = dict(visible=False)\n        ), \n        yaxis=dict(\n            title='shakeups',\n            autorange='reversed'\n        ),\n        width=800,\n        height=600)\n    \n    fig = go.Figure(data=[trace], layout=layout)    \n    iplot(fig, filename='shakeup_candlestick')\n    \n    \ndef make_scatter_competitions(comps, teams):\n\n    shakes = {}\n    COLOR_DICT = {0: 'deepskyblue', 1: 'gold', 2: 'silver', 3: 'chocolate'}\n    plt.rc('font', size=14)\n    \n    for i, df in teams.groupby('CompetitionId', sort=False):\n        fname = comps.Slug[i]\n        row = comps.loc[i]\n        shakeup = df.eval('abs(PrivateLeaderboardRank-PublicLeaderboardRank)').mean() / df.shape[0]\n        title = (f'{row.Title} — {row.TotalTeams} teams — '\n                 f'{shakeup:.3f} shake-up — {row.DeadlineText}')\n        shakes[i] = shakeup\n        df = df.sort_values('PrivateLeaderboardRank', ascending=False)  # plot gold last\n        ax = df.plot.scatter('PublicLeaderboardRank', 'PrivateLeaderboardRank', c=df.Medal.map(COLOR_DICT), figsize=(15, 15))\n        plt.title(title, fontsize=16)\n        l = np.arange(df.PrivateLeaderboardRank.max())\n        ax.plot(l, l, linestyle='--', linewidth=1, color='Black', alpha=0.5)\n        ax.set_xlabel('Public Leaderboard Rank')\n        ax.set_ylabel('Private Leaderboard Rank')\n        plt.tight_layout()\n        plt.show()\n    return shakes\n\n\ndef make_scatter_single_competition(df, tit=''):\n\n    plt.rc('font', size=14)\n    COLOR_DICT = {0: 'deepskyblue', 1: 'gold', 2: 'silver', 3: 'chocolate'}\n\n    DeadlineText = '18/02/2020'\n\n    shakeup = df['shake']\n    \n    shakeup = (abs(df.Pvt_rank - df.Public_rank)).mean() / df.shape[0]\n    \n    \n    title = f' {tit} — {len(df)} teams — {shakeup:.3f} shake-up — {DeadlineText}'\n\n    df = df.sort_values('Pvt_rank', ascending=False)  # plot gold last\n    ax = df.plot.scatter('Public_rank', 'Pvt_rank', c=df.Medal.map(COLOR_DICT), figsize=(15, 15))\n    plt.title(title, fontsize=16)\n\n    l = np.arange(df.Pvt_rank.max())\n    ax.plot(l, l, linestyle='--', linewidth=1, color='Black', alpha=0.5)\n    ax.set_xlabel('Public Leaderboard Rank')\n    ax.set_ylabel('Private Leaderboard Rank')\n    plt.tight_layout()\n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load LB data"},{"metadata":{"trusted":true},"cell_type":"code","source":"META_DIR = \"../input/meta-kaggle\"\nCSV_DIR = \"../input/cassava-lb\"\nHTML_DIR = CSV_DIR","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# read PUBLIC LB (csv)\n\npublic_lb = do_read_csv(f'{CSV_DIR}/cassava-public-lb.csv')  \n# print('Public LB (Final) shape before cleaning:', len(public_lb))\n\n# create df for future use\npublic_scores = public_lb.groupby(['TeamId'])['Score'].agg('max').sort_values(ascending=False)\npublic_scores = pd.DataFrame(public_scores)\npublic_scores = public_scores.reset_index()\n\npublic_scores = public_scores.merge(public_lb[['TeamId', 'TeamName']], on='TeamId', how='right').drop_duplicates()\n\n# drop baseline submission\npublic_scores = public_scores.drop(public_scores.loc[public_scores.TeamName=='baseline'].index, axis=0)\n\nprint('Public LB (Final) shape:', len(public_scores))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# read PVT LB (html)\n\npvt_lb = pd.read_excel(f'{CSV_DIR}/cassava-pvt-lb.xlsx', header=[0])\npvt_lb = pvt_lb[['#', '△pub', 'Team Name','Score', 'Entries', 'Last']]\npvt_lb.columns = ['Pvt_rank','pos_change', 'TeamName', 'Pvt_score', 'no_submissions', 'Last']                \n                \n# modify shake-up column\npvt_lb.pos_change = pvt_lb.pos_change.replace(to_replace='—', value=0)\npvt_lb.pos_change = pvt_lb.pos_change.astype(int)\n\n\nfile = 'cassava-pvt-lb.htm'\nshake_up = read_html(f'{HTML_DIR}/{file}')\npvt_lb['shake'] = np.array(shake_up)\nprint('No. of Teams in PVT LB (Final):', len(pvt_lb))\n# sanity check\nassert len(public_scores)==len(pvt_lb), 'Not valid shapes!'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Distribution of Shake Up"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plot_hist('Cassava: Leaf Disease Classification - Shake-up', shake_up)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plot_candle('Cassava: Leaf Disease Classification - Shake-up',  pvt_lb['shake'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n### LB Status (Final)\n\n- `Max jump-up (shake-up)`: `+1518` \n    - Team Name: `Kubilay Karakayalı` | `From`: `1775 (# Pub)` `to`: `257 (# Pvt)`\n--------------------------------------------------------------\n"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"pvt_lb[pvt_lb['shake']==1518]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- `Max jump-down (shake-down)`: `-2551` \n    - Team Name: `all-four` | `From`: `896 (# Pub)` `to`: `3448 (# Pvt)`"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"pvt_lb[pvt_lb['shake']==-2551]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- `Unshaked Team(s) - Top 3`:"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"display(pvt_lb.loc[pvt_lb['shake']==0][['Pvt_rank', 'TeamName']].head(3))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Display Final Rankings & Medals"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# modified from: https://www.kaggle.com/robikscube/ashrae-leaderboard-and-shake\n\ndf = pvt_lb[['Pvt_rank','shake','TeamName','Pvt_score','no_submissions']].copy()\n\ndf['medal'] = ''\ndf.loc[df['Pvt_rank'] <= 390, 'medal'] = '🥉'\ndf.loc[df['Pvt_rank'] <= 196, 'medal'] = '🥈'\ndf.loc[df['Pvt_rank'] <= 17, 'medal'] = '🥇'\ndf = df[['Pvt_rank','medal','shake', 'TeamName', 'Pvt_score']]   \n\ndf.head(390).style.applymap(color_negative_red).hide_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"df_new = df.copy()\ndf_new['Medal'] = 0\ndf_new.loc[df_new['Pvt_rank'] <= 390, 'Medal'] = 3\ndf_new.loc[df_new['Pvt_rank'] <= 196, 'Medal']  = 2\ndf_new.loc[df_new['Pvt_rank'] <= 17, 'Medal']  = 1\n\ndf_new['Public_rank'] = df_new['Pvt_rank']+df_new['shake']\n\n# df_new","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"make_scatter_single_competition(df_new, tit='Cassava: Leaf Disease Classification - Shake-up')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Extra: Compare with 20 most recent Competitions"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# select 20 most recent competitions\n\nteams = do_read_csv(f'{META_DIR}/Teams.csv')\ncomps = do_read_csv(f'{META_DIR}/Competitions.csv').set_index('Id')\ncomps['DeadlineText'] = comps.DeadlineDate.str.split().str[0]\ncomps['DeadlineDate'] = pd.to_datetime(comps.DeadlineDate)\n\nselected_comps = comps[(comps.HostSegmentTitle=='Featured') | (comps.HostSegmentTitle=='Research')].copy()\nselected_comps = selected_comps.sort_values('DeadlineDate')[-20:]\n\n# select teams for those competitions\nteams = teams.loc[teams.CompetitionId.isin(selected_comps.index)]\nteams = teams.assign(Medal=teams.Medal.fillna(0).astype(int))\nprint(teams.shape)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"shakes = make_scatter_competitions(selected_comps, teams)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def fmt_link(row):\n    return f'<a target=_blank href=\"https://www.kaggle.com/c/{row.Slug}\">{row.Title}</a>'\n\n\nshow_cols = ['Title', 'HostSegmentTitle', 'TotalTeams','DeadlineText', 'Shakeup']\nbars = ['TotalTeams', 'Shakeup']\n\nselected_comps['Shakeup'] = np.array([shakes[key] for key in shakes.keys()])\n\ntmp = selected_comps.assign(Title=selected_comps.apply(fmt_link, 1))\ntmp = tmp[show_cols]\n\n# add Open Vaccine\ntmp_new = {}\ntmp_new['Title'] = f'Cassava: Leaf Disease Classification'\ntmp_new['HostSegmentTitle'] = 'Research'\ntmp_new['TotalTeams'] = len(public_scores)\ntmp_new['DeadlineText'] = '18/02/2020'\ntmp_new['Shakeup'] = (abs(df_new.Pvt_rank - df_new.Public_rank)).mean() / df_new.shape[0]\ntmp_new = pd.DataFrame(tmp_new, columns=show_cols, index=[0])\n\ntmp = pd.concat([tmp, tmp_new], axis=0)\n# tmp = tmp.assign(Title=selected_comps.apply(fmt_link, 1))\n\n# display\ntmp.set_index('Title').head(30).style.bar(subset=bars)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## If you like to see similar shake-up reports from previous competitions:\n\n- [SIIM-ISIC Melanoma 2020](https://www.kaggle.com/imeintanis/melanoma-post-competition-shake-up-report-final/)\n\n- [Cornell Birdcall Identification](https://www.kaggle.com/imeintanis/cornell-shake-up-report)\n\n- [Google LR 2020](https://www.kaggle.com/imeintanis/landmark-recognition-2020-shake-up-report)\n\n- [Open Vaccine](https://www.kaggle.com/imeintanis/vaccine-degradation-prediction-shake-up-report) \n\n- [ASHRAE Great Energy Predictor III by @robikscube](https://www.kaggle.com/robikscube/ashrae-leaderboard-and-shake)\n"},{"metadata":{},"cell_type":"markdown","source":"# Credits\n\n- https://www.kaggle.com/pednoi/visualize-the-shakeups-of-10-recent-competitions\n\n- https://www.kaggle.com/jtrotman/meta-kaggle-competition-shake-up\n\n- https://www.kaggle.com/jtrotman/meta-kaggle-scatter-plot-competition-shake-up\n"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}