{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Google Landmark Recognition 2020: Shake-up Report "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import os, sys, subprocess\nimport numpy as np\nimport pandas as pd\nimport gc\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom IPython.core.display import HTML, Image\nfrom scipy.stats import skew, kurtosis\n\n# import chart_studio.plotly as py\nimport plotly.graph_objs as go\nimport plotly.express as px\nfrom plotly.offline import init_notebook_mode, iplot\ninit_notebook_mode(connected=True)\n\nfrom html.parser import HTMLParser\n\npd.set_option('display.max_rows', 150)\n\nprint('List of datasets:')\nprint(os.listdir(\"../input/\"))\nprint('LB files:')\nprint(os.listdir(\"../input/lr-2020-publicpvt-lb/\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helpers"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# see: https://www.kaggle.com/pednoi/visualize-the-shakeups-of-10-recent-competitions/data?select=Elo+Merchant+Category+Recommendation+_+Kaggle.html\n\nclass RankParser(HTMLParser):\n\n    def __init__(self):\n        self.entered = False\n        self.sign = ''\n        self.data = []\n        \n        super().__init__()\n    \n    def handle_starttag(self, tag, attrs):\n        if tag == 'td' and ('data-th', 'Change') in attrs:\n            self.entered = True\n            \n        if self.entered and tag == 'span':\n            if len(attrs) > 0 and len(attrs[0]) > 1 and attrs[0][1].startswith('position-change'):\n                direction = attrs[0][1][len('position-change__'):]\n                if direction == 'fallen':\n                    self.sign = '-'                    \n\n    def handle_endtag(self, tag):\n        if self.entered and tag == 'td':\n            self.entered = False\n            self.sign = ''\n\n    def handle_data(self, data):\n        if self.entered:\n            data = '0' if data == '—' else data\n            self.data.append(int(self.sign+data.strip()))\n    \n    def get_data(self):\n        return self.data\n\ndef read_html(file_path):\n    content = open(file_path, encoding='utf-8').read()    \n    parser = RankParser()\n    parser.feed(content)\n    return parser.get_data()\n\n\ndef do_read_csv(name):\n    df = pd.read_csv(name, low_memory=False)\n    print ('> Loaded:', name, df.shape)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def plot_hist(title, diff):\n    stats = \"\"\n    stats += \"count = %d\\n\" % len(diff)\n    stats += \"mean = %.2f\\n\" % np.mean(diff) # always zero because the data are zero-sum\n    stats += \"std = %.4f\\n\" % np.std(diff)\n    stats += \"skew = %.4f\\n\" % skew(diff)\n    stats += \"kurtosis = %.4f\\n\" % kurtosis(diff)\n    \n    print(\"Mean shake-up       \" ,np.mean(diff))\n    print(\"\\nMedian shake-up     \" ,np.median(diff))\n    print(\"\\nMax shake-up        \" ,np.max(diff))\n    print(\"\\nMin shake-down ;)   \" ,np.min(diff))\n    print(\"\\nStd shake-up        \" ,np.std(diff))\n    \n    fig = plt.figure(figsize=(16, 6))\n    #     sns.distplot(diff, bins=100)\n    plt.hist(diff, bins = 50, edgecolor = 'black', color = 'green')\n    plt.text(0.05, 0.5, stats, transform=plt.gca().transAxes)\n    plt.xlabel(\"Places Shake-up\")\n    plt.ylabel(\"Frequency\")\n    plt.title(title)\n    plt.show()\n\n\ndef color_negative_red(val):\n    \"\"\"\n    Takes a scalar and returns a string with\n    the css property `'color: red'` for negative\n    strings, black otherwise.\n    \"\"\"\n    try:\n        color = 'red' if val < 0 else 'black'\n    except:\n        color = 'black'\n    return 'color: %s' % color\n\n\ndef plot_candle(title, diff):\n    closes = np.array(range(len(diff)))+1\n    opens = closes + np.array(diff)\n    highs = np.where(np.array(diff)<0, closes, opens)\n    lows =  np.where(np.array(diff)>=0, closes, opens)\n    \n    hovertext = ['private lb: '+str(c)+'<br>public lb: '+ str(o) +'<br>TeamName: '+str(pvt_lb.iloc[c-1]['TeamName'])  for o, c in zip(opens, closes)]\n\n    trace = go.Ohlc(x=list(range(1, len(diff)+1)), open=opens, high=highs, low=lows, close=closes,\n                    increasing=dict(line=dict(color='#800000')), # '#FF6699'\n                    decreasing=dict(line=dict(color='#228B22')),          # '#66DD99'\n                    text=hovertext, \n                    hoverinfo='text')\n    \n    layout = go.Layout(\n        title = \"<b>%s</b>\" % title,\n        xaxis = dict(\n            title='Final ranks (Pvt LB)',\n            rangeslider = dict(visible=False)\n        ), \n        yaxis=dict(\n            title='shakeups',\n            autorange='reversed'\n        ),\n        width=800,\n        height=600,\n    )\n    \n    fig = go.Figure(data=[trace], layout=layout)    \n    iplot(fig, filename='shakeup_candlestick')\n    \n    \ndef make_scatter_competitions(comps, teams):\n\n    shakes = {}\n    COLOR_DICT = {0: 'deepskyblue', 1: 'gold', 2: 'silver', 3: 'chocolate'}\n    plt.rc('font', size=14)\n    for i, df in teams.groupby('CompetitionId', sort=False):\n        fname = comps.Slug[i]\n        row = comps.loc[i]\n        shakeup = df.eval('abs(PrivateLeaderboardRank-PublicLeaderboardRank)').mean() / df.shape[0]\n        title = (f'{row.Title} — {row.TotalTeams} teams — '\n                 f'{shakeup:.3f} shake-up — {row.DeadlineText}')\n        shakes[i] = shakeup\n        df = df.sort_values('PrivateLeaderboardRank', ascending=False)  # plot gold last\n        ax = df.plot.scatter('PublicLeaderboardRank', 'PrivateLeaderboardRank', c=df.Medal.map(COLOR_DICT), figsize=(15, 15))\n        plt.title(title, fontsize=16)\n        l = np.arange(df.PrivateLeaderboardRank.max())\n        ax.plot(l, l, linestyle='--', linewidth=1, color='Black', alpha=0.5)\n        ax.set_xlabel('Public Leaderboard Rank')\n        ax.set_ylabel('Private Leaderboard Rank')\n        plt.tight_layout()\n        plt.show()\n    return shakes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load LB data"},{"metadata":{"trusted":true},"cell_type":"code","source":"CSV_DIR = \"../input/lr-2020-publicpvt-lb\"\nHTML_DIR = \"../input/lr-2020-publicpvt-lb\"\nMETA_DIR = \"../input/meta-kaggle\"","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# read PUBLIC LB (csv)\n\npublic_lb = do_read_csv(f'{CSV_DIR}/LR-2020-public_LB.csv')  \nprint('Public LB (final @03/10/2020) shape before cleaning:', len(public_lb))\n\n# create df for future use\npublic_scores = public_lb.groupby(['TeamId'])['Score'].agg('max').sort_values(ascending=False)\n\npublic_Scores = pd.DataFrame(public_scores)\npublic_Scores = public_Scores.reset_index()\n\npublic_Scores = public_Scores.merge(public_lb[['TeamId', 'TeamName']], on='TeamId', how='right').drop_duplicates()\n\nprint('Public LB (final @03/10/2020) shape after cleaning:', len(public_Scores))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# read PVT LB (html)\n\npvt_lb = pd.read_excel(f'{CSV_DIR}/LR-2020-pvt_LB.xlsx', header=[0])\npvt_lb = pvt_lb[['#', '△pub', 'Team Name','Score', 'Entries', 'Last']]\npvt_lb.columns = ['Pvt_rank','pos_change', 'TeamName', 'Pvt_score', 'no_submissions', 'Last']                \n                \n\n# modify shake-up column\npvt_lb.pos_change = pvt_lb.pos_change.replace(to_replace='—', value=0)\npvt_lb.pos_change = pvt_lb.pos_change.astype(int)\n\n\n\nfile = 'LR-2020-pvt-LB.htm'\nshake_up = read_html(f'{HTML_DIR}/{file}')\npvt_lb['shake'] = np.array(shake_up)\n\nprint('No. of Teams in PVT LB (final @03/10/2020):', len(pvt_lb))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"assert len(public_Scores)==len(pvt_lb), 'Not valid shapes!'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Distribution of Number of Places Shake Up\n\n### Final LB (@03/10/2020)\n\n- Max jump-up (shake-up): +451\n\n- Max jump-down (shake-down): -474"},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"np.min(shake_up), np.max(shake_up)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_hist('Google Landmark Recognition 2020: Shake-Up', shake_up)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_candle('Google Landmark Recognition 2020: Shake-Up',  pvt_lb['shake'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Display Final Rankings & Medals"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# modified from: https://www.kaggle.com/robikscube/ashrae-leaderboard-and-shake\n\ndf = pvt_lb[['Pvt_rank','shake','TeamName','Pvt_score','no_submissions']].copy()\n\ndf['medal'] = ''\ndf.loc[df['Pvt_rank'] <= 139, 'medal'] = '🥉'\ndf.loc[df['Pvt_rank'] <= 69, 'medal'] = '🥈'\ndf.loc[df['Pvt_rank'] <= 12, 'medal'] = '🥇'\ndf = df[['Pvt_rank','medal','shake', 'TeamName', 'Pvt_score']]   # 'public_rank''Score', 'no_submissions'\n\ndf.head(100).style.applymap(color_negative_red).hide_index()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Extra: Compare with 20 most recent Competitions"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# select 20 most recent competitions\n\nteams = do_read_csv(f'{META_DIR}/Teams.csv')\ncomps = do_read_csv(f'{META_DIR}/Competitions.csv').set_index('Id')\ncomps['DeadlineText'] = comps.DeadlineDate.str.split().str[0]\ncomps['DeadlineDate'] = pd.to_datetime(comps.DeadlineDate)\n\nselected_comps = comps[(comps.HostSegmentTitle=='Featured') | (comps.HostSegmentTitle=='Research')].copy()\nselected_comps = selected_comps.sort_values('DeadlineDate')[-20:]\n\n# select teams for those competitions\nteams = teams.loc[teams.CompetitionId.isin(selected_comps.index)]\nteams = teams.assign(Medal=teams.Medal.fillna(0).astype(int))\nprint(teams.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shakes = make_scatter_competitions(selected_comps, teams)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def fmt_link(row):\n    return f'<a target=_blank href=\"https://www.kaggle.com/c/{row.Slug}\">{row.Title}</a>'\n\n\nshow_cols = ['Title', 'HostSegmentTitle', 'TotalTeams','DeadlineText', 'Shakeup']\nbars = ['TotalTeams', 'Shakeup']\n\nselected_comps['Shakeup'] = np.array([shakes[key] for key in shakes.keys()])\n\ntmp = selected_comps.assign(Title=selected_comps.apply(fmt_link, 1))\ntmp[show_cols].set_index('Title').head(20).style.bar(subset=bars)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Credits"},{"metadata":{},"cell_type":"markdown","source":"- [visualize-the-shakeups-of-10-recent-competitions by pednoi](https://www.kaggle.com/pednoi/visualize-the-shakeups-of-10-recent-competitions)\n\n- https://www.kaggle.com/jtrotman/meta-kaggle-competition-shake-up\n\n- https://www.kaggle.com/jtrotman/meta-kaggle-scatter-plot-competition-shake-up"},{"metadata":{},"cell_type":"markdown","source":"## If you like to see similar shake-up reports from previous competitions:\n\n\n- [SIIM-ISIC Melanoma 2020](https://www.kaggle.com/imeintanis/melanoma-post-competition-shake-up-report-final/)\n\n- [Cornell Birdcall Identification](https://www.kaggle.com/imeintanis/cornell-shake-up-report)\n\n- [ASHRAE Great Energy Predictor III by @robikscube](https://www.kaggle.com/robikscube/ashrae-leaderboard-and-shake)\n\n\n## Feel free to add your comments/suggestions bellow"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}