{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Dumbbell Plot with Arrow Markers\n# The arrow, angleref, and standoff properties used on the marker in this notebook needs plotly 5.11\n!pip install --upgrade plotly ","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-11-24T22:01:26.820762Z","iopub.execute_input":"2022-11-24T22:01:26.821237Z","iopub.status.idle":"2022-11-24T22:02:30.816661Z","shell.execute_reply.started":"2022-11-24T22:01:26.821141Z","shell.execute_reply":"2022-11-24T22:02:30.815012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd \n\nimport plotly.graph_objects as go\n\nprivate_LB = pd.read_csv('../input/open-problems-multimodal-2022-leaderboard/open-problems-multimodal-privateleaderboard-def.csv')\npublic_LB = pd.read_csv('../input/open-problems-multimodal-2022-leaderboard/open-problems-multimodal-publicleaderboard.csv')\n\nprivate_LB['rank'] = private_LB.index +1\npublic_LB['rank'] = public_LB.index +1\n\nLB = pd.merge(private_LB, public_LB[['TeamId','rank','Score']], on='TeamId', how='inner', suffixes=(\"_private\", \"_public\"))\nLB['diffPrivPub'] = LB.rank_private - LB.rank_public\nLB['absDiff'] = abs(LB.diffPrivPub)\n\nme = LB.query(\"TeamName == 'Jose Carmona'\").iloc[0]","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-24T22:49:56.679050Z","iopub.execute_input":"2022-11-24T22:49:56.679513Z","iopub.status.idle":"2022-11-24T22:49:56.716080Z","shell.execute_reply.started":"2022-11-24T22:49:56.679480Z","shell.execute_reply":"2022-11-24T22:49:56.714335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Open Problems Multimodal 2022 Leaderboard graphical recap","metadata":{}},{"cell_type":"code","source":"fig = go.Figure()\n\nfig.add_trace(\n    go.Scatter(\n        x = LB.rank_private, \n        y = LB.Score_private,\n        text = LB.TeamName,\n        name = 'private LB'\n    )\n)\n\nfig.add_trace(\n    go.Scatter(\n        x = LB.rank_private, \n        y = LB.Score_public,\n        text = LB.TeamName,\n        name = 'public LB'\n    )\n)\n\nannotations = [\n    {\n        'x' : me.rank_private, \n        'y' : me.Score_private,\n        'xanchor' : 'center',\n        'yanchor' : 'bottom',\n        'ax' : 20,\n        'ay' : 50,\n        'bgcolor': \"#ff7f0e\",\n        'text' : 'This is me',\n        'showarrow' : True \n    }\n]\n\nfig.update_layout(\n    title = 'Private and Public LB',\n    xaxis_title = 'Rank',\n    yaxis_title = 'Score',\n    annotations = annotations\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-24T22:50:01.563778Z","iopub.execute_input":"2022-11-24T22:50:01.564162Z","iopub.status.idle":"2022-11-24T22:50:01.597101Z","shell.execute_reply.started":"2022-11-24T22:50:01.564130Z","shell.execute_reply":"2022-11-24T22:50:01.595812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig.update_xaxes(range=[0, 100])\nfig.update_yaxes(range=[.6, .9])\nfig.update_layout(\n    title = 'Private and Public LB (zoom on top 100)'\n)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-24T22:50:08.454184Z","iopub.execute_input":"2022-11-24T22:50:08.454627Z","iopub.status.idle":"2022-11-24T22:50:08.473874Z","shell.execute_reply.started":"2022-11-24T22:50:08.454590Z","shell.execute_reply":"2022-11-24T22:50:08.472646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top100 = LB[:100]\nfig = go.Figure()\n\nfig.add_trace(\n    go.Scatter(\n        x = top100.rank_private, \n        y = (top100.Score_private - top100.Score_public),\n        text = top100.TeamName,\n        name = 'delta'\n    )\n)\n\nannotations = [\n    {\n        'x' : me.rank_private, \n        'y' : (me.Score_private - me.Score_public),\n        'xanchor' : 'center',\n        'yanchor' : 'bottom',\n        'ax' : 20,\n        'ay' : -30,\n        'bgcolor': \"#ff7f0e\",\n        'text' : 'This is me',\n        'showarrow' : True \n    }\n]\n\nfig.update_layout(\n    title = 'Overfitting (Private - Public score gap)',\n    xaxis_title = 'Rank',\n    yaxis_title = 'Private - Public score',\n    height = 600,\n    annotations = annotations\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-24T22:50:16.714616Z","iopub.execute_input":"2022-11-24T22:50:16.715063Z","iopub.status.idle":"2022-11-24T22:50:16.734211Z","shell.execute_reply.started":"2022-11-24T22:50:16.715029Z","shell.execute_reply":"2022-11-24T22:50:16.733059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Shake up\n\nshake up = mean(abs(private rank - public rank)) / len","metadata":{}},{"cell_type":"code","source":"shakeup = LB.absDiff.mean() / len(LB)\nprint(f'shake up = {shakeup}')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-24T22:50:22.037628Z","iopub.execute_input":"2022-11-24T22:50:22.038057Z","iopub.status.idle":"2022-11-24T22:50:22.047219Z","shell.execute_reply.started":"2022-11-24T22:50:22.038022Z","shell.execute_reply":"2022-11-24T22:50:22.045203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = go.Figure()\n\nfig.add_trace(\n    go.Scatter(\n        x = LB.rank_public,\n        y = LB.rank_private, \n        text = LB.TeamName,\n        mode=\"markers\",\n        name = 'private LB'\n    )\n)\n\nannotations = [\n    {\n        'x' : me.rank_public,\n        'y' : me.rank_private, \n        'xanchor' : 'center',\n        'yanchor' : 'bottom',\n        'text' : 'This is me',\n        'ax' : -30,\n        'ay' : -100,\n        'bgcolor': \"#ff7f0e\",\n        'showarrow' : True \n    }\n]\n\nfig.add_shape(\n    type = 'line',\n    x0 = 0,\n    y0 = 0,\n    x1 = len(LB),\n    y1 = len(LB),\n    line = dict(color = 'Gray',),\n    xref = 'x',\n    yref = 'y'\n)\n\nfig.update_layout(\n    title = f'Public vs Private rank. Shake up = {shakeup:.6f} ',\n    xaxis_title = 'Public rank',\n    yaxis_title = 'Private rank',\n    height = 1000,\n    annotations = annotations\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-24T22:50:56.657667Z","iopub.execute_input":"2022-11-24T22:50:56.658083Z","iopub.status.idle":"2022-11-24T22:50:56.684274Z","shell.execute_reply.started":"2022-11-24T22:50:56.658050Z","shell.execute_reply":"2022-11-24T22:50:56.683131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = go.Figure()\n\nfig.add_trace(\n    go.Scatter(\n        x = LB.Score_public,\n        y = LB.Score_private, \n        text = LB.TeamName,\n        mode=\"markers\",\n        name = 'private LB'\n    )\n)\n\nannotations = [\n    {\n        'x' : me.Score_public,\n        'y' : me.Score_private, \n        'xanchor' : 'center',\n        'yanchor' : 'bottom',\n        'text' : 'This is me',\n        'ax' : -30,\n        'ay' : -30,\n        'bgcolor': \"#ff7f0e\",\n        'showarrow' : True \n    }\n]\n\nfig.add_shape(\n    type = 'line',\n    x0 = -1,\n    y0 = -1,\n    x1 = 1,\n    y1 = 1,\n    line = dict(color = 'Gray',),\n    xref = 'x',\n    yref = 'y'\n)\n\nfig.update_layout(\n    title = f'Public vs Private Score',\n    xaxis_title = 'Public Score',\n    yaxis_title = 'Private Score',\n    height = 1000,\n    annotations = annotations\n)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-24T22:51:12.964065Z","iopub.execute_input":"2022-11-24T22:51:12.964478Z","iopub.status.idle":"2022-11-24T22:51:12.992846Z","shell.execute_reply.started":"2022-11-24T22:51:12.964443Z","shell.execute_reply":"2022-11-24T22:51:12.991124Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = {\"x\": [], \"y\": [], \"colors\": [], \"LB\": []}\n\nfor i, team in LB[:50].iterrows():\n    data[\"x\"].extend(\n        [\n            team.rank_public,\n            team.rank_private,\n            None,\n        ]\n    )\n    data[\"y\"].extend([team.TeamName, team.TeamName, None]),\n    data[\"colors\"].extend([\"silver\", \"lightskyblue\", \"white\"]),\n    data[\"LB\"].extend([\"Public\", \"Private\", None])\n\nfig = go.Figure(\n    data=[\n        go.Scatter(\n            x=data[\"x\"],\n            y=data[\"y\"],\n            mode=\"markers+lines\",\n            marker=dict(\n                symbol=\"arrow\", color=\"black\", size=16, angleref=\"previous\", standoff=8\n            ),\n        ),\n        go.Scatter(\n            x=data[\"x\"],\n            y=data[\"y\"],\n            text=data[\"LB\"],\n            mode=\"markers\",\n            marker=dict(\n                color=data[\"colors\"],\n                size=10,\n            ),\n            hovertemplate=\"\"\"Team: %{y} <br> %{text} Rank: %{x} <br><extra></extra>\"\"\",\n        ),\n    ]\n)\n\nfig.update_layout(\n    title=\"Top 50, Private vs Public shake up\",\n    height=1000,\n    showlegend=False,\n    yaxis_autorange=\"reversed\"\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-24T22:51:20.512975Z","iopub.execute_input":"2022-11-24T22:51:20.513445Z","iopub.status.idle":"2022-11-24T22:51:20.561644Z","shell.execute_reply.started":"2022-11-24T22:51:20.513412Z","shell.execute_reply":"2022-11-24T22:51:20.560111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = {\"x\": [], \"y\": [], \"colors\": [], \"LB\": []}\n\nfor i, team in LB.sort_values('diffPrivPub').head(10).iterrows():\n    data[\"x\"].extend(\n        [\n            team.rank_public,\n            team.rank_private,\n            None,\n        ]\n    )\n    data[\"y\"].extend([team.TeamName, team.TeamName, None]),\n    data[\"colors\"].extend([\"silver\", \"lightskyblue\", \"white\"]),\n    data[\"LB\"].extend([\"Public\", \"Private\", None])\n\nfig = go.Figure(\n    data=[\n        go.Scatter(\n            x=data[\"x\"],\n            y=data[\"y\"],\n            mode=\"markers+lines\",\n            marker=dict(\n                symbol=\"arrow\", color=\"black\", size=16, angleref=\"previous\", standoff=8\n            ),\n        ),\n        go.Scatter(\n            x=data[\"x\"],\n            y=data[\"y\"],\n            text=data[\"LB\"],\n            mode=\"markers\",\n            marker=dict(\n                color=data[\"colors\"],\n                size=10,\n            ),\n            hovertemplate=\"\"\"Team: %{y} <br> %{text} Rank: %{x} <br><extra></extra>\"\"\",\n        ),\n    ]\n)\n\nfig.update_layout(\n    title=\"Top 10, Best risers\",\n    showlegend=False,\n    yaxis_autorange=\"reversed\"\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-24T22:51:27.031963Z","iopub.execute_input":"2022-11-24T22:51:27.032359Z","iopub.status.idle":"2022-11-24T22:51:27.061655Z","shell.execute_reply.started":"2022-11-24T22:51:27.032327Z","shell.execute_reply":"2022-11-24T22:51:27.060388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"provisional_LB = pd.read_csv('../input/open-problems-multimodal-2022-leaderboard/open-problems-multimodal-privateleaderboard.csv')\nprivate_LB = pd.read_csv('../input/open-problems-multimodal-2022-leaderboard/open-problems-multimodal-privateleaderboard-def.csv')\npublic_LB = pd.read_csv('../input/open-problems-multimodal-2022-leaderboard/open-problems-multimodal-publicleaderboard.csv')\n\nprovisional_LB['rank'] = provisional_LB.index +1\npublic_LB['rank'] = public_LB.index +1\n\nprivate_LB['color'] = 'lightskyblue'\n\nLB = pd.merge(provisional_LB, public_LB[['TeamId','rank','Score']], on='TeamId', how='inner', suffixes=(\"_private\", \"_public\"))\nLB = pd.merge(LB, private_LB[['TeamId','color']], on='TeamId', how='left')\nLB['color'] = LB.color.fillna('red')\n\nme = LB.query(\"TeamName == 'Jose Carmona'\").iloc[0]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-24T22:51:32.407205Z","iopub.execute_input":"2022-11-24T22:51:32.407595Z","iopub.status.idle":"2022-11-24T22:51:32.448126Z","shell.execute_reply.started":"2022-11-24T22:51:32.407565Z","shell.execute_reply":"2022-11-24T22:51:32.446701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = go.Figure()\n\nfig.add_trace(\n    go.Scatter(\n        x = LB.rank_public,\n        y = LB.rank_private, \n        text = LB.TeamName,\n        mode = \"markers\",\n        name = 'private LB',\n        marker = dict(color = LB.color)\n    )\n)\n\nannotations = [\n    {\n        'x' : me.rank_public,\n        'y' : me.rank_private, \n        'xanchor' : 'center',\n        'yanchor' : 'bottom',\n        'text' : 'This is me',\n        'ax' : -30,\n        'ay' : -100,\n        'bgcolor': \"#ff7f0e\",\n        'showarrow' : True \n    }\n]\n\nfig.add_shape(\n    type = 'line',\n    x0 = 0,\n    y0 = 0,\n    x1 = len(LB),\n    y1 = len(LB),\n    line = dict(color = 'Gray',),\n    xref = 'x',\n    yref = 'y'\n)\n\nfig.update_layout(\n    title = f'Provisional Public vs Private rank (teams removed in red)',\n    xaxis_title = 'Public rank',\n    yaxis_title = 'Private rank',\n    height = 1000,\n    annotations = annotations\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-24T22:51:42.593201Z","iopub.execute_input":"2022-11-24T22:51:42.593629Z","iopub.status.idle":"2022-11-24T22:51:42.731986Z","shell.execute_reply.started":"2022-11-24T22:51:42.593598Z","shell.execute_reply":"2022-11-24T22:51:42.730744Z"},"trusted":true},"execution_count":null,"outputs":[]}]}