{"cells": [{"cell_type": "markdown", "metadata": {}, "source": "The competition is over - congratulations to the top ten prize-winners, especially the top three for their stellar scores!\nNow that all results are in, what hidden stories or patterns can we uncover from the rescored leaderboards?\n\n(You might find this [UI Hack: Widen Notebook Content View](https://www.kaggle.com/discussions/general/534303) bookmarklet helpful to fit some of the tables on screen.)"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.express as px\nimport plotly.io as pio\npio.renderers.default = 'iframe'"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "title = \"Jane Street Real-Time Market Data Forecasting\"\nsubtitle = \"Predict financial market responders using real-world data.\"\nslug = \"jane-street-real-time-market-data-forecasting\"\nmedal_colors = ['Gold', 'Silver', 'Chocolate']\nmedal_names = ['GOLD', 'SILVER', 'BRONZE']"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "MAX_TEAM_SIZE = 5\n\ndef read_lb(jfile):\n    json_dict = pd.read_json(jfile)\n    df = json_dict.publicLeaderboard.apply(pd.Series).set_index(\"teamId\").drop(\"inTheMoney\", axis=1)\n    teams_df = json_dict.teams.apply(pd.Series).set_index(\"teamId\")\n    df.displayScore = df.displayScore.astype(float)\n    df = df.join(teams_df[['teamName', 'submissionCount']])\n    return df\n\ndef read_team_members_df(jfile):\n    json_dict = pd.read_json(jfile)\n    df = json_dict.teams.apply(pd.Series).set_index(\"teamId\")\n    dfs = [df.teamMembers.str[i].apply(pd.Series).add_prefix(f'user{i}_') for i in range(MAX_TEAM_SIZE)]\n    members_df = pd.concat(dfs, axis=1)\n    # squeeze this in too:\n    members_df['lastSubmissionDate'] = df['lastSubmissionDate']\n    return members_df"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# p for public lbs\n# r for rescore (private) lbs\nbase = '/kaggle/input/jane-street-real-time-market-data-forecasting-lbs/'\ndfs = [\n    ( 'p1', read_lb(base + 'jane-street-real-time-market-data-forecasting-250202.json') ),\n    ( 'p2', read_lb(base + 'jane-street-real-time-market-data-forecasting-250205.json') ),\n    ( 'r1', read_lb(base + 'jane-street-real-time-market-data-forecasting-250211.json') ),\n    ( 'r2', read_lb(base + 'jane-street-real-time-market-data-forecasting-250310.json') ),\n    ( 'r3', read_lb(base + 'jane-street-real-time-market-data-forecasting-250408.json') ),\n    ( 'r4', read_lb(base + 'jane-street-real-time-market-data-forecasting-250512.json') ),\n    ( 'r5', read_lb(base + 'jane-street-real-time-market-data-forecasting-250616.json') ),\n    ( 'r6', read_lb(base + 'jane-street-real-time-market-data-forecasting-250714.json') ),\n]"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "team_members_df = read_team_members_df(base + 'jane-street-real-time-market-data-forecasting-250202.json')\nteam_members_df.shape"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "dfs = [ (tag, df.add_suffix(f\"_{tag}\")) for tag, df in dfs ]\nscore_cols = [ (f\"displayScore_{tag}\") for tag, df in dfs ]\nrank_cols = [ (f\"rank_{tag}\") for tag, df in dfs ]\nmedal_cols = [ (f\"medal_{tag}\") for tag, df in dfs ]\nsubCount_cols = [ (f\"submissionCount_{tag}\") for tag, df in dfs ]\nsubId_cols = [ (f\"submissionId_{tag}\") for tag, df in dfs ]\nuser_id_cols = [f'user{i}_id' for i in range(MAX_TEAM_SIZE)]"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "[df.shape for tag, df in dfs]"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "uni = pd.concat([df for tag, df in dfs] + [team_members_df], axis=1)\nuni['teamSize'] = uni[user_id_cols].count(axis=1)\nuni['label'] = uni.teamName_r6 + \" (\" + uni.rank_r6.map(lambda v: f'{v:,.0f}') + \")\"\nuni['lastSubmissionDate'] = pd.to_datetime(uni['lastSubmissionDate'], format='mixed')\nuni.shape"}, {"cell_type": "markdown", "metadata": {}, "source": "# Leaderboard Ranks\n\nThe pathway of the final top 50..."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "tmp = uni.sort_values('rank_r6').set_index('label')\n(tmp[rank_cols].head(50).style\n .format(precision=0)\n .background_gradient(subset=rank_cols[:2], axis=0, cmap='Greens_r')\n .background_gradient(subset=rank_cols[2:], axis=0, cmap='Oranges_r'))"}, {"cell_type": "markdown", "metadata": {}, "source": "# Successful Rescore Counts\n\nNumber of teams lost along the way..."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "score_counts = uni[score_cols].count()\nscore_counts.to_frame(\"Scored Teams\").assign(Diff=score_counts.diff())"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "uni[score_cols].count().plot.bar(title=\"Team Counts\")\nplt.xticks(rotation=45);"}, {"cell_type": "markdown", "metadata": {}, "source": "#\u00a0Popular Submission Combinations\n\nEach team can select two submissions, each row here could represent one model alone, or scores from both submissions that switch places over time.\n\nIf the sequence ends with NaN they will not appear on the final published leaderboard."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "(uni.groupby(list(score_cols[2:]), dropna=False).submissionId_p1.size()\n .sort_values(ascending=False)\n .to_frame(\"Num Teams\").head(55))"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "team_stats_df = uni[score_cols].max(axis=1).to_frame(\"best\")\nteam_stats_df['subCount'] = uni.submissionCount_p1\nteam_stats_df['teamSize'] = uni.teamSize\nteam_stats_df['finalRank'] = uni.rank_r6\nteam_stats_df['lastSubmissionDate'] = uni.lastSubmissionDate\nteam_stats_df['scoreCount'] = uni.groupby(list(score_cols[2:]), dropna=False).submissionId_p1.transform('size')\nteam_stats_df['teamName'] = uni['teamName_p1']\nteam_stats_df['medal'] = uni['medal_r6']\nteam_stats_df['finalSubCount'] = uni['submissionCount_r6']\nteam_stats_df['finalScore'] = uni['displayScore_r6']\nuni['scoreCount'] = uni.groupby(list(score_cols[2:]), dropna=False).submissionId_p1.transform('size')\nteam_stats_df.count()"}, {"cell_type": "markdown", "metadata": {}, "source": "# Submission Count vs Best Score relationship\n\nRed are teams where their set of rescore scores were unique."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "tmp = team_stats_df.query('best>=0')\ntmp.plot.scatter('subCount', 'best', logx=True, s=3, figsize=(8,8),\n                 title=\"Submission Count vs Best Score relationship\",\n                 c=np.where(tmp.scoreCount == 1, 'r', 'b'));"}, {"cell_type": "markdown", "metadata": {}, "source": "# Last Submission Date"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "team_stats_df.lastSubmissionDate.dt.date.value_counts().sort_index().tail()"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "team_stats_df.lastSubmissionDate.dt.date.value_counts().sort_index().plot(figsize=(8,4))\nplt.title(f'Last Submission Date for Teams')\nplt.xticks(rotation=45);"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "team_stats_df.lastSubmissionDate.dt.date.value_counts().sort_index().tail(9).plot.bar(figsize=(8,4))\nplt.title(f'{title} - lastSubmissionDate')\nplt.xticks(rotation=45);"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "tmp = team_stats_df.query('best>=0')\ntmp.plot.scatter('lastSubmissionDate', 'best', s=3, figsize=(9,6),\n                 title=f\"{title}\\nLast Submission Date vs Best Score Relationship\",\n                 c=np.where(tmp.scoreCount == 1, 'r', 'b'));\nplt.xticks(rotation=45);"}, {"cell_type": "markdown", "metadata": {}, "source": "- Popular submissions are visible as streaks, possibly backup submissions where the main submission failed (e.g. if finalSubCount is 1)\n- Strongest public solution appears 3 Jan 2025 at 15:00\n- The second half of the final day is quite a bit more busy than the (UTC) morning.\n- Victor Shlepov is an impressive outlier, managing to score a gold medal whilst last submitting in 2024.\n\nMaybe there could be some kind of new badge or meta-challenge for finishing high up yet submitting well before the deadline... (Better yet with just one submission?!)"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "tmp = team_stats_df.query('best>=0 and finalRank>0').fillna({'medal': 'N/A'})\nfig = px.scatter(tmp, 'lastSubmissionDate', 'best',\n           hover_name='teamName',\n           hover_data={\n               'finalRank': True,\n               'subCount': True,\n               'finalSubCount': True,\n               'teamSize': True,\n           },\n           symbol='medal',\n           #symbol_map={\n           #   'GOLD': 'circle',\n           #   'SILVER': 'triangle-up',\n           #   'BRONZE': 'square',\n           #   'N/A': 'x',\n           #},\n           title='Last Submission Date vs Final Score Relationship',\n           color='scoreCount')\nfig.update_traces(showlegend=False, selector=dict(mode=\"markers\"))"}, {"cell_type": "markdown", "metadata": {}, "source": "# Team Sizes"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "team_stats_df.plot.scatter('finalRank', 'teamSize', s=3, figsize=(10,2),\n                           title=\"Team Sizes\",\n                           c=np.where(team_stats_df.scoreCount == 1, 'r', 'b'));"}, {"cell_type": "markdown", "metadata": {}, "source": "# Parallel Coordinates Plots"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "tmp = uni.query(medal_cols[-1]+\"=='GOLD'\").sort_values('rank_r6')\nplt.figure(figsize=(12,7))\npd.plotting.parallel_coordinates(tmp, 'label', score_cols, colormap='tab20')\nlegend_opts = dict(bbox_to_anchor=(1.02, 0, 0.3, 1),\n                   loc=\"upper right\",\n                   ncol=1,\n                   shadow=True,\n                   edgecolor=\"black\",\n                   mode=\"expand\",\n                   borderaxespad=0.)\nplt.legend(**legend_opts)\nplt.title(f'{title} - Gold Scores Over Time')\nplt.xticks(rotation=45);"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "tmp = uni[~uni.medal_r6.isna()]\nplt.figure(figsize=(12,7))\npd.plotting.parallel_coordinates(tmp, 'medal_r6', score_cols, colormap='tab20')\nplt.legend().remove()\nplt.title(f'{title} - Medal Scores Over Time')\nplt.xticks(rotation=45);"}, {"cell_type": "markdown", "metadata": {}, "source": "# Individual Monthly Scores\n\n\n\n**The metric from the [overview page](https://www.kaggle.com/competitions/jane-street-real-time-market-data-forecasting/):**\n\nSubmissions are evaluated on a scoring function defined as the sample weighted zero-mean R-squared score (${\\rm R}^{2}$) of `responder_6`. The formula is given by:\n\n$${\\rm R^2} = 1 - \\frac{\\sum w_{i}(y_{i} - \\hat{y_{i}})^{2}}{\\sum w_{i}y_{i}^{2}}$$\n\nwhere $y$ and $\\hat{y}$ are the ground-truth and predicted value vectors of responder_6, respectively; $w$ is the sample weight vector.\n\n__________\n\nThe scores we see are accumulating both numerator and denominators over time, but the denominators are the same for all teams.\nLet's look at some training data denominator sums (${\\sum w_{i}y_{i}^{2}}$) ..."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "train = pd.read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet',\n                        columns=['responder_6','weight','date_id'])\ntrain.shape"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "day_denom_sums = (train.weight * train.responder_6**2).groupby(train.date_id).sum()"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "day_denom_sums.tail(120).plot(title='Training Data Day $w*y^2$ Sums');"}, {"cell_type": "markdown", "metadata": {}, "source": "But the rescores were on six consecutive ~20 day windows, e.g. the last six windows of training data:"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "day_denom_sums_20 = day_denom_sums.groupby(day_denom_sums.index//20).sum()\nday_denom_sums_20.index *= 20\nday_denom_sums_20.tail(6).plot.bar(title='Training Data 20-Day $w*y^2$ Sums');"}, {"cell_type": "markdown", "metadata": {}, "source": "Or for more context, the last 24:"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "day_denom_sums_20.tail(24).plot.bar(title='Training Data 20-Day $w*y^2$ Sums');"}, {"cell_type": "markdown", "metadata": {}, "source": "For simplicity, for now, assume the denominator is the same in each rescore.\n\nIt might be possible to work out something closer to the truth, maybe some of the scores at the bottom of the LB could help?\n\n__________\n\n**N.B.**: Display scores are the maximum of a teams two selected submissions so if a teams models trade places we are not computing a real monthly model performance, more a sort of model-team performance ..."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# Compute per-month R2 scores\ndef monthly_scores(R2_matrix, d):\n    D = np.cumsum(d)\n    cum_num = (1 - R2_matrix) * D[None, :]\n    num_diff = np.diff(np.concatenate([np.zeros((R2_matrix.shape[0], 1)), cum_num], axis=1), axis=1)\n    per_month_R2 = 1 - num_diff / d\n    return per_month_R2"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "tmp = uni.dropna(subset=score_cols).sort_values('rank_r6')\nmonthly_scores_df = pd.DataFrame(monthly_scores(tmp[score_cols[2:]].clip(lower=0).values, np.ones(6)))\nmonthly_scores_df.index = tmp.label.values"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "monthly_scores_df.head()"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "monthly_scores_df.stack().describe()"}, {"cell_type": "markdown", "metadata": {"execution": {"iopub.execute_input": "2025-07-18T23:20:01.658446Z", "iopub.status.busy": "2025-07-18T23:20:01.657608Z", "iopub.status.idle": "2025-07-18T23:20:01.670567Z", "shell.execute_reply": "2025-07-18T23:20:01.66692Z", "shell.execute_reply.started": "2025-07-18T23:20:01.6584Z"}}, "source": "**Second rescore month (March) is the outlier**"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "monthly_scores_df.head(20).corr().style.background_gradient()"}, {"cell_type": "markdown", "metadata": {}, "source": "Assuming uniform denominator each month makes the final score the mean of the monthly scores.\n\n**Maybe the standard deviations are a clue as to how many models each team is averaging? (More models &rarr;  more consistent &rarr; lower deviation).**"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "tmp = monthly_scores_df.head(50)\ngrad_cols = list(tmp.columns)\ntmp = tmp.assign(mean=tmp.mean(1), std=tmp.std(1))\n(tmp.style.background_gradient(axis=1, subset=grad_cols)\n .background_gradient(cmap='Wistia', subset=['mean'])\n .bar(subset=['std'], color='skyblue', height=80)\n .set_caption(\"Highlighting Best Month per Team\")\n)"}, {"cell_type": "markdown", "metadata": {}, "source": "# Correlation Between Teams\n\nA lot of correlation is expected, although for example in Numerai it's a positively rewarded thing to have a submission be uncorrelated to other submissions :)"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "vertical_headers_table_styles = [\n    dict(selector=\"th.col_heading\",\n         props=[(\"writing-mode\", \"vertical-rl\"), \n                (\"text-orientation\", \"mixed\"),   # Ensures upright text\n                (\"vertical-align\", \"bottom\"),   # So text starts from bottom\n         ])\n]\nmonthly_scores_df.head(20).T.corr().style.format(precision=2).set_table_styles(\n    vertical_headers_table_styles).background_gradient(axis=None, vmin=-1, vmax=1)"}, {"cell_type": "markdown", "metadata": {}, "source": "## Top 50 Clustermap"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "%%capture --no-display\nsns.clustermap(monthly_scores_df.head(50).T.corr(), figsize=(15,15), cmap='RdYlGn')\nfig = plt.title(\"Correlation Between Top 50 Teams\")"}, {"cell_type": "markdown", "metadata": {}, "source": "# Ranks of Monthly Scores\n\nThe top 3 were always top 3!\n\n(Note: this is not the same as the leaderboard ranks for each month...)"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "monthly_scores_df.rank(axis=0, ascending=False).head(50).astype(int)"}, {"cell_type": "markdown", "metadata": {}, "source": "### Another way: who was in a monthly top 10?"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "tmp = monthly_scores_df.rank(axis=0, ascending=False)\ntmp[tmp.min(1)<=10].astype(int)"}, {"cell_type": "markdown", "metadata": {}, "source": "### Scores That Were Top 10 in a Month"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "(monthly_scores_df[tmp.min(1)<=10].style.background_gradient(axis=1)\n .set_caption(\"Highlighting Best Month per Team\"))"}, {"cell_type": "markdown", "metadata": {}, "source": "hmmm, the last update is ~1/6th of the overall score and yet it **did** drag the score up considerably!"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "uni[uni.teamName_p1.fillna(\"\").str.contains(\"AngadY\")][score_cols[2:]]"}, {"cell_type": "markdown", "metadata": {}, "source": "similarly for displayScore_r4 here, though the score is dragged back down again afterwards"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "uni[uni.teamName_p1.fillna(\"\").str.contains(\"Japneet Singh\")][score_cols[2:]]"}, {"cell_type": "markdown", "metadata": {}, "source": "# Shake-up: Ranks\n\nSee [this notebook](https://www.kaggle.com/code/jtrotman/meta-kaggle-scatter-plot-competition-shake-up) for many more shake-up scatter plots and a comparison of competitions.\n"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "colors = uni.medal_r6.map(dict(zip(medal_names, medal_colors))).fillna('deepskyblue')\nshakeup = (uni.rank_r6 - uni.rank_p2).abs().mean() / uni.rank_r6.count()\nmax_rank = uni[rank_cols].max().max()\nplt.figure(figsize=(12, 12))\nplt.scatter(uni.rank_p1, uni.rank_r6, c=colors, s=3)\nplt.title(f'{title} - Shake-up {shakeup:.3f}')\nplt.xlabel('Public')\nplt.ylabel('Private')\nplt.plot((0, max_rank), (0, max_rank), c='k', ls='--', lw=1, alpha=.5);"}, {"cell_type": "markdown", "metadata": {}, "source": "# Shake-up: Scores\n\nShow submission count as the size of the markers..."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# Plot bronze teams last otherwise they are almost all hidden\n# Also plot smaller submission counts later so they are not covered by bigger points\nsortKey = uni.medal_r6.fillna(\"X\") + \" \" + uni.submissionCount_p1.apply(lambda v: f'{v:3.0f}')\norder = sortKey.argsort()[::-1]\ntmp = uni.iloc[order]\ncolors = tmp.medal_r6.map(dict(zip(medal_names, medal_colors))).fillna('deepskyblue')\nscoreRange = (-0.001, 0.0145)\nplt.figure(figsize=(12, 12))\nplt.scatter(tmp.displayScore_p1, tmp.displayScore_r6, c=colors, ec='k', lw=.3, s=tmp.submissionCount_p1/2)\nplt.title(f'{title} - Scores')\nplt.xlabel('Public')\nplt.ylabel('Private')\nplt.ylim(*scoreRange);\nplt.xlim(*scoreRange);\nplt.plot(scoreRange, scoreRange, c='k', ls='--', lw=1, alpha=.5);"}, {"cell_type": "markdown", "metadata": {}, "source": "# Largest Leaps &uarr;"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "tmp = uni.assign(delta=uni.rank_p2 - uni.rank_r6)\ncols = ['teamName_p1', 'teamSize', 'medal_r6',\n        'rank_p2', 'rank_r6', 'delta',\n        'displayScore_p2', 'displayScore_r6',\n        'scoreCount']\n(tmp.nlargest(30, 'delta')[cols].style\n .format(na_rep='')\n .format(precision=0, subset=['rank_p2', 'rank_r6', 'delta'])\n .bar(subset=['displayScore_r6'], height=80, color='skyblue'))"}, {"cell_type": "markdown", "metadata": {}, "source": "# Many Teams in Silver Zone\n\nThis is to investigate the discussion post [A lot of cheating in the game](https://www.kaggle.com/competitions/jane-street-real-time-market-data-forecasting/discussion/586090)...\n"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "silver_teams_df = uni[(uni.medal_r6==\"SILVER\") & (uni.teamSize>=4)].sort_values('rank_r6', ascending=True)\nsilver_teams_df.shape"}, {"cell_type": "markdown", "metadata": {}, "source": "They were all submitting in the final 3 days:"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "silver_teams_df.lastSubmissionDate.dt.date.value_counts()"}, {"cell_type": "markdown", "metadata": {}, "source": "Hundreds of submissions each on average:"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "silver_teams_df.submissionCount_p1.describe()"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "cols = ['rank_r6', 'teamName_p1', 'teamSize', 'scoreCount', 'submissionCount_p1', 'displayScore_r6', 'lastSubmissionDate', ]\n(silver_teams_df[cols].style\n .format({'lastSubmissionDate': \"{:%Y.%m.%d}\"})\n .format(precision=0, subset=['rank_r6', 'scoreCount', 'submissionCount_p1']))"}, {"cell_type": "markdown", "metadata": {}, "source": "Find the users in those teams:"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "users_df = silver_teams_df[user_id_cols].stack().astype(int).to_frame(\"user_id\")\nusers_df.shape"}, {"cell_type": "markdown", "metadata": {}, "source": "59 teams &rarr; 251 users, we can look up their activities using [this notebooks](https://www.kaggle.com/code/jtrotman/meta-kaggle-count-user-activities) output."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "user_act_counts_df = pd.read_csv('/kaggle/input/meta-kaggle-count-user-activities/ActiveUsers.csv')\nuser_act_counts_df.shape"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "user_act_counts_df.columns"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "count_cols = list(user_act_counts_df.columns[user_act_counts_df.columns.str.startswith(\"Count_\")])\n# count_cols.remove(\"Count_UserAchievements_UserId\")"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "users_df = users_df.join(user_act_counts_df.set_index(\"Id\"), on='user_id', how='left')"}, {"cell_type": "markdown", "metadata": {}, "source": "212 of the users are found in the ActiveUsers.csv.\n\nNote that the file is from January 2025 so the `PerformanceTier` still contains 0 (\"Novice\") which no longer exists."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "users_df.describe().T"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "users_df.PerformanceTier.value_counts().sort_index()"}, {"cell_type": "markdown", "metadata": {}, "source": "# Silver Teams - Novice Users\n\nThese tables fit on my screen, but only after:\n - &larr; hiding the sidebar\n - &rarr; hiding the version list\n - running my [UI Hack: Widen Notebook Content View](https://www.kaggle.com/discussions/general/534303) bookmarklet"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "show = [ 'UserName', 'PerformanceTier', 'Age', ]\n(users_df.dropna(subset=['UserName']).query(\"PerformanceTier==0\")[show + count_cols].style\n .format(precision=0)\n .set_table_styles(vertical_headers_table_styles)\n .background_gradient(axis=None))"}, {"cell_type": "markdown", "metadata": {}, "source": "# Silver Teams - Sample of Contributors"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "show = [ 'UserName', 'PerformanceTier', 'Age', ]\n(users_df.dropna(subset=['UserName'])\n .query(\"PerformanceTier==1\")\n .sample(n=50, random_state=42)[show + count_cols].style\n .format(precision=0)\n .set_table_styles(vertical_headers_table_styles)\n .background_gradient(axis=None))"}, {"cell_type": "markdown", "metadata": {}, "source": "# Silver Teams - Experts & Masters"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "show = [ 'UserName', 'PerformanceTier', 'Age', ]\n(users_df.dropna(subset=['UserName'])\n .query(\"PerformanceTier>=2\")[show + count_cols].style\n .format(precision=0)\n .set_table_styles(vertical_headers_table_styles)\n .background_gradient(axis=None))"}, {"cell_type": "markdown", "metadata": {}, "source": "# Silver Teams - Activity Count Sums per Member\n\nOK - one last look at this data: how are the team members with 0 activity counts spread out, are there many teams full of newbies?"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "teams_with_count_sums_per_user_df = users_df[count_cols].fillna(0).sum(1).unstack()\nteams_with_count_sums_per_user_df = teams_with_count_sums_per_user_df.join(silver_teams_df[['teamSize', 'rank_r6', 'teamName_p1']])\n(teams_with_count_sums_per_user_df.style\n .background_gradient(subset=user_id_cols, axis=1)\n .format(subset=user_id_cols, na_rep='')\n .format(precision=0)\n .map(lambda x: 'background: white' if pd.isnull(x) else '')\n .set_caption(\"Highlighting Activity Counts per Member\"))"}, {"cell_type": "markdown", "metadata": {}, "source": "## Summarize # Inexperienced Users per Team\n\nThey are mostly spread about; just **two** teams were quite heavy with *inexperienced* users...\n"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "((teams_with_count_sums_per_user_df[user_id_cols] == 0).sum(1).rename(\"# Inexperienced\")\n .value_counts().to_frame(\"Num Teams\"))"}, {"cell_type": "markdown", "metadata": {}, "source": "It seems most of the users in these teams have some prior activity on the site, even if it is not visible on their profile pages...\n\nMaybe some of it is quite perfunctory, for example in the old pre-2025 system, to be rated a **contributor** you had to cast *one* vote in each category (forums, notebooks).\n\nThe *Count_TeamMemberships_UserId* field is quite active: this counts the number of times a user has accepted the rules for a competition - *however* - if they do not go on to make a submission, the competition will not appear on their profile page.\nIt should be at least one for all users in the competition, however the competition's details were not in [Meta Kaggle](https://www.kaggle.com/kaggle/meta-kaggle) at the time the [notebook](https://www.kaggle.com/code/jtrotman/meta-kaggle-count-user-activities) was run.\n(And I cannot run the notebook now because KernelVotes.csv is empty!)\n\nThis does not show the date of those activities, for that you'd have to dig into [Meta Kaggle](https://www.kaggle.com/kaggle/meta-kaggle) again...\n(As for more activities, like submission dates, those are unfortunately lost, the only submission details available are for the last rescore, i.e. 1 or 2 entries per team.)\n\n**Are these teams cheating?**\nThis data is better evidence than screenshots of user profiles but the gold standard is held by Kaggle: the source code of submissions.\nSince the teams remain on the LB it suggests the submissions were different, and not indicative of private sharing.\n\nI may have missed some interesting patterns here - feel free to fork and examine further!"}, {"cell_type": "markdown", "metadata": {}, "source": "# Score Distributions"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "plt.rc(\"axes\", edgecolor='#606060')\nplt.rc(\"axes\", xmargin=0.01)"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "tmp = uni.sort_values('displayScore_r6', ascending=False)\nplt.figure(figsize=(9,5))\nplt.plot(tmp.displayScore_r6.values, c='k', lw=1)\n\nfor name, color_code in zip(medal_names, medal_colors):\n    span = np.where(tmp.medal_r6==name)[0]\n    plt.axvspan(span.min()-.5, span.max()+.5, color=color_code, ec='none', alpha=0.2)\nplt.ylim(-.001, .015);\nplt.title(f'{title} Score Distribution');"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "plt.figure(figsize=(9, 5))\nplt.plot(tmp.displayScore_r6.values, c='k', lw=1);\nfor name, color_code in zip(medal_names, medal_colors):\n    span = np.where(tmp.medal_r6==name)[0]\n    plt.axvspan(span.min()-.5, span.max()+.5, color=color_code, ec='none', alpha=0.2)\nplt.ylim(.007, .015);\nplt.xlim(-3, span.max());\nplt.title(f'{title} Score Distribution - Medal Zone');"}], "metadata": {"kaggle": {"accelerator": "none", "dataSources": [{"databundleVersionId": 11305158, "sourceId": 84493, "sourceType": "competition"}, {"datasetId": 7900239, "sourceId": 12516065, "sourceType": "datasetVersion"}, {"sourceId": 224801902, "sourceType": "kernelVersion"}], "dockerImageVersionId": 31089, "isGpuEnabled": false, "isInternetEnabled": true, "language": "python", "sourceType": "notebook"}, "kernelspec": {"display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3"}, "language_info": {"codemirror_mode": {"name": "ipython", "version": 3}, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.8"}}, "nbformat": 4, "nbformat_minor": 4}