{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":12044106,"sourceType":"datasetVersion","datasetId":7579108},{"sourceId":12067078,"sourceType":"datasetVersion","datasetId":7595409},{"sourceId":12067093,"sourceType":"datasetVersion","datasetId":7595420}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# IMPORTS\nimport numpy as np\nimport pandas as pd\nimport pandas.api.types\nimport sklearn.metrics\nfrom typing import Union\nimport ast\nimport optuna","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T07:39:16.870332Z","iopub.execute_input":"2025-06-05T07:39:16.870642Z","iopub.status.idle":"2025-06-05T07:39:18.766040Z","shell.execute_reply.started":"2025-06-05T07:39:16.870618Z","shell.execute_reply":"2025-06-05T07:39:18.765118Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# METRICS\nclass ParticipantVisibleError(Exception):\n    pass\n\n\nclass HostVisibleError(Exception):\n    pass\n\n\ndef treat_as_participant_error(error_message: str, solution: Union[pd.DataFrame, np.ndarray]) -> bool:\n    ''' Many metrics can raise more errors than can be handled manually. This function attempts\n    to identify errors that can be treated as ParticipantVisibleError without leaking any competition data.\n\n    If the solution is purely numeric, and there are no numbers in the error message,\n    then the error message is sufficiently unlikely to leak usable data and can be shown to participants.\n\n    We expect this filter to reject many safe messages. It's intended only to reduce the number of errors we need to manage manually.\n    '''\n    # This check treats bools as numeric\n    if isinstance(solution, pd.DataFrame):\n        solution_is_all_numeric = all([pandas.api.types.is_numeric_dtype(x) for x in solution.dtypes.values])\n        solution_has_bools = any([pandas.api.types.is_bool_dtype(x) for x in solution.dtypes.values])\n    elif isinstance(solution, np.ndarray):\n        solution_is_all_numeric = pandas.api.types.is_numeric_dtype(solution)\n        solution_has_bools = pandas.api.types.is_bool_dtype(solution)\n\n    if not solution_is_all_numeric:\n        return False\n\n    for char in error_message:\n        if char.isnumeric():\n            return False\n    if solution_has_bools:\n        if 'true' in error_message.lower() or 'false' in error_message.lower():\n            return False\n    return True\n\n\ndef safe_call_score(metric_function, solution, submission, **metric_func_kwargs):\n    '''\n    Call score. If that raises an error and that already been specifically handled, just raise it.\n    Otherwise make a conservative attempt to identify potential participant visible errors.\n    '''\n    try:\n        score_result = metric_function(solution, submission, **metric_func_kwargs)\n    except Exception as err:\n        error_message = str(err)\n        if err.__class__.__name__ == 'ParticipantVisibleError':\n            raise ParticipantVisibleError(error_message)\n        elif err.__class__.__name__ == 'HostVisibleError':\n            raise HostVisibleError(error_message)\n        else:\n            if treat_as_participant_error(error_message, solution):\n                raise ParticipantVisibleError(error_message)\n            else:\n                raise err\n    return score_result\n\n\ndef verify_valid_probabilities(df: pd.DataFrame, df_name: str):\n    \"\"\" Verify that the dataframe contains valid probabilities.\n\n    The dataframe must be limited to the target columns; do not pass in any ID columns.\n    \"\"\"\n    if not pandas.api.types.is_numeric_dtype(df.values):\n        raise ParticipantVisibleError(f'All target values in {df_name} must be numeric')\n\n    if df.min().min() < 0:\n        raise ParticipantVisibleError(f'All target values in {df_name} must be at least zero')\n\n    if df.max().max() > 1:\n        raise ParticipantVisibleError(f'All target values in {df_name} must be no greater than one')\n\n    if not np.allclose(df.sum(axis=1), 1):\n        raise ParticipantVisibleError(f'Target values in {df_name} do not add to one within all rows')\n\ndef score(sol: pd.DataFrame, sub: pd.DataFrame, row_id_column_name: str) -> float:\n    '''\n    Version of macro-averaged ROC-AUC score that ignores all classes that have no true positive labels.\n    '''\n    solution = sol.copy()\n    submission = sub.copy()\n    \n    del solution[row_id_column_name]\n    del submission[row_id_column_name]\n\n    if not pandas.api.types.is_numeric_dtype(submission.values):\n        bad_dtypes = {x: submission[x].dtype  for x in submission.columns if not pandas.api.types.is_numeric_dtype(submission[x])}\n        raise ParticipantVisibleError(f'Invalid submission data types found: {bad_dtypes}')\n\n    solution_sums = solution.sum(axis=0)\n    scored_columns = list(solution_sums[solution_sums > 0].index.values)\n    assert len(scored_columns) > 0\n\n    return safe_call_score(sklearn.metrics.roc_auc_score, solution[scored_columns].values, submission[scored_columns].values, average='macro')","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-06-05T07:39:18.767500Z","iopub.execute_input":"2025-06-05T07:39:18.768290Z","iopub.status.idle":"2025-06-05T07:39:18.783307Z","shell.execute_reply.started":"2025-06-05T07:39:18.768261Z","shell.execute_reply":"2025-06-05T07:39:18.782286Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_effnet = pd.read_csv('/kaggle/input/effnetfinal/submission_effnet.csv')\nsubmission_regnet = pd.read_csv('/kaggle/input/regnetfinal/submission_regnet.csv')\n\nif not submission_effnet['row_id'].equals(submission_regnet['row_id']):\n    print(\"Warning: Row IDs of submissions do not match. Blending may be incorrect.\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-05T07:39:18.784280Z","iopub.execute_input":"2025-06-05T07:39:18.784612Z","iopub.status.idle":"2025-06-05T07:39:38.298605Z","shell.execute_reply.started":"2025-06-05T07:39:18.784583Z","shell.execute_reply":"2025-06-05T07:39:38.297482Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"meansub_effnet = submission_effnet.copy()\nmeansub_effnet['row_id'] = meansub_effnet['row_id'].str.split('_').str[0]\nmeansub_effnet = meansub_effnet.groupby('row_id', as_index=False).max()\n\nmeansub_regnet = submission_regnet.copy()\nmeansub_regnet['row_id'] = meansub_regnet['row_id'].str.split('_').str[0]\nmeansub_regnet = meansub_regnet.groupby('row_id', as_index=False).max()\n\nif not meansub_effnet['row_id'].equals(meansub_regnet['row_id']):\n    print(\"Warning: Row IDs of submissions do not match. Blending may be incorrect.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T07:39:38.300501Z","iopub.execute_input":"2025-06-05T07:39:38.300809Z","iopub.status.idle":"2025-06-05T07:39:39.966419Z","shell.execute_reply.started":"2025-06-05T07:39:38.300778Z","shell.execute_reply":"2025-06-05T07:39:39.965308Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/birdclef-2025/train.csv\")\ntrain[\"row_id\"] = train[\"filename\"].str.extract(r\"/([^/.]+)\\.\")[0]\n\ntargets = meansub_effnet.copy()\nfor col in targets.columns:\n    if col != 'row_id':\n        targets[col] = 0\ntargets = targets.set_index(\"row_id\")\n\nfor _, row in train.iterrows():\n    rid = row[\"row_id\"]\n    primary = row[\"primary_label\"]\n\n    if rid in targets.index and primary in targets.columns:\n        targets.at[rid, primary] = 1\n    \n        sec_list = ast.literal_eval(row[\"secondary_labels\"])\n        for s in sec_list:\n            targets.at[rid, s] = 1\n\ntargets = targets.reset_index()\ntargets = targets.drop(columns=[\"\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T07:39:39.967510Z","iopub.execute_input":"2025-06-05T07:39:39.967770Z","iopub.status.idle":"2025-06-05T07:39:44.082325Z","shell.execute_reply.started":"2025-06-05T07:39:39.967748Z","shell.execute_reply":"2025-06-05T07:39:44.081250Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"meansub_effnet_sorted = meansub_effnet.sort_values(by='row_id').reset_index(drop=True)\nmeansub_regnet_sorted = meansub_regnet.sort_values(by='row_id').reset_index(drop=True)\ntargets_sorted = targets.sort_values(by='row_id').reset_index(drop=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T07:39:44.083204Z","iopub.execute_input":"2025-06-05T07:39:44.083453Z","iopub.status.idle":"2025-06-05T07:39:44.285842Z","shell.execute_reply.started":"2025-06-05T07:39:44.083433Z","shell.execute_reply":"2025-06-05T07:39:44.284865Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if not meansub_effnet['row_id'].equals(meansub_regnet['row_id']):\n    print(\"Warning: Row IDs of submissions do not match. Blending may be incorrect.\")\nif not meansub_effnet['row_id'].equals(targets['row_id']):\n    print(\"Warning: Row IDs of submissions do not match. Blending may be incorrect.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T07:39:44.286817Z","iopub.execute_input":"2025-06-05T07:39:44.287216Z","iopub.status.idle":"2025-06-05T07:39:44.299183Z","shell.execute_reply.started":"2025-06-05T07:39:44.287096Z","shell.execute_reply":"2025-06-05T07:39:44.298311Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(score(targets_sorted, meansub_effnet_sorted, 'row_id'))\nprint(score(targets_sorted, meansub_regnet_sorted, 'row_id'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T07:39:44.300222Z","iopub.execute_input":"2025-06-05T07:39:44.300477Z","iopub.status.idle":"2025-06-05T07:40:02.668055Z","shell.execute_reply.started":"2025-06-05T07:39:44.300456Z","shell.execute_reply":"2025-06-05T07:40:02.666754Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"weight_effnet = 1\nweight_regnet = 0\nresult = meansub_effnet_sorted.set_index('row_id').multiply(weight_effnet).add(meansub_regnet_sorted.set_index('row_id').multiply(weight_regnet), fill_value=0).reset_index()\nprint(score(targets_sorted, result, 'row_id'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T07:40:02.669096Z","iopub.execute_input":"2025-06-05T07:40:02.669452Z","iopub.status.idle":"2025-06-05T07:40:12.094394Z","shell.execute_reply.started":"2025-06-05T07:40:02.669419Z","shell.execute_reply":"2025-06-05T07:40:12.093238Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results = []\n\n# Iterate over weights for effnet from 0.0 to 1.0 in steps of 0.1\nfor w_eff in np.arange(0.0, 1.01, 0.025):\n    w_reg = 1.0 - w_eff\n\n    # Compute the weighted sum of predictions\n    combined = (\n        meansub_effnet_sorted.set_index('row_id').multiply(w_eff)\n        .add(meansub_regnet_sorted.set_index('row_id').multiply(w_reg), fill_value=0)\n        .reset_index()\n    )\n    \n    # Compute the score\n    sc = score(targets_sorted, combined, 'row_id')\n    \n    # Append to results\n    results.append({\n        'weight_effnet': round(w_eff, 1),\n        'weight_regnet': round(w_reg, 1),\n        'score': sc\n    })\n\n# Create a DataFrame from results\nscores_df = pd.DataFrame(results)\n\n# Display the table of weights vs. score\nprint(scores_df.to_string(index=False))\n\n# Find best combination\nbest_row = scores_df.loc[scores_df['score'].idxmax()]\nprint(f\"\\nBest score: {best_row['score']} at weight_effnet={best_row['weight_effnet']}, weight_regnet={best_row['weight_regnet']}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T07:52:56.567208Z","iopub.execute_input":"2025-06-05T07:52:56.567534Z","iopub.status.idle":"2025-06-05T07:59:26.272185Z","shell.execute_reply.started":"2025-06-05T07:52:56.567510Z","shell.execute_reply":"2025-06-05T07:59:26.271141Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the objective function for Optuna\ndef objective(trial):\n    # Suggest a weight for effnet between 0 and 1\n    w_eff = trial.suggest_float(\"weight_effnet\", 0.0, 1.0)\n    w_reg = 1.0 - w_eff\n\n    # Combine the two model predictions\n    combined = (\n        meansub_effnet_sorted.set_index('row_id').multiply(w_eff)\n        .add(meansub_regnet_sorted.set_index('row_id').multiply(w_reg), fill_value=0)\n        .reset_index()\n    )\n\n    # Compute and return the score (to maximize)\n    return score(targets_sorted, combined, 'row_id')\n\n# Create a study that maximizes the score\nstudy = optuna.create_study(direction=\"maximize\")\n\n# Run the optimization for a given number of trials (e.g., 50)\nstudy.optimize(objective, n_trials=50)\n\n# Extract the best trial\nbest_trial = study.best_trial\nbest_w_eff = best_trial.params[\"weight_effnet\"]\nbest_w_reg = 1.0 - best_w_eff\nbest_score = best_trial.value\n\n# Print the best result\nprint(f\"Best score: {best_score:.6f} at weight_effnet={best_w_eff:.3f}, weight_regnet={best_w_reg:.3f}\")\n\n# (Optional) If you want a DataFrame of all trials:\ntrials_df = study.trials_dataframe()\nfrom ace_tools import display_dataframe_to_user\ndisplay_dataframe_to_user(name=\"Optuna Trial Results\", dataframe=trials_df)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T07:41:55.059284Z","iopub.execute_input":"2025-06-05T07:41:55.059599Z","iopub.status.idle":"2025-06-05T07:50:13.829162Z","shell.execute_reply.started":"2025-06-05T07:41:55.059576Z","shell.execute_reply":"2025-06-05T07:50:13.827469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}