\n
![\"\"](\"https://www.googleapis.com/download/storage/v1/b/kaggle-forum-message-attachments/o/inbox%2F2224301%2Faaef2f4c2f14fc2b20e3a1cb3a64211f%2FScreenshot%202025-04-15%20170330.png?generation=1744751056220153&alt=media\")
Hi everyone. I was trying to do error analysis on my submission, I was wondering how are you doing or how this can be done for this problem.
\nWith the helps of LLM I did an error analysis on one of the notebooks shared by @shujun717
\nThe code creates individual TM-Score and rank them based on the highest to the worst performed
\n os\n re\n subprocess\n shutil\n pandas pd\n pandas.api.types\n\n\n\n\n\n\n\n\n\n () -> :\n \n work_dir = \n usalign_executable = os.path.join(work_dir, os.path.basename(usalign_path))\n\n \n os.makedirs(work_dir, exist_ok=)\n\n \n :\n os.path.exists(usalign_executable):\n shutil.copy(usalign_path, usalign_executable)\n os.chmod(usalign_executable, ) \n Exception e:\n ()\n {} \n\n \n solution.columns submission.columns:\n ValueError()\n\n \n solution.columns:\n solution[] = solution[].apply( x: x.split()[])\n submission.columns:\n submission[] = submission[].apply( x: x.split()[])\n\n \n target_scores_dict = {}\n\n \n native_pdb = os.path.join(work_dir, ) \n predicted_pdb = os.path.join(work_dir, )\n\n \n () \n target_count = (solution[].unique())\n current_target = \n\n target_id, group_native solution.groupby():\n current_target += \n () \n\n group_predicted = submission[submission[] == target_id]\n\n group_predicted.empty:\n ()\n target_scores_dict[target_id] = \n \n\n target_id_all_prediction_scores = [] \n\n \n pred_cols = [col col group_predicted.columns col.startswith()]\n available_pred_cnts = ((((col.split()[]) col pred_cols)))\n native_cols = [col col group_native.columns col.startswith()]\n available_native_cnts = ((((col.split()[]) col native_cols)))\n\n available_pred_cnts available_native_cnts:\n ()\n target_scores_dict[target_id] = \n \n\n \n pred_cnt available_pred_cnts:\n group_predicted.columns: \n\n resolved_pred_cnt = write2pdb(group_predicted, pred_cnt, predicted_pdb)\n resolved_pred_cnt == : \n\n current_prediction_vs_all_natives_scores = []\n\n \n native_cnt available_native_cnts:\n group_native.columns: \n\n resolved_native_cnt = write2pdb(group_native, native_cnt, native_pdb)\n\n resolved_native_cnt > :\n \n command = [usalign_executable, predicted_pdb, native_pdb, , ]\n :\n result = subprocess.run(command, capture_output=, text=, check=, timeout=)\n usalign_output = result.stdout\n tm_score = parse_tmscore_output(usalign_output)\n current_prediction_vs_all_natives_scores.append(tm_score)\n Exception e: \n \n current_prediction_vs_all_natives_scores.append() \n \n\n \n best_score_for_this_prediction = (current_prediction_vs_all_natives_scores) current_prediction_vs_all_natives_scores \n target_id_all_prediction_scores.append(best_score_for_this_prediction)\n\n \n best_overall_score_for_target = (target_id_all_prediction_scores) target_id_all_prediction_scores \n\n \n target_scores_dict[target_id] = best_overall_score_for_target\n \n\n \n os.path.exists(native_pdb): os.remove(native_pdb)\n os.path.exists(predicted_pdb): os.remove(predicted_pdb)\n \n\n ()\n target_scores_dict\n\n\n\n\n ():\n \n scores_dict:\n ()\n \n\n scores_df = pd.DataFrame((scores_dict.items()), columns=[, ])\n sorted_scores_df = scores_df.sort_values(, ascending=)\n\n ()\n worst_targets = sorted_scores_df.head(top_n)\n (worst_targets.to_string(index=))\n\n ()\n solution_df.columns:\n solution_df.columns:\n solution_df[] = solution_df[].apply( x: x.split()[])\n :\n ()\n\n solution_df.columns:\n target_id worst_targets[]:\n target_info = solution_df[solution_df[] == target_id]\n target_info.empty:\n length = (target_info)\n residues = target_info[].unique().tolist()\n score_val = worst_targets[worst_targets[] == target_id][].iloc[]\n ()\n ()\n ()\n ()\n ( * )\n :\n ()\n :\n ()\n\n \n :\n matplotlib.pyplot plt\n seaborn sns\n plt.figure(figsize=(, ))\n sns.histplot(scores_df[], bins=, kde=)\n plt.title()\n plt.xlabel()\n plt.ylabel()\n plt.grid(axis=, alpha=)\n plt.show()\n ImportError:\n ()\n\n sorted_scores_df\nI got this distribution:
\n
These are the individuals scores
\n![\"\"](\"https://www.googleapis.com/download/storage/v1/b/kaggle-forum-message-attachments/o/inbox%2F2224301%2F7438e44e4c7b43a86fedd25bbe7f6ab7%2Ferror_analysis.png?generation=1744751147931222&alt=media\")
What do you think of this approach and do you know other ways of performing error analysis for this dataset?
", "messages": [ { "id": "3179900", "postDate": "04/15/2025 21:14:24", "content": "Hi everyone. I was trying to do error analysis on my submission, I was wondering how are you doing or how this can be done for this problem.
\nWith the helps of LLM I did an error analysis on one of the notebooks shared by @shujun717
\nThe code creates individual TM-Score and rank them based on the highest to the worst performed
\n os\n re\n subprocess\n shutil\n pandas pd\n pandas.api.types\n\n\n\n\n\n\n\n\n\n () -> :\n \n work_dir = \n usalign_executable = os.path.join(work_dir, os.path.basename(usalign_path))\n\n \n os.makedirs(work_dir, exist_ok=)\n\n \n :\n os.path.exists(usalign_executable):\n shutil.copy(usalign_path, usalign_executable)\n os.chmod(usalign_executable, ) \n Exception e:\n ()\n {} \n\n \n solution.columns submission.columns:\n ValueError()\n\n \n solution.columns:\n solution[] = solution[].apply( x: x.split()[])\n submission.columns:\n submission[] = submission[].apply( x: x.split()[])\n\n \n target_scores_dict = {}\n\n \n native_pdb = os.path.join(work_dir, ) \n predicted_pdb = os.path.join(work_dir, )\n\n \n () \n target_count = (solution[].unique())\n current_target = \n\n target_id, group_native solution.groupby():\n current_target += \n () \n\n group_predicted = submission[submission[] == target_id]\n\n group_predicted.empty:\n ()\n target_scores_dict[target_id] = \n \n\n target_id_all_prediction_scores = [] \n\n \n pred_cols = [col col group_predicted.columns col.startswith()]\n available_pred_cnts = ((((col.split()[]) col pred_cols)))\n native_cols = [col col group_native.columns col.startswith()]\n available_native_cnts = ((((col.split()[]) col native_cols)))\n\n available_pred_cnts available_native_cnts:\n ()\n target_scores_dict[target_id] = \n \n\n \n pred_cnt available_pred_cnts:\n group_predicted.columns: \n\n resolved_pred_cnt = write2pdb(group_predicted, pred_cnt, predicted_pdb)\n resolved_pred_cnt == : \n\n current_prediction_vs_all_natives_scores = []\n\n \n native_cnt available_native_cnts:\n group_native.columns: \n\n resolved_native_cnt = write2pdb(group_native, native_cnt, native_pdb)\n\n resolved_native_cnt > :\n \n command = [usalign_executable, predicted_pdb, native_pdb, , ]\n :\n result = subprocess.run(command, capture_output=, text=, check=, timeout=)\n usalign_output = result.stdout\n tm_score = parse_tmscore_output(usalign_output)\n current_prediction_vs_all_natives_scores.append(tm_score)\n Exception e: \n \n current_prediction_vs_all_natives_scores.append() \n \n\n \n best_score_for_this_prediction = (current_prediction_vs_all_natives_scores) current_prediction_vs_all_natives_scores \n target_id_all_prediction_scores.append(best_score_for_this_prediction)\n\n \n best_overall_score_for_target = (target_id_all_prediction_scores) target_id_all_prediction_scores \n\n \n target_scores_dict[target_id] = best_overall_score_for_target\n \n\n \n os.path.exists(native_pdb): os.remove(native_pdb)\n os.path.exists(predicted_pdb): os.remove(predicted_pdb)\n \n\n ()\n target_scores_dict\n\n\n\n\n ():\n \n scores_dict:\n ()\n \n\n scores_df = pd.DataFrame((scores_dict.items()), columns=[, ])\n sorted_scores_df = scores_df.sort_values(, ascending=)\n\n ()\n worst_targets = sorted_scores_df.head(top_n)\n (worst_targets.to_string(index=))\n\n ()\n solution_df.columns:\n solution_df.columns:\n solution_df[] = solution_df[].apply( x: x.split()[])\n :\n ()\n\n solution_df.columns:\n target_id worst_targets[]:\n target_info = solution_df[solution_df[] == target_id]\n target_info.empty:\n length = (target_info)\n residues = target_info[].unique().tolist()\n score_val = worst_targets[worst_targets[] == target_id][].iloc[]\n ()\n ()\n ()\n ()\n ( * )\n :\n ()\n :\n ()\n\n \n :\n matplotlib.pyplot plt\n seaborn sns\n plt.figure(figsize=(, ))\n sns.histplot(scores_df[], bins=, kde=)\n plt.title()\n plt.xlabel()\n plt.ylabel()\n plt.grid(axis=, alpha=)\n plt.show()\n ImportError:\n ()\n\n sorted_scores_df\nI got this distribution:
\n
These are the individuals scores
\n
What do you think of this approach and do you know other ways of performing error analysis for this dataset?
", "rawMarkdown": "Hi everyone. I was trying to do error analysis on my submission, I was wondering how are you doing or how this can be done for this problem.\n\nWith the helps of LLM I did an error analysis on one of the [notebooks](https://www.kaggle.com/code/shujun717/ribonanzanet-3d-inference-add-structure-module?scriptVersionId=228564059) shared by @shujun717 \n\nThe code creates individual TM-Score and rank them based on the highest to the worst performed\n\n```\nimport os\nimport re\nimport subprocess\nimport shutil\nimport pandas as pd\nimport pandas.api.types\n\n# --- Assume your original helper functions are here ---\n# parse_tmscore_output(output) -> float\n# write_target_line(...) -> str\n# write2pdb(df, xyz_id, target_path) -> int\n# --- (Using the improved versions from the previous answer is recommended for robustness,\n# --- but you can use your original ones if preferred)\n\n# --- NEW Function: Calculate Individual Scores ---\ndef calculate_individual_scores(solution: pd.DataFrame, submission: pd.DataFrame, usalign_path: str = '/kaggle/input/usalign/USalign') -> dict:\n \"\"\"\n Calculates the best TM-score for EACH target_id individually.\n This function mirrors the logic of the original `score` function but returns\n a dictionary of scores instead of the average.\n\n Args:\n solution (pd.DataFrame): Ground truth structures.\n submission (pd.DataFrame): Predicted structures.\n usalign_path (str): Path to the USalign executable.\n\n Returns:\n dict: A dictionary mapping each target_id (str) to its highest achieved TM-score (float).\n Returns an empty dict if errors occur during setup.\n \"\"\"\n work_dir = '/kaggle/working/'\n usalign_executable = os.path.join(work_dir, os.path.basename(usalign_path))\n\n # Ensure working directory exists\n os.makedirs(work_dir, exist_ok=True)\n\n # Copy USalign and set permissions\n try:\n if not os.path.exists(usalign_executable):\n shutil.copy(usalign_path, usalign_executable)\n os.chmod(usalign_executable, 0o755) # Set execute permission\n except Exception as e:\n print(f\"Error setting up USalign: {e}\")\n return {} # Return empty if USalign setup fails\n\n # Ensure 'ID' column exists\n if 'ID' not in solution.columns or 'ID' not in submission.columns:\n raise ValueError(\"Missing 'ID' column in solution or submission DataFrame.\")\n\n # Extract target_id if not already present\n if 'target_id' not in solution.columns:\n solution['target_id'] = solution['ID'].apply(lambda x: x.split('_')[0])\n if 'target_id' not in submission.columns:\n submission['target_id'] = submission['ID'].apply(lambda x: x.split('_')[0])\n\n # --- Store results per target_id ---\n target_scores_dict = {}\n\n # Define PDB file paths\n native_pdb = os.path.join(work_dir, 'native_for_analysis.pdb') # Use different names to avoid conflict if run concurrently\n predicted_pdb = os.path.join(work_dir, 'predicted_for_analysis.pdb')\n\n # Iterate through each target_id\n print(\"Calculating individual scores for analysis...\") # Progress indicator\n target_count = len(solution['target_id'].unique())\n current_target = 0\n\n for target_id, group_native in solution.groupby('target_id'):\n current_target += 1\n print(f\"Processing target {current_target}/{target_count}: {target_id}\") # Progress indicator\n\n group_predicted = submission[submission['target_id'] == target_id]\n\n if group_predicted.empty:\n print(f\" Warning: No predictions found for target_id {target_id}. Assigning score 0.\")\n target_scores_dict[target_id] = 0.0\n continue\n\n target_id_all_prediction_scores = [] # Scores for predictions 1-5 for this target\n\n # Determine available prediction and native counts (dynamic check)\n pred_cols = [col for col in group_predicted.columns if col.startswith('x_')]\n available_pred_cnts = sorted(list(set(int(col.split('_')[1]) for col in pred_cols)))\n native_cols = [col for col in group_native.columns if col.startswith('x_')]\n available_native_cnts = sorted(list(set(int(col.split('_')[1]) for col in native_cols)))\n\n if not available_pred_cnts or not available_native_cnts:\n print(f\" Warning: Missing coordinate columns for target_id {target_id}. Assigning score 0.\")\n target_scores_dict[target_id] = 0.0\n continue\n\n # Iterate through each prediction (e.g., 1 to 5, or as available)\n for pred_cnt in available_pred_cnts:\n if f'x_{pred_cnt}' not in group_predicted.columns: continue # Skip if prediction column missing\n\n resolved_pred_cnt = write2pdb(group_predicted, pred_cnt, predicted_pdb)\n if resolved_pred_cnt == 0: continue # Skip if prediction PDB is empty\n\n current_prediction_vs_all_natives_scores = []\n\n # Iterate through each native structure (e.g., 1 to 40, or as available)\n for native_cnt in available_native_cnts:\n if f'x_{native_cnt}' not in group_native.columns: continue # Skip if native column missing\n\n resolved_native_cnt = write2pdb(group_native, native_cnt, native_pdb)\n\n if resolved_native_cnt > 0:\n # *** This is the core calculation part mirrored from score ***\n command = [usalign_executable, predicted_pdb, native_pdb, \"-atom\", \"C1'\"]\n try:\n result = subprocess.run(command, capture_output=True, text=True, check=True, timeout=60)\n usalign_output = result.stdout\n tm_score = parse_tmscore_output(usalign_output)\n current_prediction_vs_all_natives_scores.append(tm_score)\n except Exception as e: # Catch errors during USalign or parsing\n # print(f\" Error during USalign/parsing for T:{target_id} P:{pred_cnt} N:{native_cnt}: {e}\") # Verbose error\n current_prediction_vs_all_natives_scores.append(0.0) # Append 0 on error\n # else: Native PDB was empty, do nothing\n\n # Find best score for this prediction vs all natives\n best_score_for_this_prediction = max(current_prediction_vs_all_natives_scores) if current_prediction_vs_all_natives_scores else 0.0\n target_id_all_prediction_scores.append(best_score_for_this_prediction)\n\n # Find the best score among all predictions (1-5) for this target\n best_overall_score_for_target = max(target_id_all_prediction_scores) if target_id_all_prediction_scores else 0.0\n\n # *** Store the individual best score for this target_id ***\n target_scores_dict[target_id] = best_overall_score_for_target\n # print(f\" Target: {target_id}, Best Score: {best_overall_score_for_target:.4f}\") # Per-target result\n\n # Clean up temporary files (optional, could keep for debugging)\n if os.path.exists(native_pdb): os.remove(native_pdb)\n if os.path.exists(predicted_pdb): os.remove(predicted_pdb)\n # if os.path.exists(usalign_executable): os.remove(usalign_executable)\n\n print(\"Finished calculating individual scores.\")\n return target_scores_dict\n\n# --- Step 3: Use the existing Error Analysis Function (from previous answer) ---\n# This function takes the dictionary produced by calculate_individual_scores\n\ndef analyze_prediction_errors(scores_dict: dict, solution_df: pd.DataFrame, top_n: int = 10):\n \"\"\"\n Analyzes the prediction scores to identify targets with the lowest TM-scores (highest error).\n (This function remains the same as in the previous answer)\n\n Args:\n scores_dict (dict): Dictionary mapping target_id to its best TM-score.\n solution_df (pd.DataFrame): The original solution DataFrame to fetch additional info.\n top_n (int): The number of worst-performing targets to display.\n \"\"\"\n if not scores_dict:\n print(\"Scores dictionary is empty. Cannot perform error analysis.\")\n return None\n\n scores_df = pd.DataFrame(list(scores_dict.items()), columns=['target_id', 'best_tm_score'])\n sorted_scores_df = scores_df.sort_values('best_tm_score', ascending=True)\n\n print(f\"\\n--- Error Analysis: Top {top_n} Worst Performing Targets (Lowest TM-Scores) ---\")\n worst_targets = sorted_scores_df.head(top_n)\n print(worst_targets.to_string(index=False))\n\n print(f\"\\n--- Details for Worst Performing Targets ---\")\n if 'target_id' not in solution_df.columns:\n if 'ID' in solution_df.columns:\n solution_df['target_id'] = solution_df['ID'].apply(lambda x: x.split('_')[0])\n else:\n print(\"Warning: Cannot add 'target_id' to solution_df, 'ID' column missing.\")\n\n if 'target_id' in solution_df.columns:\n for target_id in worst_targets['target_id']:\n target_info = solution_df[solution_df['target_id'] == target_id]\n if not target_info.empty:\n length = len(target_info)\n residues = target_info['resname'].unique().tolist()\n score_val = worst_targets[worst_targets['target_id'] == target_id]['best_tm_score'].iloc[0]\n print(f\"Target ID: {target_id}\")\n print(f\" Best TM-Score: {score_val:.4f}\")\n print(f\" Length: {length} residues\")\n print(f\" Residue Types: {', '.join(residues)}\")\n print(\"-\" * 20)\n else:\n print(f\"Could not find info for target_id: {target_id} in the provided solution DataFrame.\")\n else:\n print(\"Cannot provide detailed info as 'target_id' column is unavailable in solution_df.\")\n\n # Optional: Plot distribution\n try:\n import matplotlib.pyplot as plt\n import seaborn as sns\n plt.figure(figsize=(10, 5))\n sns.histplot(scores_df['best_tm_score'], bins=20, kde=True)\n plt.title('Distribution of Best TM-Scores Across All Targets')\n plt.xlabel('Best TM-Score')\n plt.ylabel('Frequency')\n plt.grid(axis='y', alpha=0.5)\n plt.show()\n except ImportError:\n print(\"\\nInstall matplotlib and seaborn (`pip install matplotlib seaborn`) to see score distribution plot.\")\n\n return sorted_scores_df\n```\n\nI got this distribution:\n\n\nThese are the individuals scores\n\n\n\n\nWhat do you think of this approach and do you know other ways of performing error analysis for this dataset?", "votes": null }, { "id": "3185404", "postDate": "04/23/2025 09:15:53", "content": "Hi, have you any new idia?
", "rawMarkdown": "Hi, have you any new idia?", "votes": null } ], "comments": [ { "id": 3185404, "author_name": "stanleykroenke", "author_url": "", "post_date": "04/23/2025 09:15:53", "content": "Hi, have you any new idia?
", "votes": null, "replies": [] } ], "raw_markdown_by_id": { "3179900": "Hi everyone. I was trying to do error analysis on my submission, I was wondering how are you doing or how this can be done for this problem.\n\nWith the helps of LLM I did an error analysis on one of the [notebooks](https://www.kaggle.com/code/shujun717/ribonanzanet-3d-inference-add-structure-module?scriptVersionId=228564059) shared by @shujun717 \n\nThe code creates individual TM-Score and rank them based on the highest to the worst performed\n\n```\nimport os\nimport re\nimport subprocess\nimport shutil\nimport pandas as pd\nimport pandas.api.types\n\n# --- Assume your original helper functions are here ---\n# parse_tmscore_output(output) -> float\n# write_target_line(...) -> str\n# write2pdb(df, xyz_id, target_path) -> int\n# --- (Using the improved versions from the previous answer is recommended for robustness,\n# --- but you can use your original ones if preferred)\n\n# --- NEW Function: Calculate Individual Scores ---\ndef calculate_individual_scores(solution: pd.DataFrame, submission: pd.DataFrame, usalign_path: str = '/kaggle/input/usalign/USalign') -> dict:\n \"\"\"\n Calculates the best TM-score for EACH target_id individually.\n This function mirrors the logic of the original `score` function but returns\n a dictionary of scores instead of the average.\n\n Args:\n solution (pd.DataFrame): Ground truth structures.\n submission (pd.DataFrame): Predicted structures.\n usalign_path (str): Path to the USalign executable.\n\n Returns:\n dict: A dictionary mapping each target_id (str) to its highest achieved TM-score (float).\n Returns an empty dict if errors occur during setup.\n \"\"\"\n work_dir = '/kaggle/working/'\n usalign_executable = os.path.join(work_dir, os.path.basename(usalign_path))\n\n # Ensure working directory exists\n os.makedirs(work_dir, exist_ok=True)\n\n # Copy USalign and set permissions\n try:\n if not os.path.exists(usalign_executable):\n shutil.copy(usalign_path, usalign_executable)\n os.chmod(usalign_executable, 0o755) # Set execute permission\n except Exception as e:\n print(f\"Error setting up USalign: {e}\")\n return {} # Return empty if USalign setup fails\n\n # Ensure 'ID' column exists\n if 'ID' not in solution.columns or 'ID' not in submission.columns:\n raise ValueError(\"Missing 'ID' column in solution or submission DataFrame.\")\n\n # Extract target_id if not already present\n if 'target_id' not in solution.columns:\n solution['target_id'] = solution['ID'].apply(lambda x: x.split('_')[0])\n if 'target_id' not in submission.columns:\n submission['target_id'] = submission['ID'].apply(lambda x: x.split('_')[0])\n\n # --- Store results per target_id ---\n target_scores_dict = {}\n\n # Define PDB file paths\n native_pdb = os.path.join(work_dir, 'native_for_analysis.pdb') # Use different names to avoid conflict if run concurrently\n predicted_pdb = os.path.join(work_dir, 'predicted_for_analysis.pdb')\n\n # Iterate through each target_id\n print(\"Calculating individual scores for analysis...\") # Progress indicator\n target_count = len(solution['target_id'].unique())\n current_target = 0\n\n for target_id, group_native in solution.groupby('target_id'):\n current_target += 1\n print(f\"Processing target {current_target}/{target_count}: {target_id}\") # Progress indicator\n\n group_predicted = submission[submission['target_id'] == target_id]\n\n if group_predicted.empty:\n print(f\" Warning: No predictions found for target_id {target_id}. Assigning score 0.\")\n target_scores_dict[target_id] = 0.0\n continue\n\n target_id_all_prediction_scores = [] # Scores for predictions 1-5 for this target\n\n # Determine available prediction and native counts (dynamic check)\n pred_cols = [col for col in group_predicted.columns if col.startswith('x_')]\n available_pred_cnts = sorted(list(set(int(col.split('_')[1]) for col in pred_cols)))\n native_cols = [col for col in group_native.columns if col.startswith('x_')]\n available_native_cnts = sorted(list(set(int(col.split('_')[1]) for col in native_cols)))\n\n if not available_pred_cnts or not available_native_cnts:\n print(f\" Warning: Missing coordinate columns for target_id {target_id}. Assigning score 0.\")\n target_scores_dict[target_id] = 0.0\n continue\n\n # Iterate through each prediction (e.g., 1 to 5, or as available)\n for pred_cnt in available_pred_cnts:\n if f'x_{pred_cnt}' not in group_predicted.columns: continue # Skip if prediction column missing\n\n resolved_pred_cnt = write2pdb(group_predicted, pred_cnt, predicted_pdb)\n if resolved_pred_cnt == 0: continue # Skip if prediction PDB is empty\n\n current_prediction_vs_all_natives_scores = []\n\n # Iterate through each native structure (e.g., 1 to 40, or as available)\n for native_cnt in available_native_cnts:\n if f'x_{native_cnt}' not in group_native.columns: continue # Skip if native column missing\n\n resolved_native_cnt = write2pdb(group_native, native_cnt, native_pdb)\n\n if resolved_native_cnt > 0:\n # *** This is the core calculation part mirrored from score ***\n command = [usalign_executable, predicted_pdb, native_pdb, \"-atom\", \"C1'\"]\n try:\n result = subprocess.run(command, capture_output=True, text=True, check=True, timeout=60)\n usalign_output = result.stdout\n tm_score = parse_tmscore_output(usalign_output)\n current_prediction_vs_all_natives_scores.append(tm_score)\n except Exception as e: # Catch errors during USalign or parsing\n # print(f\" Error during USalign/parsing for T:{target_id} P:{pred_cnt} N:{native_cnt}: {e}\") # Verbose error\n current_prediction_vs_all_natives_scores.append(0.0) # Append 0 on error\n # else: Native PDB was empty, do nothing\n\n # Find best score for this prediction vs all natives\n best_score_for_this_prediction = max(current_prediction_vs_all_natives_scores) if current_prediction_vs_all_natives_scores else 0.0\n target_id_all_prediction_scores.append(best_score_for_this_prediction)\n\n # Find the best score among all predictions (1-5) for this target\n best_overall_score_for_target = max(target_id_all_prediction_scores) if target_id_all_prediction_scores else 0.0\n\n # *** Store the individual best score for this target_id ***\n target_scores_dict[target_id] = best_overall_score_for_target\n # print(f\" Target: {target_id}, Best Score: {best_overall_score_for_target:.4f}\") # Per-target result\n\n # Clean up temporary files (optional, could keep for debugging)\n if os.path.exists(native_pdb): os.remove(native_pdb)\n if os.path.exists(predicted_pdb): os.remove(predicted_pdb)\n # if os.path.exists(usalign_executable): os.remove(usalign_executable)\n\n print(\"Finished calculating individual scores.\")\n return target_scores_dict\n\n# --- Step 3: Use the existing Error Analysis Function (from previous answer) ---\n# This function takes the dictionary produced by calculate_individual_scores\n\ndef analyze_prediction_errors(scores_dict: dict, solution_df: pd.DataFrame, top_n: int = 10):\n \"\"\"\n Analyzes the prediction scores to identify targets with the lowest TM-scores (highest error).\n (This function remains the same as in the previous answer)\n\n Args:\n scores_dict (dict): Dictionary mapping target_id to its best TM-score.\n solution_df (pd.DataFrame): The original solution DataFrame to fetch additional info.\n top_n (int): The number of worst-performing targets to display.\n \"\"\"\n if not scores_dict:\n print(\"Scores dictionary is empty. Cannot perform error analysis.\")\n return None\n\n scores_df = pd.DataFrame(list(scores_dict.items()), columns=['target_id', 'best_tm_score'])\n sorted_scores_df = scores_df.sort_values('best_tm_score', ascending=True)\n\n print(f\"\\n--- Error Analysis: Top {top_n} Worst Performing Targets (Lowest TM-Scores) ---\")\n worst_targets = sorted_scores_df.head(top_n)\n print(worst_targets.to_string(index=False))\n\n print(f\"\\n--- Details for Worst Performing Targets ---\")\n if 'target_id' not in solution_df.columns:\n if 'ID' in solution_df.columns:\n solution_df['target_id'] = solution_df['ID'].apply(lambda x: x.split('_')[0])\n else:\n print(\"Warning: Cannot add 'target_id' to solution_df, 'ID' column missing.\")\n\n if 'target_id' in solution_df.columns:\n for target_id in worst_targets['target_id']:\n target_info = solution_df[solution_df['target_id'] == target_id]\n if not target_info.empty:\n length = len(target_info)\n residues = target_info['resname'].unique().tolist()\n score_val = worst_targets[worst_targets['target_id'] == target_id]['best_tm_score'].iloc[0]\n print(f\"Target ID: {target_id}\")\n print(f\" Best TM-Score: {score_val:.4f}\")\n print(f\" Length: {length} residues\")\n print(f\" Residue Types: {', '.join(residues)}\")\n print(\"-\" * 20)\n else:\n print(f\"Could not find info for target_id: {target_id} in the provided solution DataFrame.\")\n else:\n print(\"Cannot provide detailed info as 'target_id' column is unavailable in solution_df.\")\n\n # Optional: Plot distribution\n try:\n import matplotlib.pyplot as plt\n import seaborn as sns\n plt.figure(figsize=(10, 5))\n sns.histplot(scores_df['best_tm_score'], bins=20, kde=True)\n plt.title('Distribution of Best TM-Scores Across All Targets')\n plt.xlabel('Best TM-Score')\n plt.ylabel('Frequency')\n plt.grid(axis='y', alpha=0.5)\n plt.show()\n except ImportError:\n print(\"\\nInstall matplotlib and seaborn (`pip install matplotlib seaborn`) to see score distribution plot.\")\n\n return sorted_scores_df\n```\n\nI got this distribution:\n\n\nThese are the individuals scores\n\n\n\n\nWhat do you think of this approach and do you know other ways of performing error analysis for this dataset?", "3185404": "Hi, have you any new idia?" }, "source": "meta" }