{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":87793,"databundleVersionId":12276181,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":11644010,"sourceType":"datasetVersion","datasetId":7306643},{"sourceId":11969392,"sourceType":"datasetVersion","datasetId":7526656}],"dockerImageVersionId":30918,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import time\n\nimport pandas as pd\nimport numpy as np\n\nimport random\nfrom Bio import pairwise2\nfrom Bio.Seq import Seq\n\nfrom tqdm import tqdm\n\nfrom scipy.spatial.transform import Rotation as R\nfrom sklearn.preprocessing import normalize\nfrom scipy.spatial import distance_matrix\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"\\nLoading data files...\")\ntrain_seqs = pd.read_csv('/kaggle/input/stanford-rna-3d-folding/train_sequences.csv')\nvalid_seqs = pd.read_csv('/kaggle/input/stanford-rna-3d-folding/validation_sequences.csv')\ntest_seqs = pd.read_csv('/kaggle/input/stanford-rna-3d-folding/test_sequences.csv')\ntrain_labels = pd.read_csv('/kaggle/input/stanford-rna-3d-folding/train_labels.csv')\nvalid_labels = pd.read_csv('/kaggle/input/stanford-rna-3d-folding/validation_labels.csv')\n\nprint(f\"Loaded {len(train_seqs)} training sequences, {len(valid_seqs)} validation sequences, and {len(test_seqs)} test sequences\")","metadata":{"_uuid":"7e2d63ef-4429-4722-95ee-6681af62a6d5","_cell_guid":"0ec49051-b8ac-4f27-b152-0c2fe59d4b5c","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-05-27T11:59:37.435267Z","iopub.execute_input":"2025-05-27T11:59:37.435683Z","iopub.status.idle":"2025-05-27T11:59:38.507705Z","shell.execute_reply.started":"2025-05-27T11:59:37.435631Z","shell.execute_reply":"2025-05-27T11:59:38.506363Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train_seqs_v2 = pd.read_csv('/kaggle/input/extended-rna/train_sequences_v2.csv')\n# train_labels_v2 = pd.read_csv('/kaggle/input/extended-rna/train_labels_v2.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-27T11:59:38.509240Z","iopub.execute_input":"2025-05-27T11:59:38.509674Z","iopub.status.idle":"2025-05-27T11:59:44.244781Z","shell.execute_reply.started":"2025-05-27T11:59:38.509635Z","shell.execute_reply":"2025-05-27T11:59:44.243830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_seqs_v2 = pd.read_csv('/kaggle/input/rna-cif-to-csv/rna_sequences.csv')\ntrain_labels_v2 = pd.read_csv('/kaggle/input/rna-cif-to-csv/rna_coordinates.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-27T12:01:09.442977Z","iopub.execute_input":"2025-05-27T12:01:09.443289Z","iopub.status.idle":"2025-05-27T12:01:19.760385Z","shell.execute_reply.started":"2025-05-27T12:01:09.443264Z","shell.execute_reply":"2025-05-27T12:01:19.759214Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_seqs_v2.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-27T12:01:23.031949Z","iopub.execute_input":"2025-05-27T12:01:23.032306Z","iopub.status.idle":"2025-05-27T12:01:23.041278Z","shell.execute_reply.started":"2025-05-27T12:01:23.032274Z","shell.execute_reply":"2025-05-27T12:01:23.040269Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels_v2.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-27T12:01:23.239062Z","iopub.execute_input":"2025-05-27T12:01:23.239457Z","iopub.status.idle":"2025-05-27T12:01:23.269134Z","shell.execute_reply.started":"2025-05-27T12:01:23.239422Z","shell.execute_reply":"2025-05-27T12:01:23.267820Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import pandas as pd\n# import numpy as np\n\n# # 1. Check if v2 is larger than original\n# print(\"Dataset Size Comparison:\")\n# print(f\"train_seqs: {len(train_seqs)} rows → train_seqs_v2: {len(train_seqs_v2)} rows\")\n# print(f\"train_labels: {len(train_labels)} rows → train_labels_v2: {len(train_labels_v2)} rows\")\n# print()\n\n# # 2. Verify all train_seqs records exist in train_seqs_v2\n# print(\"Checking if original sequences exist in v2...\")\n# original_target_ids = set(train_seqs['target_id'])\n# extended_target_ids = set(train_seqs_v2['target_id'])\n# all_targets_included = original_target_ids.issubset(extended_target_ids)\n\n# print(f\"Original train_seqs has {len(original_target_ids)} unique target_ids\")\n# print(f\"All original target_ids found in train_seqs_v2: {all_targets_included}\")\n\n# if not all_targets_included:\n#     missing_ids = original_target_ids - extended_target_ids\n#     print(f\"Missing {len(missing_ids)} target_ids\")\n#     if len(missing_ids) <= 5:\n#         print(f\"Missing IDs: {list(missing_ids)}\")\n#     else:\n#         print(f\"First 5 missing IDs: {list(missing_ids)[:5]}\")\n# print()\n\n# # 3. Check for consistency in a sample of target_ids that exist in both datasets\n# if all_targets_included:\n#     print(\"Checking consistency of data between original and v2 sequences...\")\n#     # Sample a few target_ids that exist in both datasets\n#     sample_size = min(5, len(original_target_ids))\n#     sample_ids = np.random.choice(list(original_target_ids), sample_size, replace=False)\n    \n#     for target_id in sample_ids:\n#         original_row = train_seqs[train_seqs['target_id'] == target_id].iloc[0]\n#         extended_row = train_seqs_v2[train_seqs_v2['target_id'] == target_id].iloc[0]\n        \n#         # Compare important columns\n#         sequence_match = original_row['sequence'] == extended_row['sequence']\n#         print(f\"Target ID {target_id}: Sequences match: {sequence_match}\")\n# print()\n\n# # 4. Check if all train_labels IDs exist in train_labels_v2\n# print(\"Checking labels dataset...\")\n# # Since labels dataset is large, we'll check a sample of IDs\n# sample_size = min(1000, len(train_labels))\n# sample_indices = np.random.choice(len(train_labels), sample_size, replace=False)\n# sample_rows = train_labels.iloc[sample_indices]\n\n# # Create a composite key for comparison (ID + resid)\n# sample_rows['composite_key'] = sample_rows['ID'] + '_' + sample_rows['resid'].astype(str)\n# train_labels_v2['composite_key'] = train_labels_v2['ID'] + '_' + train_labels_v2['resid'].astype(str)\n\n# sample_keys = set(sample_rows['composite_key'])\n# extended_keys = set(train_labels_v2['composite_key'])\n\n# keys_found = sample_keys.issubset(extended_keys)\n# if keys_found:\n#     found_percentage = 100\n# else:\n#     intersection = sample_keys.intersection(extended_keys)\n#     found_percentage = (len(intersection) / len(sample_keys)) * 100\n\n# print(f\"Sampled {len(sample_keys)} keys from train_labels\")\n# print(f\"All sampled keys found in train_labels_v2: {keys_found} ({found_percentage:.2f}%)\")\n\n# if not keys_found:\n#     missing_keys = sample_keys - extended_keys\n#     print(f\"Missing {len(missing_keys)} keys out of {len(sample_keys)} sampled\")\n#     if len(missing_keys) <= 5:\n#         print(f\"Missing keys: {list(missing_keys)}\")\n#     else:\n#         print(f\"First 5 missing keys: {list(missing_keys)[:5]}\")\n# print()\n\n# # 5. Data type consistency check\n# print(\"Data type consistency check:\")\n# print(\"train_seqs vs train_seqs_v2:\")\n# for col in train_seqs.columns:\n#     print(f\"  Column '{col}': {train_seqs[col].dtype} → {train_seqs_v2[col].dtype} - Match: {train_seqs[col].dtype == train_seqs_v2[col].dtype}\")\n\n# print(\"\\ntrain_labels vs train_labels_v2:\")\n# for col in train_labels.columns:\n#     if col != 'composite_key':  # Skip the key we created\n#         print(f\"  Column '{col}': {train_labels[col].dtype} → {train_labels_v2[col].dtype} - Match: {train_labels[col].dtype == train_labels_v2[col].dtype}\")\n# print()\n\n# # 6. Check for missing values pattern\n# print(\"Missing values comparison:\")\n# print(\"train_seqs vs train_seqs_v2:\")\n# for col in train_seqs.columns:\n#     original_missing = train_seqs[col].isnull().sum() / len(train_seqs) * 100\n#     extended_missing = train_seqs_v2[col].isnull().sum() / len(train_seqs_v2) * 100\n#     print(f\"  Column '{col}': {original_missing:.2f}% → {extended_missing:.2f}%\")\n\n# print(\"\\ntrain_labels vs train_labels_v2:\")\n# for col in train_labels.columns:\n#     if col != 'composite_key':  # Skip the key we created\n#         original_missing = train_labels[col].isnull().sum() / len(train_labels) * 100\n#         extended_missing = train_labels_v2[col].isnull().sum() / len(train_labels_v2) * 100\n#         print(f\"  Column '{col}': {original_missing:.2f}% → {extended_missing:.2f}%\")\n# print()\n\n# # Clean up the temporary column we added\n# if 'composite_key' in train_labels_v2.columns:\n#     train_labels_v2.drop('composite_key', axis=1, inplace=True)\n\n# # Final assessment\n# print(\"FINAL ASSESSMENT:\")\n# print(\"-\" * 50)\n# seqs_extended_properly = all_targets_included and len(train_seqs_v2) > len(train_seqs)\n# labels_extended_properly = found_percentage > 99 and len(train_labels_v2) > len(train_labels)\n\n# if seqs_extended_properly and labels_extended_properly:\n#     print(\"✓ PASS: Both train_seqs_v2 and train_labels_v2 appear to be proper extensions of the original datasets.\")\n#     print(\"✓ It should be safe to swap them.\")\n# else:\n#     print(\"✗ ISSUES DETECTED:\")\n#     if not seqs_extended_properly:\n#         print(\"  - train_seqs_v2 may not fully contain train_seqs data\")\n#     if not labels_extended_properly:\n#         print(\"  - train_labels_v2 may not fully contain train_labels data\")\n#     print(\"✗ Recommend investigating the issues above before swapping datasets.\")\n# print(\"-\" * 50)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-27T12:01:26.591169Z","iopub.execute_input":"2025-05-27T12:01:26.591606Z","iopub.status.idle":"2025-05-27T12:01:26.597395Z","shell.execute_reply.started":"2025-05-27T12:01:26.591573Z","shell.execute_reply":"2025-05-27T12:01:26.596253Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# Function to extend the original dataset with new records from v2\ndef extend_dataset(original_df, v2_df, key_columns, dataset_name):\n    print(f\"Extending {dataset_name}...\")\n    print(f\"  Original size: {len(original_df)} rows\")\n    print(f\"  v2 size: {len(v2_df)} rows\")\n    \n    # Create a composite key for identification if multiple key columns\n    if isinstance(key_columns, list) and len(key_columns) > 1:\n        original_df['temp_key'] = original_df[key_columns].astype(str).agg('_'.join, axis=1)\n        v2_df['temp_key'] = v2_df[key_columns].astype(str).agg('_'.join, axis=1)\n        key_for_identification = 'temp_key'\n    else:\n        key_for_identification = key_columns[0] if isinstance(key_columns, list) else key_columns\n    \n    # Identify unique records in each dataset\n    original_keys = set(original_df[key_for_identification])\n    v2_keys = set(v2_df[key_for_identification])\n    \n    # Calculate stats\n    keys_only_in_original = original_keys - v2_keys\n    keys_only_in_v2 = v2_keys - original_keys \n    common_keys = original_keys.intersection(v2_keys)\n    \n    print(f\"  Keys only in original: {len(keys_only_in_original)}\")\n    print(f\"  Keys only in v2: {len(keys_only_in_v2)}\")\n    print(f\"  Common keys: {len(common_keys)}\")\n    \n    # Create a mask to filter v2 records that don't exist in original\n    new_records_mask = ~v2_df[key_for_identification].isin(original_keys)\n    new_records = v2_df[new_records_mask].copy()\n    \n    # Drop temporary key if it was created\n    if key_for_identification == 'temp_key':\n        new_records.drop('temp_key', axis=1, inplace=True)\n        original_df.drop('temp_key', axis=1, inplace=True)\n    \n    # Combine original with new records from v2\n    extended_df = pd.concat([original_df, new_records], ignore_index=True)\n    \n    # Report final sizes\n    print(f\"  New records added: {len(new_records)}\")\n    print(f\"  Extended dataset size: {len(extended_df)} rows\")\n    print(f\"  Verification - All original keys in extended dataset: {set(original_df[key_columns[0] if isinstance(key_columns, list) else key_columns]).issubset(set(extended_df[key_columns[0] if isinstance(key_columns, list) else key_columns]))}\")\n    \n    # Check for missing values in key columns\n    for col in extended_df.columns:\n        original_missing = original_df[col].isnull().sum()\n        extended_missing = extended_df[col].isnull().sum()\n        if original_missing > 0 or extended_missing > 0:\n            print(f\"  Column '{col}': Missing values - Original: {original_missing}, Extended: {extended_missing}\")\n    \n    # Clean up\n    if key_for_identification == 'temp_key' and 'temp_key' in v2_df.columns:\n        v2_df.drop('temp_key', axis=1, inplace=True)\n        \n    return extended_df\n\n# 1. Extend train_seqs with train_seqs_v2\nprint(\"\\n\" + \"=\"*50)\nprint(\"EXTENDING SEQUENCE DATASETS\")\nprint(\"=\"*50)\ntrain_seqs_extended = extend_dataset(\n    train_seqs, \n    train_seqs_v2,\n    ['target_id'],  # Using target_id as the unique identifier\n    \"train_seqs\"\n)\n\n# 2. Extend train_labels with train_labels_v2\nprint(\"\\n\" + \"=\"*50)\nprint(\"EXTENDING LABELS DATASETS\")\nprint(\"=\"*50)\n# For labels, we need a composite key of ID and resid\ntrain_labels_extended = extend_dataset(\n    train_labels,\n    train_labels_v2,\n    ['ID', 'resid'],  # Using composite key\n    \"train_labels\"\n)\n\n# Verify relationships between extended datasets\nprint(\"\\n\" + \"=\"*50)\nprint(\"VERIFYING RELATIONSHIPS\")\nprint(\"=\"*50)\n\n# Check if all sequence IDs have corresponding labels\nseq_ids = set(train_seqs_extended['target_id'].unique())\nlabel_ids = set(train_labels_extended['ID'].unique())\n\nseq_ids_with_labels = seq_ids.intersection(label_ids)\nseq_ids_without_labels = seq_ids - label_ids\n\nprint(f\"Total unique sequence IDs: {len(seq_ids)}\")\nprint(f\"Sequence IDs with corresponding labels: {len(seq_ids_with_labels)} ({len(seq_ids_with_labels)/len(seq_ids)*100:.2f}%)\")\nprint(f\"Sequence IDs without corresponding labels: {len(seq_ids_without_labels)} ({len(seq_ids_without_labels)/len(seq_ids)*100:.2f}%)\")\n\nif len(seq_ids_without_labels) > 0:\n    print(\"Sample of sequence IDs without labels (up to 5):\")\n    print(list(seq_ids_without_labels)[:5])\n\n# Print summary of extended datasets\nprint(\"\\n\" + \"=\"*50)\nprint(\"SUMMARY OF EXTENDED DATASETS\")\nprint(\"=\"*50)\nprint(f\"Original train_seqs: {len(train_seqs)} rows\")\nprint(f\"Original train_labels: {len(train_labels)} rows\")\nprint(f\"Extended train_seqs: {len(train_seqs_extended)} rows (+{len(train_seqs_extended)-len(train_seqs)})\")\nprint(f\"Extended train_labels: {len(train_labels_extended)} rows (+{len(train_labels_extended)-len(train_labels)})\")\n\n# Save the extended datasets (uncomment to save)\n# train_seqs_extended.to_csv('train_seqs_combined.csv', index=False)\n# train_labels_extended.to_csv('train_labels_combined.csv', index=False)\n\nprint(\"\\n\" + \"=\"*50)\nprint(\"DONE! Extended datasets created.\")\nprint(\"To save the datasets, uncomment the last two lines.\")\nprint(\"=\"*50)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-27T12:01:26.800317Z","iopub.execute_input":"2025-05-27T12:01:26.800758Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_seqs_extended.info()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels_extended.info()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Get the first 500 sequences\n# train_seqs_small = train_seqs_extended.iloc[:500].copy()\n\n# # Extract base IDs from train_labels_extended once\n# base_ids = train_labels_extended['ID'].str.rsplit('_', n=1).str[0]\n\n# # Filter labels where the base ID is in our sequence IDs\n# train_labels_small = train_labels_extended[base_ids.isin(train_seqs_small['target_id'])].copy()\n\n# # Verify\n# print(f\"Number of sequences in train_seqs_small: {len(train_seqs_small)}\")\n# print(f\"Total rows in train_labels_small: {len(train_labels_small)}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_labels(labels_df):\n    coords_dict = {}\n    \n    # Group by target ID and wrap with tqdm for progress tracking\n    id_groups = labels_df.groupby(lambda x: labels_df['ID'][x].rsplit('_', 1)[0])\n    for id_prefix, group in tqdm(id_groups, desc=\"Processing structures\"):\n        # Extract just the coordinates columns for the first structure (x_1, y_1, z_1)\n        coords = []\n        for _, row in group.sort_values('resid').iterrows():\n            coords.append([row['x_1'], row['y_1'], row['z_1']])\n        \n        coords_dict[id_prefix] = np.array(coords)\n    \n    return coords_dict\n\ntrain_coords_dict = process_labels(train_labels_extended)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from Bio.Seq import Seq\nfrom Bio import pairwise2\nimport numpy as np\nfrom sklearn.cluster import KMeans\nfrom sklearn.metrics.pairwise import cosine_similarity\n\ndef find_similar_sequences(query_seq, train_seqs_df, train_coords_dict, top_n=5):\n    \"\"\"\n    Find similar RNA sequences using enhanced scoring and clustering for diversity.\n    \n    Improvements:\n    - Multi-tier length filtering\n    - Enhanced alignment scoring with multiple algorithms\n    - RNA-specific structural features\n    - Adaptive clustering\n    \"\"\"\n    similar_seqs = []\n    query_seq_obj = Seq(query_seq)\n    query_features = _extract_enhanced_rna_features(query_seq)\n    \n    # Step 1: Enhanced candidate selection with multi-tier filtering\n    for _, row in train_seqs_df.iterrows():\n        target_id = row['target_id']\n        train_seq = row['sequence']\n        \n        # Skip if coordinates not available\n        if target_id not in train_coords_dict:\n            continue\n        \n        # Multi-tier length filtering (more permissive for very short/long sequences)\n        len_ratio = abs(len(train_seq) - len(query_seq)) / max(len(train_seq), len(query_seq))\n        if len(query_seq) < 50 or len(train_seq) < 50:  # Short sequences - more permissive\n            if len_ratio > 0.6:\n                continue\n        elif len(query_seq) > 1000 or len(train_seq) > 1000:  # Long sequences - stricter\n            if len_ratio > 0.2:\n                continue\n        else:  # Medium sequences - original threshold\n            if len_ratio > 0.4:\n                continue\n        \n        # Calculate composite similarity score\n        composite_score = _calculate_composite_similarity(query_seq, train_seq, query_features)\n        \n        if composite_score > 0:  # Only keep sequences with positive similarity\n            similar_seqs.append((target_id, train_seq, composite_score, train_coords_dict[target_id]))\n    \n    # Sort by composite score and take top candidates\n    similar_seqs.sort(key=lambda x: x[2], reverse=True)\n    \n    # Adaptive candidate selection based on score distribution\n    candidate_count = min(50, len(similar_seqs))  # Increased initial pool\n    if len(similar_seqs) > 10:\n        # Filter out sequences with very low scores (bottom 20%)\n        score_threshold = np.percentile([x[2] for x in similar_seqs], 80)\n        filtered_candidates = [x for x in similar_seqs if x[2] >= score_threshold]\n        candidate_count = min(candidate_count, len(filtered_candidates))\n        top_candidates = filtered_candidates[:candidate_count]\n    else:\n        top_candidates = similar_seqs[:candidate_count]\n    \n    # If we have fewer sequences than requested clusters, return all\n    if len(top_candidates) <= top_n:\n        return top_candidates[:top_n]\n    \n    # Step 2: Enhanced feature matrix for better clustering\n    feature_matrix = []\n    for _, seq, _, _ in top_candidates:\n        features = _extract_enhanced_rna_features(seq)\n        feature_matrix.append(features)\n    \n    feature_matrix = np.array(feature_matrix)\n    \n    # Step 3: Adaptive clustering\n    n_clusters = min(top_n, len(top_candidates))\n    \n    # Use different clustering approach based on dataset size\n    if len(top_candidates) >= 15:\n        # K-means for larger datasets\n        kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init=10)\n        cluster_labels = kmeans.fit_predict(feature_matrix)\n    else:\n        # Simple diversity-based selection for smaller datasets\n        cluster_labels = _diversity_based_clustering(feature_matrix, n_clusters)\n    \n    # Step 4: Select best representative from each cluster\n    final_results = []\n    for cluster_id in range(n_clusters):\n        cluster_sequences = [top_candidates[i] for i in range(len(top_candidates)) \n                           if cluster_labels[i] == cluster_id]\n        \n        if cluster_sequences:\n            # Sort by composite score and take the best one\n            cluster_sequences.sort(key=lambda x: x[2], reverse=True)\n            final_results.append(cluster_sequences[0])\n    \n    # Sort final results by similarity score\n    final_results.sort(key=lambda x: x[2], reverse=True)\n    \n    return final_results[:top_n]\n\ndef _calculate_composite_similarity(query_seq, train_seq, query_features):\n    \"\"\"\n    Calculate composite similarity using multiple alignment methods and features.\n    \"\"\"\n    query_seq_obj = Seq(query_seq)\n    \n    # 1. Global alignment (original method)\n    global_alignments = pairwise2.align.globalms(query_seq_obj, train_seq, 2.9, -1, -10, -0.5, one_alignment_only=True)\n    global_score = 0\n    if global_alignments:\n        alignment = global_alignments[0]\n        global_score = alignment.score / (2 * min(len(query_seq), len(train_seq)))\n    \n    # 2. Local alignment for finding similar regions\n    local_alignments = pairwise2.align.localms(query_seq_obj, train_seq, 2.9, -1, -10, -0.5, one_alignment_only=True)\n    local_score = 0\n    if local_alignments:\n        alignment = local_alignments[0]\n        local_score = alignment.score / (2 * min(len(query_seq), len(train_seq)))\n    \n    # 3. Feature-based similarity\n    train_features = _extract_enhanced_rna_features(train_seq)\n    feature_similarity = cosine_similarity([query_features], [train_features])[0][0]\n    \n    # 4. K-mer similarity for sequence motifs\n    kmer_similarity = _calculate_kmer_similarity(query_seq, train_seq, k=3)\n    \n    # Weighted composite score\n    composite_score = (\n        0.4 * global_score + \n        0.3 * local_score + \n        0.2 * feature_similarity + \n        0.1 * kmer_similarity\n    )\n    \n    return composite_score\n\ndef _calculate_kmer_similarity(seq1, seq2, k=3):\n    \"\"\"Calculate k-mer based similarity between sequences.\"\"\"\n    def get_kmers(seq, k):\n        return set(seq[i:i+k] for i in range(len(seq) - k + 1))\n    \n    kmers1 = get_kmers(seq1.upper(), k)\n    kmers2 = get_kmers(seq2.upper(), k)\n    \n    if not kmers1 or not kmers2:\n        return 0\n    \n    intersection = len(kmers1.intersection(kmers2))\n    union = len(kmers1.union(kmers2))\n    \n    return intersection / union if union > 0 else 0\n\ndef _diversity_based_clustering(feature_matrix, n_clusters):\n    \"\"\"Simple diversity-based clustering for small datasets.\"\"\"\n    n_samples = len(feature_matrix)\n    cluster_labels = np.zeros(n_samples, dtype=int)\n    \n    if n_samples <= n_clusters:\n        return np.arange(n_samples)\n    \n    # Select diverse representatives\n    selected_indices = [0]  # Start with first sequence\n    \n    for cluster_id in range(1, n_clusters):\n        max_min_distance = -1\n        best_idx = -1\n        \n        for i in range(n_samples):\n            if i in selected_indices:\n                continue\n            \n            # Find minimum distance to already selected sequences\n            min_distance = min(\n                np.linalg.norm(feature_matrix[i] - feature_matrix[j]) \n                for j in selected_indices\n            )\n            \n            if min_distance > max_min_distance:\n                max_min_distance = min_distance\n                best_idx = i\n        \n        if best_idx != -1:\n            selected_indices.append(best_idx)\n    \n    # Assign remaining sequences to closest cluster centers\n    for i in range(n_samples):\n        if i not in selected_indices:\n            distances = [\n                np.linalg.norm(feature_matrix[i] - feature_matrix[j]) \n                for j in selected_indices\n            ]\n            cluster_labels[i] = np.argmin(distances)\n        else:\n            cluster_labels[i] = selected_indices.index(i)\n    \n    return cluster_labels\n\ndef _extract_enhanced_rna_features(sequence):\n    \"\"\"\n    Extract comprehensive RNA-specific features for better clustering and similarity.\n    \"\"\"\n    seq = sequence.upper()\n    features = []\n    \n    # 1. Basic nucleotide frequencies\n    nucleotides = ['A', 'U', 'G', 'C']\n    for nuc in nucleotides:\n        freq = seq.count(nuc) / len(seq) if len(seq) > 0 else 0\n        features.append(freq)\n    \n    # 2. Dinucleotide frequencies (reduced set - most important for RNA)\n    important_dinucs = ['AU', 'UA', 'GC', 'CG', 'GU', 'UG', 'AA', 'UU', 'GG', 'CC']\n    for dinuc in important_dinucs:\n        count = 0\n        for i in range(len(seq) - 1):\n            if seq[i:i+2] == dinuc:\n                count += 1\n        freq = count / (len(seq) - 1) if len(seq) > 1 else 0\n        features.append(freq)\n    \n    # 3. RNA secondary structure indicators\n    gc_content = (seq.count('G') + seq.count('C')) / len(seq) if len(seq) > 0 else 0\n    au_content = (seq.count('A') + seq.count('U')) / len(seq) if len(seq) > 0 else 0\n    purine_content = (seq.count('A') + seq.count('G')) / len(seq) if len(seq) > 0 else 0\n    pyrimidine_content = (seq.count('U') + seq.count('C')) / len(seq) if len(seq) > 0 else 0\n    \n    features.extend([gc_content, au_content, purine_content, pyrimidine_content])\n    \n    # 4. Sequence complexity measures\n    length_normalized = min(len(seq) / 1000.0, 1.0)  # Capped normalization\n    \n    # Simple entropy calculation\n    entropy = 0\n    for nuc in nucleotides:\n        freq = seq.count(nuc) / len(seq) if len(seq) > 0 else 0\n        if freq > 0:\n            entropy -= freq * np.log2(freq)\n    entropy_normalized = entropy / 2.0  # Max entropy for 4 nucleotides is 2\n    \n    features.extend([length_normalized, entropy_normalized])\n    \n    # 5. Repetitive pattern detection\n    repeat_content = _calculate_repeat_content(seq)\n    features.append(repeat_content)\n    \n    return features\n\ndef _calculate_repeat_content(sequence):\n    \"\"\"Calculate the proportion of repetitive content in the sequence.\"\"\"\n    if len(sequence) < 6:\n        return 0\n    \n    repeat_count = 0\n    window_size = 3\n    \n    for i in range(len(sequence) - window_size + 1):\n        motif = sequence[i:i + window_size]\n        # Look for the same motif in the rest of the sequence\n        for j in range(i + window_size, len(sequence) - window_size + 1):\n            if sequence[j:j + window_size] == motif:\n                repeat_count += 1\n                break\n    \n    return repeat_count / (len(sequence) - window_size + 1) if len(sequence) > window_size else 0","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def adaptive_rna_constraints(coordinates, sequence, confidence=1.0):\n    # Make a copy of coordinates to refine\n    refined_coords = coordinates.copy()\n    n_residues = len(sequence)\n    \n    # Calculate constraint strength (inverse of confidence)\n    # High confidence templates receive gentler constraints\n    constraint_strength = 0.8 * (1.0 - min(confidence, 0.8))\n    \n    # 1. Sequential distance constraints (consecutive nucleotides)\n    # More flexible distance range (statistical distribution from PDB)\n    seq_min_dist = 5.5  # Minimum sequential distance\n    seq_max_dist = 6.5  # Maximum sequential distance\n    \n    for i in range(n_residues - 1):\n        current_pos = refined_coords[i]\n        next_pos = refined_coords[i+1]\n        \n        # Calculate current distance\n        current_dist = np.linalg.norm(next_pos - current_pos)\n        \n        # Only adjust if significantly outside expected range\n        if current_dist < seq_min_dist or current_dist > seq_max_dist:\n            # Calculate target distance (midpoint of range)\n            target_dist = (seq_min_dist + seq_max_dist) / 2\n            \n            # Get direction vector\n            direction = next_pos - current_pos\n            direction = direction / (np.linalg.norm(direction) + 1e-10)\n            \n            # Apply partial adjustment based on constraint strength\n            adjustment = (target_dist - current_dist) * constraint_strength\n            \n            # Only adjust the next position to preserve the overall fold\n            refined_coords[i+1] = current_pos + direction * (current_dist + adjustment)\n    \n    # 2. Steric clash prevention (more conservative)\n    min_allowed_distance = 3.8  # Minimum distance between non-consecutive C1' atoms\n    \n    # Calculate all pairwise distances\n    dist_matrix = distance_matrix(refined_coords, refined_coords)\n    \n    # Find severe clashes (atoms too close)\n    severe_clashes = np.where((dist_matrix < min_allowed_distance) & (dist_matrix > 0))\n    \n    # Fix severe clashes\n    for idx in range(len(severe_clashes[0])):\n        i, j = severe_clashes[0][idx], severe_clashes[1][idx]\n        \n        # Skip consecutive nucleotides and previously processed pairs\n        if abs(i - j) <= 1 or i >= j:\n            continue\n            \n        # Get current positions and distance\n        pos_i = refined_coords[i]\n        pos_j = refined_coords[j]\n        current_dist = dist_matrix[i, j]\n        \n        # Calculate necessary adjustment but scale by constraint strength\n        direction = pos_j - pos_i\n        direction = direction / (np.linalg.norm(direction) + 1e-10)\n        \n        # Calculate partial adjustment\n        adjustment = (min_allowed_distance - current_dist) * constraint_strength\n        \n        # Move points apart\n        refined_coords[i] = pos_i - direction * (adjustment / 2)\n        refined_coords[j] = pos_j + direction * (adjustment / 2)\n    \n    # 3. Very light base-pair constraining (if confidence is low)\n    if constraint_strength > 0.3:  # Only apply if template confidence is low\n        # Simple Watson-Crick base pairs\n        pairs = {'A': 'U', 'U': 'A', 'G': 'C', 'C': 'G'}\n        \n        # Scan for potential base pairs\n        for i in range(n_residues):\n            base_i = sequence[i]\n            complement = pairs.get(base_i)\n            \n            if not complement:\n                continue\n                \n            # Look for complementary bases within a reasonable range\n            for j in range(i + 3, min(i + 20, n_residues)):\n                if sequence[j] == complement:\n                    # Calculate current distance\n                    current_dist = np.linalg.norm(refined_coords[i] - refined_coords[j])\n                    \n                    # Only consider if distance suggests potential pairing\n                    if 8.0 < current_dist < 14.0:\n                        # Target 10.5Å as generic base-pair C1'-C1' distance\n                        target_dist = 10.5\n                        \n                        # Calculate very gentle adjustment (scaled by constraint_strength)\n                        adjustment = (target_dist - current_dist) * (constraint_strength * 0.3)\n                        \n                        # Get direction vector\n                        direction = refined_coords[j] - refined_coords[i]\n                        direction = direction / (np.linalg.norm(direction) + 1e-10)\n                        \n                        # Apply very gentle adjustment to both positions\n                        refined_coords[i] = refined_coords[i] - direction * (adjustment / 2)\n                        refined_coords[j] = refined_coords[j] + direction * (adjustment / 2)\n                        \n                        # Only consider one potential pair per base (closest match)\n                        break\n    \n    return refined_coords","metadata":{"_uuid":"0d532205-a881-4262-86ec-36588b58b8f8","_cell_guid":"aaf3f747-0607-4470-a425-dbe714562373","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def adapt_template_to_query(query_seq, template_seq, template_coords, alignment=None):\n    if alignment is None:\n        from Bio.Seq import Seq\n        from Bio import pairwise2\n        \n        query_seq_obj = Seq(query_seq)\n        template_seq_obj = Seq(template_seq)\n        alignments = pairwise2.align.globalms(query_seq_obj, template_seq_obj, 2.9, -1, -10, -0.5, one_alignment_only=True)\n        \n        if not alignments:\n            return generate_improved_rna_structure(query_seq)\n            \n        alignment = alignments[0]\n    \n    aligned_query = alignment.seqA\n    aligned_template = alignment.seqB\n    \n    query_coords = np.zeros((len(query_seq), 3))\n    query_coords.fill(np.nan)\n    \n    # Map template coordinates to query\n    query_idx = 0\n    template_idx = 0\n    \n    for i in range(len(aligned_query)):\n        query_char = aligned_query[i]\n        template_char = aligned_template[i]\n        \n        if query_char != '-' and template_char != '-':\n            if template_idx < len(template_coords):\n                query_coords[query_idx] = template_coords[template_idx]\n            template_idx += 1\n            query_idx += 1\n        elif query_char != '-' and template_char == '-':\n            query_idx += 1\n        elif query_char == '-' and template_char != '-':\n            template_idx += 1\n    \n    # IMPROVED GAP FILLING - maintains RNA backbone geometry\n    backbone_distance = 5.9  # Typical C1'-C1' distance\n    \n    # Fill gaps by maintaining realistic backbone connectivity\n    for i in range(len(query_coords)):\n        if np.isnan(query_coords[i, 0]):\n            # Find nearest valid neighbors\n            prev_valid = next_valid = None\n            \n            for j in range(i-1, -1, -1):\n                if not np.isnan(query_coords[j, 0]):\n                    prev_valid = j\n                    break\n                    \n            for j in range(i+1, len(query_coords)):\n                if not np.isnan(query_coords[j, 0]):\n                    next_valid = j\n                    break\n            \n            if prev_valid is not None and next_valid is not None:\n                # Interpolate along realistic RNA backbone path\n                gap_size = next_valid - prev_valid\n                total_distance = np.linalg.norm(query_coords[next_valid] - query_coords[prev_valid])\n                expected_distance = gap_size * backbone_distance\n                \n                # If gap is compressed, extend it realistically\n                if total_distance < expected_distance * 0.7:\n                    direction = query_coords[next_valid] - query_coords[prev_valid]\n                    direction = direction / (np.linalg.norm(direction) + 1e-10)\n                    \n                    # Place intermediate points along extended path\n                    for k, idx in enumerate(range(prev_valid + 1, next_valid)):\n                        progress = (k + 1) / gap_size\n                        base_pos = query_coords[prev_valid] + direction * expected_distance * progress\n                        \n                        # Add slight curvature for realism\n                        perpendicular = np.cross(direction, [0, 0, 1])\n                        if np.linalg.norm(perpendicular) < 1e-6:\n                            perpendicular = np.cross(direction, [1, 0, 0])\n                        perpendicular = perpendicular / (np.linalg.norm(perpendicular) + 1e-10)\n                        \n                        curve_amplitude = 2.0 * np.sin(progress * np.pi)\n                        query_coords[idx] = base_pos + perpendicular * curve_amplitude\n                else:\n                    # Linear interpolation for normal gaps\n                    for k, idx in enumerate(range(prev_valid + 1, next_valid)):\n                        weight = (k + 1) / gap_size\n                        query_coords[idx] = (1 - weight) * query_coords[prev_valid] + weight * query_coords[next_valid]\n            \n            elif prev_valid is not None:\n                # Extend from previous position\n                if prev_valid > 0 and not np.isnan(query_coords[prev_valid-1, 0]):\n                    direction = query_coords[prev_valid] - query_coords[prev_valid-1]\n                    direction = direction / (np.linalg.norm(direction) + 1e-10)\n                else:\n                    direction = np.array([1.0, 0.0, 0.0])\n                \n                steps_needed = i - prev_valid\n                for step in range(1, steps_needed + 1):\n                    pos_idx = prev_valid + step\n                    if pos_idx < len(query_coords):\n                        query_coords[pos_idx] = query_coords[prev_valid] + direction * backbone_distance * step\n            \n            elif next_valid is not None:\n                # Work backwards from next position\n                direction = np.array([-1.0, 0.0, 0.0])  # Default backward direction\n                steps_needed = next_valid - i\n                for step in range(steps_needed, 0, -1):\n                    pos_idx = next_valid - step\n                    if pos_idx >= 0:\n                        query_coords[pos_idx] = query_coords[next_valid] - direction * backbone_distance * step\n    \n    # Final cleanup\n    query_coords = np.nan_to_num(query_coords)\n    return query_coords","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def generate_improved_rna_structure(sequence):\n    \"\"\"\n    Generate a more realistic RNA structure fallback based on sequence patterns\n    and basic RNA structure principles.\n    \n    Args:\n        sequence: RNA sequence string\n        \n    Returns:\n        Array of 3D coordinates\n    \"\"\"\n    n_residues = len(sequence)\n    coordinates = np.zeros((n_residues, 3))\n    \n    # Analyze sequence to predict structural elements\n    # Look for complementary regions that could form base pairs\n    potential_stems = identify_potential_stems(sequence)\n    \n    # Default parameters\n    radius_helix = 10.0\n    radius_loop = 15.0\n    rise_per_residue_helix = 2.5\n    rise_per_residue_loop = 1.5\n    angle_per_residue_helix = 0.6\n    angle_per_residue_loop = 0.3\n    \n    # Assign structural classifications\n    structure_types = assign_structure_types(sequence, potential_stems)\n    \n    # Generate coordinates based on predicted structure\n    current_pos = np.array([0.0, 0.0, 0.0])\n    current_direction = np.array([0.0, 0.0, 1.0])\n    current_angle = 0.0\n    \n    for i in range(n_residues):\n        if structure_types[i] == 'stem':\n            # Part of a helical stem\n            current_angle += angle_per_residue_helix\n            coordinates[i] = [\n                radius_helix * np.cos(current_angle), \n                radius_helix * np.sin(current_angle), \n                current_pos[2] + rise_per_residue_helix\n            ]\n            current_pos = coordinates[i]\n        elif structure_types[i] == 'loop':\n            # Part of a loop\n            current_angle += angle_per_residue_loop\n            z_shift = rise_per_residue_loop * np.sin(current_angle * 0.5)\n            coordinates[i] = [\n                radius_loop * np.cos(current_angle), \n                radius_loop * np.sin(current_angle), \n                current_pos[2] + z_shift\n            ]\n            current_pos = coordinates[i]\n        else:\n            # Single-stranded region\n            # Add some randomness to make it look more realistic\n            jitter = np.random.normal(0, 1, 3) * 2.0\n            coordinates[i] = current_pos + jitter\n            current_pos = coordinates[i]\n            \n    return coordinates\n\ndef identify_potential_stems(sequence):\n    \"\"\"\n    Identify potential stem regions by looking for self-complementary segments.\n    \n    Args:\n        sequence: RNA sequence string\n        \n    Returns:\n        List of tuples (start1, end1, start2, end2) representing potentially paired regions\n    \"\"\"\n    complementary_bases = {'A': 'U', 'U': 'A', 'G': 'C', 'C': 'G'}\n    min_stem_length = 3\n    potential_stems = []\n    \n    # Simple stem identification\n    for i in range(len(sequence) - min_stem_length):\n        for j in range(i + min_stem_length + 3, len(sequence) - min_stem_length + 1):\n            # Check if regions could form a stem\n            potential_stem_len = min(min_stem_length, len(sequence) - j)\n            is_stem = True\n            \n            for k in range(potential_stem_len):\n                if sequence[i+k] not in complementary_bases or \\\n                   complementary_bases[sequence[i+k]] != sequence[j+potential_stem_len-k-1]:\n                    is_stem = False\n                    break\n            \n            if is_stem:\n                potential_stems.append((i, i+potential_stem_len-1, j, j+potential_stem_len-1))\n    \n    return potential_stems\n\ndef assign_structure_types(sequence, potential_stems):\n    \"\"\"\n    Assign each nucleotide to a structural element type.\n    \n    Args:\n        sequence: RNA sequence string\n        potential_stems: List of tuples representing stem regions\n        \n    Returns:\n        List of structure types ('stem', 'loop', 'single')\n    \"\"\"\n    structure_types = ['single'] * len(sequence)\n    \n    # Mark stem regions\n    for stem in potential_stems:\n        start1, end1, start2, end2 = stem\n        for i in range(end1 - start1 + 1):\n            structure_types[start1 + i] = 'stem'\n            structure_types[end2 - i] = 'stem'\n    \n    # Mark loop regions (regions between paired regions)\n    for i in range(len(potential_stems) - 1):\n        _, end1, start2, _ = potential_stems[i]\n        next_start1, _, _, _ = potential_stems[i+1]\n        \n        if next_start1 > end1 + 1 and start2 > next_start1:\n            for j in range(end1 + 1, next_start1):\n                structure_types[j] = 'loop'\n    \n    return structure_types","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to create a more realistic RNA structure when no good templates are found\ndef generate_rna_structure(sequence, seed=None):\n    if seed is not None:\n        np.random.seed(seed)\n        random.seed(seed)\n    \n    n_residues = len(sequence)\n    coordinates = np.zeros((n_residues, 3))\n    \n    # Initialize the first few residues in a helix\n    for i in range(min(3, n_residues)):\n        angle = i * 0.6\n        coordinates[i] = [10.0 * np.cos(angle), 10.0 * np.sin(angle), i * 2.5]\n    \n    # Add more complex folding patterns\n    current_direction = np.array([0.0, 0.0, 1.0])  # Start moving along z-axis\n    \n    # Define base-pairing tendencies (G-C and A-U pairs)\n    for i in range(3, n_residues):\n        # Check for potential base-pairing in the sequence\n        has_pair = False\n        pair_idx = -1\n        \n        # Simple detection of complementary bases (G-C, A-U)\n        complementary = {'G': 'C', 'C': 'G', 'A': 'U', 'U': 'A'}\n        current_base = sequence[i]\n        \n        # Look for potential base-pairing within a window before the current position\n        window_size = min(i, 15)  # Look back up to 15 bases\n        for j in range(i-window_size, i):\n            if j >= 0 and sequence[j] == complementary.get(current_base, 'X'):\n                # Found a potential pair\n                has_pair = True\n                pair_idx = j\n                break\n        \n        if has_pair and i - pair_idx <= 10 and random.random() < 0.7:\n            # Try to create a base-pair by positioning this nucleotide near its pair\n            pair_pos = coordinates[pair_idx]\n            \n            # Create a position that's roughly opposite to the pair\n            random_offset = np.random.normal(0, 1, 3) * 2.0\n            base_pair_distance = 10.0 + random.uniform(-1.0, 1.0)\n            \n            # Calculate a vector from base-pair toward center of structure\n            center = np.mean(coordinates[:i], axis=0)\n            direction = center - pair_pos\n            direction = direction / (np.linalg.norm(direction) + 1e-10)\n            \n            # Position new nucleotide in the general direction of the \"center\"\n            coordinates[i] = pair_pos + direction * base_pair_distance + random_offset\n            \n            # Update direction for next nucleotide\n            current_direction = np.random.normal(0, 0.3, 3)\n            current_direction = current_direction / (np.linalg.norm(current_direction) + 1e-10)\n            \n        else:\n            # No base-pairing detected, continue with the current fold direction\n            # Randomly rotate current direction to simulate RNA flexibility\n            if random.random() < 0.3:\n                # More significant direction change\n                angle = random.uniform(0.2, 0.6)\n                axis = np.random.normal(0, 1, 3)\n                axis = axis / (np.linalg.norm(axis) + 1e-10)\n                rotation = R.from_rotvec(angle * axis)\n                current_direction = rotation.apply(current_direction)\n            else:\n                # Small random changes in direction\n                current_direction += np.random.normal(0, 0.15, 3)\n                current_direction = current_direction / (np.linalg.norm(current_direction) + 1e-10)\n            \n            # Distance between consecutive nucleotides (3.5-4.5Å is typical)\n            step_size = random.uniform(3.5, 4.5)\n            \n            # Update position\n            coordinates[i] = coordinates[i-1] + step_size * current_direction\n    \n    return coordinates","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict_rna_structures(sequence, target_id, train_seqs_df, train_coords_dict, n_predictions=5):\n    predictions = []\n    \n    # Find similar sequences in the training data\n    similar_seqs = find_similar_sequences(sequence, train_seqs_df, train_coords_dict, top_n=n_predictions)\n    \n    # If we found any similar sequences, use them as templates\n    if similar_seqs:\n        for i, (template_id, template_seq, similarity_score, template_coords) in enumerate(similar_seqs):\n            # Adapt template coordinates to the query sequence\n            adapted_coords = adapt_template_to_query(sequence, template_seq, template_coords)\n            \n            if adapted_coords is not None:\n                # Apply adaptive constraints based on template similarity\n                # For high similarity templates, apply very gentle constraints\n                refined_coords = adaptive_rna_constraints(adapted_coords, sequence, confidence=similarity_score)\n                \n                # Add some randomness (less for better templates)\n                random_scale = max(0.05, 0.8 - similarity_score)  # Reduced randomness\n                randomized_coords = refined_coords.copy()\n                randomized_coords += np.random.normal(0, random_scale, randomized_coords.shape)\n                \n                predictions.append(randomized_coords)\n                \n                if len(predictions) >= n_predictions:\n                    break\n    \n    # If we don't have enough predictions from templates, generate de novo structures\n    while len(predictions) < n_predictions:\n        seed_value = hash(target_id) % 10000 + len(predictions) * 1000\n        de_novo_coords = generate_rna_structure(sequence, seed=seed_value)\n        \n        # Apply stronger constraints to de novo structures (lower confidence)\n        refined_de_novo = adaptive_rna_constraints(de_novo_coords, sequence, confidence=0.2)\n        \n        predictions.append(refined_de_novo)\n    \n    return predictions[:n_predictions]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# List to store all prediction records\nall_predictions = []\n\n# Set up time tracking\nstart_time = time.time()\ntotal_targets = len(test_seqs)\n\n# For each sequence in the test set\nfor idx, row in test_seqs.iterrows():\n    target_id = row['target_id']\n    sequence = row['sequence']\n    \n    # Progress tracking\n    if idx % 5 == 0:\n        elapsed = time.time() - start_time\n        targets_processed = idx + 1\n        if targets_processed > 0:\n            avg_time_per_target = elapsed / targets_processed\n            est_time_remaining = avg_time_per_target * (total_targets - targets_processed)\n            print(f\"Processing target {targets_processed}/{total_targets}: {target_id} ({len(sequence)} nt), \"\n                  f\"elapsed: {elapsed:.1f}s, est. remaining: {est_time_remaining:.1f}s\")\n    \n    # Generate 5 different structure predictions\n    predictions = predict_rna_structures(sequence, target_id, train_seqs_extended, train_coords_dict, n_predictions=5)\n    \n    # For each residue in the sequence\n    for j in range(len(sequence)):\n        pred_row = {\n            'ID': f\"{target_id}_{j+1}\",\n            'resname': sequence[j],\n            'resid': j + 1\n        }\n        \n        # Add coordinates from all 5 predictions\n        for i in range(5):\n            pred_row[f'x_{i+1}'] = predictions[i][j][0]\n            pred_row[f'y_{i+1}'] = predictions[i][j][1]\n            pred_row[f'z_{i+1}'] = predictions[i][j][2]\n        \n        all_predictions.append(pred_row)\n\n# Create DataFrame with predictions\nsubmission_df = pd.DataFrame(all_predictions)\n\n# Ensure the submission file has the correct format\ncolumn_order = ['ID', 'resname', 'resid']\nfor i in range(1, 6):\n    for coord in ['x', 'y', 'z']:\n        column_order.append(f'{coord}_{i}')\nsubmission_df = submission_df[column_order]\n\n# Save the submission file\nsubmission_df.to_csv('submission.csv', index=False)\nprint(f\"Generated predictions for {len(test_seqs)} RNA sequences\")\nprint(f\"Total runtime: {time.time() - start_time:.1f} seconds\")","metadata":{"trusted":true,"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}