{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":87793,"databundleVersionId":11403143,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":44.614522,"end_time":"2025-03-16T03:39:17.34772","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-03-16T03:38:32.733198","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## RNA3D Data Pipeline","metadata":{"papermill":{"duration":0.008439,"end_time":"2025-03-16T03:38:35.617034","exception":false,"start_time":"2025-03-16T03:38:35.608595","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Credit goes to this notebook and author --> https://www.kaggle.com/code/fernandosr85/rna-3d-structure\n","metadata":{}},{"cell_type":"code","source":"import os\nimport time\nimport gc\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport warnings\nimport os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport traceback  # Add this import\nfrom sklearn.model_selection import train_test_split\n\n# Inicializando a semente para controlar a aleatoriedade\nnp.random.seed(0)\n\n# Diretórios e arquivos ajustados para a nova competição\nDATA_DIR = os.getenv('DATA_DIR', '/kaggle/input/stanford-rna-3d-folding/')\nmain_files = [\n    \"train_sequences.csv\", \n    \"train_labels.csv\", \n    \"validation_sequences.csv\", \n    \"validation_labels.csv\", \n    \"test_sequences.csv\",\n    \"sample_submission.csv\"\n]\n\nDEFAULT_THRESHOLD = 0.45# Limiar padrão após análise\n\ndef optimize_dataframe(df, inplace=False, category_threshold=DEFAULT_THRESHOLD):\n    \"\"\"\n    Otimiza o DataFrame para economizar memória.\n    \"\"\"\n    if category_threshold < 0 or category_threshold > 1:\n        raise ValueError(\"category_threshold deve estar entre 0 e 1.\")\n    \n    if not inplace:\n        df = df.copy()\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if np.issubdtype(col_type, np.integer):\n            c_min, c_max = df[col].min(), df[col].max()\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                df[col] = df[col].astype(np.int8)\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                df[col] = df[col].astype(np.int16)\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                df[col] = df[col].astype(np.int32)\n        elif np.issubdtype(col_type, np.floating):\n            if df[col].min() > np.finfo(np.float32).min and df[col].max() < np.finfo(np.float32).max:\n                df[col] = df[col].astype(np.float32)\n        if col_type == object:\n            unique_vals = len(df[col].unique())\n            if unique_vals / len(df) < category_threshold:\n                df[col] = df[col].astype('category')\n    \n    return df\n\ndef load_main_data(chunksize=50000):\n    \"\"\"\n    Carrega os arquivos principais.\n    \"\"\"\n    data = {}\n    for file_name in main_files:\n        file_path = os.path.join(DATA_DIR, file_name)\n        if os.path.exists(file_path):\n            chunks = pd.read_csv(file_path, on_bad_lines='skip', low_memory=False, chunksize=chunksize)\n            dataframes = [optimize_dataframe(chunk, category_threshold=DEFAULT_THRESHOLD) for chunk in chunks]\n            data[file_name] = pd.concat(dataframes, ignore_index=True)\n        else:\n            print(f\"Arquivo {file_path} não encontrado!\")\n    return data\n\ndef check_data_integrity(original_df, optimized_df):\n    \"\"\"\n    Verifica se a otimização não alterou os dados.\n    \"\"\"\n    try:\n        pd.testing.assert_frame_equal(original_df, optimized_df, check_like=True)\n        print(\"Integrity check passed: No changes in data after optimization.\")\n    except AssertionError as e:\n        print(f\"Data integrity check failed: {e}\")\n\ndef check_duplicates(df):\n    \"\"\"\n    Verifica duplicatas no DataFrame.\n    \"\"\"\n    duplicates = df[df.duplicated(keep=False)]\n    if not duplicates.empty:\n        print(f\"Warning: Duplicates found in the dataset. Number of duplicates: {duplicates.shape[0]}\")\n        return duplicates\n    else:\n        print(\"No duplicates found.\")\n    return None\n\ndef test_thresholds(df):\n    \"\"\"\n    Testa diferentes limiares para otimização do DataFrame.\n    \"\"\"\n    thresholds = np.linspace(0.1, 0.9, 9)\n    memory_usages = []\n    for threshold in thresholds:\n        optimized_df = optimize_dataframe(df.copy(), category_threshold=threshold)\n        memory_usages.append(optimized_df.memory_usage(deep=True).sum() / 1024**2)\n    return thresholds, memory_usages\n\ndef plot_memory_usage(thresholds, memory_usages):\n    \"\"\"\n    Plota o uso de memória versus limiares.\n    \"\"\"\n    plt.figure(figsize=(10, 6))\n    plt.plot(thresholds, memory_usages, marker='o', linestyle='-')\n    plt.title(\"Memory Usage vs. Threshold\")\n    plt.xlabel(\"Threshold\")\n    plt.ylabel(\"Memory Usage (MB)\")\n    plt.grid(True)\n    plt.show()\n\ndef analyze_sequence_data(df_sequences):\n    \"\"\"\n    Analisa os dados de sequências RNA.\n    \"\"\"\n    # Informações básicas\n    print(f\"Total de sequências: {len(df_sequences)}\")\n    print(f\"Colunas disponíveis: {df_sequences.columns.tolist()}\")\n    \n    # Análise das sequências\n    if 'sequence' in df_sequences.columns:\n        # Distribuição de tamanho das sequências\n        seq_lengths = df_sequences['sequence'].apply(len)\n        print(f\"\\nEstatísticas de comprimento das sequências:\")\n        print(f\"Mínimo: {seq_lengths.min()}\")\n        print(f\"Máximo: {seq_lengths.max()}\")\n        print(f\"Média: {seq_lengths.mean():.2f}\")\n        \n        # Contagem de nucleotídeos\n        nucleotides = ['A', 'C', 'G', 'U']\n        nucleotide_counts = {n: df_sequences['sequence'].str.count(n).sum() for n in nucleotides}\n        total_nucleotides = sum(nucleotide_counts.values())\n        \n        print(\"\\nDistribuição de nucleotídeos:\")\n        for n, count in nucleotide_counts.items():\n            print(f\"{n}: {count} ({count/total_nucleotides*100:.2f}%)\")\n    \n    return df_sequences\n\ndef analyze_label_data(df_labels):\n    \"\"\"\n    Analisa os dados de coordenadas 3D (labels).\n    \"\"\"\n    print(f\"Total de entradas nas labels: {len(df_labels)}\")\n    print(f\"Colunas disponíveis: {df_labels.columns.tolist()}\")\n    \n    # Análise das coordenadas 3D se disponíveis\n    coord_columns = [col for col in df_labels.columns if col.startswith(('x_', 'y_', 'z_'))]\n    if coord_columns:\n        print(f\"\\nColunas de coordenadas encontradas: {len(coord_columns)}\")\n        \n        # Estatísticas básicas das coordenadas\n        for i in range(1, 6):  # Para as 5 estruturas possíveis\n            x_col = f'x_{i}'\n            y_col = f'y_{i}'\n            z_col = f'z_{i}'\n            \n            if x_col in df_labels.columns and y_col in df_labels.columns and z_col in df_labels.columns:\n                print(f\"\\nEstatísticas para estrutura {i}:\")\n                print(f\"X - Média: {df_labels[x_col].mean():.2f}, Desvio: {df_labels[x_col].std():.2f}\")\n                print(f\"Y - Média: {df_labels[y_col].mean():.2f}, Desvio: {df_labels[y_col].std():.2f}\")\n                print(f\"Z - Média: {df_labels[z_col].mean():.2f}, Desvio: {df_labels[z_col].std():.2f}\")\n    \n    return df_labels\n\ndef create_submission_template(test_df, sample_submission_df):\n    \"\"\"\n    Cria um template para submissão com base nos dados de teste.\n    \"\"\"\n    # Verifica se o arquivo sample_submission.csv está disponível\n    if sample_submission_df is None:\n        print(\"Arquivo de submissão de exemplo não encontrado. Criando um novo template.\")\n        \n        # Cria um novo DataFrame para submissão\n        submission_df = pd.DataFrame()\n        \n        # Exemplo de código para preencher o template (ajustar conforme necessário)\n        ids = []\n        resnames = []\n        resids = []\n        \n        for _, row in test_df.iterrows():\n            sequence = row['sequence']\n            target_id = row['target_id']\n            \n            for i, nucleotide in enumerate(sequence, 1):\n                ids.append(f\"{target_id}_{i}\")\n                resnames.append(nucleotide)\n                resids.append(i)\n        \n        submission_df['ID'] = ids\n        submission_df['resname'] = resnames\n        submission_df['resid'] = resids\n        \n        # Adiciona colunas de coordenadas (5 estruturas)\n        for i in range(1, 6):\n            submission_df[f'x_{i}'] = 0.0\n            submission_df[f'y_{i}'] = 0.0\n            submission_df[f'z_{i}'] = 0.0\n    else:\n        submission_df = sample_submission_df.copy()\n        print(\"Template de submissão criado com base no exemplo fornecido.\")\n    \n    return submission_df\n\ndef main():\n    start_time = time.time()\n    \n    # Carregar dados principais\n    print(\"Carregando dados principais...\")\n    main_data = load_main_data()\n    \n    # Verificar quais arquivos foram carregados\n    print(\"\\nArquivos carregados:\")\n    for file_name, df in main_data.items():\n        print(f\"- {file_name}: {df.shape if df is not None else 'Não encontrado'}\")\n    \n    # Analisar dados de sequências de treinamento\n    if \"train_sequences.csv\" in main_data:\n        print(\"\\n===== Análise das Sequências de Treinamento =====\")\n        analyze_sequence_data(main_data[\"train_sequences.csv\"])\n    \n    # Analisar dados de labels de treinamento\n    if \"train_labels.csv\" in main_data:\n        print(\"\\n===== Análise das Labels de Treinamento =====\")\n        analyze_label_data(main_data[\"train_labels.csv\"])\n    \n    # Verificação de duplicatas nos dados de treinamento\n    if \"train_sequences.csv\" in main_data:\n        print(\"\\nVerificando duplicatas nas sequências de treinamento...\")\n        check_duplicates(main_data[\"train_sequences.csv\"])\n    \n    # Criar template para submissão\n    if \"test_sequences.csv\" in main_data:\n        print(\"\\nCriando template para submissão...\")\n        submission_template = create_submission_template(\n            main_data[\"test_sequences.csv\"],\n            main_data.get(\"sample_submission.csv\")\n        )\n        print(f\"Shape do template de submissão: {submission_template.shape}\")\n        print(f\"Primeiras linhas do template de submissão:\")\n        print(submission_template.head())\n    \n    # Calcular tempo de execução\n    end_time = time.time()\n    print(f\"\\nRuntime: {end_time - start_time:.2f} seconds\")\n    \n    return main_data\n\nif __name__ == '__main__':\n    main_data = main()","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2025-03-18T03:15:38.792956Z","iopub.execute_input":"2025-03-18T03:15:38.793303Z","iopub.status.idle":"2025-03-18T03:15:56.218536Z","shell.execute_reply.started":"2025-03-18T03:15:38.793273Z","shell.execute_reply":"2025-03-18T03:15:56.217585Z"},"papermill":{"duration":1.773557,"end_time":"2025-03-16T03:38:37.398516","exception":false,"start_time":"2025-03-16T03:38:35.624959","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Explorador de Diretórios e Verificação de CSV RNA3D","metadata":{"papermill":{"duration":0.006493,"end_time":"2025-03-16T03:38:37.412261","exception":false,"start_time":"2025-03-16T03:38:37.405768","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\n\n# Diretório principal atualizado\ndir_main = \"/kaggle/input/stanford-rna-3d-folding/\"\n\n# Lista todos os arquivos e diretórios no diretório principal\ntry:\n    todos_arquivos = os.listdir(dir_main)\n    print(f\"Todos os arquivos e diretórios em '{dir_main}':\")\n    for arquivo in todos_arquivos:\n        # Verifica se é um arquivo ou diretório\n        path_completo = os.path.join(dir_main, arquivo)\n        tipo = \"diretório\" if os.path.isdir(path_completo) else \"arquivo\"\n        tamanho = os.path.getsize(path_completo) / 1024  # Tamanho em KB\n        print(f\" - {arquivo} ({tipo}, {tamanho:.2f} KB)\")\n        \n        # Se for um diretório, lista até 5 arquivos dentro dele\n        if os.path.isdir(path_completo):\n            try:\n                arquivos_internos = os.listdir(path_completo)[:5]  # Limita a 5 arquivos\n                if arquivos_internos:\n                    print(f\"   Primeiros arquivos em '{arquivo}':\")\n                    for arq_interno in arquivos_internos:\n                        print(f\"    * {arq_interno}\")\n                    if len(os.listdir(path_completo)) > 5:\n                        print(f\"    * ... e mais {len(os.listdir(path_completo)) - 5} arquivo(s)\")\n                else:\n                    print(f\"   '{arquivo}' está vazio\")\n            except Exception as e:\n                print(f\"   Erro ao listar conteúdo de '{arquivo}': {e}\")\n\nexcept Exception as e:\n    print(f\"Erro ao listar o diretório {dir_main}: {e}\")\n\n# Verifica a estrutura dos principais arquivos CSV\nprincipais_arquivos = [\n    \"train_sequences.csv\", \n    \"train_labels.csv\", \n    \"validation_sequences.csv\", \n    \"validation_labels.csv\", \n    \"test_sequences.csv\",\n    \"sample_submission.csv\"\n]\n\nprint(\"\\nVerificando os principais arquivos CSV:\")\nfor arquivo in principais_arquivos:\n    path_completo = os.path.join(dir_main, arquivo)\n    if os.path.exists(path_completo):\n        # Obtém o tamanho do arquivo\n        tamanho_mb = os.path.getsize(path_completo) / (1024 * 1024)  # Tamanho em MB\n        \n        # Lê as primeiras linhas para verificar a estrutura\n        try:\n            import pandas as pd\n            df = pd.read_csv(path_completo, nrows=1)\n            print(f\"\\n{arquivo} ({tamanho_mb:.2f} MB):\")\n            print(f\"Colunas: {df.columns.tolist()}\")\n            print(f\"Exemplo:\")\n            print(df.head())\n        except Exception as e:\n            print(f\"Erro ao ler {arquivo}: {e}\")\n    else:\n        print(f\"{arquivo} não encontrado.\")","metadata":{"execution":{"iopub.status.busy":"2025-03-18T03:15:56.219822Z","iopub.execute_input":"2025-03-18T03:15:56.220203Z","iopub.status.idle":"2025-03-18T03:15:56.3163Z","shell.execute_reply.started":"2025-03-18T03:15:56.22016Z","shell.execute_reply":"2025-03-18T03:15:56.315219Z"},"papermill":{"duration":0.13333,"end_time":"2025-03-16T03:38:37.552345","exception":false,"start_time":"2025-03-16T03:38:37.419015","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Checador de Dados RNA3D","metadata":{"papermill":{"duration":0.00698,"end_time":"2025-03-16T03:38:37.566945","exception":false,"start_time":"2025-03-16T03:38:37.559965","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Diretório principal atualizado\ndir_main = \"/kaggle/input/stanford-rna-3d-folding/\"\n\ndef load_data():\n    \"\"\"\n    Carrega os principais arquivos CSV da competição Stanford RNA 3D Folding.\n    Retorna um dicionário com os DataFrames.\n    \"\"\"\n    main_files = [\n        \"train_sequences.csv\", \n        \"train_labels.csv\", \n        \"validation_sequences.csv\", \n        \"validation_labels.csv\", \n        \"test_sequences.csv\",\n        \"sample_submission.csv\"\n    ]\n    \n    data = {}\n    for file_name in main_files:\n        file_path = os.path.join(dir_main, file_name)\n        if os.path.exists(file_path):\n            try:\n                data[file_name] = pd.read_csv(file_path)\n                print(f\"Arquivo {file_name} carregado com sucesso. Shape: {data[file_name].shape}\")\n            except Exception as e:\n                print(f\"Erro ao carregar {file_name}: {e}\")\n        else:\n            print(f\"Arquivo {file_name} não encontrado.\")\n            data[file_name] = None\n    \n    return data\n\ndef compare_columns(main_data):\n    \"\"\"\n    Compara as colunas entre diferentes DataFrames.\n    \"\"\"\n    # Listar todas as chaves disponíveis\n    print(\"\\nArquivos carregados:\")\n    print(list(main_data.keys()))\n    \n    # Comparar colunas entre train_sequences.csv e test_sequences.csv\n    if \"train_sequences.csv\" in main_data and \"test_sequences.csv\" in main_data:\n        train_cols = set(main_data[\"train_sequences.csv\"].columns)\n        test_cols = set(main_data[\"test_sequences.csv\"].columns)\n        \n        print(\"\\nColunas em train_sequences.csv:\")\n        print(list(main_data[\"train_sequences.csv\"].columns))\n        \n        print(\"\\nColunas exclusivas em train_sequences.csv (não presentes em test_sequences.csv):\")\n        print(train_cols - test_cols)\n        \n        print(\"\\nColunas exclusivas em test_sequences.csv (não presentes em train_sequences.csv):\")\n        print(test_cols - train_cols)\n    \n    # Comparar colunas entre train_labels.csv e validation_labels.csv\n    if \"train_labels.csv\" in main_data and \"validation_labels.csv\" in main_data:\n        train_label_cols = set(main_data[\"train_labels.csv\"].columns)\n        val_label_cols = set(main_data[\"validation_labels.csv\"].columns)\n        \n        print(\"\\nColunas em train_labels.csv:\")\n        print(list(main_data[\"train_labels.csv\"].columns))\n        \n        print(\"\\nColunas em validation_labels.csv:\")\n        print(list(main_data[\"validation_labels.csv\"].columns))\n        \n        print(\"\\nColunas exclusivas em validation_labels.csv (não presentes em train_labels.csv):\")\n        print(val_label_cols - train_label_cols)\n    \n    # Comparar colunas entre validation_labels.csv e sample_submission.csv\n    if \"validation_labels.csv\" in main_data and \"sample_submission.csv\" in main_data:\n        val_label_cols = set(main_data[\"validation_labels.csv\"].columns)\n        sample_cols = set(main_data[\"sample_submission.csv\"].columns)\n        \n        print(\"\\nColunas em sample_submission.csv:\")\n        print(list(main_data[\"sample_submission.csv\"].columns))\n        \n        print(\"\\nColunas exclusivas em validation_labels.csv (não presentes em sample_submission.csv):\")\n        print(val_label_cols - sample_cols)\n        \n        print(\"\\nColunas exclusivas em sample_submission.csv (não presentes em validation_labels.csv):\")\n        print(sample_cols - val_label_cols)\n\ndef analyze_structure_format(main_data):\n    \"\"\"\n    Analisa o formato das estruturas 3D (coordenadas).\n    \"\"\"\n    if \"validation_labels.csv\" in main_data and main_data[\"validation_labels.csv\"] is not None:\n        df = main_data[\"validation_labels.csv\"]\n        \n        # Encontrar todas as colunas de coordenadas (x_1, y_1, z_1, etc.)\n        coord_cols = [col for col in df.columns if col.startswith(('x_', 'y_', 'z_'))]\n        \n        # Agrupar por estrutura\n        structures = {}\n        for col in coord_cols:\n            # Extrair número da estrutura (e.g., \"x_1\" -> 1)\n            parts = col.split('_')\n            if len(parts) == 2:\n                struct_num = int(parts[1])\n                coord_type = parts[0]\n                \n                if struct_num not in structures:\n                    structures[struct_num] = []\n                \n                structures[struct_num].append(col)\n        \n        print(\"\\nEstrutura do arquivo de labels:\")\n        print(f\"Total de estruturas encontradas: {len(structures)}\")\n        \n        # Mostrar detalhes da primeira estrutura\n        if structures:\n            first_struct = min(structures.keys())\n            print(f\"\\nDetalhe da estrutura {first_struct}:\")\n            print(f\"Colunas: {sorted(structures[first_struct])}\")\n            \n            # Verificar se há valores ausentes\n            for col in structures[first_struct]:\n                missing = df[col].isna().sum()\n                total = len(df)\n                print(f\"{col}: {missing} valores ausentes ({missing/total*100:.2f}%)\")\n            \n            # Verificar o range dos valores não-ausentes para a primeira estrutura\n            for col in structures[first_struct]:\n                non_null = df[col][df[col] != -1.0e+18]  # Valores que não são -1.0e+18\n                if not non_null.empty:\n                    print(f\"{col} - Range: [{non_null.min():.3f}, {non_null.max():.3f}]\")\n\ndef main():\n    # Carregar os dados\n    main_data = load_data()\n    \n    # Comparar colunas entre diferentes arquivos\n    compare_columns(main_data)\n    \n    # Analisar o formato das estruturas 3D\n    analyze_structure_format(main_data)\n    \n    return main_data\n\nif __name__ == '__main__':\n    main_data = main()","metadata":{"execution":{"iopub.status.busy":"2025-03-18T03:15:56.318432Z","iopub.execute_input":"2025-03-18T03:15:56.318707Z","iopub.status.idle":"2025-03-18T03:15:56.658032Z","shell.execute_reply.started":"2025-03-18T03:15:56.318685Z","shell.execute_reply":"2025-03-18T03:15:56.656862Z"},"papermill":{"duration":0.397916,"end_time":"2025-03-16T03:38:37.972088","exception":false,"start_time":"2025-03-16T03:38:37.574172","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Analisador Integrado de Sequências e Estruturas RNA3D","metadata":{"papermill":{"duration":0.006997,"end_time":"2025-03-16T03:38:37.98682","exception":false,"start_time":"2025-03-16T03:38:37.979823","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom collections import Counter\nimport gc\n\n# Inicializando a semente para controlar a aleatoriedade\nnp.random.seed(0)\n\n# Diretórios e arquivos ajustados para a nova competição\nDATA_DIR = os.getenv('DATA_DIR', '/kaggle/input/stanford-rna-3d-folding/')\nmain_files = [\n    \"train_sequences.csv\", \n    \"train_labels.csv\", \n    \"validation_sequences.csv\", \n    \"validation_labels.csv\", \n    \"test_sequences.csv\",\n    \"sample_submission.csv\"\n]\n\nDEFAULT_THRESHOLD = 0.4  # Limiar padrão após análise\n\ndef optimize_dataframe(df, inplace=False, category_threshold=DEFAULT_THRESHOLD):\n    \"\"\"\n    Otimiza o DataFrame para economizar memória.\n    \"\"\"\n    if category_threshold < 0 or category_threshold > 1:\n        raise ValueError(\"category_threshold deve estar entre 0 e 1.\")\n    \n    if not inplace:\n        df = df.copy()\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if np.issubdtype(col_type, np.integer):\n            c_min, c_max = df[col].min(), df[col].max()\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                df[col] = df[col].astype(np.int8)\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                df[col] = df[col].astype(np.int16)\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                df[col] = df[col].astype(np.int32)\n        elif np.issubdtype(col_type, np.floating):\n            # Verificar primeiro se não é o valor especial -1.0e+18\n            if df[col].min() > np.finfo(np.float32).min and df[col].max() < np.finfo(np.float32).max:\n                df[col] = df[col].astype(np.float32)\n        if col_type == object:\n            unique_vals = len(df[col].unique())\n            if unique_vals / len(df) < category_threshold:\n                df[col] = df[col].astype('category')\n    \n    return df\n\ndef load_main_data(chunksize=50000):\n    \"\"\"\n    Carrega os arquivos principais.\n    \"\"\"\n    data = {}\n    for file_name in main_files:\n        file_path = os.path.join(DATA_DIR, file_name)\n        if os.path.exists(file_path):\n            chunks = pd.read_csv(file_path, on_bad_lines='skip', low_memory=False, chunksize=chunksize)\n            dataframes = [optimize_dataframe(chunk, category_threshold=DEFAULT_THRESHOLD) for chunk in chunks]\n            data[file_name] = pd.concat(dataframes, ignore_index=True)\n            print(f\"Arquivo {file_name} carregado com sucesso. Shape: {data[file_name].shape}\")\n        else:\n            print(f\"Arquivo {file_path} não encontrado!\")\n            data[file_name] = None\n    return data\n\ndef filter_columns_by_prefix(df, prefix=\"x_\"):\n    \"\"\"\n    Filtra e conta o número de colunas em um DataFrame com base em um prefixo fornecido.\n    \n    :param df: DataFrame onde a filtragem será aplicada.\n    :param prefix: Prefixo a ser usado para a filtragem. Ex: \"x_\", \"y_\", \"z_\".\n    :return: Lista das colunas filtradas.\n    \"\"\"\n    filtered_columns = [col for col in df.columns if col.startswith(prefix)]\n    return filtered_columns\n\ndef count_nucleotides(df, column_name='sequence'):\n    \"\"\"\n    Conta a frequência de cada nucleotídeo em uma coluna específica de um DataFrame.\n    \n    :param df: DataFrame que contém as sequências.\n    :param column_name: Nome da coluna que contém as sequências. Padrão é 'sequence'.\n    :return: Counter object com a contagem dos nucleotídeos.\n    \"\"\"\n    from collections import Counter\n\n    # Verifica se a coluna existe no DataFrame\n    if column_name not in df.columns:\n        raise ValueError(f\"Column '{column_name}' not found in DataFrame.\")\n    \n    # Concatena todas as sequências e conta os nucleotídeos\n    all_sequences = ''.join(df[column_name].tolist())\n    nucleotide_counts = Counter(all_sequences)\n    \n    return nucleotide_counts\n\ndef get_columns_without_missing_values(df):\n    \"\"\"\n    Retorna colunas sem nenhum valor ausente no DataFrame.\n    \n    :param df: DataFrame a ser verificado.\n    :return: Lista de colunas sem valores ausentes.\n    \"\"\"\n    missing_values = df.isnull().sum()\n    return missing_values[missing_values == 0].index.tolist()\n\ndef get_empty_columns(df):\n    \"\"\"\n    Retorna colunas que estão completamente vazias no DataFrame.\n    \n    :param df: DataFrame a ser verificado.\n    :return: Lista de colunas vazias.\n    \"\"\"\n    missing_values = df.isnull().sum()\n    return missing_values[missing_values == df.shape[0]].index.tolist()\n\ndef plot_coord_distributions(df_labels, prefix='x_', max_structures=5):\n    \"\"\"\n    Plota a distribuição das coordenadas (x, y ou z) para até max_structures estruturas.\n    \n    :param df_labels: DataFrame contendo as coordenadas.\n    :param prefix: Prefixo das colunas a serem plotadas ('x_', 'y_' ou 'z_').\n    :param max_structures: Número máximo de estruturas a mostrar.\n    \"\"\"\n    # Encontrar colunas de coordenadas com o prefixo especificado\n    coord_cols = filter_columns_by_prefix(df_labels, prefix)\n    \n    # Limitar ao número máximo de estruturas\n    coord_cols = sorted(coord_cols)[:max_structures]\n    \n    if not coord_cols:\n        print(f\"Nenhuma coluna com prefixo '{prefix}' encontrada.\")\n        return\n    \n    # Configurar o gráfico\n    fig, axes = plt.subplots(1, len(coord_cols), figsize=(16, 4))\n    if len(coord_cols) == 1:\n        axes = [axes]  # Garantir que axes seja iterável mesmo com uma única subplot\n    \n    # Plotar histogramas para cada coluna\n    for i, col in enumerate(coord_cols):\n        # Filtrar valores especiais (-1.0e+18) se presentes\n        values = df_labels[col]\n        filtered_values = values[values > -1.0e+17]  # Valor de corte para filtrar -1.0e+18\n        \n        axes[i].hist(filtered_values, bins=30, alpha=0.7)\n        axes[i].set_title(f'Distribuição de {col}')\n        axes[i].set_xlabel('Valor')\n        axes[i].set_ylabel('Frequência')\n    \n    plt.tight_layout()\n    plt.show()\n\ndef analyze_3d_structure(df_labels):\n    \"\"\"\n    Analisa as coordenadas 3D das estruturas RNA.\n    \n    :param df_labels: DataFrame contendo as coordenadas 3D.\n    \"\"\"\n    # Encontrar todas as colunas de coordenadas\n    x_cols = filter_columns_by_prefix(df_labels, 'x_')\n    y_cols = filter_columns_by_prefix(df_labels, 'y_')\n    z_cols = filter_columns_by_prefix(df_labels, 'z_')\n    \n    print(f\"Número de colunas x: {len(x_cols)}\")\n    print(f\"Número de colunas y: {len(y_cols)}\")\n    print(f\"Número de colunas z: {len(z_cols)}\")\n    \n    # Verificar valores ausentes ou especiais nas coordenadas\n    special_value = -1.0e+18  # Valor especial observado nos dados\n    \n    for i, (x_col, y_col, z_col) in enumerate(zip(x_cols, y_cols, z_cols), 1):\n        # Contar valores ausentes ou especiais\n        x_special = (df_labels[x_col] == special_value).sum()\n        y_special = (df_labels[y_col] == special_value).sum()\n        z_special = (df_labels[z_col] == special_value).sum()\n        \n        x_null = df_labels[x_col].isnull().sum()\n        y_null = df_labels[y_col].isnull().sum()\n        z_null = df_labels[z_col].isnull().sum()\n        \n        # Contar quantas estruturas completas existem (todos x, y, z não são especiais nem nulos)\n        valid_structures = ((df_labels[x_col] != special_value) & \n                           (df_labels[y_col] != special_value) & \n                           (df_labels[z_col] != special_value) &\n                           df_labels[x_col].notnull() & \n                           df_labels[y_col].notnull() & \n                           df_labels[z_col].notnull()).sum()\n        \n        total_rows = len(df_labels)\n        \n        print(f\"\\nEstrutura {i}:\")\n        print(f\"  Valores especiais: x={x_special} ({x_special/total_rows*100:.2f}%), y={y_special} ({y_special/total_rows*100:.2f}%), z={z_special} ({z_special/total_rows*100:.2f}%)\")\n        print(f\"  Valores nulos: x={x_null} ({x_null/total_rows*100:.2f}%), y={y_null} ({y_null/total_rows*100:.2f}%), z={z_null} ({z_null/total_rows*100:.2f}%)\")\n        print(f\"  Estruturas completas: {valid_structures} ({valid_structures/total_rows*100:.2f}%)\")\n        \n        # Limitar a análise às primeiras 5 estruturas\n        if i >= 5:\n            print(\"\\nAnalise limitada às primeiras 5 estruturas.\")\n            break\n\ndef analyze_sequences(df_sequences):\n    \"\"\"\n    Analisa as sequências de RNA.\n    \n    :param df_sequences: DataFrame contendo a coluna 'sequence'.\n    \"\"\"\n    # Estatísticas básicas da coluna sequence\n    print(\"\\nEstatísticas básicas da coluna 'sequence':\")\n    print(df_sequences['sequence'].describe())\n    \n    # Comprimento das sequências\n    seq_lengths = df_sequences['sequence'].apply(len)\n    print(\"\\nEstatísticas de comprimento das sequências:\")\n    print(f\"Mínimo: {seq_lengths.min()}\")\n    print(f\"Máximo: {seq_lengths.max()}\")\n    print(f\"Média: {seq_lengths.mean():.2f}\")\n    print(f\"Mediana: {seq_lengths.median()}\")\n    \n    # Contagem de nucleotídeos\n    nucleotide_counts = count_nucleotides(df_sequences)\n    total_nucleotides = sum(nucleotide_counts.values())\n    \n    print(\"\\nDistribuição de nucleotídeos:\")\n    for nucleotide, count in sorted(nucleotide_counts.items()):\n        print(f\"{nucleotide}: {count} ({count/total_nucleotides*100:.2f}%)\")\n    \n    # Plotar distribuição de comprimento\n    plt.figure(figsize=(10, 6))\n    plt.hist(seq_lengths, bins=30, alpha=0.7)\n    plt.title('Distribuição de Comprimento das Sequências')\n    plt.xlabel('Comprimento')\n    plt.ylabel('Frequência')\n    plt.grid(True, alpha=0.3)\n    plt.show()\n\ndef main():\n    # Carregar dados principais\n    main_data = load_main_data()\n\n    # Verificar quais arquivos foram carregados\n    print(\"\\nArquivos carregados:\")\n    for file_name, df in main_data.items():\n        if df is not None:\n            print(f\"- {file_name}: {df.shape}\")\n    \n    # Analisar estruturas 3D no arquivo validation_labels.csv\n    if \"validation_labels.csv\" in main_data and main_data[\"validation_labels.csv\"] is not None:\n        print(\"\\n===== Análise das Estruturas 3D (validation_labels.csv) =====\")\n        df_labels = main_data[\"validation_labels.csv\"]\n        \n        # Contar colunas de coordenadas\n        x_cols = filter_columns_by_prefix(df_labels, 'x_')\n        y_cols = filter_columns_by_prefix(df_labels, 'y_')\n        z_cols = filter_columns_by_prefix(df_labels, 'z_')\n        \n        print(f\"Há {len(x_cols)} colunas x_ no DataFrame.\")\n        print(f\"Há {len(y_cols)} colunas y_ no DataFrame.\")\n        print(f\"Há {len(z_cols)} colunas z_ no DataFrame.\")\n        \n        # Identificar colunas sem valores ausentes\n        columns_without_missing = get_columns_without_missing_values(df_labels)\n        print(f\"\\nColunas sem valores ausentes: {len(columns_without_missing)}\")\n        \n        # Identificar colunas completamente vazias\n        empty_columns = get_empty_columns(df_labels)\n        print(f\"Colunas completamente vazias: {len(empty_columns)}\")\n        \n        # Analisar coordenadas 3D em detalhes\n        analyze_3d_structure(df_labels)\n        \n        # Plotar distribuição das coordenadas x, y, z para as primeiras estruturas\n        print(\"\\nDistribuição das coordenadas X:\")\n        plot_coord_distributions(df_labels, 'x_', max_structures=3)\n        print(\"\\nDistribuição das coordenadas Y:\")\n        plot_coord_distributions(df_labels, 'y_', max_structures=3)\n        print(\"\\nDistribuição das coordenadas Z:\")\n        plot_coord_distributions(df_labels, 'z_', max_structures=3)\n    \n    # Analisar sequências no arquivo train_sequences.csv\n    if \"train_sequences.csv\" in main_data and main_data[\"train_sequences.csv\"] is not None:\n        print(\"\\n===== Análise das Sequências (train_sequences.csv) =====\")\n        df_sequences = main_data[\"train_sequences.csv\"]\n        \n        # Primeiras linhas da coluna sequence\n        print(\"\\nPrimeiras linhas da coluna 'sequence':\")\n        print(df_sequences['sequence'].head())\n        \n        # Tipo de dados da coluna sequence\n        print(\"\\nTipo de dados da coluna 'sequence':\")\n        print(df_sequences['sequence'].dtype)\n        \n        # Análise completa das sequências\n        analyze_sequences(df_sequences)\n    \n    return main_data\n\nif __name__ == '__main__':\n    main_data = main()","metadata":{"execution":{"iopub.status.busy":"2025-03-18T03:15:56.659852Z","iopub.execute_input":"2025-03-18T03:15:56.6603Z","iopub.status.idle":"2025-03-18T03:15:59.759005Z","shell.execute_reply.started":"2025-03-18T03:15:56.660255Z","shell.execute_reply":"2025-03-18T03:15:59.757774Z"},"papermill":{"duration":3.261654,"end_time":"2025-03-16T03:38:41.255723","exception":false,"start_time":"2025-03-16T03:38:37.994069","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Preparação de Dados para Predição de Estrutura 3D de RNA","metadata":{"papermill":{"duration":0.010926,"end_time":"2025-03-16T03:38:41.27785","exception":false,"start_time":"2025-03-16T03:38:41.266924","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom collections import Counter\n\n# Caminhos de arquivos\nDATA_DIR = \"/kaggle/input/stanford-rna-3d-folding/\"\nOUTPUT_DIR = \"/kaggle/working/\"\nos.makedirs(OUTPUT_DIR, exist_ok=True)\n\ndef load_data():\n    \"\"\"\n    Carrega os dados necessários para a competição.\n    \"\"\"\n    data = {}\n    \n    # Carregar sequências\n    data['train_seq'] = pd.read_csv(os.path.join(DATA_DIR, \"train_sequences.csv\"))\n    data['valid_seq'] = pd.read_csv(os.path.join(DATA_DIR, \"validation_sequences.csv\"))\n    data['test_seq'] = pd.read_csv(os.path.join(DATA_DIR, \"test_sequences.csv\"))\n    \n    # Carregar estruturas (labels)\n    data['train_labels'] = pd.read_csv(os.path.join(DATA_DIR, \"train_labels.csv\"))\n    data['valid_labels'] = pd.read_csv(os.path.join(DATA_DIR, \"validation_labels.csv\"))\n    \n    # Carregar formato de submissão\n    data['sample_submission'] = pd.read_csv(os.path.join(DATA_DIR, \"sample_submission.csv\"))\n    \n    return data\n\ndef analyze_id_structure(data_dict):\n    \"\"\"\n    Analisa a estrutura dos IDs nos diferentes arquivos para entender o mapeamento correto.\n    \"\"\"\n    # Vamos analisar os formatos específicos para train e valid\n    \n    # 1. Análise das labels de treinamento\n    train_label_ids = data_dict['train_labels']['ID'].tolist()\n    print(f\"Total de IDs nas labels de treinamento: {len(train_label_ids)}\")\n    print(f\"Número de IDs únicos: {len(set(train_label_ids))}\")\n    \n    # Tentar entender o formato de ID no arquivo de labels\n    train_id_parts = {}\n    for id_str in train_label_ids[:100]:  # Analisa os primeiros 100\n        parts = id_str.split('_')\n        num_parts = len(parts)\n        if num_parts not in train_id_parts:\n            train_id_parts[num_parts] = []\n        train_id_parts[num_parts].append(parts)\n    \n    print(\"\\nFormatos de ID encontrados em train_labels:\")\n    for num_parts, examples in train_id_parts.items():\n        print(f\"\\nFormato com {num_parts} partes:\")\n        for i, parts in enumerate(examples[:3]):\n            print(f\"  Exemplo {i+1}: {parts}\")\n    \n    # 2. Análise das sequências de treinamento\n    train_seq_ids = data_dict['train_seq']['target_id'].tolist()\n    print(f\"\\nTotal de IDs nas sequências de treinamento: {len(train_seq_ids)}\")\n    print(f\"Número de IDs únicos: {len(set(train_seq_ids))}\")\n    \n    # Tentar entender o formato de ID no arquivo de sequências\n    train_seq_id_parts = {}\n    for id_str in train_seq_ids[:100]:  # Analisa os primeiros 100\n        parts = id_str.split('_')\n        num_parts = len(parts)\n        if num_parts not in train_seq_id_parts:\n            train_seq_id_parts[num_parts] = []\n        train_seq_id_parts[num_parts].append(parts)\n    \n    print(\"\\nFormatos de ID encontrados em train_sequences:\")\n    for num_parts, examples in train_seq_id_parts.items():\n        print(f\"\\nFormato com {num_parts} partes:\")\n        for i, parts in enumerate(examples[:3]):\n            print(f\"  Exemplo {i+1}: {parts}\")\n    \n    # 3. Análise das labels de validação\n    valid_label_ids = data_dict['valid_labels']['ID'].tolist()\n    print(f\"\\nTotal de IDs nas labels de validação: {len(valid_label_ids)}\")\n    print(f\"Número de IDs únicos: {len(set(valid_label_ids))}\")\n    \n    # Contar IDs únicos de sequência nas labels de validação\n    valid_seq_ids_from_labels = set([id_str.split('_')[0] for id_str in valid_label_ids])\n    print(f\"Número de IDs únicos de sequência nas labels de validação: {len(valid_seq_ids_from_labels)}\")\n    print(f\"Exemplos: {list(valid_seq_ids_from_labels)[:5]}\")\n    \n    # 4. Análise das sequências de validação\n    valid_seq_ids = data_dict['valid_seq']['target_id'].tolist()\n    print(f\"\\nTotal de IDs nas sequências de validação: {len(valid_seq_ids)}\")\n    print(f\"Número de IDs únicos: {len(set(valid_seq_ids))}\")\n    print(f\"Exemplos: {valid_seq_ids[:5]}\")\n    \n    # 5. Verificar a correspondência entre os IDs únicos\n    overlap_valid = set(valid_seq_ids).intersection(valid_seq_ids_from_labels)\n    print(f\"\\nCorrespondência entre sequências e labels de validação: {len(overlap_valid)} de {len(valid_seq_ids)}\")\n    \n    # 6. Verificar como as sequências e resíduos se relacionam\n    if len(overlap_valid) > 0:\n        sample_id = list(overlap_valid)[0]\n        sample_seq = data_dict['valid_seq'][data_dict['valid_seq']['target_id'] == sample_id]['sequence'].iloc[0]\n        sample_labels = data_dict['valid_labels'][data_dict['valid_labels']['ID'].str.startswith(f\"{sample_id}_\")]\n        \n        print(f\"\\nAnálise para o ID de sequência: {sample_id}\")\n        print(f\"Comprimento da sequência: {len(sample_seq)}\")\n        print(f\"Número de resíduos nas labels: {len(sample_labels)}\")\n        \n        # Verificar como os números de resíduos estão relacionados\n        residue_numbers = sample_labels['resid'].sort_values().tolist()\n        print(f\"Primeiros números de resíduos: {residue_numbers[:10]}\")\n        print(f\"Últimos números de resíduos: {residue_numbers[-10:]}\")\n        \n    return train_id_parts, train_seq_id_parts, overlap_valid\n\ndef fix_train_mapping(train_seq_df, train_labels_df):\n    \"\"\"\n    Identifica um mapeamento correto entre train_sequences.csv e train_labels.csv\n    usando o formato de ID do arquivo de validação como referência.\n    \n    Isso é necessário porque não existe uma correspondência direta óbvia entre os IDs.\n    \"\"\"\n    # Primeiro, vamos extrair o prefixo do ID dos labels (formato: XX_Y_Z)\n    train_labels_df['seq_id'] = train_labels_df['ID'].apply(lambda x: x.split('_')[0] + '_' + x.split('_')[1])\n    \n    # Verificar se este formato corresponde ao formato dos IDs das sequências\n    seq_ids_set = set(train_seq_df['target_id'])\n    label_seq_ids_set = set(train_labels_df['seq_id'])\n    \n    overlap = seq_ids_set.intersection(label_seq_ids_set)\n    print(f\"Overlap após ajuste do formato: {len(overlap)} de {len(seq_ids_set)}\")\n    \n    if len(overlap) > 0:\n        print(f\"Exemplos de IDs correspondentes: {list(overlap)[:5]}\")\n        return overlap\n    \n    # Se ainda não funcionar, precisamos analisar a estrutura mais detalhadamente\n    print(\"Nenhuma correspondência encontrada, verificando outros formatos...\")\n    \n    # Tentar outros formatos possíveis\n    formats_to_try = [\n        lambda x: x.split('_')[0],                             # Apenas primeira parte\n        lambda x: '_'.join(x.split('_')[:2]),                  # Primeiras duas partes\n        lambda x: x.split('_')[0] + '_' + x.split('_')[1][0],  # Primeira parte + primeira letra da segunda parte\n    ]\n    \n    for i, format_func in enumerate(formats_to_try):\n        train_labels_df[f'seq_id_{i}'] = train_labels_df['ID'].apply(format_func)\n        label_seq_ids_set = set(train_labels_df[f'seq_id_{i}'])\n        overlap = seq_ids_set.intersection(label_seq_ids_set)\n        print(f\"Formato {i}: Overlap = {len(overlap)} de {len(seq_ids_set)}\")\n        \n        if len(overlap) > 0:\n            print(f\"Exemplos de IDs correspondentes: {list(overlap)[:5]}\")\n            return overlap, f'seq_id_{i}'\n    \n    # Se nenhuma correspondência for encontrada, vamos criar um mapeamento baseado nos padrões observados\n    print(\"Nenhuma correspondência encontrada usando padrões simples.\")\n    print(\"Criando um mapeamento manual baseado na estrutura de dados...\")\n    \n    # Agrupar labels por primeiras partes do ID\n    train_labels_df['prefix'] = train_labels_df['ID'].apply(lambda x: x.split('_')[0])\n    label_groups = train_labels_df.groupby('prefix')\n    \n    # Para cada sequência, encontrar a melhor correspondência baseada no número de resíduos\n    mapping = {}\n    for _, seq_row in train_seq_df.iterrows():\n        seq_id = seq_row['target_id']\n        seq_length = len(seq_row['sequence'])\n        \n        best_match = None\n        best_diff = float('inf')\n        \n        for prefix, group in label_groups:\n            residue_count = len(group)\n            diff = abs(residue_count - seq_length)\n            \n            if diff < best_diff:\n                best_diff = diff\n                best_match = prefix\n        \n        # Considerar uma correspondência apenas se o número de resíduos for próximo\n        if best_diff <= 10:  # Tolerância de 10 resíduos\n            mapping[seq_id] = best_match\n    \n    print(f\"Mapeamento manual criado com {len(mapping)} correspondências\")\n    return mapping\n\ndef create_mapping_valid(valid_seq_df, valid_labels_df):\n    \"\"\"\n    Cria um mapeamento entre sequências de validação e suas coordenadas.\n    \n    Neste caso, os IDs já correspondem diretamente (R1107 -> R1107_1, R1107_2, etc.)\n    \"\"\"\n    # Verificar qual formato de ID é usado no conjunto de validação\n    valid_labels_df['seq_id'] = valid_labels_df['ID'].apply(lambda x: x.split('_')[0])\n    \n    # Verificar sobreposição\n    seq_ids = set(valid_seq_df['target_id'])\n    label_seq_ids = set(valid_labels_df['seq_id'])\n    \n    overlap = seq_ids.intersection(label_seq_ids)\n    print(f\"Correspondência para validação: {len(overlap)} de {len(seq_ids)}\")\n    \n    mapping = {}\n    for seq_id in overlap:\n        # Obter sequência\n        seq = valid_seq_df[valid_seq_df['target_id'] == seq_id]['sequence'].iloc[0]\n        \n        # Obter todos os resíduos para esta sequência\n        residues = valid_labels_df[valid_labels_df['seq_id'] == seq_id].sort_values('resid')\n        \n        # Extrair coordenadas para todas as estruturas\n        num_structures = 1\n        for col in residues.columns:\n            if col.startswith('x_'):\n                struct_num = int(col.split('_')[1])\n                num_structures = max(num_structures, struct_num)\n        \n        # Inicializar estruturas\n        structures = []\n        \n        for struct_idx in range(1, num_structures + 1):\n            coords = []\n            has_valid_coords = False\n            \n            # Verificar se esta estrutura tem coordenadas\n            if f'x_{struct_idx}' in residues.columns:\n                for _, row in residues.iterrows():\n                    x = row[f'x_{struct_idx}']\n                    y = row[f'y_{struct_idx}']\n                    z = row[f'z_{struct_idx}']\n                    \n                    # Verificar se são valores válidos\n                    if abs(x) < 1.0e+17 and abs(y) < 1.0e+17 and abs(z) < 1.0e+17:\n                        coords.append([x, y, z])\n                        has_valid_coords = True\n                    else:\n                        coords.append([np.nan, np.nan, np.nan])\n            \n            if has_valid_coords:\n                structures.append(coords)\n        \n        # Adicionar ao mapeamento se houver estruturas válidas\n        if structures:\n            mapping[seq_id] = {\n                'sequence': seq,\n                'structures': structures\n            }\n    \n    print(f\"Mapeamento criado com {len(mapping)} sequências válidas\")\n    return mapping\n\ndef create_processed_data(mapping, output_prefix):\n    \"\"\"\n    Cria e salva dados processados a partir do mapeamento.\n    \n    Parâmetros:\n    mapping: Dicionário com o mapeamento de sequências para estruturas\n    output_prefix: Prefixo para os arquivos de saída ('train' ou 'valid')\n    \n    Retorna:\n    X, y: Arrays para treinamento\n    \"\"\"\n    if not mapping:\n        print(f\"AVISO: Nenhum mapeamento válido para {output_prefix}\")\n        return None, None\n    \n    X_data = []\n    y_data = []\n    ids = []\n    \n    for seq_id, data in mapping.items():\n        seq = data['sequence']\n        structures = data['structures']\n        \n        # Pular se não houver estruturas\n        if not structures:\n            continue\n        \n        # Usar a primeira estrutura válida\n        structure = structures[0]\n        \n        # Verificar se a estrutura tem coordenadas válidas para todos os resíduos\n        if len(structure) != len(seq):\n            print(f\"AVISO: Diferença entre comprimento da sequência ({len(seq)}) e coordenadas ({len(structure)}) para {seq_id}\")\n            # Se necessário, podemos considerar padding ou truncamento aqui\n            continue\n        \n        # Criar matriz de características (one-hot encoding)\n        features = []\n        for nucleotide in seq:\n            if nucleotide == 'A':\n                features.append([1, 0, 0, 0, 0])\n            elif nucleotide == 'C':\n                features.append([0, 1, 0, 0, 0])\n            elif nucleotide == 'G':\n                features.append([0, 0, 1, 0, 0])\n            elif nucleotide == 'U':\n                features.append([0, 0, 0, 1, 0])\n            else:\n                features.append([0, 0, 0, 0, 1])  # Para nucleotídeos desconhecidos\n        \n        X_data.append(np.array(features))\n        y_data.append(np.array(structure))\n        ids.append(seq_id)\n    \n    if not X_data:\n        print(f\"AVISO: Nenhum dado processado válido para {output_prefix}\")\n        return None, None, []\n    \n    # Padding para garantir que todas as sequências tenham o mesmo comprimento\n    max_length = max(len(x) for x in X_data)\n    X_padded = []\n    y_padded = []\n    \n    for x, y in zip(X_data, y_data):\n        if len(x) < max_length:\n            x_pad = np.zeros((max_length, 5))\n            x_pad[:len(x), :] = x\n            \n            y_pad = np.zeros((max_length, 3))\n            y_pad[:len(y), :] = y\n            \n            X_padded.append(x_pad)\n            y_padded.append(y_pad)\n        else:\n            X_padded.append(x)\n            y_padded.append(y)\n    \n    X = np.array(X_padded)\n    y = np.array(y_padded)\n    \n    # Salvar os dados processados\n    np.save(os.path.join(OUTPUT_DIR, f'X_{output_prefix}.npy'), X)\n    np.save(os.path.join(OUTPUT_DIR, f'y_{output_prefix}.npy'), y)\n    \n    with open(os.path.join(OUTPUT_DIR, f'{output_prefix}_ids.txt'), 'w') as f:\n        for id in ids:\n            f.write(f\"{id}\\n\")\n    \n    print(f\"Dados processados para {output_prefix}: X.shape = {X.shape}, y.shape = {y.shape}\")\n    return X, y, ids\n\ndef explore_sequence_mapping(seq_id, mapping, data_dict):\n    \"\"\"\n    Explora em detalhes um exemplo de mapeamento para diagnóstico.\n    \"\"\"\n    if seq_id not in mapping:\n        print(f\"AVISO: ID de sequência {seq_id} não encontrado no mapeamento\")\n        return\n    \n    data = mapping[seq_id]\n    seq = data['sequence']\n    structures = data['structures']\n    \n    print(f\"Explorando mapeamento para sequência: {seq_id}\")\n    print(f\"Comprimento da sequência: {len(seq)}\")\n    print(f\"Número de estruturas disponíveis: {len(structures)}\")\n    \n    # Detalhar cada estrutura\n    for i, structure in enumerate(structures):\n        print(f\"\\nEstrutura {i+1}:\")\n        print(f\"  Número de coordenadas: {len(structure)}\")\n        if len(structure) > 0:\n            print(f\"  Primeiras coordenadas: {structure[:3]}\")\n            print(f\"  Últimas coordenadas: {structure[-3:]}\")\n        \n        # Verificar correspondência com a sequência\n        if len(structure) != len(seq):\n            print(f\"  AVISO: Diferença entre comprimento da sequência ({len(seq)}) e coordenadas ({len(structure)})\")\n        else:\n            print(f\"  Correspondência perfeita entre sequência e coordenadas\")\n\ndef main():\n    # Carregar os dados\n    print(\"Carregando dados...\")\n    data_dict = load_data()\n    \n    # Analisar estrutura dos IDs para entender o mapeamento\n    print(\"\\nAnalisando estrutura dos IDs...\")\n    train_id_parts, train_seq_id_parts, overlap_valid = analyze_id_structure(data_dict)\n    \n    # Para validação, o mapeamento é direto (R1107 -> R1107_1, R1107_2, etc.)\n    print(\"\\nCriando mapeamento para dados de validação...\")\n    valid_mapping = create_mapping_valid(data_dict['valid_seq'], data_dict['valid_labels'])\n    \n    # Explorar um exemplo do mapeamento de validação para verificar\n    if valid_mapping:\n        sample_id = list(valid_mapping.keys())[0]\n        print(f\"\\nExplorando um exemplo de mapeamento de validação ({sample_id}):\")\n        explore_sequence_mapping(sample_id, valid_mapping, data_dict)\n    \n    # Criar e salvar dados processados para validação\n    X_valid, y_valid, valid_ids = create_processed_data(valid_mapping, 'valid')\n    \n    # Como não conseguimos estabelecer um mapeamento para treinamento,\n    # vamos usar os dados de validação para treinamento também (transfer learning)\n    print(\"\\nUsando dados de validação como treinamento (devido à falta de mapeamento direto)...\")\n    X_train = X_valid\n    y_train = y_valid\n    train_ids = valid_ids\n    \n    if X_train is not None:\n        np.save(os.path.join(OUTPUT_DIR, 'X_train.npy'), X_train)\n        np.save(os.path.join(OUTPUT_DIR, 'y_train.npy'), y_train)\n        \n        with open(os.path.join(OUTPUT_DIR, 'train_ids.txt'), 'w') as f:\n            for id in train_ids:\n                f.write(f\"{id}\\n\")\n    \n    # Retornar os dados processados\n    return {\n        'X_train': X_train,\n        'y_train': y_train,\n        'X_valid': X_valid,\n        'y_valid': y_valid,\n        'valid_mapping': valid_mapping,\n        'valid_ids': valid_ids\n    }\n\nif __name__ == \"__main__\":\n    processed_data = main()","metadata":{"execution":{"iopub.status.busy":"2025-03-18T03:15:59.760228Z","iopub.execute_input":"2025-03-18T03:15:59.760562Z","iopub.status.idle":"2025-03-18T03:16:06.533266Z","shell.execute_reply.started":"2025-03-18T03:15:59.760535Z","shell.execute_reply":"2025-03-18T03:16:06.531843Z"},"papermill":{"duration":6.289358,"end_time":"2025-03-16T03:38:47.577093","exception":false,"start_time":"2025-03-16T03:38:41.287735","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Visualizador de Heatmap para Sequências de RNA","metadata":{"papermill":{"duration":0.009766,"end_time":"2025-03-16T03:38:47.597662","exception":false,"start_time":"2025-03-16T03:38:47.587896","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.colors as mcolors\nimport os\n\ndef visualize_rna_heatmap_from_processed_data(processed_data, num_samples=12):\n    \"\"\"\n    Visualiza um heatmap para sequências de RNA usando dados processados.\n    \n    Parâmetros:\n    processed_data: Dicionário com dados processados retornado pela função main()\n    num_samples: Número de sequências a serem visualizadas\n    \"\"\"\n    try:\n        # Verificar se temos os dados necessários\n        if 'X_valid' not in processed_data or processed_data['X_valid'] is None:\n            print(\"Dados de validação não encontrados no objeto processed_data\")\n            return None\n        \n        # Obter os dados\n        X_valid = processed_data['X_valid']\n        print(f\"Dados encontrados com formato: {X_valid.shape}\")\n        \n        # Limitar ao número de amostras\n        X_valid_subset = X_valid[:num_samples]\n        \n        # Se temos IDs, usar eles\n        if 'valid_ids' in processed_data and processed_data['valid_ids']:\n            valid_ids = processed_data['valid_ids'][:num_samples]\n        else:\n            valid_ids = [f\"Seq_{i+1}\" for i in range(X_valid_subset.shape[0])]\n        \n        # Converter one-hot encoding para índices de nucleotídeos\n        # Formato esperado: A=[1,0,0,0,0], C=[0,1,0,0,0], G=[0,0,1,0,0], U=[0,0,0,1,0], N=[0,0,0,0,1]\n        sequences_matrix = np.argmax(X_valid_subset, axis=2)\n        \n        # Substituir zeros (padding) por 4 (N/Desconhecido) quando todos os valores são zero\n        is_padding = np.all(X_valid_subset == 0, axis=2)\n        sequences_matrix[is_padding] = 4\n        \n        # Definir um colormap categórico (cores distintas por nucleotídeo)\n        cmap = mcolors.ListedColormap(['#3498db', '#2ecc71', '#e74c3c', '#9b59b6', '#95a5a6'])\n        bounds = [0, 1, 2, 3, 4, 5]\n        norm = mcolors.BoundaryNorm(bounds, cmap.N)\n        \n        # Criar figura\n        plt.figure(figsize=(20, 10))\n        im = plt.imshow(sequences_matrix, cmap=cmap, norm=norm, aspect='auto')\n        \n        # Adicionar barra de cores\n        cbar = plt.colorbar(im, ticks=[0.5, 1.5, 2.5, 3.5, 4.5])\n        cbar.set_label('Nucleotídeos', fontsize=14)\n        cbar.set_ticklabels(['A', 'C', 'G', 'U', 'N/Padding'])\n        \n        # Adicionar rótulos dos eixos\n        plt.xlabel(\"Posição na sequência\", fontsize=14)\n        plt.ylabel(\"Sequências de RNA\", fontsize=14)\n        \n        # Adicionar título\n        plt.title(\"Heatmap de Sequências de RNA\", fontsize=16)\n        \n        # Adicionar id das sequências como rótulos do eixo y\n        plt.yticks(range(len(valid_ids)), valid_ids, fontsize=10)\n        \n        # Mostrar apenas alguns rótulos no eixo x para não ficar muito lotado\n        sequence_length = sequences_matrix.shape[1]\n        step = max(1, sequence_length // 20)  # Mostrar no máximo 20 rótulos\n        plt.xticks(range(0, sequence_length, step), range(1, sequence_length + 1, step))\n        \n        # Adicionar grade\n        plt.grid(False)\n        \n        # Adicionar informações sobre distribuição de nucleotídeos\n        all_nucleotides = sequences_matrix.flatten()\n        nucleotide_counts = {\n            'A': np.sum(all_nucleotides == 0),\n            'C': np.sum(all_nucleotides == 1),\n            'G': np.sum(all_nucleotides == 2),\n            'U': np.sum(all_nucleotides == 3),\n            'N': np.sum(all_nucleotides == 4)\n        }\n        \n        total_nucleotides = sum(nucleotide_counts.values())\n        nucleotide_percentages = {k: (v / total_nucleotides) * 100 for k, v in nucleotide_counts.items()}\n        \n        # Adicionar texto com estatísticas\n        info_text = \"\\n\".join([\n            f\"Total de sequências visualizadas: {num_samples}\",\n            f\"Comprimento máximo: {sequence_length}\",\n            f\"A: {nucleotide_percentages['A']:.1f}%\",\n            f\"C: {nucleotide_percentages['C']:.1f}%\",\n            f\"G: {nucleotide_percentages['G']:.1f}%\",\n            f\"U: {nucleotide_percentages['U']:.1f}%\",\n            f\"N/Padding: {nucleotide_percentages['N']:.1f}%\"\n        ])\n        \n        plt.figtext(0.02, 0.02, info_text, fontsize=10, bbox=dict(facecolor='white', alpha=0.8))\n        \n        # Mostrar o gráfico\n        plt.tight_layout()\n        plt.show()\n        \n        # Opcionalmente, salvar o gráfico\n        output_dir = '/kaggle/working/'\n        plt.savefig(os.path.join(output_dir, 'rna_heatmap.png'), dpi=300)\n        print(f\"Gráfico de calor salvo em {os.path.join(output_dir, 'rna_heatmap.png')}\")\n        \n        return sequences_matrix\n    except Exception as e:\n        print(f\"Erro ao processar dados: {e}\")\n        return None\n\n# Usar a função (assumindo que processed_data está disponível)\nvisualize_rna_heatmap_from_processed_data(processed_data)","metadata":{"execution":{"iopub.status.busy":"2025-03-18T03:16:06.53442Z","iopub.execute_input":"2025-03-18T03:16:06.534786Z","iopub.status.idle":"2025-03-18T03:16:07.541728Z","shell.execute_reply.started":"2025-03-18T03:16:06.534718Z","shell.execute_reply":"2025-03-18T03:16:07.540717Z"},"papermill":{"duration":1.153477,"end_time":"2025-03-16T03:38:48.761534","exception":false,"start_time":"2025-03-16T03:38:47.608057","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Previsão de Estruturas 3D de RNA com Abordagem Baseada em Referência e Amostragem Estrutural","metadata":{"papermill":{"duration":0.014083,"end_time":"2025-03-16T03:38:48.790636","exception":false,"start_time":"2025-03-16T03:38:48.776553","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\n# Caminhos de arquivos\nDATA_DIR = \"/kaggle/input/stanford-rna-3d-folding/\"\nOUTPUT_DIR = \"/kaggle/working/\"\nos.makedirs(OUTPUT_DIR, exist_ok=True)\n\n##############################################\n# 1. Função para gerar variação estrutural\n##############################################\n\ndef sample_structural_variation(coords, noise_level=0.2, preserve_distance=True, \n                               use_global_movement=False, correlation=0.7):\n    \"\"\"\n    Enhanced version of structural variation sampling with better\n    handling of large RNAs and improved noise distribution.\n    \"\"\"\n    new_coords = coords.copy()\n    valid_mask = ~np.all(coords == 0, axis=1)\n    valid_indices = np.where(valid_mask)[0]\n    \n    if len(valid_indices) < 3:\n        return new_coords\n    \n    # Parameters optimized for RNA structure\n    typical_bond_length = 3.8  # Angstroms - typical RNA backbone distance\n    \n    # Add global domain movements if requested\n    if use_global_movement and len(valid_indices) > 20:\n        # More natural domain identification - try to find natural hinge points\n        # For RNA, these often occur at junctions between helices\n        \n        # Calculate distance between consecutive residues as a heuristic\n        # for finding potential hinge points (larger distances often indicate junctions)\n        distances = []\n        for i in range(1, len(valid_indices)):\n            idx1 = valid_indices[i-1]\n            idx2 = valid_indices[i]\n            dist = np.linalg.norm(coords[idx1] - coords[idx2])\n            distances.append((i, dist))\n        \n        # Sort by distance to find potential hinges\n        distances.sort(key=lambda x: x[1], reverse=True)\n        \n        # Take top 2 potential hinge points (if we have enough points)\n        num_hinges = min(2, len(distances)//3)\n        \n        for h in range(num_hinges):\n            if h < len(distances):\n                hinge_point = distances[h][0]\n                if hinge_point < 5 or hinge_point > len(valid_indices) - 5:\n                    continue\n                    \n                hinge_idx = valid_indices[hinge_point]\n                \n                # Angle of rotation with natural distribution\n                # More small movements than large ones\n                angle = np.random.exponential(0.2)  # Mostly small angles with occasional larger ones\n                if np.random.random() < 0.5:\n                    angle = -angle  # Allow both directions\n                \n                # Create a more natural rotation matrix with slight 3D component\n                # RNAs often bend and twist in 3D\n                sin_a, cos_a = np.sin(angle), np.cos(angle)\n                tilt = np.random.normal(0, 0.1)  # Small tilt in 3D\n                rotation_matrix = np.array([\n                    [cos_a, -sin_a, 0],\n                    [sin_a, cos_a, tilt],\n                    [0, -tilt, 1]\n                ])\n                \n                # Apply rotation around hinge point\n                ref_point = new_coords[hinge_idx]\n                for i in valid_indices[hinge_point+1:]:\n                    vector = new_coords[i] - ref_point\n                    rotated = np.dot(vector, rotation_matrix)\n                    new_coords[i] = ref_point + rotated\n    \n    # Propagate variation residue by residue, with correlation\n    # RNA has strong local correlations in structure\n    prev_noise = np.zeros(3)\n    \n    correlation = 0.5  # High correlation for smoother variations\n    \n    for i in range(1, len(coords)):\n        if not valid_mask[i] or not valid_mask[i-1]:\n            continue\n            \n        vec = new_coords[i-1] - new_coords[i]\n        vec_length = np.linalg.norm(vec)\n        \n        # Generate correlated noise (smoother transitions)\n        new_noise = np.random.normal(0, noise_level, size=3)\n        noise_vec = correlation * prev_noise + (1 - correlation) * new_noise\n        prev_noise = noise_vec.copy()\n        \n        noise_norm = np.linalg.norm(noise_vec)\n        if noise_norm > 0:\n            # Scale noise proportionally\n            noise_vec = noise_vec / noise_norm * (noise_level * vec_length)\n        \n        # Add noise to the direction\n        new_vec = vec + noise_vec\n        \n        # Preserve distance if requested\n        if preserve_distance:\n            current_length = np.linalg.norm(new_vec)\n            if current_length > 0:\n                # Allow slight variation in bond length (RNA is not rigid)\n                target_length = typical_bond_length * (1 + np.random.normal(0, 0.05))\n                new_vec = new_vec / current_length * target_length\n        \n        new_coords[i] = new_coords[i-1] - new_vec\n    \n    return new_coords\n\ndef normalize_structure(coords):\n    \"\"\"\n    Centraliza e normaliza a estrutura.\n    \"\"\"\n    # Remover padding\n    valid_mask = ~np.all(coords == 0, axis=1)\n    valid_coords = coords[valid_mask]\n    \n    # Centralizar no centro de massa\n    center = np.mean(valid_coords, axis=0)\n    centered_coords = coords.copy()\n    centered_coords[valid_mask] = valid_coords - center\n    \n    return centered_coords\n\ndef check_structure_validity(coords, min_distance=0.8, max_distance=7.0, allow_clashes=0.05):\n    \"\"\"\n    Verificação biofísica mais refinada e realista.\n    \"\"\"\n    valid = True\n    valid_mask = ~np.all(coords == 0, axis=1)\n    valid_coords = coords[valid_mask]\n    \n    if len(valid_coords) < 3:\n        return True\n    \n    # Verificar distâncias entre resíduos consecutivos\n    invalid_bonds = 0\n    for i in range(1, len(valid_coords)):\n        dist = np.linalg.norm(valid_coords[i] - valid_coords[i-1])\n        if dist < min_distance or dist > max_distance:\n            invalid_bonds += 1\n    \n    # Permitir uma pequena porcentagem de ligações inválidas\n    if invalid_bonds / len(valid_coords) > 0.1:  # Mais de 10% de ligações inválidas\n        valid = False\n    \n    # Verificar colisões, permitindo algumas\n    clashes = 0\n    total_pairs = 0\n    for i in range(len(valid_coords)):\n        for j in range(i+3, len(valid_coords)):  # Pular adjacentes\n            total_pairs += 1\n            dist = np.linalg.norm(valid_coords[i] - valid_coords[j])\n            if dist < min_distance:\n                clashes += 1\n    \n    # Permitir uma pequena porcentagem de colisões\n    if total_pairs > 0 and clashes / total_pairs > allow_clashes:\n        valid = False\n    \n    return valid\n\n##############################################\n# 2. Função robusta para cálculo do TM-score\n##############################################\ndef calculate_tm_score(pred_coords, true_coords, d0_scale=1.24):\n    \"\"\"\n    Calcula uma aproximação robusta do TM-score entre coordenadas preditas e verdadeiras.\n    Adiciona proteções contra divisão por zero e NaN.\n    \"\"\"\n    # Remover padding (linhas com zeros) das estruturas verdadeiras\n    mask = ~np.all(true_coords == 0, axis=1)\n    pred = pred_coords[mask]\n    true = true_coords[mask]\n    \n    L = len(true)\n    if L < 3:\n        return 0.0\n    \n    # Definir d0 baseado em L (valores adaptados para RNA)\n    if L >= 30:\n        d0 = 0.6 * np.sqrt(L - 0.5) - 2.5\n        d0 = max(0.1, d0)\n    elif L >= 24:\n        d0 = 0.7\n    elif L >= 20:\n        d0 = 0.6\n    elif L >= 16:\n        d0 = 0.5\n    elif L >= 12:\n        d0 = 0.4\n    else:\n        d0 = 0.3\n    \n    distances = np.sqrt(np.sum((pred - true) ** 2, axis=1))\n    tm_terms = 1.0 / (1.0 + (distances / (d0 + 1e-8)) ** 2)\n    tm_score = np.sum(tm_terms) / L\n    return float(tm_score)\n\ndef calculate_tm_score_exact(pred_coords, true_coords):\n    \"\"\"\n    Implementation more closely matching US-align with sequence-independent alignment.\n    Includes multiple rotation schemes to find the optimal structural alignment.\n    \"\"\"\n    # Remove padding\n    mask = ~np.all(true_coords == 0, axis=1)\n    pred = pred_coords[mask]\n    true = true_coords[mask]\n    \n    Lref = len(true)\n    if Lref < 3:\n        return 0.0\n    \n    # Define d0 exactly as in the evaluation formula\n    if Lref >= 30:\n        d0 = 0.6 * np.sqrt(Lref - 0.5) - 2.5\n    elif Lref >= 24:\n        d0 = 0.7\n    elif Lref >= 20:\n        d0 = 0.6\n    elif Lref >= 16:\n        d0 = 0.5\n    elif Lref >= 12:\n        d0 = 0.4\n    else:\n        d0 = 0.3\n    \n    # Normalize structures\n    pred_centered = pred - np.mean(pred, axis=0)\n    true_centered = true - np.mean(true, axis=0)\n    \n    # Try multiple fragment lengths for sequence-independent alignment\n    # This mimics US-align's approach to find the best fragment alignment\n    best_tm_score = 0.0\n    fragment_lengths = [Lref, max(5, Lref//2), max(5, Lref//4)]\n    \n    for frag_len in fragment_lengths:\n        # Try different fragment start positions\n        for i in range(0, Lref - frag_len + 1, max(1, frag_len//2)):\n            pred_frag = pred_centered[i:i+frag_len]\n            \n            # Try aligning with different parts of the true structure\n            for j in range(0, Lref - frag_len + 1, max(1, frag_len//2)):\n                true_frag = true_centered[j:j+frag_len]\n                \n                # Covariance matrix for optimal rotation\n                covariance = np.dot(pred_frag.T, true_frag)\n                U, S, Vt = np.linalg.svd(covariance)\n                rotation = np.dot(U, Vt)\n                \n                # Try different rotation schemes - this is the new part\n                rotations_to_try = [\n                    rotation,  # Original rotation from SVD\n                    np.dot(rotation, np.array([[0, 1, 0], [-1, 0, 0], [0, 0, 1]])),  # 90 degree Z rotation\n                    np.dot(rotation, np.array([[-1, 0, 0], [0, -1, 0], [0, 0, 1]]))  # 180 degree Z rotation\n                ]\n                \n                for rot in rotations_to_try:\n                    # Apply rotation to the full structure\n                    pred_aligned = np.dot(pred_centered, rot)\n                    \n                    # Calculate distances\n                    distances = np.sqrt(np.sum((pred_aligned - true_centered) ** 2, axis=1))\n                    \n                    # Calculate TM-score terms\n                    tm_terms = 1.0 / (1.0 + (distances / d0) ** 2)\n                    tm_score = np.sum(tm_terms) / Lref\n                    \n                    best_tm_score = max(best_tm_score, tm_score)\n    \n    return float(best_tm_score)\n\n##############################################\n# 3. Função para carregar dados processados\n##############################################\ndef load_processed_data():\n    \"\"\"\n    Carrega os dados processados para treinamento.\n    \"\"\"\n    X_train = np.load(os.path.join(OUTPUT_DIR, 'X_train.npy'))\n    y_train = np.load(os.path.join(OUTPUT_DIR, 'y_train.npy'))\n    X_valid = np.load(os.path.join(OUTPUT_DIR, 'X_valid.npy'))\n    y_valid = np.load(os.path.join(OUTPUT_DIR, 'y_valid.npy'))\n    \n    print(f\"Dados carregados - X_train: {X_train.shape}, y_train: {y_train.shape}\")\n    print(f\"Dados carregados - X_valid: {X_valid.shape}, y_valid: {y_valid.shape}\")\n    \n    return X_train, y_train, X_valid, y_valid\n\n##############################################\n# 4. Modelo de Referência (Baseline)\n##############################################\ndef reference_based_approach(X_ref, y_ref, geometric_sampling=False, noise_level=0.2, correlation=0.7):\n    try:\n        class ReferenceModel:\n            def __init__(self, geometric_sampling=False, base_noise_level=0.2, correlation=0.7):\n                self.geometric_sampling = geometric_sampling\n                self.base_noise_level = base_noise_level\n                self.correlation = correlation\n                \n            def fit(self, X, y):\n                # First, handle NaN values in the reference structures\n                self.reference_structures = np.nan_to_num(y, nan=0.0)\n                self.global_mean = np.nanmean(y, axis=(0, 1))\n                self.global_std = np.nanstd(y, axis=(0, 1))\n                \n                # Replace potential NaN values in statistics\n                self.global_mean = np.nan_to_num(self.global_mean, nan=0.0)\n                self.global_std = np.nan_to_num(self.global_std, nan=1.0)\n                \n                # Calculate size statistics\n                self.size_groups = {}\n                # Group reference structures by size\n                for i in range(len(self.reference_structures)):\n                    valid_mask = ~np.all(self.reference_structures[i] == 0, axis=1)\n                    size = np.sum(valid_mask)\n                    \n                    if size < 120:\n                        group = \"small\"\n                    elif size < 200:\n                        group = \"medium\"\n                    else:\n                        group = \"large\"\n                        \n                    if group not in self.size_groups:\n                        self.size_groups[group] = []\n                    self.size_groups[group].append(i)\n                    \n                print(f\"Size distribution - Small: {len(self.size_groups.get('small', []))}, \"\n                      f\"Medium: {len(self.size_groups.get('medium', []))}, \"\n                      f\"Large: {len(self.size_groups.get('large', []))}\")\n                      \n                # Store the correlation parameter for use in sample_structural_variation\n                global_correlation = self.correlation\n                print(f\"Using noise level: {self.base_noise_level}, correlation: {global_correlation}\")\n                \n                return self\n                \n            def predict(self, X):\n                batch_size = X.shape[0]\n                seq_length = X.shape[1]\n                predictions = np.zeros((batch_size, seq_length, 3))\n                \n                for i in range(batch_size):\n                    # Determine the RNA size group\n                    valid_mask = ~np.all(X[i] == 0, axis=1)\n                    size = np.sum(valid_mask)\n                    if size < 120:\n                        group = \"small\"\n                        # Size-specific noise scaling\n                        noise_level = self.base_noise_level * 0.6\n                    elif size < 200:\n                        group = \"medium\"\n                        noise_level = self.base_noise_level * 1.0\n                    else:\n                        group = \"large\"\n                        noise_level = self.base_noise_level * 0.4\n                    \n                    # If we have reference structures in this size group, use them\n                    if group in self.size_groups and self.size_groups[group]:\n                        # Randomly pick a reference structure from the same size group\n                        ref_idx = np.random.choice(self.size_groups[group])\n                        base_struct = self.reference_structures[ref_idx].copy()\n                        \n                        if self.geometric_sampling:\n                            # Pass the correlation parameter to the variation function\n                            predictions[i] = sample_structural_variation(\n                                base_struct, \n                                noise_level=noise_level,\n                                preserve_distance=True,\n                                use_global_movement=(group == \"small\"),\n                                correlation=self.correlation\n                            )\n                        else:\n                            noise = np.random.normal(0, noise_level, base_struct.shape)\n                            predictions[i] = base_struct + noise\n                    else:\n                        # Fall back to the original method if no size match\n                        sample = np.random.normal(self.global_mean, self.global_std, size=(seq_length, 3))\n                        if self.geometric_sampling:\n                            predictions[i] = sample_structural_variation(\n                                sample, \n                                noise_level=noise_level,\n                                preserve_distance=True,\n                                use_global_movement=(group == \"small\"),\n                                correlation=self.correlation\n                            )\n                        else:\n                            predictions[i] = sample\n                        \n                return predictions\n        \n        # Create and return model with specific parameters\n        model = ReferenceModel(geometric_sampling=geometric_sampling, \n                              base_noise_level=noise_level,\n                              correlation=correlation)\n        model.fit(X_ref, y_ref)\n        return model\n    \n    except Exception as e:\n        print(f\"Error in reference_based_approach: {str(e)}\")\n        import traceback\n        traceback.print_exc()\n        return None\n\n##############################################\n# 5. Função para visualizar estruturas 3D\n##############################################\ndef visualize_3d_structure(true_coords, pred_coords, sample_idx=0, title=\"Comparação de Estruturas 3D\"):\n    \"\"\"\n    Visualiza as estruturas 3D verdadeiras e preditas para uma amostra.\n    \"\"\"\n    true = true_coords[sample_idx]\n    pred = pred_coords[sample_idx]\n    mask = ~np.all(true == 0, axis=1)\n    true = true[mask]\n    pred = pred[mask]\n    \n    fig = plt.figure(figsize=(15, 7))\n    ax1 = fig.add_subplot(121, projection='3d')\n    ax1.plot(true[:, 0], true[:, 1], true[:, 2], 'b-', label='Verdadeira')\n    ax1.scatter(true[:, 0], true[:, 1], true[:, 2], c='b', s=20, alpha=0.5)\n    ax1.set_title('Estrutura Verdadeira')\n    ax1.set_xlabel('X')\n    ax1.set_ylabel('Y')\n    ax1.set_zlabel('Z')\n    ax1.grid(True)\n    \n    ax2 = fig.add_subplot(122, projection='3d')\n    ax2.plot(pred[:, 0], pred[:, 1], pred[:, 2], 'r-', label='Predita')\n    ax2.scatter(pred[:, 0], pred[:, 1], pred[:, 2], c='r', s=20, alpha=0.5)\n    ax2.set_title('Estrutura Predita')\n    ax2.set_xlabel('X')\n    ax2.set_ylabel('Y')\n    ax2.set_zlabel('Z')\n    ax2.grid(True)\n    \n    plt.suptitle(title)\n    plt.tight_layout()\n    plt.savefig(os.path.join(OUTPUT_DIR, f'structure_comparison_{sample_idx}.png'))\n    plt.show()\n\n##############################################\n# 6. Função para avaliar o modelo\n##############################################\ndef evaluate_model(model, X_valid, y_valid):\n    \"\"\"\n    Avalia o modelo calculando MAE, MSE e TM-score para cada estrutura.\n    Também plota a distribuição dos TM-scores.\n    \"\"\"\n    y_valid = np.nan_to_num(y_valid, nan=0.0)\n    y_pred = model.predict(X_valid)\n    y_pred = np.nan_to_num(y_pred, nan=0.0)\n    \n    mae = np.mean(np.abs(y_pred - y_valid))\n    mse = np.mean((y_pred - y_valid)**2)\n    print(f\"MAE geral: {mae:.4f}\")\n    print(f\"MSE geral: {mse:.4f}\")\n    \n    tm_scores = []\n    for i in range(len(X_valid)):\n        tm = calculate_tm_score(y_pred[i], y_valid[i])\n        tm_scores.append(tm)\n    avg_tm_score = np.mean(tm_scores)\n    print(f\"TM-score médio aproximado: {avg_tm_score:.4f}\")\n    \n    plt.figure(figsize=(10,6))\n    plt.hist(tm_scores, bins=10, alpha=0.7)\n    plt.axvline(avg_tm_score, color='r', linestyle='--', label=f'Média: {avg_tm_score:.4f}')\n    plt.title('Distribuição dos TM-scores')\n    plt.xlabel('TM-score')\n    plt.ylabel('Frequência')\n    plt.legend()\n    plt.grid(True, alpha=0.3)\n    plt.savefig(os.path.join(OUTPUT_DIR, 'tm_score_distribution.png'))\n    plt.show()\n    \n    return {\n        'mae': mae,\n        'mse': mse,\n        'tm_scores': tm_scores,\n        'avg_tm_score': avg_tm_score\n    }\n\n##############################################\n# 7. Função para gerar submissão\n##############################################\ndef prepare_test_features(test_seq_df, max_length=720):\n    \"\"\"\n    Prepara as features de teste (one-hot encoding da sequência).\n    \"\"\"\n    X_test = []\n    for _, row in test_seq_df.iterrows():\n        seq = row['sequence']\n        features = []\n        for nucleotide in seq:\n            if nucleotide == 'A':\n                features.append([1, 0, 0, 0, 0])\n            elif nucleotide == 'C':\n                features.append([0, 1, 0, 0, 0])\n            elif nucleotide == 'G':\n                features.append([0, 0, 1, 0, 0])\n            elif nucleotide == 'U':\n                features.append([0, 0, 0, 1, 0])\n            else:\n                features.append([0, 0, 0, 0, 1])\n        if len(features) < max_length:\n            padding = [[0, 0, 0, 0, 0]] * (max_length - len(features))\n            features.extend(padding)\n        else:\n            features = features[:max_length]\n        X_test.append(features)\n    return np.array(X_test)\n\ndef model_rna_specific_features(seq, base_structure):\n    \"\"\"\n    Modelo refinado baseado em características específicas de RNA.\n    \"\"\"\n    result = base_structure.copy()\n    valid_mask = ~np.all(base_structure == 0, axis=1)\n    \n    # Análise da sequência\n    seq_length = len(seq)\n    gc_content = (seq.count('G') + seq.count('C')) / seq_length\n    au_content = (seq.count('A') + seq.count('U')) / seq_length\n    \n    # Detecção de motivos conhecidos\n    hairpin_motifs = ['GNRA', 'UNCG', 'CUYG', 'ANYA']  # Tetraloops comuns\n    has_motif = False\n    \n    for motif in hairpin_motifs:\n        if motif in seq:\n            has_motif = True\n    \n    # Aplicar conhecimento sobre RNA\n    if gc_content > 0.7:\n        # RNAs ricos em GC tendem a formar estruturas mais rígidas e compactas\n        result = sample_structural_variation(result, noise_level=0.3, preserve_distance=True)\n    elif au_content > 0.6:\n        # RNAs ricos em AU tendem a formar estruturas mais flexíveis\n        result = sample_structural_variation(result, noise_level=0.8, preserve_distance=True, use_global_movement=True)\n    \n    # Sequências longas têm maior chance de formar estruturas complexas\n    if seq_length > 100:\n        # Aplicar dobras globais para simular domínios\n        result = sample_structural_variation(result, noise_level=0.5, preserve_distance=True, use_global_movement=True)\n    \n    return result\n\ndef generate_simple_diverse_structures(base_structure, seq_length, num_structures=5):\n    \"\"\"\n    Generate diverse structures with a simpler approach, focusing on \n    effective exploration of conformational space without complexity.\n    \"\"\"\n    structures = []\n    \n    # Add the base structure\n    structures.append(normalize_structure(base_structure))\n    \n    # Size-specific parameter tuning\n    if seq_length < 120:\n        # For small RNAs, use higher noise and more global movements\n        noise_levels = [0.1, 0.3, 0.6, 0.9]\n        use_global = [True, True, True, False]\n    elif seq_length < 200:\n        # For medium RNAs, balanced approach\n        noise_levels = [0.2, 0.4, 0.7, 1.0]\n        use_global = [False, True, True, False]\n    else:\n        # For large RNAs, more conservative variations\n        noise_levels = [0.1, 0.2, 0.3, 0.5]\n        use_global = [False, False, True, False]\n    \n    # Generate variations with different parameters\n    for i in range(len(noise_levels)):\n        candidate = sample_structural_variation(\n            base_structure,\n            noise_level=noise_levels[i],\n            preserve_distance=True,\n            use_global_movement=use_global[i]\n        )\n        \n        # Add slight random rotations for diversity\n        angle = np.random.uniform(0, np.pi/2)  # 0-90 degrees\n        rotation_matrix = np.array([\n            [np.cos(angle), -np.sin(angle), 0],\n            [np.sin(angle), np.cos(angle), 0],\n            [0, 0, 1]\n        ])\n        \n        rotated = np.zeros_like(candidate)\n        for j in range(len(candidate)):\n            rotated[j] = np.dot(candidate[j], rotation_matrix)\n        \n        structures.append(normalize_structure(rotated))\n    \n    # Ensure we have exactly 5 structures\n    while len(structures) < 5:\n        i = len(structures) - 1\n        noise = noise_levels[i % len(noise_levels)] * 1.1  # Slightly higher noise\n        structures.append(sample_structural_variation(structures[0], noise_level=noise))\n    \n    return structures[:5]  # Return exactly 5 structures\n\ndef generate_hybrid_structures(base_structure, seq_length, num_structures=5):\n    \"\"\"\n    Generate structures using a hybrid of approaches that combines the best elements\n    of multiple generation strategies. This function adapts its strategy based on \n    RNA sequence length to optimize performance across different RNA sizes.\n    \"\"\"\n    structures = []\n    \n    # Add normalized base structure as the first candidate\n    structures.append(normalize_structure(base_structure))\n    \n    # Implement size-specific strategies\n    if seq_length < 120:\n        # For small RNAs: Use higher diversity with controlled variations\n        # Small RNAs benefit from exploring more of the conformational space\n        structures.append(sample_structural_variation(\n            base_structure, \n            noise_level=0.2, \n            preserve_distance=True\n        ))\n        structures.append(sample_structural_variation(\n            base_structure, \n            noise_level=0.4, \n            preserve_distance=True,\n            use_global_movement=True\n        ))\n        \n        # Add more diverse structures with global rotations\n        for i in range(2):\n            # Create a random rotation to explore different orientations\n            angle = np.random.uniform(0, np.pi)\n            rotation_matrix = np.array([\n                [np.cos(angle), -np.sin(angle), 0],\n                [np.sin(angle), np.cos(angle), 0],\n                [0, 0, 1]\n            ])\n            \n            # Apply rotation to the base structure\n            rotated = np.zeros_like(base_structure)\n            for j in range(len(base_structure)):\n                rotated[j] = np.dot(base_structure[j], rotation_matrix)\n            \n            # Add structural variation to the rotated structure\n            structures.append(sample_structural_variation(\n                rotated, \n                noise_level=0.5, \n                preserve_distance=True,\n                use_global_movement=True\n            ))\n    elif seq_length < 200:\n        # For medium RNAs: Use balanced approach with moderate variations\n        noise_levels = [0.1, 0.2, 0.3, 0.4]\n        global_movements = [False, True, False, True]\n        \n        for i in range(4):\n            structures.append(sample_structural_variation(\n                base_structure, \n                noise_level=noise_levels[i], \n                preserve_distance=True,\n                use_global_movement=global_movements[i]\n            ))\n    else:\n        # For large RNAs: Use conservative variations to preserve global structure\n        # Large RNAs performed well in previous tests with subtle variations\n        noise_levels = [0.05, 0.1, 0.15, 0.2]\n        for noise in noise_levels:\n            structures.append(sample_structural_variation(\n                base_structure, \n                noise_level=noise, \n                preserve_distance=True,\n                use_global_movement=False\n            ))\n    \n    # Ensure we have exactly num_structures (default 5)\n    while len(structures) < num_structures:\n        # If we need more structures, create additional ones with small variations\n        i = len(structures) - 1\n        noise = 0.1 + 0.1 * i  # Gradually increase noise for diversity\n        new_struct = sample_structural_variation(\n            structures[0],  # Base on the normalized structure\n            noise_level=noise,\n            preserve_distance=True\n        )\n        structures.append(new_struct)\n    \n    # If we have too many, keep only the first num_structures\n    return structures[:num_structures]\n\ndef generate_simplified_submission(model, test_seq_df, sample_submission_df):\n    \"\"\"\n    Enhanced submission generation that uses ensemble prediction averaging\n    and hybrid structure generation for improved performance.\n    \"\"\"\n    X_test = prepare_test_features(test_seq_df)\n    \n    # Generate multiple predictions and average them for more stability\n    print(\"Generating ensemble predictions...\")\n    base_predictions = []\n    ensemble_size = 3  # Number of predictions to average\n    \n    for i in range(ensemble_size):\n        print(f\"  Generating prediction set {i+1}/{ensemble_size}\")\n        pred = model.predict(X_test)\n        base_predictions.append(pred)\n    \n    # Average the predictions for more stability\n    avg_predictions = np.mean(base_predictions, axis=0)\n    print(\"Ensemble prediction complete\")\n    \n    seq_to_coords = {}\n    for i, (_, row) in enumerate(test_seq_df.iterrows()):\n        target_id = row['target_id']\n        seq = row['sequence']\n        seq_length = len(seq)\n        \n        # Get base coordinates from ensemble average prediction\n        base_coords = avg_predictions[i][:seq_length]\n        \n        # Generate diverse structures with our hybrid approach\n        print(f\"Generating structures for sequence {i+1}/{len(test_seq_df)}, \" +\n              f\"length: {seq_length}\")\n        structures = generate_hybrid_structures(base_coords, seq_length)\n        \n        # Store the structures\n        seq_to_coords[target_id] = structures\n    \n    # Create submission DataFrame\n    print(\"Creating submission file...\")\n    submission_df = sample_submission_df.copy()\n    for i, row in submission_df.iterrows():\n        id_parts = row['ID'].split('_')\n        seq_id = id_parts[0]\n        residue_idx = int(id_parts[1]) - 1\n        if seq_id in seq_to_coords:\n            structures = seq_to_coords[seq_id]\n            if residue_idx < len(structures[0]):\n                for struct_idx in range(5):\n                    submission_df.at[i, f'x_{struct_idx+1}'] = structures[struct_idx][residue_idx][0]\n                    submission_df.at[i, f'y_{struct_idx+1}'] = structures[struct_idx][residue_idx][1]\n                    submission_df.at[i, f'z_{struct_idx+1}'] = structures[struct_idx][residue_idx][2]\n    \n    submission_file = os.path.join(OUTPUT_DIR, 'submission.csv')\n    submission_df.to_csv(submission_file, index=False)\n    print(f\"Enhanced submission file saved to {submission_file}\")\n    return submission_df\n\ndef advanced_search_best_model(X_train, y_train, X_valid, y_valid, \n                              target_score=0.20, max_iterations=30):\n    \"\"\"\n    Advanced search for the best model using multi-phase parameter tuning\n    and ensemble modeling.\n    \"\"\"\n    # Initialize tracking variables\n    best_params = None\n    best_model = None\n    best_score = 0.0\n    top_models = []  # For ensemble modeling\n    \n    # Phase 1: Broad parameter search\n    print(\"Phase 1: Broad parameter search\")\n    noise_levels = [0.15, 0.2, 0.25, 0.3,0.35]\n    correlations = [0.5, 0.7, 0.85,0.95]\n    \n    # Create a grid of parameters to try\n    param_combinations = []\n    for noise in noise_levels:\n        for corr in correlations:\n            param_combinations.append((noise, corr))\n    \n    # Shuffle the parameter combinations for better exploration\n    np.random.shuffle(param_combinations)\n    \n    # Limit the number of combinations to try in Phase 1\n    phase1_iterations = min(len(param_combinations), 12)\n    \n    for i in range(phase1_iterations):\n        noise, corr = param_combinations[i]\n        print(f\"\\nPhase 1 - Iteration {i+1}/{phase1_iterations}\")\n        print(f\"Trying noise_level={noise}, correlation={corr}\")\n        \n        # Set different random seed each iteration\n        np.random.seed(i * 42)\n        \n        # Create model with these parameters\n        model = reference_based_approach(X_valid, y_valid, \n                                        geometric_sampling=True,\n                                        noise_level=noise, \n                                        correlation=corr)\n        \n        if model is None:\n            print(\"Model creation failed, continuing...\")\n            continue\n        \n        # Evaluate the model\n        y_pred = model.predict(X_valid)\n        metrics = evaluate_model(model, X_valid, y_valid)\n        current_score = metrics['avg_tm_score']\n        \n        print(f\"TM-score: {current_score:.4f}\")\n        \n        # Track for ensemble modeling\n        top_models.append((model, current_score, noise, corr))\n        top_models.sort(key=lambda x: x[1], reverse=True)\n        top_models = top_models[:3]  # Keep only top 3 models\n        \n        # Update best parameters if this is the best model\n        if current_score > best_score:\n            best_score = current_score\n            best_model = model\n            best_params = {'noise': noise, 'corr': corr}\n            print(f\"New best model! TM-score: {best_score:.4f}, params: {best_params}\")\n            \n            # Save the best predictions\n            np.save(os.path.join(OUTPUT_DIR, 'best_phase1_predictions.npy'), y_pred)\n        \n        # Check if we've reached the target score\n        if current_score >= target_score:\n            print(f\"Target score {target_score} reached! Stopping search.\")\n            return best_model, {'avg_tm_score': best_score}, top_models\n    \n    # If we found good parameters, proceed to Phase 2\n    if best_params is not None:\n        print(f\"\\nPhase 1 complete. Best parameters: {best_params}\")\n        print(f\"Best TM-score so far: {best_score:.4f}\")\n        \n        # Phase 2: Refined parameter search around the best parameters\n        print(\"\\nPhase 2: Refined parameter search\")\n        \n        # Create refined parameter ranges centered around best parameters\n        refined_noise = [\n            max(0.05, best_params['noise'] - 0.05),\n            best_params['noise'],\n            min(0.5, best_params['noise'] + 0.05)\n        ]\n        \n        refined_corr = [\n            max(0.1, best_params['corr'] - 0.1),\n            best_params['corr'],\n            min(0.95, best_params['corr'] + 0.1)\n        ]\n        \n        # Create refined parameter grid\n        refined_combinations = []\n        for noise in refined_noise:\n            for corr in refined_corr:\n                # Skip the exact combination we already tried\n                if noise == best_params['noise'] and corr == best_params['corr']:\n                    continue\n                refined_combinations.append((noise, corr))\n        \n        # Try the refined parameters\n        phase2_iterations = min(len(refined_combinations), 8)\n        for i in range(phase2_iterations):\n            noise, corr = refined_combinations[i]\n            print(f\"\\nPhase 2 - Iteration {i+1}/{phase2_iterations}\")\n            print(f\"Trying refined params: noise_level={noise}, correlation={corr}\")\n            \n            # Set different random seed\n            np.random.seed((i+100) * 42)\n            \n            # Create model with refined parameters\n            model = reference_based_approach(X_valid, y_valid, \n                                            geometric_sampling=True,\n                                            noise_level=noise, \n                                            correlation=corr)\n            \n            if model is None:\n                continue\n            \n            # Evaluate the model\n            y_pred = model.predict(X_valid)\n            metrics = evaluate_model(model, X_valid, y_valid)\n            current_score = metrics['avg_tm_score']\n            \n            print(f\"TM-score with refined params: {current_score:.4f}\")\n            \n            # Update top models for ensemble\n            top_models.append((model, current_score, noise, corr))\n            top_models.sort(key=lambda x: x[1], reverse=True)\n            top_models = top_models[:3]\n            \n            # Update best model if improved\n            if current_score > best_score:\n                best_score = current_score\n                best_model = model\n                best_params = {'noise': noise, 'corr': corr}\n                print(f\"New best model in Phase 2! TM-score: {best_score:.4f}\")\n                \n                # Save the best predictions\n                np.save(os.path.join(OUTPUT_DIR, 'best_phase2_predictions.npy'), y_pred)\n            \n            if current_score >= target_score:\n                print(f\"Target score {target_score} reached in Phase 2!\")\n                break\n    \n    # Final report\n    print(\"\\nSearch complete!\")\n    print(f\"Best model parameters: Noise={best_params['noise']}, Correlation={best_params['corr']}\")\n    print(f\"Best individual model TM-score: {best_score:.4f}\")\n    \n    # Report on top models for ensemble\n    print(\"\\nTop models for ensemble:\")\n    for i, (model, score, noise, corr) in enumerate(top_models):\n        print(f\"Model {i+1}: TM-score={score:.4f}, Noise={noise}, Correlation={corr}\")\n    \n    return best_model, {'avg_tm_score': best_score}, top_models\n\ndef generate_ensemble_submission(top_models, test_seq_df, sample_submission_df):\n    \"\"\"\n    Generate a submission using an ensemble of the top performing models.\n    \"\"\"\n    X_test = prepare_test_features(test_seq_df)\n    \n    print(\"Generating ensemble predictions from top models...\")\n    ensemble_predictions = []\n    \n    # Generate predictions from each top model\n    for i, (model, score, _, _) in enumerate(top_models):\n        print(f\"Generating predictions from model {i+1} (TM-score: {score:.4f})\")\n        model_predictions = model.predict(X_test)\n        ensemble_predictions.append(model_predictions)\n    \n    # Average the predictions\n    avg_predictions = np.mean(ensemble_predictions, axis=0)\n    print(\"Ensemble averaging complete\")\n    \n    # Generate diverse structures for each sequence\n    seq_to_coords = {}\n    for i, (_, row) in enumerate(test_seq_df.iterrows()):\n        target_id = row['target_id']\n        seq = row['sequence']\n        seq_length = len(seq)\n        \n        # Get base coordinates from ensemble prediction\n        base_coords = avg_predictions[i][:seq_length]\n        \n        # Generate diverse structures with hybrid approach\n        print(f\"Generating structures for sequence {i+1}/{len(test_seq_df)}, length: {seq_length}\")\n        structures = generate_hybrid_structures(base_coords, seq_length)\n        \n        # Store the structures\n        seq_to_coords[target_id] = structures\n    \n    # Create submission DataFrame\n    print(\"Creating submission file...\")\n    submission_df = sample_submission_df.copy()\n    for i, row in submission_df.iterrows():\n        id_parts = row['ID'].split('_')\n        seq_id = id_parts[0]\n        residue_idx = int(id_parts[1]) - 1\n        if seq_id in seq_to_coords:\n            structures = seq_to_coords[seq_id]\n            if residue_idx < len(structures[0]):\n                for struct_idx in range(5):\n                    submission_df.at[i, f'x_{struct_idx+1}'] = structures[struct_idx][residue_idx][0]\n                    submission_df.at[i, f'y_{struct_idx+1}'] = structures[struct_idx][residue_idx][1]\n                    submission_df.at[i, f'z_{struct_idx+1}'] = structures[struct_idx][residue_idx][2]\n    \n    # Changed filename to submission.csv\n    submission_file = os.path.join(OUTPUT_DIR, 'submission.csv')\n    submission_df.to_csv(submission_file, index=False)\n    print(f\"Ensemble submission file saved to {submission_file}\")\n    return submission_df\n\ndef refine_parameter_search(base_noise=0.2, base_corr=0.85, X_valid=None, y_valid=None):\n    \"\"\"\n    Realiza uma busca refinada em torno dos parâmetros ótimos já identificados.\n    \n    Parameters:\n    -----------\n    base_noise: float\n        Valor de ruído base que demonstrou bons resultados (0.2)\n    base_corr: float\n        Valor de correlação base que demonstrou bons resultados (0.85)\n    \"\"\"\n    # Defina pequenas variações em torno dos valores ótimos\n    noise_variations = [\n        base_noise - 0.03, \n        base_noise - 0.01, \n        base_noise, \n        base_noise + 0.01, \n        base_noise + 0.03\n    ]\n    \n    corr_variations = [\n        max(0.1, base_corr - 0.05),\n        base_corr - 0.02,\n        base_corr,\n        min(0.98, base_corr + 0.02),\n        min(0.98, base_corr + 0.05)\n    ]\n    \n    # Armazene o melhor modelo e seu escore\n    best_model = None\n    best_score = 0.0\n    best_params = None\n    \n    # Execute uma busca em grade refinada\n    print(\"Iniciando busca refinada de parâmetros:\")\n    for noise in noise_variations:\n        for corr in corr_variations:\n            # Pule combinação exata já testada\n            if noise == base_noise and corr == base_corr:\n                continue\n                \n            print(f\"Testando noise={noise:.3f}, correlation={corr:.3f}\")\n            \n            # Use uma semente aleatória diferente para cada iteração\n            np.random.seed(int(noise*1000 + corr*100))\n            \n            # Crie modelo com estes parâmetros\n            model = reference_based_approach(\n                X_valid, y_valid,\n                geometric_sampling=True,\n                noise_level=noise,\n                correlation=corr\n            )\n            \n            if model is None:\n                continue\n                \n            # Avalie o modelo\n            metrics = evaluate_model(model, X_valid, y_valid)\n            current_score = metrics['avg_tm_score']\n            \n            print(f\"TM-score: {current_score:.4f}\")\n            \n            # Atualize o melhor modelo se for superior\n            if current_score > best_score:\n                best_score = current_score\n                best_model = model\n                best_params = {'noise': noise, 'corr': corr}\n                print(f\"Novo melhor modelo! TM-score: {best_score:.4f}, params: {best_params}\")\n    \n    return best_model, best_score, best_params\n\ndef ablation_analysis(X_valid, y_valid, base_params=None):\n    \"\"\"\n    Realiza uma análise de ablação para identificar os componentes críticos.\n    \"\"\"\n    if base_params is None:\n        base_params = {'noise': 0.2, 'corr': 0.85}\n    \n    # Crie o modelo base com todos os componentes\n    print(\"Criando modelo base com todos os componentes\")\n    base_model = reference_based_approach(\n        X_valid, y_valid,\n        geometric_sampling=True,  # Componente 1: Amostragem geométrica\n        noise_level=base_params['noise'],\n        correlation=base_params['corr']\n    )\n    \n    base_metrics = evaluate_model(base_model, X_valid, y_valid)\n    base_score = base_metrics['avg_tm_score']\n    print(f\"Modelo base - TM-score: {base_score:.4f}\")\n    \n    # Teste sem amostragem geométrica\n    print(\"\\nTestando sem amostragem geométrica\")\n    no_geom_model = reference_based_approach(\n        X_valid, y_valid,\n        geometric_sampling=False,  # Removido componente 1\n        noise_level=base_params['noise'],\n        correlation=base_params['corr']\n    )\n    \n    no_geom_metrics = evaluate_model(no_geom_model, X_valid, y_valid)\n    no_geom_score = no_geom_metrics['avg_tm_score']\n    print(f\"Sem amostragem geométrica - TM-score: {no_geom_score:.4f}\")\n    print(f\"Impacto: {(no_geom_score - base_score) / base_score * 100:.2f}%\")\n    \n    # Teste sem preservação de distância - abordagem simplificada\n    print(\"\\nTestando sem preservação de distância (abordagem simplificada)\")\n    \n    # Em vez de criar uma classe complexa, vamos usar o modelo base e modificar\n    # a função sample_structural_variation temporariamente durante a predição\n    original_sample_fn = sample_structural_variation\n    \n    # Criar versão modificada da função que não preserva distância\n    def modified_sample_fn(coords, noise_level=0.5, preserve_distance=True, \n                          use_global_movement=False, correlation=0.7):\n        # Versão da função com preserve_distance=False\n        return original_sample_fn(coords, noise_level, False, use_global_movement, correlation)\n    \n    # Substituir temporariamente a função global\n    globals()['sample_structural_variation'] = modified_sample_fn\n    \n    # Criar modelo para teste\n    no_distance_model = reference_based_approach(\n        X_valid, y_valid,\n        geometric_sampling=True,\n        noise_level=base_params['noise'],\n        correlation=base_params['corr']\n    )\n    \n    # Avaliar com a função modificada\n    no_distance_metrics = evaluate_model(no_distance_model, X_valid, y_valid)\n    no_distance_score = no_distance_metrics['avg_tm_score']\n    \n    # Restaurar a função original\n    globals()['sample_structural_variation'] = original_sample_fn\n    \n    print(f\"Sem preservação de distância - TM-score: {no_distance_score:.4f}\")\n    print(f\"Impacto: {(no_distance_score - base_score) / base_score * 100:.2f}%\")\n    \n    # Retornar resultados da análise\n    return {\n        'base': base_score,\n        'no_geometric_sampling': no_geom_score,\n        'no_distance_preservation': no_distance_score\n    }\n\ndef test_specific_improvements(X_valid, y_valid, base_params=None):\n    \"\"\"\n    Testa melhorias específicas individualmente para avaliar seu impacto.\n    \n    Parameters:\n    -----------\n    base_params: dict\n        Parâmetros base para comparação (ex: {'noise': 0.2, 'corr': 0.85})\n    \"\"\"\n    if base_params is None:\n        base_params = {'noise': 0.2, 'corr': 0.85}\n    \n    # Crie o modelo base com configuração padrão\n    print(\"Criando modelo base\")\n    base_model = reference_based_approach(\n        X_valid, y_valid,\n        geometric_sampling=True,\n        noise_level=base_params['noise'],\n        correlation=base_params['corr']\n    )\n    \n    base_metrics = evaluate_model(base_model, X_valid, y_valid)\n    base_score = base_metrics['avg_tm_score']\n    print(f\"Modelo base - TM-score: {base_score:.4f}\")\n    \n    # Melhoria 1: Normalização de estruturas\n    print(\"\\nTestando melhoria: Normalização de estruturas\")\n    # Para este teste, precisamos modificar a função predict do modelo\n    \n    class ImprovedNormalizationModel(base_model.__class__):\n        def __init__(self):\n            # Copy all attributes from base_model\n            for attr_name in dir(base_model):\n                if not attr_name.startswith('__') and not callable(getattr(base_model, attr_name)):\n                    setattr(self, attr_name, getattr(base_model, attr_name))\n    \n        def predict(self, X):\n            # Obtain normal predictions\n            predictions = super().predict(X)\n        \n            # Apply additional normalization to each structure\n            for i in range(len(predictions)):\n                predictions[i] = normalize_structure(predictions[i])\n        \n            return predictions\n    \n    norm_model = ImprovedNormalizationModel()\n    norm_metrics = evaluate_model(norm_model, X_valid, y_valid)\n    norm_score = norm_metrics['avg_tm_score']\n    print(f\"Com normalização melhorada - TM-score: {norm_score:.4f}\")\n    print(f\"Impacto: {(norm_score - base_score) / base_score * 100:.2f}%\")\n    \n    # Melhoria 2: Adaptação de parâmetros por tamanho de RNA\n    print(\"\\nTestando melhoria: Adaptação refinada de parâmetros por tamanho\")\n    \n    class SizeRefinedModel(base_model.__class__):\n        def predict(self, X):\n            batch_size = X.shape[0]\n            seq_length = X.shape[1]\n            predictions = np.zeros((batch_size, seq_length, 3))\n            \n            for i in range(batch_size):\n                valid_mask = ~np.all(X[i] == 0, axis=1)\n                size = np.sum(valid_mask)\n                \n                # Refinamento mais detalhado por tamanho\n                if size < 50:  # Muito pequenos\n                    group = \"small\"\n                    noise_level = self.base_noise_level * 0.8\n                    use_global = True\n                elif size < 120:  # Pequenos\n                    group = \"small\"\n                    noise_level = self.base_noise_level * 0.6\n                    use_global = True\n                elif size < 160:  # Médios pequenos\n                    group = \"medium\"\n                    noise_level = self.base_noise_level * 0.9\n                    use_global = True\n                elif size < 200:  # Médios grandes\n                    group = \"medium\"\n                    noise_level = self.base_noise_level * 1.1\n                    use_global = False\n                elif size < 300:  # Grandes pequenos\n                    group = \"large\"\n                    noise_level = self.base_noise_level * 0.5\n                    use_global = False\n                else:  # Muito grandes\n                    group = \"large\"\n                    noise_level = self.base_noise_level * 0.3\n                    use_global = False\n                \n                # Lógica adaptada para selecionar referências\n                group_to_use = group\n                if group in self.size_groups and self.size_groups[group]:\n                    ref_indices = self.size_groups[group]\n                else:\n                    # Caso não tenha referências exatas, use grupo mais próximo\n                    available_groups = [g for g in self.size_groups if self.size_groups[g]]\n                    if available_groups:\n                        group_to_use = available_groups[0]\n                        ref_indices = self.size_groups[group_to_use]\n                    else:\n                        # Fallback para média global\n                        sample = np.random.normal(self.global_mean, self.global_std, size=(seq_length, 3))\n                        predictions[i] = sample_structural_variation(\n                            sample, \n                            noise_level=noise_level,\n                            preserve_distance=True,\n                            use_global_movement=use_global,\n                            correlation=self.correlation\n                        )\n                        continue\n                \n                # Selecione referência e aplique variação\n                ref_idx = np.random.choice(ref_indices)\n                base_struct = self.reference_structures[ref_idx].copy()\n                \n                predictions[i] = sample_structural_variation(\n                    base_struct, \n                    noise_level=noise_level,\n                    preserve_distance=True,\n                    use_global_movement=use_global,\n                    correlation=self.correlation\n                )\n                    \n            return predictions\n    \n    size_refined_model = SizeRefinedModel()\n    size_refined_metrics = evaluate_model(size_refined_model, X_valid, y_valid)\n    size_refined_score = size_refined_metrics['avg_tm_score']\n    print(f\"Com adaptação refinada por tamanho - TM-score: {size_refined_score:.4f}\")\n    print(f\"Impacto: {(size_refined_score - base_score) / base_score * 100:.2f}%\")\n    \n    # Retornar resultados das melhorias\n    return {\n        'base': base_score,\n        'improved_normalization': norm_score,\n        'size_refined_adaptation': size_refined_score\n    }\n\ndef create_optimized_model(X_valid, y_valid, optimal_params, improvement_results):\n    \"\"\"\n    Cria um modelo otimizado combinando os componentes mais impactantes\n    identificados através das análises anteriores.\n    \n    Parameters:\n    -----------\n    optimal_params: dict\n        Parâmetros otimizados da busca refinada\n    improvement_results: dict\n        Resultados das análises de ablação e melhorias específicas\n    \"\"\"\n    print(\"Criando modelo final otimizado\")\n    \n    # Determine quais melhorias foram mais impactantes\n    use_improved_normalization = (improvement_results.get('improved_normalization', 0) > \n                                 improvement_results.get('base', 0))\n    \n    use_size_refinement = (improvement_results.get('size_refined_adaptation', 0) > \n                          improvement_results.get('base', 0))\n    \n    # Crie o modelo base com parâmetros otimizados\n    base_model = reference_based_approach(\n        X_valid, y_valid,\n        geometric_sampling=True,  # Assumimos que is se mostrou importante\n        noise_level=optimal_params['noise'],\n        correlation=optimal_params['corr']\n    )\n    \n    # Se nenhuma melhoria foi impactante, retorne o modelo base otimizado\n    if not use_improved_normalization and not use_size_refinement:\n        print(\"Nenhuma melhoria adicional teve impacto positivo. Usando modelo base otimizado.\")\n        return base_model\n    \n    # Construa classe de modelo final com as melhorias que foram úteis\n    class OptimizedModel(base_model.__class__):\n        def predict(self, X):\n            batch_size = X.shape[0]\n            seq_length = X.shape[1]\n            predictions = np.zeros((batch_size, seq_length, 3))\n            \n            for i in range(batch_size):\n                valid_mask = ~np.all(X[i] == 0, axis=1)\n                size = np.sum(valid_mask)\n                \n                # Aplicar refinamento por tamanho se for benéfico\n                if use_size_refinement:\n                    if size < 50:  # Muito pequenos\n                        group = \"small\"\n                        noise_level = self.base_noise_level * 0.8\n                        use_global = True\n                    elif size < 120:  # Pequenos\n                        group = \"small\"\n                        noise_level = self.base_noise_level * 0.6\n                        use_global = True\n                    elif size < 160:  # Médios pequenos\n                        group = \"medium\"\n                        noise_level = self.base_noise_level * 0.9\n                        use_global = True\n                    elif size < 200:  # Médios grandes\n                        group = \"medium\"\n                        noise_level = self.base_noise_level * 1.1\n                        use_global = False\n                    elif size < 300:  # Grandes pequenos\n                        group = \"large\"\n                        noise_level = self.base_noise_level * 0.5\n                        use_global = False\n                    else:  # Muito grandes\n                        group = \"large\"\n                        noise_level = self.base_noise_level * 0.3\n                        use_global = False\n                else:\n                    # Usar categorização original\n                    if size < 120:\n                        group = \"small\"\n                        noise_level = self.base_noise_level * 0.6\n                    elif size < 200:\n                        group = \"medium\"\n                        noise_level = self.base_noise_level * 1.0\n                    else:\n                        group = \"large\"\n                        noise_level = self.base_noise_level * 0.4\n                    use_global = (group == \"small\")\n                \n                # Lógica de seleção de referência e geração de estrutura\n                group_to_use = group\n                if group in self.size_groups and self.size_groups[group]:\n                    ref_indices = self.size_groups[group]\n                else:\n                    # Caso não tenha referências exatas, use grupo mais próximo\n                    available_groups = [g for g in self.size_groups if self.size_groups[g]]\n                    if available_groups:\n                        group_to_use = available_groups[0]\n                        ref_indices = self.size_groups[group_to_use]\n                    else:\n                        # Fallback para média global\n                        sample = np.random.normal(self.global_mean, self.global_std, size=(seq_length, 3))\n                        pred = sample_structural_variation(\n                            sample, \n                            noise_level=noise_level,\n                            preserve_distance=True,\n                            use_global_movement=use_global,\n                            correlation=self.correlation\n                        )\n                        predictions[i] = pred\n                        continue\n                \n                # Selecione referência e aplique variação\n                ref_idx = np.random.choice(ref_indices)\n                base_struct = self.reference_structures[ref_idx].copy()\n                \n                pred = sample_structural_variation(\n                    base_struct, \n                    noise_level=noise_level,\n                    preserve_distance=True,\n                    use_global_movement=use_global,\n                    correlation=self.correlation\n                )\n                \n                # Aplicar normalização melhorada se for benéfica\n                if use_improved_normalization:\n                    pred = normalize_structure(pred)\n                \n                predictions[i] = pred\n                    \n            return predictions\n    \n    optimized_model = OptimizedModel()\n    \n    # Avalie o modelo otimizado final\n    final_metrics = evaluate_model(optimized_model, X_valid, y_valid)\n    final_score = final_metrics['avg_tm_score']\n    \n    print(f\"Modelo otimizado final - TM-score: {final_score:.4f}\")\n    print(f\"Melhorias aplicadas:\")\n    print(f\"- Normalização melhorada: {'Sim' if use_improved_normalization else 'Não'}\")\n    print(f\"- Refinamento por tamanho: {'Sim' if use_size_refinement else 'Não'}\")\n    print(f\"- Parâmetros otimizados: noise={optimal_params['noise']}, corr={optimal_params['corr']}\")\n    \n    return optimized_model\n\n##############################################\n# MAIN – Uso do Modelo de Referência\n##############################################\n\ndef search_best_model(X_train, y_train, X_valid, y_valid, max_iterations=10, target_score=0.20):\n    \"\"\"\n    Search for the best performing model by running multiple iterations\n    and keeping track of the best result.\n    \n    Parameters:\n    -----------\n    X_train, y_train: Training data\n    X_valid, y_valid: Validation data\n    max_iterations: Maximum number of search iterations\n    target_score: Target TM-score to stop the search\n    \n    Returns:\n    --------\n    best_model: The model with highest TM-score\n    best_metrics: Metrics for the best model\n    best_predictions: Predictions from the best model\n    \"\"\"\n    best_model = None\n    best_metrics = None\n    best_predictions = None\n    best_score = 0.0\n    \n    print(f\"Starting model search (max {max_iterations} iterations, target score: {target_score})\")\n    \n    for iteration in range(max_iterations):\n        print(f\"\\n----- Iteration {iteration+1}/{max_iterations} -----\")\n        \n        # Create a new model with random seed based on iteration\n        np.random.seed(iteration * 42)  # Different seed each iteration\n        model = reference_based_approach(X_valid, y_valid, geometric_sampling=True)\n        \n        if model is None:\n            print(\"Model creation failed in this iteration, continuing...\")\n            continue\n            \n        # Evaluate the model\n        y_pred = model.predict(X_valid)\n        metrics = evaluate_model(model, X_valid, y_valid)\n        \n        # Check if this is the best model so far\n        current_score = metrics['avg_tm_score']\n        print(f\"Iteration {iteration+1} TM-score: {current_score:.4f} (best so far: {best_score:.4f})\")\n        \n        if current_score > best_score:\n            print(f\"New best model found! TM-score improved: {best_score:.4f} -> {current_score:.4f}\")\n            best_model = model\n            best_metrics = metrics\n            best_predictions = y_pred\n            best_score = current_score\n            \n            # Save the best model's predictions\n            np.save(os.path.join(OUTPUT_DIR, 'best_predictions.npy'), best_predictions)\n            \n            # Optional: Visualize the best model's results\n            for i in range(min(3, len(X_valid))):\n                visualize_3d_structure(\n                    y_valid, best_predictions, sample_idx=i,\n                    title=f\"Best Model Structure (TM-score: {best_metrics['tm_scores'][i]:.4f})\"\n                )\n        \n        # Check if we've reached the target score\n        if current_score >= target_score:\n            print(f\"Target TM-score of {target_score} reached! Stopping search.\")\n            break\n    \n    print(f\"\\nSearch completed. Best TM-score: {best_score:.4f}\")\n    return best_model, best_metrics, best_predictions\n\n##############################################\n# 8. Funções otimizadas com base na análise de ablação\n##############################################\n\ndef create_optimized_model_based_on_ablation(X_valid, y_valid, optimal_params):\n    \"\"\"\n    Creates an optimized model based on ablation study results.\n    \"\"\"\n    print(\"Criando modelo otimizado com base nos resultados da análise de ablação\")\n    \n    # Criar modelo com parâmetros ótimos mas SEM amostragem geométrica\n    model = reference_based_approach(\n        X_valid, y_valid,\n        geometric_sampling=False,  # Desabilitar amostragem geométrica com base nos resultados da ablação\n        noise_level=optimal_params['noise'],\n        correlation=optimal_params['corr']\n    )\n    \n    print(f\"Modelo criado com noise={optimal_params['noise']}, correlation={optimal_params['corr']}, geometric_sampling=False\")\n    \n    return model\n\ndef simplified_submission_generator(model, test_seq_df, sample_submission_df, output_dir):\n    \"\"\"\n    Simplified submission generation to ensure a file is created.\n    \"\"\"\n    X_test = prepare_test_features(test_seq_df)\n    y_pred = model.predict(X_test)\n    \n    # Mapear predições para o formato de submissão\n    submission_df = sample_submission_df.copy()\n    seq_to_coords = {}\n    \n    # Processar cada sequência de teste\n    for i, (_, row) in enumerate(test_seq_df.iterrows()):\n        target_id = row['target_id']\n        seq_length = len(row['sequence'])\n        \n        # Gerar 5 estruturas diversas\n        base_coords = y_pred[i][:seq_length]\n        structures = []\n        \n        # Adicionar a predição base\n        structures.append(normalize_structure(base_coords))\n        \n        # Adicionar 4 variações com diferentes níveis de ruído\n        for noise in [0.1, 0.2, 0.3, 0.4]:\n            variation = base_coords + np.random.normal(0, noise, base_coords.shape)\n            structures.append(normalize_structure(variation))\n        \n        seq_to_coords[target_id] = structures\n        print(f\"Processada sequência {i+1}/{len(test_seq_df)}, ID: {target_id}, comprimento: {seq_length}\")\n    \n    # Preencher o dataframe de submissão\n    for i, row in submission_df.iterrows():\n        if i % 1000 == 0:\n            print(f\"Processando linha {i}/{len(submission_df)} da submissão\")\n            \n        id_parts = row['ID'].split('_')\n        seq_id = id_parts[0]\n        residue_idx = int(id_parts[1]) - 1\n        \n        if seq_id in seq_to_coords and residue_idx < len(seq_to_coords[seq_id][0]):\n            for struct_idx in range(5):\n                submission_df.at[i, f'x_{struct_idx+1}'] = seq_to_coords[seq_id][struct_idx][residue_idx][0]\n                submission_df.at[i, f'y_{struct_idx+1}'] = seq_to_coords[seq_id][struct_idx][residue_idx][1]\n                submission_df.at[i, f'z_{struct_idx+1}'] = seq_to_coords[seq_id][struct_idx][residue_idx][2]\n    \n    # Salvar arquivo e verificar\n    submission_file = os.path.join(output_dir, 'submission.csv')\n    submission_df.to_csv(submission_file, index=False)\n    print(f\"Submissão salva em {submission_file}\")\n    \n    # Verificar se o arquivo existe\n    if os.path.exists(submission_file):\n        print(f\"Arquivo verificado: {os.path.getsize(submission_file)} bytes\")\n    else:\n        print(\"AVISO: Arquivo não encontrado após o salvamento!\")\n    \n    return submission_df\n\ndef simplified_main():\n    \"\"\"\n    Simplified main function with better error handling.\n    \"\"\"\n    try:\n        print(\"Carregando dados processados...\")\n        X_train, y_train, X_valid, y_valid = load_processed_data()\n        \n        print(\"\\nVerificando validade dos dados...\")\n        print(f\"X_valid shape: {X_valid.shape}, has NaN: {np.isnan(X_valid).any()}\")\n        print(f\"y_valid shape: {y_valid.shape}, has NaN: {np.isnan(y_valid).any()}\")\n        \n        print(\"\\nCarregando dados de teste...\")\n        try:\n            test_seq_df = pd.read_csv(os.path.join(DATA_DIR, \"test_sequences.csv\"))\n            sample_submission_df = pd.read_csv(os.path.join(DATA_DIR, \"sample_submission.csv\"))\n            print(f\"Dados de teste carregados: {len(test_seq_df)} sequências\")\n        except Exception as e:\n            print(f\"Erro ao carregar dados de teste: {e}\")\n            traceback.print_exc()\n            return None, None\n        \n        # Usar parâmetros ótimos da busca anterior\n        optimal_params = {'noise': 0.21, 'corr': 0.83}\n        \n        # Criar modelo sem amostragem geométrica (com base nos resultados da ablação)\n        print(\"\\nCriando modelo otimizado...\")\n        model = create_optimized_model_based_on_ablation(\n            X_valid, y_valid,\n            optimal_params\n        )\n        \n        # Avaliar modelo\n        print(\"\\nAvaliando modelo...\")\n        metrics = evaluate_model(model, X_valid, y_valid)\n        \n        # Garantir que o diretório de saída exista\n        os.makedirs(OUTPUT_DIR, exist_ok=True)\n        \n        # Gerar submissão\n        print(\"\\nGerando submissão...\")\n        submission_df = simplified_submission_generator(\n            model, test_seq_df, sample_submission_df, OUTPUT_DIR\n        )\n        \n        print(\"\\nProcesso concluído com sucesso!\")\n        return model, metrics\n        \n    except Exception as e:\n        print(f\"ERRO em simplified_main: {str(e)}\")\n        import traceback\n        traceback.print_exc()\n        return None, None\n\n##############################################\n# 9. Funções para ensemble com múltiplas sementes\n##############################################\n\ndef ensemble_with_multiple_seeds(X_valid, y_valid, test_seq_df, sample_submission_df, output_dir, \n                                num_runs=10, optimal_params={'noise': 0.21, 'corr': 0.83}):\n    \"\"\"\n    Executa o modelo com múltiplas sementes aleatórias e cria um ensemble das melhores execuções.\n    \"\"\"\n    import numpy as np\n    import os\n    import time\n    import traceback\n    \n    # Lista para armazenar os resultados de cada execução\n    all_results = []\n    \n    print(f\"Iniciando ensemble com {num_runs} execuções diferentes...\")\n    \n    # Executar o modelo várias vezes com sementes diferentes\n    for run in range(num_runs):\n        try:\n            # Semente baseada no run e no timestamp para garantir aleatoriedade\n            seed = run * 100 + int(time.time()) % 1000\n            np.random.seed(seed)\n            \n            print(f\"\\nExecução {run+1}/{num_runs} - Semente: {seed}\")\n            \n            # Criar e avaliar o modelo\n            model = reference_based_approach(\n                X_valid, y_valid,\n                geometric_sampling=False,  # Com base na ablação\n                noise_level=optimal_params['noise'],\n                correlation=optimal_params['corr']\n            )\n            \n            if model is None:\n                print(f\"Falha na criação do modelo na execução {run+1}, continuando...\")\n                continue\n            \n            # Avaliar o modelo\n            print(\"Avaliando modelo...\")\n            metrics = evaluate_model(model, X_valid, y_valid)\n            tm_score = metrics['avg_tm_score']\n            print(f\"TM-score desta execução: {tm_score:.4f}\")\n            \n            # Gerar predições para teste\n            X_test = prepare_test_features(test_seq_df)\n            y_pred = model.predict(X_test)\n            \n            # Armazenar os resultados desta execução\n            all_results.append({\n                'seed': seed,\n                'tm_score': tm_score,\n                'predictions': y_pred,\n                'model': model\n            })\n            \n            # Salvar predições intermediárias para segurança\n            np.save(os.path.join(output_dir, f'predictions_run_{run+1}_tmscore_{tm_score:.4f}.npy'), y_pred)\n            \n        except Exception as e:\n            print(f\"Erro na execução {run+1}: {str(e)}\")\n            traceback.print_exc()\n            continue\n    \n    if not all_results:\n        print(\"Nenhuma execução foi bem-sucedida. Não é possível criar ensemble.\")\n        return None, all_results\n        \n    # Ordenar resultados pelo TM-score\n    all_results.sort(key=lambda x: x['tm_score'], reverse=True)\n    \n    print(\"\\nTodas as execuções completadas. TM-scores:\")\n    for i, result in enumerate(all_results):\n        print(f\"Execução com semente {result['seed']}: TM-score = {result['tm_score']:.4f}\")\n    \n    # Selecionar as N melhores execuções para o ensemble\n    num_best = min(5, len(all_results))  # Usar no máximo as 5 melhores\n    best_results = all_results[:num_best]\n    \n    print(f\"\\nUsando as {num_best} melhores execuções para o ensemble:\")\n    for i, result in enumerate(best_results):\n        print(f\"{i+1}. TM-score: {result['tm_score']:.4f} (semente: {result['seed']})\")\n    \n    # Criar ensemble a partir das melhores execuções\n    print(\"\\nCriando ensemble das melhores execuções...\")\n    \n    # Inicializar dicionário para armazenar estruturas por sequência\n    seq_to_coords = {}\n    \n    # Para cada sequência de teste\n    for i, (_, row) in enumerate(test_seq_df.iterrows()):\n        target_id = row['target_id']\n        seq_length = len(row['sequence'])\n        print(f\"Processando sequência {i+1}/{len(test_seq_df)}, ID: {target_id}\")\n        \n        # Coletar predições das melhores execuções para esta sequência\n        sequence_predictions = []\n        for result in best_results:\n            pred = result['predictions'][i][:seq_length]\n            sequence_predictions.append(pred)\n        \n        # Usar as 5 melhores estruturas para submissão\n        # Começar com a média das predições como base\n        avg_pred = np.mean(sequence_predictions, axis=0)\n        \n        # Criar estruturas usando a média e pequenas variações\n        structures = []\n        \n        # Adicionar a estrutura média normalizada\n        structures.append(normalize_structure(avg_pred))\n        \n        # Adicionar estruturas das melhores execuções\n        for j in range(min(4, len(best_results))):\n            best_pred = best_results[j]['predictions'][i][:seq_length]\n            structures.append(normalize_structure(best_pred))\n            \n        # Garantir que temos exatamente 5 estruturas\n        while len(structures) < 5:\n            # Adicionar pequenas variações da média\n            noise = 0.1 * (len(structures) - 1)\n            variation = avg_pred + np.random.normal(0, noise, avg_pred.shape)\n            structures.append(normalize_structure(variation))\n        \n        # Armazenar as estruturas para esta sequência\n        seq_to_coords[target_id] = structures[:5]  # Exatamente 5 estruturas\n    \n    # Criar DataFrame de submissão\n    print(\"\\nCriando arquivo de submissão do ensemble...\")\n    submission_df = sample_submission_df.copy()\n    \n    # Preencher o DataFrame\n    for i, row in submission_df.iterrows():\n        if i % 1000 == 0:\n            print(f\"Processando linha {i}/{len(submission_df)}\")\n            \n        id_parts = row['ID'].split('_')\n        seq_id = id_parts[0]\n        residue_idx = int(id_parts[1]) - 1\n        \n        if seq_id in seq_to_coords and residue_idx < len(seq_to_coords[seq_id][0]):\n            for struct_idx in range(5):\n                submission_df.at[i, f'x_{struct_idx+1}'] = seq_to_coords[seq_id][struct_idx][residue_idx][0]\n                submission_df.at[i, f'y_{struct_idx+1}'] = seq_to_coords[seq_id][struct_idx][residue_idx][1]\n                submission_df.at[i, f'z_{struct_idx+1}'] = seq_to_coords[seq_id][struct_idx][residue_idx][2]\n    \n    # Salvar submissão\n    ensemble_submission_file = os.path.join(output_dir, 'submission.csv')\n    submission_df.to_csv(ensemble_submission_file, index=False)\n    print(f\"Submissão do ensemble salva em {ensemble_submission_file}\")\n    \n    # Verificar arquivo\n    if os.path.exists(ensemble_submission_file):\n        print(f\"Arquivo verificado: {os.path.getsize(ensemble_submission_file)} bytes\")\n    else:\n        print(\"AVISO: Arquivo não encontrado após o salvamento!\")\n    \n    # Também salvar a versão normal submission.csv (para compatibilidade)\n    standard_submission_file = os.path.join(output_dir, 'submission.csv')\n    submission_df.to_csv(standard_submission_file, index=False)\n    \n    return submission_df, all_results\n\ndef run_ensemble_main(num_runs=50):\n    \"\"\"\n    Executa o fluxo principal com ensemble de múltiplas sementes.\n    \"\"\"\n    try:\n        print(\"Carregando dados processados...\")\n        X_train, y_train, X_valid, y_valid = load_processed_data()\n        \n        print(\"\\nVerificando validade dos dados...\")\n        print(f\"X_valid shape: {X_valid.shape}, has NaN: {np.isnan(X_valid).any()}\")\n        print(f\"y_valid shape: {y_valid.shape}, has NaN: {np.isnan(y_valid).any()}\")\n        \n        print(\"\\nCarregando dados de teste...\")\n        try:\n            test_seq_df = pd.read_csv(os.path.join(DATA_DIR, \"test_sequences.csv\"))\n            sample_submission_df = pd.read_csv(os.path.join(DATA_DIR, \"sample_submission.csv\"))\n            print(f\"Dados de teste carregados: {len(test_seq_df)} sequências\")\n        except Exception as e:\n            print(f\"Erro ao carregar dados de teste: {e}\")\n            import traceback\n            traceback.print_exc()\n            return None, None\n        \n        # Garantir que o diretório de saída existe\n        os.makedirs(OUTPUT_DIR, exist_ok=True)\n        \n        # Parâmetros ótimos baseados nas execuções anteriores\n        optimal_params = {'noise': 0.21, 'corr': 0.83}\n        \n        # Executar ensemble com múltiplas sementes\n        print(\"\\nIniciando processo de ensemble...\")\n        submission_df, all_results = ensemble_with_multiple_seeds(\n            X_valid, y_valid, \n            test_seq_df, sample_submission_df, \n            OUTPUT_DIR,\n            num_runs=num_runs,\n            optimal_params=optimal_params\n        )\n        \n        if submission_df is None:\n            print(\"Falha na criação do ensemble. Tentando abordagem simplificada...\")\n            return simplified_main()\n        \n        print(\"\\nProcesso de ensemble concluído com sucesso!\")\n        return submission_df, all_results\n        \n    except Exception as e:\n        print(f\"ERRO em run_ensemble_main: {str(e)}\")\n        import traceback\n        traceback.print_exc()\n        print(\"\\nTentando abordagem simplificada após erro...\")\n        return simplified_main()\n\nif __name__ == \"__main__\":\n    use_ensemble = True  # Defina como True para usar o ensemble, False para usar a implementação simplificada\n    \n    if use_ensemble:\n        print(\"Usando implementação com ensemble de múltiplas sementes...\")\n        submission_df, all_results = run_ensemble_main(num_runs=7)  # Ajuste o número de execuções conforme necessário\n    else:\n        print(\"Usando implementação simplificada...\")\n        model, metrics = simplified_main()","metadata":{"execution":{"iopub.status.busy":"2025-03-18T03:16:07.543613Z","iopub.execute_input":"2025-03-18T03:16:07.543937Z","iopub.status.idle":"2025-03-18T03:16:10.748928Z","shell.execute_reply.started":"2025-03-18T03:16:07.543908Z","shell.execute_reply":"2025-03-18T03:16:10.747876Z"},"papermill":{"duration":25.719433,"end_time":"2025-03-16T03:39:14.524317","exception":false,"start_time":"2025-03-16T03:38:48.804884","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df = pd.read_csv('/kaggle/working/submission.csv')\nprint(\"Visão geral do DataFrame:\")\nprint(submission_df.shape)  \nprint(submission_df.head())  ","metadata":{"execution":{"iopub.status.busy":"2025-03-18T03:16:10.750896Z","iopub.execute_input":"2025-03-18T03:16:10.751188Z","iopub.status.idle":"2025-03-18T03:16:10.779447Z","shell.execute_reply.started":"2025-03-18T03:16:10.751163Z","shell.execute_reply":"2025-03-18T03:16:10.778395Z"},"papermill":{"duration":0.055324,"end_time":"2025-03-16T03:39:14.60706","exception":false,"start_time":"2025-03-16T03:39:14.551736","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}