{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":87793,"databundleVersionId":11228175,"isSourceIdPinned":false,"sourceType":"competition"}],"dockerImageVersionId":30919,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 1. Importing necessary libraaries","metadata":{}},{"cell_type":"code","source":"import os\nimport warnings\nwarnings.filterwarnings('ignore') # Suppress warnings for cleaner output\n\nimport numpy as np\nimport pandas as pd\n\n# Visualization\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.express as px  # For interactive plots\nimport plotly.graph_objects as go # For interactive plots\n\n# Deep Learning \nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nimport numpy as np\nimport pandas as pd\nfrom sklearn.preprocessing import LabelEncoder","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-03-01T15:26:14.217134Z","iopub.execute_input":"2025-03-01T15:26:14.217402Z","iopub.status.idle":"2025-03-01T15:26:20.843002Z","shell.execute_reply.started":"2025-03-01T15:26:14.217373Z","shell.execute_reply":"2025-03-01T15:26:20.842019Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Data Loading and Exploration","metadata":{}},{"cell_type":"code","source":"print(\"Loading datasets...\")\ntrain_sequences = pd.read_csv('/kaggle/input/stanford-rna-3d-folding/train_sequences.csv')\ntrain_labels = pd.read_csv('/kaggle/input/stanford-rna-3d-folding/train_labels.csv')\nvalidation_sequences = pd.read_csv('/kaggle/input/stanford-rna-3d-folding/validation_sequences.csv')\nvalidation_labels = pd.read_csv('/kaggle/input/stanford-rna-3d-folding/validation_labels.csv')\ntest_sequences = pd.read_csv('/kaggle/input/stanford-rna-3d-folding/test_sequences.csv')\nsample_submission = pd.read_csv('/kaggle/input/stanford-rna-3d-folding/sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-01T15:26:20.844293Z","iopub.execute_input":"2025-03-01T15:26:20.844754Z","iopub.status.idle":"2025-03-01T15:26:21.353482Z","shell.execute_reply.started":"2025-03-01T15:26:20.844731Z","shell.execute_reply":"2025-03-01T15:26:21.352454Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fill missing coordinate values to avoid NaNs.\ntrain_labels.fillna(0, inplace=True)\nvalidation_labels.fillna(0, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-01T15:26:21.354730Z","iopub.execute_input":"2025-03-01T15:26:21.354976Z","iopub.status.idle":"2025-03-01T15:26:21.382005Z","shell.execute_reply.started":"2025-03-01T15:26:21.354954Z","shell.execute_reply":"2025-03-01T15:26:21.381316Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\nBasic dataset information:\")\nprint(f\"Training sequences: {train_sequences.shape}\")\nprint(f\"Training labels: {train_labels.shape}\")\nprint(f\"Validation sequences: {validation_sequences.shape}\")\nprint(f\"Validation labels: {validation_labels.shape}\")\nprint(f\"Test sequences: {test_sequences.shape}\")\nprint(f\"Sample submission: {sample_submission.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-01T15:26:21.382816Z","iopub.execute_input":"2025-03-01T15:26:21.383134Z","iopub.status.idle":"2025-03-01T15:26:21.390352Z","shell.execute_reply.started":"2025-03-01T15:26:21.383110Z","shell.execute_reply":"2025-03-01T15:26:21.389555Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 2. Data Analysis (optional visualizations)\nprint(\"\\nAnalyzing RNA sequence lengths...\")\ntrain_sequences['length'] = train_sequences['sequence'].str.len()\nplt.figure(figsize=(12, 6))\nsns.histplot(train_sequences['length'], bins=50)\nplt.title('Distribution of RNA Sequence Lengths')\nplt.xlabel('Sequence Length')\nplt.ylabel('Count')\nplt.savefig('sequence_length_distribution.png')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-01T15:26:21.392908Z","iopub.execute_input":"2025-03-01T15:26:21.393164Z","iopub.status.idle":"2025-03-01T15:26:21.862810Z","shell.execute_reply.started":"2025-03-01T15:26:21.393143Z","shell.execute_reply":"2025-03-01T15:26:21.861769Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_sequences.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-01T15:26:21.864003Z","iopub.execute_input":"2025-03-01T15:26:21.864313Z","iopub.status.idle":"2025-03-01T15:26:21.884044Z","shell.execute_reply.started":"2025-03-01T15:26:21.864282Z","shell.execute_reply":"2025-03-01T15:26:21.883228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_submission.head(3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-01T15:26:21.884873Z","iopub.execute_input":"2025-03-01T15:26:21.885132Z","iopub.status.idle":"2025-03-01T15:26:21.905837Z","shell.execute_reply.started":"2025-03-01T15:26:21.885099Z","shell.execute_reply":"2025-03-01T15:26:21.905009Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get the first 10 rows\n#ten_sequences = train_sequences.head(10)\n\n# Download as CSV\n#ten_sequences.to_csv('first_10_sequences.csv', index=False)\n\n#print(\"first_10_sequences.csv has been created.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-01T15:26:21.906667Z","iopub.execute_input":"2025-03-01T15:26:21.906995Z","iopub.status.idle":"2025-03-01T15:26:21.924164Z","shell.execute_reply.started":"2025-03-01T15:26:21.906970Z","shell.execute_reply":"2025-03-01T15:26:21.923323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get the first 10 rows\n#sample_submission = sample_submission.head(10)\n\n# Download as CSV\n#sample_submission.to_csv('first_10_sample_sub.csv', index=False)\n\n#print(\"first_10_sample_sub.csv has been created.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-01T15:26:21.925006Z","iopub.execute_input":"2025-03-01T15:26:21.925366Z","iopub.status.idle":"2025-03-01T15:26:21.939717Z","shell.execute_reply.started":"2025-03-01T15:26:21.925336Z","shell.execute_reply":"2025-03-01T15:26:21.938889Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get the first 10 rows\ntrain_labels= train_labels.head(10)\n\n# Download as CSV\ntrain_labels.to_csv('first_10_train_labels.csv', index=False)\n\nprint(\"first_10_sequences.csv has been created.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-01T16:18:25.447568Z","iopub.execute_input":"2025-03-01T16:18:25.447966Z","iopub.status.idle":"2025-03-01T16:18:25.459128Z","shell.execute_reply.started":"2025-03-01T16:18:25.447938Z","shell.execute_reply":"2025-03-01T16:18:25.458178Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Encoding and Model training ","metadata":{}},{"cell_type":"code","source":"import os\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\n\n# 1. Data Loading\nprint(\"Starting Stanford RNA 3D Folding notebook...\")\nprint(\"Loading datasets...\")\ntrain_sequences = pd.read_csv('/kaggle/input/stanford-rna-3d-folding/train_sequences.csv')\ntrain_labels = pd.read_csv('/kaggle/input/stanford-rna-3d-folding/train_labels.csv')\nvalidation_sequences = pd.read_csv('/kaggle/input/stanford-rna-3d-folding/validation_sequences.csv')\nvalidation_labels = pd.read_csv('/kaggle/input/stanford-rna-3d-folding/validation_labels.csv')\ntest_sequences = pd.read_csv('/kaggle/input/stanford-rna-3d-folding/test_sequences.csv')\nsample_submission = pd.read_csv('/kaggle/input/stanford-rna-3d-folding/sample_submission.csv')\n\n# Fill missing coordinate values to avoid NaNs.\ntrain_labels.fillna(0, inplace=True)\nvalidation_labels.fillna(0, inplace=True)\n\nprint(\"\\nBasic dataset information:\")\nprint(f\"Training sequences: {train_sequences.shape}\")\nprint(f\"Training labels: {train_labels.shape}\")\nprint(f\"Validation sequences: {validation_sequences.shape}\")\nprint(f\"Validation labels: {validation_labels.shape}\")\nprint(f\"Test sequences: {test_sequences.shape}\")\nprint(f\"Sample submission: {sample_submission.shape}\")\n\n# 2. Analyze sequence lengths (optional)\ntrain_sequences['length'] = train_sequences['sequence'].str.len()\nplt.figure(figsize=(12, 6))\nsns.histplot(train_sequences['length'], bins=50)\nplt.title('Distribution of RNA Sequence Lengths')\nplt.xlabel('Sequence Length')\nplt.ylabel('Count')\nplt.savefig('sequence_length_distribution.png')\nplt.close()\n\n# 3. Data Preprocessing\ndef preprocess_sequence_data(sequences_df, labels_df=None, is_train=True):\n    nucleotide_map = {'A': 0, 'C': 1, 'G': 2, 'U': 3, 'T': 3}\n    processed_data = []\n    for idx, row in sequences_df.iterrows():\n        seq_id = row['target_id']\n        sequence = row['sequence']\n        numerical_seq = [nucleotide_map.get(nuc, 4) for nuc in sequence]\n        structures = None\n        if is_train and labels_df is not None:\n            sequence_labels = labels_df[labels_df['ID'].str.startswith(seq_id + '_')]\n            if not sequence_labels.empty:\n                coords = []\n                for _, label_row in sequence_labels.iterrows():\n                    x = label_row['x_1']\n                    y = label_row['y_1']\n                    z = label_row['z_1']\n                    coords.append([x, y, z])\n                coords = np.array(coords)\n                mean = np.mean(coords, axis=0)\n                std = np.std(coords, axis=0) + 1e-8\n                coords_norm = (coords - mean) / std\n                structures = [coords_norm]\n        processed_data.append({\n            'id': seq_id,\n            'sequence': numerical_seq,\n            'structures': structures\n        })\n    return processed_data\n\nprint(\"Preprocessing training data...\")\ntrain_data = preprocess_sequence_data(train_sequences, train_labels, is_train=True)\nprint(\"Preprocessing validation data...\")\nvalidation_data = preprocess_sequence_data(validation_sequences, validation_labels, is_train=True)\nprint(\"Preprocessing test data...\")\ntest_data = preprocess_sequence_data(test_sequences, is_train=False)\n\n# 4. Feature Engineering: one-hot encoding, positional info, GC-content\ndef extract_sequence_features(sequence):\n    one_hot = np.zeros((len(sequence), 5))\n    for i, nuc in enumerate(sequence):\n        one_hot[i, nuc] = 1\n    gc_content = []\n    window_size = 5\n    for i in range(len(sequence)):\n        start = max(0, i - window_size // 2)\n        end = min(len(sequence), i + window_size // 2 + 1)\n        window = sequence[start:end]\n        gc_count = sum(1 for n in window if n in [1, 2])\n        gc_content.append(gc_count / len(window))\n    positions = np.array([[i / len(sequence)] for i in range(len(sequence))])\n    features = np.hstack((one_hot, positions, np.array(gc_content).reshape(-1, 1)))\n    return features\n\nprint(\"Extracting sequence features...\")\nfor i, data in enumerate(train_data):\n    train_data[i]['features'] = extract_sequence_features(data['sequence'])\nfor i, data in enumerate(validation_data):\n    validation_data[i]['features'] = extract_sequence_features(data['sequence'])\nfor i, data in enumerate(test_data):\n    test_data[i]['features'] = extract_sequence_features(data['sequence'])\n\n# 5. Enhance features with RNA Secondary Structure information\ndef predict_rna_secondary_structure(sequence):\n    nucleotide_map_inv = {0: 'A', 1: 'C', 2: 'G', 3: 'U', 4: 'X'}\n    seq_chars = [nucleotide_map_inv[n] for n in sequence]\n    structure = ['.' for _ in range(len(seq_chars))]\n    complementary = {'A': 'U', 'U': 'A', 'G': 'C', 'C': 'G', 'X': None}\n    for i in range(len(seq_chars)):\n        if structure[i] != '.':\n            continue\n        for j in range(len(seq_chars) - 1, i + 3, -1):\n            if structure[j] != '.':\n                continue\n            if complementary.get(seq_chars[i]) == seq_chars[j]:\n                structure[i] = '('\n                structure[j] = ')'\n                break\n    return ''.join(structure)\n\ndef enhance_features_with_ss(data):\n    for i, item in enumerate(data):\n        seq = item['sequence']\n        ss = predict_rna_secondary_structure(seq)\n        ss_features = np.zeros((len(ss), 3))\n        for j, char in enumerate(ss):\n            if char == '.':\n                ss_features[j, 0] = 1\n            elif char == '(':\n                ss_features[j, 1] = 1\n            elif char == ')':\n                ss_features[j, 2] = 1\n        data[i]['features'] = np.hstack((item['features'], ss_features))\n    return data\n\nprint(\"Enhancing features with secondary structure information...\")\ntrain_data = enhance_features_with_ss(train_data)\nvalidation_data = enhance_features_with_ss(validation_data)\ntest_data = enhance_features_with_ss(test_data)\n\n# 6. Custom Dataset and DataLoader\nclass RNADataset(Dataset):\n    def __init__(self, data):\n        self.data = data\n    def __len__(self):\n        return len(self.data)\n    def __getitem__(self, idx):\n        item = self.data[idx]\n        features = torch.tensor(item['features'], dtype=torch.float32)\n        if item['structures'] is not None:\n            target = torch.tensor(item['structures'][0], dtype=torch.float32)\n            return features, target, item['id']\n        else:\n            return features, None, item['id']\n\ndef collate_fn(batch):\n    batch = sorted(batch, key=lambda x: x[0].shape[0], reverse=True)\n    features = [item[0] for item in batch]\n    targets = [item[1] for item in batch]\n    ids = [item[2] for item in batch]\n    max_length = features[0].shape[0]\n    feature_dim = features[0].shape[1]\n    padded_features = []\n    padded_targets = []\n    lengths = []\n    for i, feat in enumerate(features):\n        length = feat.shape[0]\n        lengths.append(length)\n        pad_feat = torch.cat([feat, torch.zeros((max_length - length, feature_dim))], dim=0)\n        padded_features.append(pad_feat)\n        if targets[i] is not None:\n            pad_target = torch.cat([targets[i], torch.zeros((max_length - length, 3))], dim=0)\n            padded_targets.append(pad_target)\n    features_tensor = torch.stack(padded_features)\n    if all(t is not None for t in targets):\n        targets_tensor = torch.stack(padded_targets)\n        return features_tensor, targets_tensor, ids, lengths\n    else:\n        return features_tensor, None, ids, lengths\n\ntrain_dataset = RNADataset(train_data)\nvalidation_dataset = RNADataset(validation_data)\ntest_dataset = RNADataset(test_data)\n\nbatch_size = 4\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_fn)\nvalidation_loader = DataLoader(validation_dataset, batch_size=batch_size, shuffle=False, collate_fn=collate_fn)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=collate_fn)\n\n# 7. Define a Transformer-based Model (improved capacity)\nclass PositionalEncoding(nn.Module):\n    def __init__(self, d_model, max_len=5000):\n        super(PositionalEncoding, self).__init__()\n        pe = torch.zeros(max_len, d_model)\n        pos = torch.arange(0, max_len, dtype=torch.float32).unsqueeze(1)\n        div_term = torch.exp(torch.arange(0, d_model, 2, dtype=torch.float32) * (-np.log(10000.0) / d_model))\n        pe[:, 0::2] = torch.sin(pos * div_term)\n        pe[:, 1::2] = torch.cos(pos * div_term)\n        pe = pe.unsqueeze(0)\n        self.register_buffer('pe', pe)\n    \n    def forward(self, x):\n        return x + self.pe[:, :x.size(1)]\n\nclass RNAFoldingTransformer(nn.Module):\n    def __init__(self, input_dim, d_model=512, nhead=8, num_layers=4, dim_feedforward=2048, dropout=0.3):\n        super(RNAFoldingTransformer, self).__init__()\n        self.input_linear = nn.Linear(input_dim, d_model)\n        self.pos_encoder = PositionalEncoding(d_model)\n        encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, dim_feedforward=dim_feedforward, dropout=dropout)\n        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)\n        self.dropout = nn.Dropout(dropout)\n        self.fc_out = nn.Linear(d_model, 3)\n    \n    def forward(self, x, seq_lengths=None):\n        # x: (B, L, input_dim)\n        x = self.input_linear(x)  # (B, L, d_model)\n        x = self.pos_encoder(x)\n        x = x.transpose(0, 1)  # (L, B, d_model)\n        x = self.transformer_encoder(x)  # (L, B, d_model)\n        x = x.transpose(0, 1)  # (B, L, d_model)\n        x = self.dropout(x)\n        x = self.fc_out(x)  # (B, L, 3)\n        return x\n\n# 8. Loss Function: Smooth L1 Loss over valid timesteps\ndef smooth_l1_loss(output, target, seq_lengths):\n    mask = torch.zeros_like(target, dtype=torch.bool)\n    for i, length in enumerate(seq_lengths):\n        mask[i, :length, :] = True\n    loss = nn.SmoothL1Loss(reduction='none')(output, target)\n    masked_loss = loss * mask.float()\n    return masked_loss.sum() / mask.sum() if mask.sum() > 0 else 0\n\n# 9. Training and Evaluation Functions\ndef train_epoch(model, dataloader, optimizer, device):\n    model.train()\n    epoch_loss = 0\n    batches = 0\n    for features, targets, _, seq_lengths in dataloader:\n        if targets is None:\n            continue\n        optimizer.zero_grad()\n        features = features.to(device)\n        targets = targets.to(device)\n        outputs = model(features, seq_lengths)\n        loss = smooth_l1_loss(outputs, targets, seq_lengths)\n        loss.backward()\n        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=5.0)\n        optimizer.step()\n        epoch_loss += loss.item()\n        batches += 1\n    return epoch_loss / batches if batches > 0 else float('inf')\n\ndef validate(model, dataloader, device):\n    model.eval()\n    val_loss = 0\n    batches = 0\n    with torch.no_grad():\n        for features, targets, _, seq_lengths in dataloader:\n            if targets is None:\n                continue\n            features = features.to(device)\n            targets = targets.to(device)\n            outputs = model(features, seq_lengths)\n            loss = smooth_l1_loss(outputs, targets, seq_lengths)\n            val_loss += loss.item()\n            batches += 1\n    return val_loss / batches if batches > 0 else float('inf')\n\ndef train_model(model, train_loader, val_loader, num_epochs=40, lr=0.0002, device='cpu'):\n    optimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-4)\n    scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=num_epochs)\n    \n    train_losses = []\n    val_losses = []\n    best_model_state = model.state_dict()\n    best_val_loss = float('inf')\n    \n    print(\"Starting training...\")\n    for epoch in range(num_epochs):\n        train_loss = train_epoch(model, train_loader, optimizer, device)\n        val_loss = validate(model, val_loader, device)\n        train_losses.append(train_loss)\n        val_losses.append(val_loss)\n        scheduler.step()\n        \n        print(f\"Epoch {epoch+1}/{num_epochs}: Train Loss = {train_loss:.4f}, Val Loss = {val_loss:.4f}\")\n        if val_loss < best_val_loss:\n            best_val_loss = val_loss\n            best_model_state = model.state_dict().copy()\n            print(\"  New best model saved.\")\n    \n    model.load_state_dict(best_model_state)\n    plt.figure(figsize=(10, 4))\n    plt.plot(train_losses, label=\"Train Loss\")\n    plt.plot(val_losses, label=\"Val Loss\")\n    plt.xlabel(\"Epoch\")\n    plt.ylabel(\"Loss\")\n    plt.legend()\n    plt.title(\"Training History\")\n    plt.savefig(\"training_history.png\")\n    plt.close()\n    \n    print(f\"Training complete. Best Val Loss: {best_val_loss:.4f}\")\n    return model\n\n# 10. Ensemble: Train multiple models and average predictions\nclass ModelEnsemble:\n    def __init__(self, models):\n        self.models = models\n    def predict(self, features, seq_lengths=None, device='cpu'):\n        preds = []\n        for model in self.models:\n            model.eval()\n            with torch.no_grad():\n                out = model(features.to(device), seq_lengths)\n                preds.append(out)\n        # Average predictions\n        return torch.mean(torch.stack(preds), dim=0)\n\ndef generate_ensemble_predictions(ensemble, dataloader, device, num_predictions=5):\n    ensemble.models[0].eval()\n    all_predictions = {}\n    with torch.no_grad():\n        for features, _, ids, seq_lengths in dataloader:\n            features = features.to(device)\n            for i, (seq_id, length) in enumerate(zip(ids, seq_lengths)):\n                seq_features = features[i, :length, :].unsqueeze(0)\n                # Get ensemble prediction (averaged output)\n                avg_pred = ensemble.predict(seq_features, [length], device=device)\n                # Generate diverse predictions via noise perturbation\n                preds = []\n                for _ in range(num_predictions):\n                    noise = torch.randn_like(seq_features) * 0.03\n                    noisy_features = seq_features + noise\n                    noisy_pred = ensemble.predict(noisy_features, [length], device=device)\n                    preds.append(noisy_pred[0].cpu().numpy())\n                all_predictions[seq_id] = preds\n    return all_predictions\n\n# 11. Submission File Generation (same as before)\ndef create_submission_file(predictions, test_sequences_df, output_file='submission.csv'):\n    submission_rows = []\n    for _, row in test_sequences_df.iterrows():\n        seq_id = row['target_id']\n        sequence = row['sequence']\n        if seq_id in predictions:\n            pred_structures = predictions[seq_id]\n            for i in range(len(sequence)):\n                submission_row = {\n                    'ID': f\"{seq_id}_{i+1}\",\n                    'resname': sequence[i],\n                    'resid': i+1\n                }\n                for j in range(5):\n                    coords = pred_structures[j][i]\n                    submission_row[f'x_{j+1}'] = coords[0]\n                    submission_row[f'y_{j+1}'] = coords[1]\n                    submission_row[f'z_{j+1}'] = coords[2]\n                submission_rows.append(submission_row)\n    submission_df = pd.DataFrame(submission_rows)\n    submission_df.to_csv(output_file, index=False)\n    return submission_df\n\n# 12. Visualization Function for 3D Structures (same as before)\ndef visualize_3d_structure(coords, title=\"RNA 3D Structure\"):\n    from mpl_toolkits.mplot3d import Axes3D\n    fig = plt.figure(figsize=(10, 8))\n    ax = fig.add_subplot(111, projection='3d')\n    ax.scatter(coords[:, 0], coords[:, 1], coords[:, 2], c='blue', s=30, label=\"C1' atoms\")\n    for i in range(len(coords)-1):\n        ax.plot([coords[i, 0], coords[i+1, 0]], \n                [coords[i, 1], coords[i+1, 1]], \n                [coords[i, 2], coords[i+1, 2]], 'k-', lw=1)\n    ax.set_title(title)\n    ax.set_xlabel('X (Å)')\n    ax.set_ylabel('Y (Å)')\n    ax.set_zlabel('Z (Å)')\n    ax.legend()\n    plt.savefig(f\"{title.replace(' ', '_')}.png\")\n    plt.close()\n\n# 13. Main Execution: Train two transformer models and ensemble them\ndef main():\n    print(\"\\n--- Main execution ---\")\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    print(f\"Using device: {device}\")\n    \n    input_dim = train_data[0]['features'].shape[1]\n    # Train two transformer models with different random seeds\n    models = []\n    for seed in [42, 2023]:\n        torch.manual_seed(seed)\n        model = RNAFoldingTransformer(input_dim=input_dim, d_model=512, nhead=8, num_layers=4, dim_feedforward=2048, dropout=0.3).to(device)\n        print(f\"Training model with seed {seed}...\")\n        model = train_model(model, train_loader, validation_loader, num_epochs=40, lr=0.0002, device=device)\n        models.append(model)\n    \n    ensemble = ModelEnsemble(models)\n    print(\"Ensemble created.\")\n    \n    print(\"Generating ensemble predictions on test data...\")\n    test_predictions = generate_ensemble_predictions(ensemble, test_loader, device, num_predictions=5)\n    print(\"Predictions generated.\")\n    \n    print(\"Creating submission file...\")\n    submission_df = create_submission_file(test_predictions, test_sequences, output_file='submission.csv')\n    print(\"Submission file created. Sample:\")\n    print(submission_df.head())\n    \n    print(\"Visualizing a sample prediction (first test sequence)...\")\n    sample_seq_id = test_sequences['target_id'].iloc[0]\n    if sample_seq_id in test_predictions:\n        sample_prediction = test_predictions[sample_seq_id][0]\n        visualize_3d_structure(sample_prediction, title=f\"Predicted 3D Structure - {sample_seq_id}\")\n        print(f\"Visualization saved for {sample_seq_id}.\")\n    else:\n        print(\"No prediction found for the first test sequence for visualization.\")\n    \n    print(\"\\n--- Main execution completed ---\")\n\nif __name__ == '__main__':\n    main()\n\nprint(\"\\nNotebook execution finished.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-02T08:00:50.815106Z","iopub.execute_input":"2025-03-02T08:00:50.815429Z","iopub.status.idle":"2025-03-02T08:37:39.554538Z","shell.execute_reply.started":"2025-03-02T08:00:50.815402Z","shell.execute_reply":"2025-03-02T08:37:39.553820Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df = pd.read_csv(\"/kaggle/working/submission.csv\")\nsub_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-02T09:07:07.639225Z","iopub.execute_input":"2025-03-02T09:07:07.639534Z","iopub.status.idle":"2025-03-02T09:07:07.665250Z","shell.execute_reply.started":"2025-03-02T09:07:07.639511Z","shell.execute_reply":"2025-03-02T09:07:07.664262Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}