{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":87793,"databundleVersionId":11228175,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 0. Preparation","metadata":{}},{"cell_type":"markdown","source":"## 0-1. Import","metadata":{}},{"cell_type":"code","source":"import polars as pl\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\n\nimport plotly.io as pio\npio.renderers.default = \"notebook\"\n\nimport plotly.offline as pyo\npyo.init_notebook_mode(connected=True)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2025-03-05T13:45:13.496425Z","iopub.execute_input":"2025-03-05T13:45:13.496756Z","iopub.status.idle":"2025-03-05T13:45:13.503637Z","shell.execute_reply.started":"2025-03-05T13:45:13.496731Z","shell.execute_reply":"2025-03-05T13:45:13.502579Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 0-2. Data Loading","metadata":{}},{"cell_type":"code","source":"# Path\nCOMPETITION_NAME = \"stanford-rna-3d-folding\"\nROOT = Path(\".\").resolve().parent\nINPUT_ROOT = ROOT / \"input\"\nRAW_DATA = INPUT_ROOT / COMPETITION_NAME\n\n# read_csv\ntrain_labels    = pl.read_csv(RAW_DATA / \"train_labels.csv\")\ntrain_sequences = pl.read_csv(RAW_DATA / \"train_sequences.csv\")\nvalid_labels      = pl.read_csv(RAW_DATA / \"validation_labels.csv\") \nvalid_sequences   = pl.read_csv(RAW_DATA / \"validation_sequences.csv\")","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2025-03-05T13:41:51.668567Z","iopub.execute_input":"2025-03-05T13:41:51.668926Z","iopub.status.idle":"2025-03-05T13:41:51.740307Z","shell.execute_reply.started":"2025-03-05T13:41:51.668896Z","shell.execute_reply":"2025-03-05T13:41:51.739289Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 0-3. Data Processing","metadata":{}},{"cell_type":"code","source":"# train data\ntrain_labels = train_labels.with_columns(\n    ('train_' + pl.col(\"ID\").str.replace(r\"_[^_]*$\", \"\")).alias(\"target_id\") # Remove suffix\n)\ntrain_sequences = train_sequences.with_columns(\n    ('train_' + pl.col(\"target_id\")).alias('target_id') # Remove suffix\n)\n\nvalid_labels = valid_labels.with_columns(\n    pl.col(\"ID\").str.replace(r\"_[^_]*$\", \"\").alias(\"target_id\") # Remove suffix\n)\n\n# validation data\ndfs = []\nfor target_id in valid_sequences.select('target_id').to_series():\n    for i in range(1,41):\n        dfs.append(valid_labels\n            .filter(pl.col('target_id')==target_id)\n            .select(\n                pl.col('ID'),\n                pl.col('resname'),\n                pl.col('resid'),\n                pl.col(f'x_{i}').alias('x_1'),\n                pl.col(f'y_{i}').alias('y_1'),\n                pl.col(f'z_{i}').alias('z_1'),\n                pl.col('target_id'),\n                ('valid_' + pl.col('target_id') + f'_{i}').alias('target_id_n')\n            )\n        )\n\nvalid_labels = pl.concat(dfs).filter(pl.col('x_1')>-1.0e18)\n\nvalid_sequences = valid_sequences.join(\n    valid_labels.select(['target_id', 'target_id_n']).unique(),\n    on='target_id',\n    how='left')\n\nvalid_sequences = valid_sequences.with_columns(pl.col('target_id_n').alias('target_id')).sort('target_id_n')\nvalid_labels = valid_labels.with_columns(pl.col('target_id_n').alias('target_id')).sort('target_id_n')\n\ncols = ['target_id','sequence']\ntrain_sequences = train_sequences.select(pl.col(cols))\nvalid_sequences = valid_sequences.select(pl.col(cols))\nsequences = pl.concat([\n    train_sequences,\n    valid_sequences,\n    ])\ncols = ['target_id', 'resname', 'resid', 'x_1', 'y_1', 'z_1']\ntrain_labels = train_labels.select(pl.col(cols))\nvalid_labels = valid_labels.select(pl.col(cols))\nlabels = pl.concat([\n    train_labels,\n    valid_labels,\n    ])","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-03-05T13:41:51.741860Z","iopub.execute_input":"2025-03-05T13:41:51.742136Z","iopub.status.idle":"2025-03-05T13:41:52.503616Z","shell.execute_reply.started":"2025-03-05T13:41:51.742106Z","shell.execute_reply":"2025-03-05T13:41:52.502380Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 0-4. Function Definitions for Visualization","metadata":{}},{"cell_type":"code","source":"def plotlyRNAStructure(df_labels, target_ids):\n    \"\"\"\n    Plots the 3D structure of RNA molecules for the given target IDs.\n    This function visualizes the RNA structure in a 3D space using Plotly.\n    Each nucleotide (A, C, G, U, X, and -) is represented by a different color.\n    The function retrieves structural data from `train_labels` and sequence data from `train_sequences`,\n    then creates a subplot for each target ID.\n\n    Parameters:\n        df_labels: [train/validation]_labels.\n        target_ids (list of str): A list of target IDs for which the RNA structures will be plotted.\n    \"\"\"\n    sequence = sequences.filter(pl.col('target_id')==target_ids[0]).select('sequence').to_series()[0]\n    print('sequence:', sequence)\n    \n    color_map = {\n        'A': 'red',\n        'C': 'blue',\n        'G': 'green',\n        'U': 'orange',\n        'X': 'black',\n        '-': 'gray'\n    }\n    column_num = len(target_ids)\n    \n    fig = make_subplots(\n        rows=1, cols=column_num,\n        specs=[[{'type': 'scene'}]*column_num],\n        subplot_titles=target_ids\n    )\n\n    # dummy plot for legend\n    for base, color in color_map.items():\n        fig.add_trace(go.Scatter3d(\n            x=[None], y=[None], z=[None],\n            mode='markers',\n            marker=dict(\n                size=10,\n                color=color,\n                opacity=0.8\n            ),\n            name=base\n        ))\n\n    # plot RNA structure\n    for i in range(column_num):\n        df = df_labels.filter(pl.col('target_id')==target_ids[i])\n        colors = [color_map[base] for base in df['resname']]\n        hover_texts = [f'{base}{id}' for base, id in zip(df['resname'], df['resid'])]\n        \n        # fig = go.Figure()\n    \n        fig.add_trace(\n            go.Scatter3d(\n                x=df['x_1'],\n                y=df['y_1'],\n                z=df['z_1'],\n                mode='markers+text',\n                marker=dict(\n                    size=6,\n                    color=colors,\n                    opacity=0.8,\n                    symbol='circle',\n                    line=dict(width=1, color='DarkSlateGrey')\n                ),\n                text=hover_texts,\n                hoverinfo='text',\n                hovertext=hover_texts,\n                name='residue',\n                showlegend=False\n            ),\n            row=1, col=i+1\n        )\n    \n        fig.add_trace(\n            go.Scatter3d(\n                x=df['x_1'],\n                y=df['y_1'],\n                z=df['z_1'],\n                mode='lines',\n                line=dict(\n                    color='gray',\n                    width=2,\n                    dash='solid'\n                ),\n                opacity=0.5,\n                name='bone',\n                showlegend=False\n            ),\n            row=1, col=i+1\n        )\n\n    fig.update_layout(\n        title='',\n        scene=dict(\n            xaxis_title='X',\n            yaxis_title='Y',\n            zaxis_title='Z',\n            aspectmode='cube'\n        ),\n        margin=dict(r=20, l=10, b=10, t=50),\n        legend=dict(\n            title='resname',\n            x=0,\n            y=1,\n            bgcolor='rgba(255, 255, 255, 0.5)'\n        ),\n        hovermode='closest',\n        template='plotly_white'\n    )\n    \n    fig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-03-05T13:41:52.506033Z","iopub.execute_input":"2025-03-05T13:41:52.506311Z","iopub.status.idle":"2025-03-05T13:41:52.518942Z","shell.execute_reply.started":"2025-03-05T13:41:52.506288Z","shell.execute_reply":"2025-03-05T13:41:52.517536Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def matplotRNAStructure(df_labels, target_ids):\n    sequence = sequences.filter(pl.col('target_id')==target_ids[0]).select('sequence').to_series()[0]\n    print('sequence:', sequence)\n    \n    color_map = {\n        'A': 'red',\n        'C': 'blue',\n        'G': 'green',\n        'U': 'orange',\n        'X': 'black',\n        '-': 'gray'\n    }\n    column_num = len(target_ids)\n    row_num = 1\n    if column_num > 5:\n        row_num += (column_num-1) // 5\n        column_num = 5\n\n    fig, axes = plt.subplots(row_num, column_num, figsize=(5 * column_num, 5 * row_num),\n                            subplot_kw={'projection': '3d'})\n\n    if len(target_ids) == 1:\n        axes = [[axes]]\n    elif len(target_ids) < 6:\n        axes = [axes]\n        \n    for i in range(len(target_ids)):\n        ax = axes[i//5][i%5]\n        target_id = target_ids[i]\n        df = df_labels.filter(pl.col('target_id')==target_id)\n        colors = [color_map[base] for base in df['resname']]\n        ax.scatter(df['x_1'], df['y_1'], df['z_1'], c=colors, s=100, alpha=0.7, marker='o')\n        ax.plot(df['x_1'], df['y_1'], df['z_1'], 'k-', alpha=0.3)\n\n        for i, txt in enumerate(df['resname']):\n            if df['x_1'][i] is not None:\n                ax.text(df['x_1'][i], df['y_1'][i], df['z_1'][i], \n                        f'{txt}{df[\"resid\"][i]}', size=6, color='black')\n\n        ax.set_xlabel('X')\n        ax.set_ylabel('Y')\n        ax.set_zlabel('Z')\n        ax.set_title(target_id)\n        ax.view_init(elev=20, azim=30)\n\n    legend_elements = [plt.Line2D([0], [0], marker='o', color='w', \n                                label=base, markerfacecolor=color, markersize=10) \n                    for base, color in color_map.items()]\n    fig.legend(handles=legend_elements, title='Nucleotides', loc='lower left')\n    # sequence = sequences.filter(pl.col('target_id')==target_ids[0]).select('sequence').to_series()[0]\n    # plt.suptitle(sequence, fontsize=6)\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-03-05T13:41:52.520696Z","iopub.execute_input":"2025-03-05T13:41:52.520961Z","iopub.status.idle":"2025-03-05T13:41:52.544888Z","shell.execute_reply.started":"2025-03-05T13:41:52.520940Z","shell.execute_reply":"2025-03-05T13:41:52.543658Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. Analysis of Training Data","metadata":{}},{"cell_type":"markdown","source":"## 1-1. Extracting Data with the Same Sequence","metadata":{}},{"cell_type":"code","source":"cnt_sequence = (\n    train_sequences\n    .group_by('sequence')\n    .agg(\n        pl.col(\"target_id\").count().alias('duplicates')\n    )\n    .sort(['duplicates', 'sequence'])\n)\n\ncnt_duplicates = (cnt_sequence\n    .group_by('duplicates')\n    .agg(\n        pl.col(\"sequence\").count().alias('count')\n    )\n    .sort(['duplicates'])\n)\nplt.figure(figsize=(8, 5))\nplt.bar(cnt_duplicates[\"duplicates\"], cnt_duplicates[\"count\"], color=\"skyblue\")\nplt.title(\"Number of data points with duplicate sequences\")\nplt.ylabel('num of sequences')\nplt.xlabel('duplicates')\nplt.show()\n\nsame_sequence = cnt_sequence.filter(pl.col('duplicates')>1)\nprint('Number of duplicate sequences:', same_sequence.height)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T13:41:52.545929Z","iopub.execute_input":"2025-03-05T13:41:52.546313Z","iopub.status.idle":"2025-03-05T13:41:52.790431Z","shell.execute_reply.started":"2025-03-05T13:41:52.546278Z","shell.execute_reply":"2025-03-05T13:41:52.788708Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- There are 45 duplicate sequences.\n- The most frequent sequence appears in 5 instances.","metadata":{}},{"cell_type":"markdown","source":"## 1-2. Example Visualization Using Plotly\nLet's check the RNA structures of a sequence (ACGAGUGUCGUACCAAG).","metadata":{}},{"cell_type":"code","source":"seq = 'ACGAGUGUCGUACCAAG'\ntarget_ids = train_sequences.filter(pl.col('sequence')==seq).select(pl.col('target_id')).to_series()\nplotlyRNAStructure(labels, target_ids)","metadata":{"execution":{"iopub.status.busy":"2025-03-05T13:41:52.791732Z","iopub.execute_input":"2025-03-05T13:41:52.792357Z","iopub.status.idle":"2025-03-05T13:41:52.912865Z","shell.execute_reply.started":"2025-03-05T13:41:52.792287Z","shell.execute_reply":"2025-03-05T13:41:52.911833Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Note that bases without coordinate data are not displayed.\n- By moving the viewpoint with the mouse and comparing them, **you can observe similar structures**, such as the helices at the ends and the constricted positions.","metadata":{}},{"cell_type":"markdown","source":"## 1-3. Visualization of All Data Using Matplotlib\nLet's display all the sequences as well.<br>\nSince Plotly is computationally heavy, all duplicate sequence data will be visualized using Matplotlib.<br>\nIf you find any sequences of interest, you can use the Plotly function to examine them in detail.","metadata":{}},{"cell_type":"code","source":"for seq in same_sequence['sequence']:\n    target_ids = train_sequences.filter(pl.col('sequence')==seq).select(pl.col('target_id')).to_series()\n    matplotRNAStructure(labels, target_ids)","metadata":{"execution":{"iopub.status.busy":"2025-03-05T13:41:52.913678Z","iopub.execute_input":"2025-03-05T13:41:52.914165Z","iopub.status.idle":"2025-03-05T13:42:25.770856Z","shell.execute_reply.started":"2025-03-05T13:41:52.914129Z","shell.execute_reply":"2025-03-05T13:42:25.769585Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Analysis of Validation Data","metadata":{}},{"cell_type":"markdown","source":"## 2-1. Extracting Data with the Same Sequence","metadata":{}},{"cell_type":"code","source":"cnt_sequence = (\n    valid_sequences\n    .group_by('sequence')\n    .agg(\n        pl.col(\"target_id\").count().alias('duplicates')\n    )\n    .sort(['duplicates', 'sequence'])\n)\n\ncnt_duplicates = (cnt_sequence\n    .group_by('duplicates')\n    .agg(\n        pl.col(\"sequence\").count().alias('count')\n    )\n    .sort(['duplicates'])\n)\nplt.figure(figsize=(8, 5))\nplt.bar(cnt_duplicates[\"duplicates\"], cnt_duplicates[\"count\"], color=\"skyblue\")\nplt.title(\"Number of data points with duplicate sequences\")\nplt.ylabel('num of sequences')\nplt.xlabel('duplicates')\nplt.show()\n\nsame_sequence = cnt_sequence.filter(pl.col('duplicates')>1)\nprint('Number of duplicate sequences:', same_sequence.height)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T13:42:25.771962Z","iopub.execute_input":"2025-03-05T13:42:25.772325Z","iopub.status.idle":"2025-03-05T13:42:25.978161Z","shell.execute_reply.started":"2025-03-05T13:42:25.772292Z","shell.execute_reply":"2025-03-05T13:42:25.977012Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2-2. Example Visualization Using Plotly","metadata":{}},{"cell_type":"code","source":"seq = 'GCGUACAGGGAACACGCAACCCCGAAGGAUCGGGGAAGGGACGUCGCCAGGGAGGCGAUUCCAUCAGGAUGAUGACGAGGGACUGAAGAGUGGGCGGGGUAAUACCCCGCCCCUUUUU'\ntarget_ids = valid_sequences.filter(pl.col('sequence')==seq).select(pl.col('target_id')).to_series()\nplotlyRNAStructure(labels, target_ids)","metadata":{"execution":{"iopub.status.busy":"2025-03-05T13:42:25.980253Z","iopub.execute_input":"2025-03-05T13:42:25.980581Z","iopub.status.idle":"2025-03-05T13:42:26.061756Z","shell.execute_reply.started":"2025-03-05T13:42:25.980553Z","shell.execute_reply":"2025-03-05T13:42:26.060678Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2-3. Visualization of All Data Using Matplotlib","metadata":{}},{"cell_type":"code","source":"for seq in same_sequence['sequence']:\n    target_ids = valid_sequences.filter(pl.col('sequence')==seq).select(pl.col('target_id')).to_series()\n    matplotRNAStructure(labels, target_ids)","metadata":{"execution":{"iopub.status.busy":"2025-03-05T13:42:26.062812Z","iopub.execute_input":"2025-03-05T13:42:26.063060Z","iopub.status.idle":"2025-03-05T13:42:48.046750Z","shell.execute_reply.started":"2025-03-05T13:42:26.063039Z","shell.execute_reply":"2025-03-05T13:42:48.043591Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. Analysis of Common Sequences Between Train and Validation","metadata":{}},{"cell_type":"markdown","source":"## 3-1. Extracting Data with the Same Sequence","metadata":{}},{"cell_type":"code","source":"same_sequence = list(set(train_sequences.select(pl.col('sequence')).to_series()) & set(valid_sequences.select(pl.col('sequence')).to_series()))\nsame_sequence.sort()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T13:42:48.048143Z","iopub.execute_input":"2025-03-05T13:42:48.048551Z","iopub.status.idle":"2025-03-05T13:42:48.055103Z","shell.execute_reply.started":"2025-03-05T13:42:48.048522Z","shell.execute_reply":"2025-03-05T13:42:48.054004Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-2. Visualization of All Data Using Matplotlib","metadata":{}},{"cell_type":"code","source":"for seq in same_sequence:\n    target_ids = sequences.filter(pl.col('sequence')==seq).select(pl.col('target_id')).to_series()\n    matplotRNAStructure(labels, target_ids)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T13:42:48.055953Z","iopub.execute_input":"2025-03-05T13:42:48.056222Z","iopub.status.idle":"2025-03-05T13:43:15.369671Z","shell.execute_reply.started":"2025-03-05T13:42:48.056198Z","shell.execute_reply":"2025-03-05T13:43:15.368310Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Both the train and validation data show similar structures when they share the same sequence.","metadata":{}}]}