{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":51294,"databundleVersionId":6512296,"sourceType":"competition"}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# EDA of RNA Main Dataset for Stanford Ribonanza RNA Folding","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\nfrom warnings import simplefilter\nimport gc\n\n# Too much warning\nsimplefilter(action=\"ignore\", category=FutureWarning)\nsimplefilter(action=\"ignore\", category=RuntimeWarning)\n\npath_DATADIR = Path(\"../input/stanford-ribonanza-rna-folding\")\ndisplay(\n    list(path_DATADIR.iterdir()),\n)","metadata":{"editable":true,"slideshow":{"slide_type":""},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Data","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\n\nnp.__version__, matplotlib.__version__, sns.__version__, pd.__version__","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reactivity_cols = [f\"reactivity_{n:04d}\" for n in range(1, 207)]\nreactivity_error_cols = [f\"reactivity_error_{n:04d}\" for n in range(1, 207)]\nusecols = [\"sequence_id\", \"sequence\", \"experiment_type\", \"dataset_name\",\n           \"reads\", \"signal_to_noise\", \"SN_filter\"] + \\\n            reactivity_cols + reactivity_error_cols\n\ndf = pd.read_csv(path_DATADIR / \"train_data.csv\", usecols=usecols,\n                 dtype={k: np.float32 for k in reactivity_cols + reactivity_error_cols})\ndf.dtypes, df.memory_usage().sum()","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Train DataFrame manipulation\nif \"sequence_len\" not in df.columns:\n    df = pd.concat([df, df.sequence.apply(len).rename(\"sequence_len\")], axis=1)\n    df_reactivity = df.loc[:, reactivity_cols].rename(columns=lambda s: int(s[-4:])-1)\n    df_reactivity_notna = ~df_reactivity.isna()\n\n    df = pd.concat([df, df_reactivity_notna.sum(axis=1)\n                    .rename(\"reactivity_len\")], axis=1)\n    df = pd.concat([df, df_reactivity.isna().idxmin(axis=1)\n                    .rename(\"reactivity_idx_begin\")], axis=1)\n    df = pd.concat([df, (df[\"reactivity_idx_begin\"]+df[\"reactivity_len\"])\n                    .rename(\"reactivity_idx_end\")], axis=1)\n    df = pd.concat([df, (df[\"sequence_len\"]-df[\"reactivity_len\"])\n                    .rename(\"delta_seq_react\")], axis=1)\n\n    del df_reactivity, df_reactivity_notna\n\n# Column selection after manipulation\nother_cols = [c for c in df.columns\n              if c not in reactivity_cols + reactivity_error_cols]\ngc.collect()","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head(5)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_2a3 = df[df[\"experiment_type\"] == \"2A3_MaP\"]\ndf_dms = df[df[\"experiment_type\"] == \"DMS_MaP\"]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Sequence distinctiveness","metadata":{}},{"cell_type":"code","source":"sequence_value_counts_2a3 = df_2a3.sequence_id.value_counts()\nsequence_value_counts_dms = df_dms.sequence_id.value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(sequence_value_counts_2a3, sequence_value_counts_dms)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### N/A in Reactivity Columns","metadata":{}},{"cell_type":"code","source":"df_2a3_na = df_2a3.loc[:, reactivity_cols].isna().mean()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_2a3_na.loc[df_2a3_na < 1]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dms_na = df_dms.loc[:, reactivity_cols].isna().mean()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dms_na.loc[df_dms_na < 1]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Number of valid Reactivity values","metadata":{}},{"cell_type":"code","source":"sns.displot(df_2a3[\"reactivity_len\"])\nplt.show()","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(df_dms[\"reactivity_len\"])\nplt.show()","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_2a3_actlt20 = df_2a3.loc[df_2a3[\"reactivity_len\"].lt(20), other_cols]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_2a3_actlt20.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_2a3_act60to80 = df_2a3.loc[df_2a3[\"reactivity_len\"].between(60, 80), other_cols]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_2a3_act60to80.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Continuity of Reactivity valid values","metadata":{}},{"cell_type":"code","source":"df_2a3.loc[:, reactivity_cols].isna().astype(np.float32).diff(axis=1).abs().sum(axis=1) \\\n                              .value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dms.loc[:, reactivity_cols].isna().astype(np.float32).diff(axis=1).abs().sum(axis=1) \\\n                              .value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Reactivity valid value sequences are continuous, sequences may look like:\n\n* \\[NaN, NaN, 0., 1., ..., 0.5, 0.1, NaN, NaN\\] -> isna.diff.abs.sum == 2\n* \\[NaN, NaN, ..., NaN, NaN\\] -> isna.diff.abs.sum == 0 , [See Section of Unlabeled Rows](#We-have-Unlabeled-Rows)\n* x \\[0., 1., ..., 0.5, 0.1\\] -> isna.diff.abs.sum == 0 , however, we don't have fully labeled rows\n\nNot like:\n* \\[NaN, NaN, 0., 1., NaN, 0.5, NaN, ..., NaN, 0.2\\] -> isna.diff.abs.sum > 2","metadata":{}},{"cell_type":"markdown","source":"### Where the reactivity label begins/ends","metadata":{}},{"cell_type":"code","source":"df_2a3[\"reactivity_idx_begin\"].value_counts()","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dms[\"reactivity_idx_begin\"].value_counts()","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_2a3[\"reactivity_idx_end\"].value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dms[\"reactivity_idx_end\"].value_counts()","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We have Unlabeled Rows","metadata":{}},{"cell_type":"code","source":"df_unlabeled = df.loc[df.reactivity_len == 0]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_unlabeled.experiment_type.value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_unlabeled.loc[df_unlabeled.experiment_type == \"2A3_MaP\"].head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_unlabeled.loc[df_unlabeled.experiment_type == \"DMS_MaP\"].head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Look duplication in slice of sequence","metadata":{}},{"cell_type":"code","source":"df_2a3_major = df_2a3.query(\"reactivity_idx_begin == 26 and reactivity_idx_end == 126\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_2a3_major.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dms_major = df_dms.query(\"reactivity_idx_begin == 26 and reactivity_idx_end == 126\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dms_major.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_2a3_major[\"sequence\"].map(lambda s: s[:26]).value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dms_major[\"sequence\"].map(lambda s: s[:26]).value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_2a3_major[\"sequence\"].map(lambda s: s[126:]).value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dms_major[\"sequence\"].map(lambda s: s[126:]).value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_2a3_major[\"sequence\"].map(lambda s: s[-21:]).value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dms_major[\"sequence\"].map(lambda s: s[-21:]).value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Length of sequence","metadata":{}},{"cell_type":"code","source":"df_2a3[\"sequence_len\"].quantile([0., 0.25, 0.5, 0.75, 1.])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dms[\"sequence_len\"].quantile([0., 0.25, 0.5, 0.75, 1.])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_2a3[\"delta_seq_react\"].value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dms[\"delta_seq_react\"].value_counts()","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Signal to Noise","metadata":{}},{"cell_type":"code","source":"sns.displot(df_2a3[\"signal_to_noise\"].clip(upper=5))\nplt.show()","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(df_dms[\"signal_to_noise\"].clip(upper=5))\nplt.show()","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_2a3_sn_lt02 = df_2a3.loc[df_2a3[\"signal_to_noise\"].lt(.2), other_cols] \\\n    .sort_values(by=\"signal_to_noise\", ascending=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_2a3_sn_lt02.head()","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dms_sn_lt02 = df_dms.loc[df_dms[\"signal_to_noise\"].lt(.2), other_cols] \\\n    .sort_values(by=\"signal_to_noise\", ascending=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dms_sn_lt02.head()","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Relationship","metadata":{}},{"cell_type":"code","source":"df_2a3.loc[:, [\"reads\", \"signal_to_noise\",\n               \"sequence_len\", \"reactivity_len\",\n               \"delta_seq_react\"]].corr()","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dms.loc[:, [\"reads\", \"signal_to_noise\",\n               \"sequence_len\", \"reactivity_len\",\n               \"delta_seq_react\"]].corr()","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Distribution of Reactivity","metadata":{}},{"cell_type":"code","source":"df_reactivity_clipped = pd.concat([df.loc[:, other_cols], df.loc[:, reactivity_cols].clip(0., 1.)], axis=1)","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in reactivity_cols[26:87:5]:\n    plt.figure(figsize=(8, 4), tight_layout=True)\n    sns.boxplot(data=df_reactivity_clipped, x=col, y=\"experiment_type\", hue=\"SN_filter\")\n    plt.show()","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Distribution of Reactivity Error","metadata":{}},{"cell_type":"code","source":"df_reactivity_error = pd.concat([df.loc[:, other_cols], df.loc[:, reactivity_error_cols]], axis=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in reactivity_error_cols[26:87:5]:\n    plt.figure(figsize=(8, 4), tight_layout=True)\n    sns.boxplot(data=df_reactivity_error, x=col, y=\"experiment_type\", hue=\"SN_filter\")\n    plt.show()","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### How many Reads will be","metadata":{}},{"cell_type":"code","source":"df_reads_clipped = df[\"reads\"].clip(0, 1e3)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8, 2), tight_layout=True)\nsns.boxplot(data=df_reads_clipped, orient=\"h\")\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test Sequence","metadata":{}},{"cell_type":"code","source":"df_testseq = pd.read_csv(path_DATADIR / \"test_sequences.csv\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if \"sequence_len\" not in df_testseq:\n    df_testseq = pd.concat([df_testseq, df_testseq[\"sequence\"].map(len).rename(\"sequence_len\")], axis=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_testseq.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_testseq[\"sequence_len\"].value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_testseq_seq_first26 = df_testseq[\"sequence\"].map(lambda s: s[:26])\ndf_testseq_seq_first26.value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_testseq_seq_last21 = df_testseq[\"sequence\"].map(lambda s: s[-21:])\ndf_testseq_seq_last21.value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_testseq_457 = df_testseq.loc[df_testseq[\"sequence_len\"] == 457]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_testseq_457.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## How to preprocess Train Dataset\n\n* We have a lot of unlabeled rows, drop them before training to reduce memory usage.\n* May consider that drop rows whose value of signal_to_noise are too low. Or filter by SN_filter\n* May consider drop first 26 padded elements and/or last 21 padded elements of each sequence as well as reactivity sequence in train dataset. Or do nothing with them.","metadata":{}}]}