{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# What is about ?\n\nFilter/clean/prepare data for  Stanford Ribonanza RNA Folding. \n\nOutcomes are mainly stored into dataset: https://www.kaggle.com/datasets/alexandervc/stanford-ribonanza-rna-folding-data\n        \n        \nVersions:\n\n\n    1 select \"small,clean\" subpart of rna - choose: SN_filter = 1, seqence is NOT duplicated - only ONE 2A3 and DMS data for it, no extra nan in reactivity. Create files for 2A3 part and DMS parts - ensure that ordering of sequences is the SAME is both file sets. We save both csv and npy (for fast load). Both (2A3,DMS) sets  contain 163428 rna. ","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport time\nt0start = time.time() \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\ncc = 0\nfor dirname, _, filenames in os.walk('/kaggle/input/stanford-ribonanza-rna-folding-converted/'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\nfor dirname, _, filenames in os.walk('/kaggle/input/stanford-ribonanza-rna-folding-data/'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    if 'stanford-ribonanza-rna-folding-converted' in dirname: continue \n    if 'stanford-ribonanza-rna-folding-data' in dirname: continue \n    for filename in filenames:\n        cc += 1\n        if cc < 20:\n            print(os.path.join(dirname, filename))\n        else:\n            break\n    if cc < 20:\n        pass\n    else:\n        break\n\n        \n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-22T16:14:33.165238Z","iopub.execute_input":"2023-09-22T16:14:33.165707Z","iopub.status.idle":"2023-09-22T16:14:33.196322Z","shell.execute_reply.started":"2023-09-22T16:14:33.165674Z","shell.execute_reply":"2023-09-22T16:14:33.195172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \nPATH = '/kaggle/input/stanford-ribonanza-rna-folding-converted/'\ndf = pd.read_parquet(os.path.join(PATH,'train_data.parquet'))\nprint( df.shape )\ndf","metadata":{"execution":{"iopub.status.busy":"2023-09-22T15:19:18.958652Z","iopub.execute_input":"2023-09-22T15:19:18.959105Z","iopub.status.idle":"2023-09-22T15:19:31.428712Z","shell.execute_reply.started":"2023-09-22T15:19:18.959070Z","shell.execute_reply":"2023-09-22T15:19:31.427429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprint(df.shape, df['sequence_id'].nunique() )\n\nv0 = df.iloc[:,33:133].isnull().sum(axis = 1)\nm0 = v0 == 0 \nprint(m0.sum())\nprint()\n\nm1 = (df['SN_filter'] == 1 ) & (m0)\nprint(m1.sum(), df['sequence_id'][m1].nunique() )\nprint()\n\nd2 = df[m1][['sequence_id','experiment_type']].groupby('sequence_id')['experiment_type'].nunique()\ndisplay(d2.head(2))\nm2a =  d2 == 2\nm2 = df['sequence_id'].isin(d2[m2a].index )\nprint(m2.sum() , m2a.sum())\nprint()\n\nd3 = df[m1][['sequence_id','sequence']].groupby('sequence_id')['sequence'].count()\ndisplay(d3.head(2))\nm3a =  d3 == 2\nm3 = df['sequence_id'].isin(d3[m3a].index )\nprint(m3.sum() , m3a.sum())\nprint()\n\nm_full = m1 & m2 & m3\nprint(m_full.sum())","metadata":{"execution":{"iopub.status.busy":"2023-09-22T15:20:41.703624Z","iopub.execute_input":"2023-09-22T15:20:41.704211Z","iopub.status.idle":"2023-09-22T15:20:47.527627Z","shell.execute_reply.started":"2023-09-22T15:20:41.704169Z","shell.execute_reply":"2023-09-22T15:20:47.526079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[m_full]['experiment_type'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-09-22T15:20:55.659225Z","iopub.execute_input":"2023-09-22T15:20:55.659643Z","iopub.status.idle":"2023-09-22T15:20:56.189684Z","shell.execute_reply.started":"2023-09-22T15:20:55.659609Z","shell.execute_reply":"2023-09-22T15:20:56.188294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[m_full].iloc[:,33:133].isnull().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2023-09-22T15:21:23.103020Z","iopub.execute_input":"2023-09-22T15:21:23.103846Z","iopub.status.idle":"2023-09-22T15:21:23.683925Z","shell.execute_reply.started":"2023-09-22T15:21:23.103800Z","shell.execute_reply":"2023-09-22T15:21:23.682701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nl = [ len(s) for s in df[m_full]['sequence'] ]\n","metadata":{"execution":{"iopub.status.busy":"2023-09-22T15:22:48.095682Z","iopub.execute_input":"2023-09-22T15:22:48.096118Z","iopub.status.idle":"2023-09-22T15:22:48.708121Z","shell.execute_reply.started":"2023-09-22T15:22:48.096084Z","shell.execute_reply":"2023-09-22T15:22:48.706819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.Series(l).value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-09-22T15:23:04.326419Z","iopub.execute_input":"2023-09-22T15:23:04.326899Z","iopub.status.idle":"2023-09-22T15:23:04.503848Z","shell.execute_reply.started":"2023-09-22T15:23:04.326860Z","shell.execute_reply":"2023-09-22T15:23:04.502435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head(1)","metadata":{"execution":{"iopub.status.busy":"2023-09-22T15:25:03.945413Z","iopub.execute_input":"2023-09-22T15:25:03.946020Z","iopub.status.idle":"2023-09-22T15:25:03.972157Z","shell.execute_reply.started":"2023-09-22T15:25:03.945984Z","shell.execute_reply":"2023-09-22T15:25:03.970966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = df['experiment_type'] == '2A3_MaP'\nprint( (m&m_full).sum() )\n\nd1 = df[m&m_full]\nd1.index.name = 'index'\nprint(d1.shape)\ndisplay(d1.head(2))\nprint()\n\nm = df['experiment_type'] == 'DMS_MaP'\nprint( (m&m_full).sum() )\ns = set(df[m&m_full]['sequence_id']) & set(d1['sequence_id'])\nprint( len(s) )\n\nd2 = pd.DataFrame()\nd2['IX'] = range(len(d2))\nd2['sequence_id'] = d1['sequence_id']\nd2 = d2.merge(df[m&m_full].reset_index(), on = 'sequence_id' ).sort_values('IX')\nd2 = d2.iloc[:,1:]\nd2 = d2.set_index('index')\nprint(d2.shape)\ndisplay(d2.head(2))\n\n\n\nd1['sequence'].tolist() == d2['sequence'].tolist()","metadata":{"execution":{"iopub.status.busy":"2023-09-22T15:58:32.930058Z","iopub.execute_input":"2023-09-22T15:58:32.930509Z","iopub.status.idle":"2023-09-22T15:58:35.605846Z","shell.execute_reply.started":"2023-09-22T15:58:32.930473Z","shell.execute_reply":"2023-09-22T15:58:35.604663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nd1.to_csv('train_cleaned_2A3_MaP_163428.csv')\nd2.to_csv('train_cleaned_DMS_MaP_163428.csv')\nprint(d1.shape,d2.shape)","metadata":{"execution":{"iopub.status.busy":"2023-09-22T15:58:42.185890Z","iopub.execute_input":"2023-09-22T15:58:42.186350Z","iopub.status.idle":"2023-09-22T16:00:32.491804Z","shell.execute_reply.started":"2023-09-22T15:58:42.186314Z","shell.execute_reply":"2023-09-22T16:00:32.490379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d1.iloc[:2,:7]","metadata":{"execution":{"iopub.status.busy":"2023-09-22T16:00:32.494499Z","iopub.execute_input":"2023-09-22T16:00:32.495361Z","iopub.status.idle":"2023-09-22T16:00:32.510277Z","shell.execute_reply.started":"2023-09-22T16:00:32.495320Z","shell.execute_reply":"2023-09-22T16:00:32.509201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nnp.save('train_cleaned_2A3_MaP_163428_reactivity.npy', d1.iloc[:,7:].values )\nnp.save('train_cleaned_DMS_MaP_163428_reactivity.npy', d2.iloc[:,7:].values )","metadata":{"execution":{"iopub.status.busy":"2023-09-22T16:00:32.511489Z","iopub.execute_input":"2023-09-22T16:00:32.512569Z","iopub.status.idle":"2023-09-22T16:00:33.251325Z","shell.execute_reply.started":"2023-09-22T16:00:32.512522Z","shell.execute_reply":"2023-09-22T16:00:33.249978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nd1.iloc[:,:7].to_csv('train_cleaned_2A3_MaP_163428_info_only.csv')\nd2.iloc[:,:7].to_csv('train_cleaned_DMS_MaP_163428_info_only.csv')","metadata":{"execution":{"iopub.status.busy":"2023-09-22T16:00:33.253426Z","iopub.execute_input":"2023-09-22T16:00:33.253744Z","iopub.status.idle":"2023-09-22T16:00:38.953118Z","shell.execute_reply.started":"2023-09-22T16:00:33.253717Z","shell.execute_reply":"2023-09-22T16:00:38.951854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('%.1f seconds passed total '%(time.time()-t0start) )\nprint('%.1f minutes passed total '%( (time.time()-t0start)/60)  )\nprint('%.2f hours passed total '%( (time.time()-t0start)/3600)  )","metadata":{},"execution_count":null,"outputs":[]}]}