{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\nimport os\nimport sys\nimport shutil\nimport glob\nfrom shutil import copyfile\nimport gc\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-02T07:02:12.304586Z","iopub.execute_input":"2023-11-02T07:02:12.304990Z","iopub.status.idle":"2023-11-02T07:02:12.736746Z","shell.execute_reply.started":"2023-11-02T07:02:12.304955Z","shell.execute_reply":"2023-11-02T07:02:12.735750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! python --version","metadata":{"execution":{"iopub.status.busy":"2023-11-02T07:02:19.867931Z","iopub.execute_input":"2023-11-02T07:02:19.868494Z","iopub.status.idle":"2023-11-02T07:02:20.969693Z","shell.execute_reply.started":"2023-11-02T07:02:19.868455Z","shell.execute_reply":"2023-11-02T07:02:20.968378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### make some directories in /tmp to work with","metadata":{}},{"cell_type":"code","source":"!mkdir /tmp/ribonanza\n!mkdir /tmp/ribonanza/PDBs","metadata":{"execution":{"iopub.status.busy":"2023-11-02T07:02:24.890717Z","iopub.execute_input":"2023-11-02T07:02:24.891177Z","iopub.status.idle":"2023-11-02T07:02:25.978575Z","shell.execute_reply.started":"2023-11-02T07:02:24.891139Z","shell.execute_reply":"2023-11-02T07:02:25.977352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### copy input RhoFold_PDBs_zip to /tmp/ribonanza folder and rename with .zip ","metadata":{"execution":{"iopub.status.busy":"2023-11-02T07:02:34.932050Z","iopub.execute_input":"2023-11-02T07:02:34.932491Z","iopub.status.idle":"2023-11-02T07:02:36.025515Z","shell.execute_reply.started":"2023-11-02T07:02:34.932456Z","shell.execute_reply":"2023-11-02T07:02:36.024348Z"}}},{"cell_type":"code","source":"!cp -r /kaggle/input/ribonanza-3d-coords/RhoFold_PDBs_zip  /tmp/ribonanza/RhoFold_PDBs_zip.zip","metadata":{"execution":{"iopub.status.busy":"2023-11-02T07:03:01.470899Z","iopub.execute_input":"2023-11-02T07:03:01.471387Z","iopub.status.idle":"2023-11-02T07:04:18.204656Z","shell.execute_reply.started":"2023-11-02T07:03:01.471346Z","shell.execute_reply":"2023-11-02T07:04:18.203280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### list /tmp/ribonanza directory","metadata":{}},{"cell_type":"code","source":"!ls /tmp/ribonanza","metadata":{"execution":{"iopub.status.busy":"2023-11-02T07:04:53.521601Z","iopub.execute_input":"2023-11-02T07:04:53.522008Z","iopub.status.idle":"2023-11-02T07:04:54.621688Z","shell.execute_reply.started":"2023-11-02T07:04:53.521976Z","shell.execute_reply":"2023-11-02T07:04:54.620616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### use shutil to extract renamed RhoFold_PDBs_zip.zip to /tmp/ribonanza/PDBs ","metadata":{}},{"cell_type":"code","source":"shutil.unpack_archive('/tmp/ribonanza/RhoFold_PDBs_zip.zip', '/tmp/ribonanza/PDBs')\nprint('unpack archive complete!')","metadata":{"execution":{"iopub.status.busy":"2023-11-02T07:05:35.177798Z","iopub.execute_input":"2023-11-02T07:05:35.178241Z","iopub.status.idle":"2023-11-02T07:11:24.707507Z","shell.execute_reply.started":"2023-11-02T07:05:35.178184Z","shell.execute_reply":"2023-11-02T07:11:24.705551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### list /tmp/ribonanza/PDBs now has directory RhoFold_PDBs","metadata":{}},{"cell_type":"code","source":"!ls /tmp/ribonanza/PDBs","metadata":{"execution":{"iopub.status.busy":"2023-11-02T07:11:45.329424Z","iopub.execute_input":"2023-11-02T07:11:45.329874Z","iopub.status.idle":"2023-11-02T07:11:46.421920Z","shell.execute_reply.started":"2023-11-02T07:11:45.329834Z","shell.execute_reply":"2023-11-02T07:11:46.420618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### save a version of the extracted PDBs for future use","metadata":{}},{"cell_type":"code","source":"archn = 'PDBs'\nshutil.make_archive(archn, 'zip', '/tmp/ribonanza/PDBs/RhoFold_PDBs/')","metadata":{"execution":{"iopub.status.busy":"2023-11-02T08:50:17.919625Z","iopub.execute_input":"2023-11-02T08:50:17.920038Z","iopub.status.idle":"2023-11-02T08:50:17.925365Z","shell.execute_reply.started":"2023-11-02T08:50:17.920008Z","shell.execute_reply":"2023-11-02T08:50:17.924149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# to extract in another notebook e.g.\n# datasetname = Ribonanza_3D_coords_prep\n#shutil.unpack_archive(f'/kaggle/input/{datasetname}/PDBs.zip', '/tmp/ribonanza/PDBs)\n#print('unpack archive complete!')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### walk through RhoFold_PDBs to get directories which are sequence ids","metadata":{}},{"cell_type":"code","source":"seqIDs = []\nfor dirname, _, filenames in os.walk('/tmp/ribonanza/PDBs/RhoFold_PDBs'):\n    seqIDs.append(dirname)\n    #for filename in filenames:\n        #print(os.path.join(dirname, filename))\nlen(seqIDs)","metadata":{"execution":{"iopub.status.busy":"2023-11-02T07:28:11.179570Z","iopub.execute_input":"2023-11-02T07:28:11.180032Z","iopub.status.idle":"2023-11-02T07:28:15.561483Z","shell.execute_reply.started":"2023-11-02T07:28:11.179997Z","shell.execute_reply":"2023-11-02T07:28:15.560630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seqIDs[:5],seqIDs[-5:]","metadata":{"execution":{"iopub.status.busy":"2023-11-02T07:28:21.829487Z","iopub.execute_input":"2023-11-02T07:28:21.829903Z","iopub.status.idle":"2023-11-02T07:28:21.836852Z","shell.execute_reply.started":"2023-11-02T07:28:21.829870Z","shell.execute_reply":"2023-11-02T07:28:21.835925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### list a sequence id directory name to view contents - each will have unrelaxed_model.pdb","metadata":{}},{"cell_type":"code","source":"!ls /tmp/ribonanza/PDBs/RhoFold_PDBs/b9e74f0b44be","metadata":{"execution":{"iopub.status.busy":"2023-11-02T07:28:47.349498Z","iopub.execute_input":"2023-11-02T07:28:47.349928Z","iopub.status.idle":"2023-11-02T07:28:48.448142Z","shell.execute_reply.started":"2023-11-02T07:28:47.349898Z","shell.execute_reply":"2023-11-02T07:28:48.446666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### inspect each using head","metadata":{}},{"cell_type":"code","source":"!head /tmp/ribonanza/PDBs/RhoFold_PDBs/b9e74f0b44be/log.txt","metadata":{"execution":{"iopub.status.busy":"2023-11-02T07:28:56.689725Z","iopub.execute_input":"2023-11-02T07:28:56.690179Z","iopub.status.idle":"2023-11-02T07:28:57.800401Z","shell.execute_reply.started":"2023-11-02T07:28:56.690140Z","shell.execute_reply":"2023-11-02T07:28:57.799226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head /tmp/ribonanza/PDBs/RhoFold_PDBs/b9e74f0b44be/ss.ct","metadata":{"execution":{"iopub.status.busy":"2023-11-02T07:29:05.919292Z","iopub.execute_input":"2023-11-02T07:29:05.919689Z","iopub.status.idle":"2023-11-02T07:29:07.014996Z","shell.execute_reply.started":"2023-11-02T07:29:05.919657Z","shell.execute_reply":"2023-11-02T07:29:07.013857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head /tmp/ribonanza/PDBs/RhoFold_PDBs/b9e74f0b44be/unrelaxed_model.pdb","metadata":{"execution":{"iopub.status.busy":"2023-11-02T07:29:11.739650Z","iopub.execute_input":"2023-11-02T07:29:11.740093Z","iopub.status.idle":"2023-11-02T07:29:12.841194Z","shell.execute_reply.started":"2023-11-02T07:29:11.740050Z","shell.execute_reply":"2023-11-02T07:29:12.839492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### create a DataFrame for the RhoFold_PDBs folders names and sequence id","metadata":{}},{"cell_type":"code","source":"df_PDBs  = pd.DataFrame({ 'RhoFold_PDBs_folder' : seqIDs[1:],\n                   'sequence_id'  : '',\n      \n  } )\ndf_PDBs.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-02T07:32:49.458712Z","iopub.execute_input":"2023-11-02T07:32:49.459463Z","iopub.status.idle":"2023-11-02T07:32:49.509350Z","shell.execute_reply.started":"2023-11-02T07:32:49.459419Z","shell.execute_reply":"2023-11-02T07:32:49.508268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### extract the sequence id using split by '/' taking the last entry","metadata":{}},{"cell_type":"code","source":"df_PDBs['sequence_id'] = df_PDBs.RhoFold_PDBs_folder.apply( lambda x : x.split('/')[-1])\ndf_PDBs.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-02T07:34:41.671301Z","iopub.execute_input":"2023-11-02T07:34:41.671727Z","iopub.status.idle":"2023-11-02T07:34:41.781443Z","shell.execute_reply.started":"2023-11-02T07:34:41.671688Z","shell.execute_reply":"2023-11-02T07:34:41.780138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### get competition train.csv with selected columns and add a column for length of sequence ","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/stanford-ribonanza-rna-folding/train_data.csv', usecols=['sequence_id','sequence','experiment_type','reads','signal_to_noise','SN_filter' ],)\ndf_train['sequence_len'] = df_train.sequence.apply(len)","metadata":{"execution":{"iopub.status.busy":"2023-11-02T07:47:21.131195Z","iopub.execute_input":"2023-11-02T07:47:21.131675Z","iopub.status.idle":"2023-11-02T07:48:15.810380Z","shell.execute_reply.started":"2023-11-02T07:47:21.131644Z","shell.execute_reply":"2023-11-02T07:48:15.809102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-02T07:48:49.018343Z","iopub.execute_input":"2023-11-02T07:48:49.019072Z","iopub.status.idle":"2023-11-02T07:48:49.035320Z","shell.execute_reply.started":"2023-11-02T07:48:49.019029Z","shell.execute_reply":"2023-11-02T07:48:49.033858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### merge competition train with the DataFrame for the RhoFold_PDBs folders names and sequence ids","metadata":{}},{"cell_type":"code","source":"df_train = df_train.merge(df_PDBs, how='left', left_on='sequence_id', right_on='sequence_id',)","metadata":{"execution":{"iopub.status.busy":"2023-11-02T08:02:54.129594Z","iopub.execute_input":"2023-11-02T08:02:54.130463Z","iopub.status.idle":"2023-11-02T08:02:55.531897Z","shell.execute_reply.started":"2023-11-02T08:02:54.130423Z","shell.execute_reply":"2023-11-02T08:02:55.530872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[df_train.sequence_id=='b9e74f0b44be']","metadata":{"execution":{"iopub.status.busy":"2023-11-02T08:03:13.889283Z","iopub.execute_input":"2023-11-02T08:03:13.889735Z","iopub.status.idle":"2023-11-02T08:03:14.224983Z","shell.execute_reply.started":"2023-11-02T08:03:13.889699Z","shell.execute_reply":"2023-11-02T08:03:14.222638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_train[pd.notna(df_train.RhoFold_PDBs_folder)])   # how many train entries have PDBs","metadata":{"execution":{"iopub.status.busy":"2023-11-02T08:04:16.145524Z","iopub.execute_input":"2023-11-02T08:04:16.145913Z","iopub.status.idle":"2023-11-02T08:04:16.304779Z","shell.execute_reply.started":"2023-11-02T08:04:16.145879Z","shell.execute_reply":"2023-11-02T08:04:16.303723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train entries with PDBS, also SN filter 1 and 0\nlen(df_train[pd.notna(df_train.RhoFold_PDBs_folder) & (df_train.SN_filter==1)]), len(df_train[pd.notna(df_train.RhoFold_PDBs_folder) & (df_train.SN_filter==0)])","metadata":{"execution":{"iopub.status.busy":"2023-11-02T08:48:00.637183Z","iopub.execute_input":"2023-11-02T08:48:00.637650Z","iopub.status.idle":"2023-11-02T08:48:00.901903Z","shell.execute_reply.started":"2023-11-02T08:48:00.637616Z","shell.execute_reply":"2023-11-02T08:48:00.900721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### take a copy of train for rows with PDBs and save a csv for future use","metadata":{}},{"cell_type":"code","source":"df_train_PDBs   = df_train[pd.notna(df_train.RhoFold_PDBs_folder)].copy().reset_index(drop=True)\ndf_train_PDBs.tail()","metadata":{"execution":{"iopub.status.busy":"2023-11-02T08:08:36.167006Z","iopub.execute_input":"2023-11-02T08:08:36.167445Z","iopub.status.idle":"2023-11-02T08:08:36.417710Z","shell.execute_reply.started":"2023-11-02T08:08:36.167411Z","shell.execute_reply":"2023-11-02T08:08:36.416805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_PDBs.to_csv('df_train_PDBs.csv', index=False)  # save csv of train with PDBs","metadata":{"execution":{"iopub.status.busy":"2023-11-02T08:10:26.342707Z","iopub.execute_input":"2023-11-02T08:10:26.343166Z","iopub.status.idle":"2023-11-02T08:10:30.950647Z","shell.execute_reply.started":"2023-11-02T08:10:26.343137Z","shell.execute_reply":"2023-11-02T08:10:30.949417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train entries without PDBs, also SN filter 1 and 0\nlen(df_train[pd.isna(df_train.RhoFold_PDBs_folder)]),len(df_train[pd.isna(df_train.RhoFold_PDBs_folder) & (df_train.SN_filter==1)]) ,len(df_train[pd.isna(df_train.RhoFold_PDBs_folder) & (df_train.SN_filter==0)]) \n","metadata":{"execution":{"iopub.status.busy":"2023-11-02T08:48:40.326941Z","iopub.execute_input":"2023-11-02T08:48:40.327380Z","iopub.status.idle":"2023-11-02T08:48:41.374893Z","shell.execute_reply.started":"2023-11-02T08:48:40.327346Z","shell.execute_reply":"2023-11-02T08:48:41.373502Z"},"trusted":true},"execution_count":null,"outputs":[]}]}