{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":51294,"databundleVersionId":7331882,"sourceType":"competition"},{"sourceId":7268701,"sourceType":"datasetVersion","datasetId":4213511},{"sourceId":156217852,"sourceType":"kernelVersion"}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"The second step for reproducing my solution: turning the data into TFRecords. Check [my github](https://github.com/shlomoron/Stanford-Ribonanza-RNA-Folding-10th-place-solution) for more details. I will admit, this notebook is not the most beautiful of them all, but oh well...maybe in the future I will put some more work on it (doubtful).","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pickle\nimport os\nimport matplotlib.pyplot as plt\nimport tensorflow as tf","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-23T21:30:04.703243Z","iopub.execute_input":"2023-12-23T21:30:04.703883Z","iopub.status.idle":"2023-12-23T21:30:13.934038Z","shell.execute_reply.started":"2023-12-23T21:30:04.703849Z","shell.execute_reply":"2023-12-23T21:30:13.933147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_id = pickle.load(open('/kaggle/input/srrf-solution-1-csv-to-np/sequence_id.p', 'br'))\nsequence = pickle.load(open('/kaggle/input/srrf-solution-1-csv-to-np/sequence.p', 'br'))\nexperiment_type = pickle.load(open('/kaggle/input/srrf-solution-1-csv-to-np/experiment_type.p', 'br'))\ndataset_name = pickle.load(open('/kaggle/input/srrf-solution-1-csv-to-np/dataset_name.p', 'br'))\nreads = pickle.load(open('/kaggle/input/srrf-solution-1-csv-to-np/reads.p', 'br'))\nsignal_to_noise = pickle.load(open('/kaggle/input/srrf-solution-1-csv-to-np/signal_to_noise.p', 'br'))\nSN_filter = pickle.load(open('/kaggle/input/srrf-solution-1-csv-to-np/SN_filter.p', 'br'))\nreactivity = pickle.load(open('/kaggle/input/srrf-solution-1-csv-to-np/reactivity.p', 'br'))\nreactivity_error = pickle.load(open('/kaggle/input/srrf-solution-1-csv-to-np/reactivity_error.p', 'br'))","metadata":{"execution":{"iopub.status.busy":"2023-12-23T22:07:56.067288Z","iopub.execute_input":"2023-12-23T22:07:56.068539Z","iopub.status.idle":"2023-12-23T22:08:40.896182Z","shell.execute_reply.started":"2023-12-23T22:07:56.068473Z","shell.execute_reply":"2023-12-23T22:08:40.894759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(experiment_type)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:30:51.238367Z","iopub.execute_input":"2023-12-23T21:30:51.238722Z","iopub.status.idle":"2023-12-23T21:30:52.692256Z","shell.execute_reply.started":"2023-12-23T21:30:51.238693Z","shell.execute_reply":"2023-12-23T21:30:52.690723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indices_2A3 = experiment_type == '2A3_MaP'\nindices_DMS = experiment_type == 'DMS_MaP'","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:34:22.588679Z","iopub.execute_input":"2023-12-23T21:34:22.589088Z","iopub.status.idle":"2023-12-23T21:34:22.674015Z","shell.execute_reply.started":"2023-12-23T21:34:22.589055Z","shell.execute_reply":"2023-12-23T21:34:22.672750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_id_2A3 = sequence_id[indices_2A3]\nsequence_id_DMS = sequence_id[indices_DMS]\n\nsequence_2A3 = sequence[indices_2A3]\nsequence_DMS = sequence[indices_DMS]\n\nexperiment_type_2A3 = experiment_type[indices_2A3]\nexperiment_type_DMS = experiment_type[indices_DMS]\n\ndataset_name_2A3 = dataset_name[indices_2A3]\ndataset_name_DMS = dataset_name[indices_DMS]\n\nreads_2A3 = reads[indices_2A3]\nreads_DMS = reads[indices_DMS]\n\nsignal_to_noise_2A3 = signal_to_noise[indices_2A3]\nsignal_to_noise_DMS = signal_to_noise[indices_DMS]\n\nSN_filter_2A3 = SN_filter[indices_2A3]\nSN_filter_DMS = SN_filter[indices_DMS]\n\nreactivity_2A3 = reactivity[indices_2A3]\nreactivity_DMS = reactivity[indices_DMS]\n\nreactivity_error_2A3 = reactivity_error[indices_2A3]\nreactivity_error_DMS = reactivity_error[indices_DMS]","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:30:52.800527Z","iopub.execute_input":"2023-12-23T21:30:52.801118Z","iopub.status.idle":"2023-12-23T21:30:56.092311Z","shell.execute_reply.started":"2023-12-23T21:30:52.801086Z","shell.execute_reply":"2023-12-23T21:30:56.091340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(np.unique(sequence_2A3)) == len(np.unique(sequence_id_2A3)) ","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:30:56.093493Z","iopub.execute_input":"2023-12-23T21:30:56.093866Z","iopub.status.idle":"2023-12-23T21:30:58.742238Z","shell.execute_reply.started":"2023-12-23T21:30:56.093832Z","shell.execute_reply":"2023-12-23T21:30:58.741222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(reactivity_error_DMS)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:34:40.841886Z","iopub.execute_input":"2023-12-23T21:34:40.842338Z","iopub.status.idle":"2023-12-23T21:34:40.848963Z","shell.execute_reply.started":"2023-12-23T21:34:40.842306Z","shell.execute_reply":"2023-12-23T21:34:40.847887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#In order to shuffle the sequences before creating the TFRecords. It ensure that the the validation and training splits would be similar.\nnp.random.seed(42)\nsamples_num = len(np.unique(sequence_2A3))\nprint(samples_num)\nrandomize_order_indices = np.random.choice(range(samples_num), size=samples_num, replace=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:34:45.610688Z","iopub.execute_input":"2023-12-23T21:34:45.611034Z","iopub.status.idle":"2023-12-23T21:34:47.174373Z","shell.execute_reply.started":"2023-12-23T21:34:45.611008Z","shell.execute_reply":"2023-12-23T21:34:47.173320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = randomize_order_indices\ndata_chunked = []\nchunk_size = 5000\nfor i in range(len(data)//chunk_size+1):\n    data_chunked.append(data[i*chunk_size:(i+1)*chunk_size])\n\nprint(len(data))\nprint(np.sum([len(x) for x in data_chunked]))\n\nif not os.path.isdir(\"tfds\"): os.mkdir(\"tfds\")\ntffile_names = [f\"tfds/{file_id}.tfrecord\" for file_id in range(len(data_chunked))]","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:36:25.751789Z","iopub.execute_input":"2023-12-23T21:36:25.753001Z","iopub.status.idle":"2023-12-23T21:36:25.760534Z","shell.execute_reply.started":"2023-12-23T21:36:25.752951Z","shell.execute_reply":"2023-12-23T21:36:25.759781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.mean(sequence_id_2A3 == sequence_id_DMS) == 1)\nprint(np.mean(sequence_2A3 == sequence_DMS) == 1)\nprint(np.mean([x == '2A3_MaP' for x in experiment_type_2A3]))\nprint(np.mean([x == 'DMS_MaP' for x in experiment_type_DMS]))","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:36:29.921387Z","iopub.execute_input":"2023-12-23T21:36:29.922095Z","iopub.status.idle":"2023-12-23T21:36:30.248038Z","shell.execute_reply.started":"2023-12-23T21:36:29.922063Z","shell.execute_reply":"2023-12-23T21:36:30.246969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoding_dict = {'A':1, 'C': 2, 'G': 3, 'U': 4}\nencoding_dict","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:36:37.448592Z","iopub.execute_input":"2023-12-23T21:36:37.449007Z","iopub.status.idle":"2023-12-23T21:36:37.455652Z","shell.execute_reply.started":"2023-12-23T21:36:37.448976Z","shell.execute_reply":"2023-12-23T21:36:37.454667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_seq, y = np.unique(sequence_2A3, return_counts=True)\nunique_seq = np.sort(unique_seq)\nm, n = np.unique(y, return_counts=True)\nprint(m)\nprint(n)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:36:39.490833Z","iopub.execute_input":"2023-12-23T21:36:39.491232Z","iopub.status.idle":"2023-12-23T21:36:41.733002Z","shell.execute_reply.started":"2023-12-23T21:36:39.491177Z","shell.execute_reply":"2023-12-23T21:36:41.731899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c, v = np.unique([len(x) for x in sequence_2A3], return_counts=True)\nprint(c)\nprint(v)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:36:48.326704Z","iopub.execute_input":"2023-12-23T21:36:48.327113Z","iopub.status.idle":"2023-12-23T21:36:48.506446Z","shell.execute_reply.started":"2023-12-23T21:36:48.327081Z","shell.execute_reply":"2023-12-23T21:36:48.505419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c, v = np.unique([len(x) for x in unique_seq], return_counts=True)\nprint(c)\nprint(v)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:37:02.137114Z","iopub.execute_input":"2023-12-23T21:37:02.137526Z","iopub.status.idle":"2023-12-23T21:37:02.405919Z","shell.execute_reply.started":"2023-12-23T21:37:02.137493Z","shell.execute_reply":"2023-12-23T21:37:02.404900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c, v = np.unique([len(x) for x in unique_seq[np.where(y>1)]], return_counts=True)\nprint(c)\nprint(v)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:39:05.011256Z","iopub.execute_input":"2023-12-23T21:39:05.011691Z","iopub.status.idle":"2023-12-23T21:39:05.020570Z","shell.execute_reply.started":"2023-12-23T21:39:05.011659Z","shell.execute_reply":"2023-12-23T21:39:05.019642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seq_indices = {}\nfor i in range(len(sequence_2A3)):\n    seq = sequence_2A3[i]\n    if seq not in seq_indices:\n        seq_indices[seq] = [i]\n    else:\n        seq_indices[seq].append(i)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:41:04.296356Z","iopub.execute_input":"2023-12-23T21:41:04.296704Z","iopub.status.idle":"2023-12-23T21:41:05.902813Z","shell.execute_reply.started":"2023-12-23T21:41:04.296678Z","shell.execute_reply":"2023-12-23T21:41:05.901688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"caprs = pickle.load(open('/kaggle/input/srrf-train-set-capr/results.p', 'br'))\nprint(np.mean(np.asarray([len(x[1][0]) for x in caprs]) == np.asarray([len(x) for x in unique_seq])))","metadata":{"execution":{"iopub.status.busy":"2023-12-23T22:08:40.898283Z","iopub.execute_input":"2023-12-23T22:08:40.898654Z","iopub.status.idle":"2023-12-23T22:09:17.251756Z","shell.execute_reply.started":"2023-12-23T22:08:40.898623Z","shell.execute_reply":"2023-12-23T22:09:17.250587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nbpp_files_path = {}\nfor dirpath, dirnames, filenames in os.walk(\n    '/kaggle/input/stanford-ribonanza-rna-folding/Ribonanza_bpp_files/extra_data'):\n    for filename in [f for f in filenames if f.endswith(\".txt\")]:\n        bpp_files_path[filename[:-4]] = dirpath","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:44:12.458089Z","iopub.execute_input":"2023-12-23T21:44:12.458636Z","iopub.status.idle":"2023-12-23T21:59:56.324906Z","shell.execute_reply.started":"2023-12-23T21:44:12.458594Z","shell.execute_reply":"2023-12-23T21:59:56.323705Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_bpps_data(seq_id):\n    path = bpp_files_path[seq_id]\n    file_name = seq_id+'.txt'\n    with open(os.path.join(path, file_name)) as f:\n        lines = f.readlines()\n        processed_lines = []\n        for i,line in enumerate(lines):\n            line = line.replace('\\n', '').split()\n            if len(line)>0:\n                processed_lines.append(line)\n        processed_lines = np.asarray(processed_lines)\n    indices = processed_lines[:, :2].astype(np.uint8)\n    values = processed_lines[:, 2].astype(np.float32)\n    return indices, values","metadata":{"execution":{"iopub.status.busy":"2023-12-23T22:00:35.009312Z","iopub.execute_input":"2023-12-23T22:00:35.009985Z","iopub.status.idle":"2023-12-23T22:00:35.017519Z","shell.execute_reply.started":"2023-12-23T22:00:35.009950Z","shell.execute_reply":"2023-12-23T22:00:35.016177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_to_tfrecord(file_name, randomize_order_indices, batch_index):\n    with tf.io.TFRecordWriter(file_name, 'GZIP') as file_writer:\n        for index in range(len(randomize_order_indices)):\n            random_index = randomize_order_indices[index]\n\n            seq_sample = unique_seq[random_index]\n            seq_indices_sample = np.asarray(seq_indices[seq_sample])\n            sequence_id_2A3_sample = sequence_id_2A3[seq_indices_sample]\n            dataset_name_2A3_sample = dataset_name_2A3[seq_indices_sample]\n            dataset_name_DMS_sample = dataset_name_DMS[seq_indices_sample]\n\n            reads_2A3_sample = reads_2A3[seq_indices_sample]\n            reads_DMS_sample = reads_DMS[seq_indices_sample]\n\n            signal_to_noise_2A3_sample = signal_to_noise_2A3[seq_indices_sample]\n            signal_to_noise_DMS_sample = signal_to_noise_DMS[seq_indices_sample]\n\n            SN_filter_2A3_sample = SN_filter_2A3[seq_indices_sample]\n            SN_filter_DMS_sample = SN_filter_DMS[seq_indices_sample]\n\n            reactivity_2A3_sample = reactivity_2A3[seq_indices_sample]\n            reactivity_DMS_sample = reactivity_DMS[seq_indices_sample]\n\n            reactivity_error_2A3_sample = reactivity_error_2A3[seq_indices_sample]\n            reactivity_error_DMS_sample = reactivity_error_DMS[seq_indices_sample]\n\n            samples_num = len(seq_indices_sample)\n            \n            seq_id = sequence_id_2A3_sample[0]\n            encoded_id = seq_id.encode(\"utf-8\")\n            encoded_seq = [encoding_dict[x] for x in seq_sample]\n\n            dataset_name_2A3_sample_lens = np.asarray([len(x) for x in dataset_name_2A3_sample])\n            encoded_dataset_name_2A3_sample = ''.join(dataset_name_2A3_sample).encode(\"utf-8\")\n\n            dataset_name_DMS_sample_lens = np.asarray([len(x) for x in dataset_name_DMS_sample])\n            encoded_dataset_name_DMS_sample = ''.join(dataset_name_DMS_sample).encode(\"utf-8\")\n            \n            capr = tf.io.serialize_tensor(caprs[random_index][1]).numpy()\n            \n            bpp_indices, bpp_values = get_bpps_data(seq_id)\n            bpp_indices = tf.io.serialize_tensor(bpp_indices).numpy()\n\n            features = {}\n            features['id'] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[encoded_id]))\n            features['seq'] = tf.train.Feature(float_list=tf.train.FloatList(value=encoded_seq))\n            features['dataset_name_2A3'] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[encoded_dataset_name_2A3_sample]))\n            features['dataset_name_DMS'] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[encoded_dataset_name_DMS_sample]))\n            features['reads_2A3'] =tf.train.Feature(float_list=tf.train.FloatList(value=reads_2A3_sample))\n            features['reads_DMS'] =tf.train.Feature(float_list=tf.train.FloatList(value=reads_DMS_sample))\n            features['signal_to_noise_2A3'] =tf.train.Feature(float_list=tf.train.FloatList(value=signal_to_noise_2A3_sample))\n            features['signal_to_noise_DMS'] =tf.train.Feature(float_list=tf.train.FloatList(value=signal_to_noise_DMS_sample))\n            \n            features['SN_filter_2A3'] =tf.train.Feature(float_list=tf.train.FloatList(value=SN_filter_2A3_sample))\n            features['SN_filter_DMS'] =tf.train.Feature(float_list=tf.train.FloatList(value=SN_filter_DMS_sample))\n            \n            features['reactivity_2A3'] =tf.train.Feature(float_list=tf.train.FloatList(value=np.concatenate(reactivity_2A3_sample)))\n            features['reactivity_DMS'] =tf.train.Feature(float_list=tf.train.FloatList(value=np.concatenate(reactivity_DMS_sample)))\n            \n            features['reactivity_error_2A3'] =tf.train.Feature(float_list=tf.train.FloatList(value=np.concatenate(reactivity_error_2A3_sample)))\n            features['reactivity_error_DMS'] =tf.train.Feature(float_list=tf.train.FloatList(value=np.concatenate(reactivity_error_DMS_sample)))\n\n            features['random_index'] =tf.train.Feature(float_list=tf.train.FloatList(value=[random_index]))\n            features['seq_indices'] =tf.train.Feature(float_list=tf.train.FloatList(value=seq_indices_sample))\n            features['samples_num'] =tf.train.Feature(float_list=tf.train.FloatList(value=[samples_num]))\n            features['dataset_name_2A3_sample_lens'] =tf.train.Feature(float_list=tf.train.FloatList(value=dataset_name_2A3_sample_lens))\n            features['dataset_name_DMS_sample_lens'] =tf.train.Feature(float_list=tf.train.FloatList(value=dataset_name_DMS_sample_lens))\n            \n            features['capr'] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[capr]))\n            \n            features['batch_index'] = tf.train.Feature(float_list=tf.train.FloatList(value=[batch_index]))\n            \n            features['bpp_indices'] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[bpp_indices]))\n            features['bpp_values'] = tf.train.Feature(float_list=tf.train.FloatList(value=bpp_values))\n            \n            record_bytes = tf.train.Example(features=tf.train.Features(feature=features)).SerializeToString()\n            file_writer.write(record_bytes)\n            \n        print(file_name)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T22:02:30.400966Z","iopub.execute_input":"2023-12-23T22:02:30.401417Z","iopub.status.idle":"2023-12-23T22:02:30.427362Z","shell.execute_reply.started":"2023-12-23T22:02:30.401382Z","shell.execute_reply":"2023-12-23T22:02:30.426109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor i in range(len(data_chunked)):\n    save_to_tfrecord(tffile_names[i], data_chunked[i], i)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:30:59.434308Z","iopub.status.idle":"2023-12-23T21:30:59.435135Z","shell.execute_reply.started":"2023-12-23T21:30:59.434833Z","shell.execute_reply":"2023-12-23T21:30:59.434872Z"},"trusted":true},"execution_count":null,"outputs":[]}]}