{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Convert original CSV file to TFrecord\n\n### Prepares data for notebook use: [greySnow - \"SRRF: Transformer (TPU) training\"](https://www.kaggle.com/code/shlomoron/srrf-transformer-tpu-training/comments)\n\n## Warning. This notebook requires at least 48 GB of RAM to run.","metadata":{}},{"cell_type":"markdown","source":"## Installing additional libraries","metadata":{}},{"cell_type":"code","source":"%%capture","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install object_detection","metadata":{"execution":{"iopub.execute_input":"2023-11-12T21:20:13.496682Z","iopub.status.busy":"2023-11-12T21:20:13.496191Z","iopub.status.idle":"2023-11-12T21:20:29.978631Z","shell.execute_reply":"2023-11-12T21:20:29.977787Z","shell.execute_reply.started":"2023-11-12T21:20:13.496626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os, gc\nimport tensorflow as tf\n\nimport contextlib2\nfrom object_detection.dataset_tools import tf_record_creation_util\n\nfrom tqdm import tqdm\ntqdm.pandas()\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.execute_input":"2023-11-12T21:20:29.981761Z","iopub.status.busy":"2023-11-12T21:20:29.981206Z","iopub.status.idle":"2023-11-12T21:20:42.188733Z","shell.execute_reply":"2023-11-12T21:20:42.187621Z","shell.execute_reply.started":"2023-11-12T21:20:29.981727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(tf.__version__)","metadata":{"execution":{"iopub.execute_input":"2023-11-12T21:20:42.191084Z","iopub.status.busy":"2023-11-12T21:20:42.190023Z","iopub.status.idle":"2023-11-12T21:20:42.197137Z","shell.execute_reply":"2023-11-12T21:20:42.196068Z","shell.execute_reply.started":"2023-11-12T21:20:42.191051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading data into a DataSet from a CSV file","metadata":{}},{"cell_type":"code","source":"path_file_train = \"/kaggle/input/stanford-ribonanza-rna-folding/train_data.csv\"\ndf_train = pd.read_csv(filepath_or_buffer=path_file_train)\n\nlo_react_cols = df_train.filter(like='reactivity_0').columns\nlo_error_cols = df_train.filter(like='reactivity_error_0').columns\n\nlst_react_cols = lo_react_cols.to_list()\nlst_error_cols = lo_error_cols.to_list()\n\ndel lo_react_cols, lo_error_cols\n\ndf_train['reactivity'] = df_train[lst_react_cols].values.tolist()\ndf_train['error'] = df_train[lst_error_cols].values.tolist()\n\n# Remove unnecessary columns\ndf_train.drop(columns=lst_react_cols,inplace=True)\ndf_train.drop(columns=lst_error_cols,inplace=True)","metadata":{"execution":{"iopub.execute_input":"2023-11-12T21:20:42.202935Z","iopub.status.busy":"2023-11-12T21:20:42.198628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Getting column numbers by column names","metadata":{}},{"cell_type":"code","source":"all_cols = df_train.columns\n\nidx_seq_id          = all_cols.get_loc('sequence_id')\nidx_sequence        = all_cols.get_loc('sequence')\nidx_dataset_name    = all_cols.get_loc('dataset_name')\nidx_reads           = all_cols.get_loc('reads')\nidx_signal_to_noise = all_cols.get_loc('signal_to_noise')\nidx_SN_filter       = all_cols.get_loc('SN_filter')\nidx_reactivity      = all_cols.get_loc('reactivity')\nidx_error           = all_cols.get_loc('error')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Splitting into two data sets","metadata":{}},{"cell_type":"code","source":"df_2A3 = df_train.loc[df_train.experiment_type=='2A3_MaP'].reset_index(drop=True)\ndf_DMS = df_train.loc[df_train.experiment_type=='DMS_MaP'].reset_index(drop=True)\n\ndel df_train\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Conversion functions for serializing a record","metadata":{}},{"cell_type":"code","source":"def int64_feature(value):\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef int64_list_feature(value):\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=value))\n\ndef bytes_feature(value):\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef bytes_list_feature(value):\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))\n\ndef float_feature(value):\n  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef float_list_feature(value):\n  return tf.train.Feature(float_list=tf.train.FloatList(value=value))\n\ndef string_feature(value):\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value.encode()]))\n\ndef string_list_feature(value):\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=value.encode()))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seq_map = {'A':1,'C':2,'G':3,'U':4}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Row Serialization","metadata":{}},{"cell_type":"code","source":"def serialize_row(row_2A3, row_DMS):\n  \"\"\"\n  Creates a tf.train.Example message ready to be written to a file.\n  \"\"\"\n  \n  assert row_2A3[idx_seq_id] == row_DMS[idx_seq_id] and row_2A3[idx_sequence] == row_DMS[idx_sequence]\n    \n  # Create a dictionary mapping the feature name to the tf.train.Example-compatible data type.\n  feature = {\n      \"id\": string_feature(row_2A3[idx_seq_id]),\n      \"seq\": float_list_feature([seq_map[char] for char in row_2A3[idx_sequence]]), \n\n      \"dataset_name_2A3\": string_feature(row_2A3[idx_dataset_name]),\n      \"reads_2A3\": float_feature(row_2A3[idx_reads]),\n      \"signal_to_noise_2A3\": float_feature(row_2A3[idx_signal_to_noise]),\n      \"SN_filter_2A3\": float_feature(row_2A3[idx_SN_filter]),\n      \"reactivity_2A3\": float_list_feature(row_2A3[idx_reactivity]),\n      \"error_2A3\": float_list_feature(row_2A3[idx_error]),\n\n      \"dataset_name_DMS\": string_feature(row_DMS[idx_dataset_name]),\n      \"reads_DMS\": float_feature(row_DMS[idx_reads]),\n      \"signal_to_noise_DMS\": float_feature(row_DMS[idx_signal_to_noise]),\n      \"SN_filter_DMS\": float_feature(row_DMS[idx_SN_filter]),\n      \"reactivity_DMS\": float_list_feature(row_DMS[idx_reactivity]),\n      \"error_DMS\": float_list_feature(row_DMS[idx_error])\n  }\n\n  # Create a Features message using tf.train.Example.\n  features=tf.train.Features(feature=feature)\n  proto_example = tf.train.Example(features=features)\n  return proto_example","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r\"\"\"Utilities for creating TFRecords of TF examples for the Open Images dataset.\n\"\"\"\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\ndef open_sharded_output_tfrecords(exit_stack, base_path, num_shards):\n  \"\"\"Opens all TFRecord shards for writing and adds them to an exit stack.\n\n  Args:\n    exit_stack: A context2.ExitStack used to automatically closed the TFRecords\n      opened in this function.\n    base_path: The base path for all shards\n    num_shards: The number of shards\n\n  Returns:\n    The list of opened TFRecords. Position k in the list corresponds to shard k.\n  \"\"\"\n  tf_record_output_filenames = ['{}/{:03d}.tfrecord'.format(base_path, idx) for idx in range(num_shards)]\n\n  # options = tf.io.TFRecordOptions(compression_type=\"GZIP\")\n  tfrecords = [\n      exit_stack.enter_context(tf.io.TFRecordWriter(file_name, options=\"GZIP\"))\n          for file_name in tf_record_output_filenames\n  ]\n\n  return tfrecords","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Saving a dataset in 164 shards","metadata":{}},{"cell_type":"code","source":"num_shards=164\noutput_filebase='/kaggle/working'\n\nwith contextlib2.ExitStack() as tf_record_close_stack:\n    output_tfrecords = open_sharded_output_tfrecords(tf_record_close_stack, output_filebase, num_shards)\n    for index, (row_2A3, row_DMS) in enumerate(zip(df_2A3.itertuples(index=False), df_DMS.itertuples(index=False))):\n        tf_example = serialize_row(row_2A3, row_DMS)\n        shard_index = index % num_shards\n        output_tfrecords[shard_index].write(tf_example.SerializeToString())","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}