{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Transform training data to parquet file\n\nReading from large `.csv` files can be slow. \n\nLet's turn the training and testing data into parquet files:","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"from pathlib import Path\nimport polars as pl","metadata":{"execution":{"iopub.status.busy":"2023-11-04T11:01:48.368215Z","iopub.execute_input":"2023-11-04T11:01:48.368613Z","iopub.status.idle":"2023-11-04T11:01:48.596216Z","shell.execute_reply.started":"2023-11-04T11:01:48.368585Z","shell.execute_reply":"2023-11-04T11:01:48.595123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = Path(\"/kaggle/input/stanford-ribonanza-rna-folding/\")\nTRAIN_CSV = DATA_DIR / \"train_data.csv\"\nTRAIN_PARQUET_FILE = \"train_data.parquet\"\nTEST_CSV = DATA_DIR / \"test_sequences.csv\"\nTEST_PARQUET_FILE = \"test_sequences.parquet\"\nPRED_CSV = \"submission.csv\"","metadata":{"execution":{"iopub.status.busy":"2023-11-04T11:01:49.142500Z","iopub.execute_input":"2023-11-04T11:01:49.142912Z","iopub.status.idle":"2023-11-04T11:01:49.148715Z","shell.execute_reply.started":"2023-11-04T11:01:49.142880Z","shell.execute_reply":"2023-11-04T11:01:49.147828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_parquet(csv_file, parquet_file):\n    dummy_df = pl.scan_csv(csv_file)\n\n    new_schema = {}\n    for key, value in dummy_df.schema.items():\n        if key.startswith(\"reactivity\"):\n            new_schema[key] = pl.Float32\n        else:\n            new_schema[key] = value\n\n    df = pl.scan_csv(csv_file, schema=new_schema)\n    \n    df.sink_parquet(\n            parquet_file,\n            compression='uncompressed',\n            row_group_size=10,\n    )\n\nto_parquet(TRAIN_CSV, TRAIN_PARQUET_FILE)\nto_parquet(TEST_CSV, TEST_PARQUET_FILE)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T11:01:51.268804Z","iopub.execute_input":"2023-11-04T11:01:51.269458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}