{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Improve .csv save time with changing from pandas backend to polars\n\nPolars is approximately 3x faster than pandas when using single core, - check it out,\n\nIt also appears that Polars supports multi-threaded .csv write.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\n\nimport numpy as np\n\nsub_path = 'sub.csv'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-19T08:22:32.400564Z","iopub.execute_input":"2023-10-19T08:22:32.400937Z","iopub.status.idle":"2023-10-19T08:22:33.008727Z","shell.execute_reply.started":"2023-10-19T08:22:32.400897Z","shell.execute_reply":"2023-10-19T08:22:33.007201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_preds = 100_000_000\n# num_preds = 10_000\nids = np.arange(num_preds, dtype=np.int64)\npreds_dms = np.random.rand(num_preds)\npreds_2a3 = np.random.rand(num_preds)\nids.shape, preds_dms.shape, preds_2a3.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-19T08:22:38.816063Z","iopub.execute_input":"2023-10-19T08:22:38.817148Z","iopub.status.idle":"2023-10-19T08:22:38.826007Z","shell.execute_reply.started":"2023-10-19T08:22:38.817106Z","shell.execute_reply":"2023-10-19T08:22:38.824851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nschema={k:pl.Float32 for k in ['id', 'reactivity_DMS_MaP', 'reactivity_2A3_MaP']}\nschema['id'] = pl.Int64\n\ndf = pl.DataFrame(\n    data=[ids, preds_dms, preds_2a3],\n    schema=schema\n)\ndf.write_csv(sub_path, float_precision=4) \nprint(df.shape)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-19T08:22:39.696795Z","iopub.execute_input":"2023-10-19T08:22:39.697160Z","iopub.status.idle":"2023-10-19T08:22:39.787363Z","shell.execute_reply.started":"2023-10-19T08:22:39.697133Z","shell.execute_reply":"2023-10-19T08:22:39.786590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndf = pd.DataFrame(\n    {\n        'id': ids,\n        'reativity_DMS_MaP' : preds_dms,\n        'reactivity_2A3_MaP' : preds_2a3\n    }\n)\ndf.to_csv(sub_path, float_format='%.4f', index=False)\nprint(df.shape)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-19T08:22:40.054303Z","iopub.execute_input":"2023-10-19T08:22:40.054731Z","iopub.status.idle":"2023-10-19T08:22:40.130915Z","shell.execute_reply.started":"2023-10-19T08:22:40.054700Z","shell.execute_reply":"2023-10-19T08:22:40.129492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save parquet\nIt's also possible to submit parquets which is an appropriate format for the competition","metadata":{}},{"cell_type":"code","source":"%%time\n\nschema={k:pl.Float32 for k in ['id', 'reactivity_DMS_MaP', 'reactivity_2A3_MaP']}\nschema['id'] = pl.Int64\n\ndf = pl.DataFrame(\n    data=[ids, preds_dms, preds_2a3],\n    schema=schema\n)\ndf.write_parquet(sub_path.replace('.csv', '.parquet')) \nprint(df.shape)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-19T08:22:46.032981Z","iopub.execute_input":"2023-10-19T08:22:46.034257Z","iopub.status.idle":"2023-10-19T08:22:46.078443Z","shell.execute_reply.started":"2023-10-19T08:22:46.034209Z","shell.execute_reply":"2023-10-19T08:22:46.077690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndf = pd.DataFrame(\n    {\n        'id': ids,\n        'reativity_DMS_MaP' : preds_dms,\n        'reactivity_2A3_MaP' : preds_2a3\n    }\n)\ndf.to_parquet(sub_path.replace('.csv', '.parquet'))\nprint(df.shape)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-19T08:22:56.185341Z","iopub.execute_input":"2023-10-19T08:22:56.185727Z","iopub.status.idle":"2023-10-19T08:22:56.245217Z","shell.execute_reply.started":"2023-10-19T08:22:56.185697Z","shell.execute_reply":"2023-10-19T08:22:56.244090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}