{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import feather\nimport numpy as np\nimport pandas as pd\nimport os\nfrom pathlib import Path","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-01T15:11:57.591158Z","iopub.execute_input":"2022-10-01T15:11:57.591771Z","iopub.status.idle":"2022-10-01T15:11:57.620846Z","shell.execute_reply.started":"2022-10-01T15:11:57.591664Z","shell.execute_reply":"2022-10-01T15:11:57.619216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Feather data format is a lightweight as well as very fast binary format for storing data frames. Feather file, takes less than half of the space, than the corresponding CSV file, having same data. Feather files are 100 times faster, while reading and writing from the disk, as compared to CSV file","metadata":{}},{"cell_type":"code","source":"data_dir ='../input/tabular-playground-series-oct-2022/'\ndisplay(os.listdir(data_dir))","metadata":{"execution":{"iopub.status.busy":"2022-10-01T15:11:57.623925Z","iopub.execute_input":"2022-10-01T15:11:57.624453Z","iopub.status.idle":"2022-10-01T15:11:57.639544Z","shell.execute_reply.started":"2022-10-01T15:11:57.624402Z","shell.execute_reply":"2022-10-01T15:11:57.638095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Convert data into feather","metadata":{}},{"cell_type":"code","source":"\n%%time\nn=10\nfor i in range(n):\n    data = pd.read_csv(data_dir + f'/train_{i}.csv')\n    data.to_feather(f'train_{i}.feather')\n    print('Complete Transfer on Feather :', i)\ntest =pd.read_csv(data_dir + \"test.csv\")\ntest.to_feather('test.feather')\nprint(\"Done\")","metadata":{"execution":{"iopub.status.busy":"2022-10-01T15:11:57.641778Z","iopub.execute_input":"2022-10-01T15:11:57.642288Z","iopub.status.idle":"2022-10-01T15:21:26.725095Z","shell.execute_reply.started":"2022-10-01T15:11:57.642239Z","shell.execute_reply":"2022-10-01T15:21:26.722875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check SpeeD","metadata":{}},{"cell_type":"code","source":"outdir = Path('.')\nos.makedirs(str(outdir), exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-01T15:23:01.243767Z","iopub.execute_input":"2022-10-01T15:23:01.244314Z","iopub.status.idle":"2022-10-01T15:23:01.254161Z","shell.execute_reply.started":"2022-10-01T15:23:01.244267Z","shell.execute_reply":"2022-10-01T15:23:01.252289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#load with feather data\ntrain_0 = pd.read_feather(outdir / 'train_0.feather')\ntrain_0\n","metadata":{"execution":{"iopub.status.busy":"2022-10-01T15:23:07.336990Z","iopub.execute_input":"2022-10-01T15:23:07.337919Z","iopub.status.idle":"2022-10-01T15:23:10.022930Z","shell.execute_reply.started":"2022-10-01T15:23:07.337857Z","shell.execute_reply":"2022-10-01T15:23:10.021542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, Lets read the csv files and see the difference between. feather data take 1.39 second wall time and what about CSV file? 54.1 second, can you Imagine?","metadata":{}},{"cell_type":"code","source":"%%time\n#load csv data\ntrain_0 = pd.read_csv(data_dir + 'train_0.csv')\ntrain_0","metadata":{"execution":{"iopub.status.busy":"2022-10-01T15:23:51.692852Z","iopub.execute_input":"2022-10-01T15:23:51.693299Z","iopub.status.idle":"2022-10-01T15:24:46.433081Z","shell.execute_reply.started":"2022-10-01T15:23:51.693262Z","shell.execute_reply":"2022-10-01T15:24:46.430692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\nI save the dataset to <a href=\"https://www.kaggle.com/datasets/gazu468/tpsoct22-feather-files\"> click here</a>.Thanks for read this notebook.","metadata":{"execution":{"iopub.status.busy":"2022-10-01T06:11:50.155148Z","iopub.execute_input":"2022-10-01T06:11:50.155619Z","iopub.status.idle":"2022-10-01T06:11:50.162943Z","shell.execute_reply.started":"2022-10-01T06:11:50.155583Z","shell.execute_reply":"2022-10-01T06:11:50.161259Z"}}}]}