{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\n\nchunksize = 100000 # write 1 million rows at a time\nfor i, chunk in enumerate(pd.read_csv('/kaggle/input/amex-default-prediction/train_data.csv', chunksize=chunksize)):\n    chunk.to_csv(f'output_{i}.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-03-04T17:11:24.775975Z","iopub.execute_input":"2023-03-04T17:11:24.776383Z","iopub.status.idle":"2023-03-04T17:12:47.835493Z","shell.execute_reply.started":"2023-03-04T17:11:24.776351Z","shell.execute_reply":"2023-03-04T17:12:47.834036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-31T15:44:53.993080Z","iopub.execute_input":"2023-01-31T15:44:53.993569Z","iopub.status.idle":"2023-01-31T15:44:54.018789Z","shell.execute_reply.started":"2023-01-31T15:44:53.993465Z","shell.execute_reply":"2023-01-31T15:44:54.017988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\nchunk_size = 100000 # number of rows per chunk\ndata = []\nfor chunk in pd.read_csv('/kaggle/input/amex-default-prediction/train_data.csv', chunksize=chunk_size):\n    dataframe = chunk\n    break\n#     data.append(chunk)\n    \n# concatenate the results\n# data = pd.concat(data)","metadata":{"execution":{"iopub.status.busy":"2023-02-18T12:22:39.773916Z","iopub.execute_input":"2023-02-18T12:22:39.774385Z","iopub.status.idle":"2023-02-18T12:22:43.532417Z","shell.execute_reply.started":"2023-02-18T12:22:39.774349Z","shell.execute_reply":"2023-02-18T12:22:43.531039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataframe","metadata":{"execution":{"iopub.status.busy":"2023-02-18T12:22:46.737529Z","iopub.execute_input":"2023-02-18T12:22:46.738014Z","iopub.status.idle":"2023-02-18T12:22:46.807589Z","shell.execute_reply.started":"2023-02-18T12:22:46.737972Z","shell.execute_reply":"2023-02-18T12:22:46.806077Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from dask import dataframe as dd\nimport time\nstart = time.time()\ndask_df = dd.read_csv('/kaggle/input/amex-default-prediction/train_data.csv')\nend = time.time()\nprint(\"Read csv with dask: \",(end-start),\"sec\")\n# Read csv with dask:  0.07900428771972656 sec","metadata":{"execution":{"iopub.status.busy":"2023-02-18T12:42:41.245074Z","iopub.execute_input":"2023-02-18T12:42:41.246489Z","iopub.status.idle":"2023-02-18T12:42:41.774137Z","shell.execute_reply.started":"2023-02-18T12:42:41.246425Z","shell.execute_reply":"2023-02-18T12:42:41.772658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dask_df.memory_usage_per_partition(deep=True).compute()","metadata":{"execution":{"iopub.status.busy":"2023-02-18T12:54:39.715497Z","iopub.execute_input":"2023-02-18T12:54:39.716001Z","iopub.status.idle":"2023-02-18T12:56:57.038346Z","shell.execute_reply.started":"2023-02-18T12:54:39.715965Z","shell.execute_reply":"2023-02-18T12:56:57.037364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}