{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport dask.dataframe as dd\nfrom time import time\nimport gc #Coletor de lixo\ngc.enable()\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-14T05:15:15.140181Z","iopub.execute_input":"2022-06-14T05:15:15.140917Z","iopub.status.idle":"2022-06-14T05:15:16.394644Z","shell.execute_reply.started":"2022-06-14T05:15:15.14078Z","shell.execute_reply":"2022-06-14T05:15:16.393048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Os tipos de dados foram tirados daqui: https://www.kaggle.com/code/rohanrao/tutorial-on-reading-large-datasets/notebook\ndtypes = {\n    \"row_id\": \"int64\",\n    \"timestamp\": \"int64\",\n    \"user_id\": \"int32\",\n    \"content_id\": \"int16\",\n    \"content_type_id\": \"boolean\",\n    \"task_container_id\": \"int16\",\n    \"user_answer\": \"int8\",\n    \"answered_correctly\": \"int8\",\n    \"prior_question_elapsed_time\": \"float32\", \n    \"prior_question_had_explanation\": \"boolean\"\n}","metadata":{"execution":{"iopub.status.busy":"2022-06-14T05:15:16.397156Z","iopub.execute_input":"2022-06-14T05:15:16.398761Z","iopub.status.idle":"2022-06-14T05:15:16.406883Z","shell.execute_reply.started":"2022-06-14T05:15:16.3987Z","shell.execute_reply":"2022-06-14T05:15:16.405489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TAREFA 1: LENDO O DATAFRAME EM CSV","metadata":{}},{"cell_type":"markdown","source":"* Pandas","metadata":{}},{"cell_type":"code","source":"inicio = time()\n\npdf = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", nrows = 10000, dtype=dtypes)\nfinal = time()\n\nprint(f\"Lendo o dataframe com pandas (10000 linhas): {(final-inicio):.3f}s\")","metadata":{"execution":{"iopub.status.busy":"2022-06-14T05:15:16.409867Z","iopub.execute_input":"2022-06-14T05:15:16.410472Z","iopub.status.idle":"2022-06-14T05:15:17.292974Z","shell.execute_reply.started":"2022-06-14T05:15:16.410423Z","shell.execute_reply":"2022-06-14T05:15:17.291454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inicio = time()\n\npdf = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", nrows = 100000, dtype=dtypes)\nfinal = time()\n\nprint(f\"Lendo o dataframe com pandas (100000 linhas): {(final-inicio):.3f}s\")","metadata":{"execution":{"iopub.status.busy":"2022-06-14T05:15:17.296973Z","iopub.execute_input":"2022-06-14T05:15:17.297409Z","iopub.status.idle":"2022-06-14T05:15:24.470685Z","shell.execute_reply.started":"2022-06-14T05:15:17.297375Z","shell.execute_reply":"2022-06-14T05:15:24.469595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inicio = time()\n\npdf = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", nrows = 1000000, dtype=dtypes)\nfinal = time()\n\nprint(f\"Lendo o dataframe com pandas (1000000 linhas): {(final-inicio):.3f}s\")","metadata":{"execution":{"iopub.status.busy":"2022-06-14T05:15:24.472245Z","iopub.execute_input":"2022-06-14T05:15:24.475096Z","iopub.status.idle":"2022-06-14T05:16:34.754183Z","shell.execute_reply.started":"2022-06-14T05:15:24.475043Z","shell.execute_reply":"2022-06-14T05:16:34.753057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Dask","metadata":{}},{"cell_type":"code","source":"inicio = time()\n\npdf = dd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", blocksize = \"30MB\", dtype=dtypes).head(n = 10000)\nfinal = time()\n\nprint(f\"Lendo o dataframe com dask (10000 linhas): {(final-inicio):.3f}s\")","metadata":{"execution":{"iopub.status.busy":"2022-06-14T05:16:34.756023Z","iopub.execute_input":"2022-06-14T05:16:34.756602Z","iopub.status.idle":"2022-06-14T05:16:35.51471Z","shell.execute_reply.started":"2022-06-14T05:16:34.756565Z","shell.execute_reply":"2022-06-14T05:16:35.513372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inicio = time()\n\npdf = dd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", blocksize = \"64MB\", dtype=dtypes).head(n = 100000, npartitions = 20)\nfinal = time()\n\nprint(f\"Lendo o dataframe com dask (100000 linhas): {(final-inicio):.3f}s\")","metadata":{"execution":{"iopub.status.busy":"2022-06-14T05:16:35.516545Z","iopub.execute_input":"2022-06-14T05:16:35.517792Z","iopub.status.idle":"2022-06-14T05:16:46.645561Z","shell.execute_reply.started":"2022-06-14T05:16:35.517739Z","shell.execute_reply":"2022-06-14T05:16:46.644763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Ao tentar rodar as linhas abaixo, obtenho o erro: \"Your notebook tried to allocate more memory than is available. It has restarted.\"\n#inicio = time()\n\n#pdf = dd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", blocksize = None).head(n = 1000000)\n#final = time()\n\n#print(f\"Lendo o dataframe com dask (1000000 linhas): {(final-inicio):.3f}s\")","metadata":{"execution":{"iopub.status.busy":"2022-06-14T05:16:46.649045Z","iopub.execute_input":"2022-06-14T05:16:46.649484Z","iopub.status.idle":"2022-06-14T05:16:46.656221Z","shell.execute_reply.started":"2022-06-14T05:16:46.649447Z","shell.execute_reply":"2022-06-14T05:16:46.654842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inicio = time()\n\npdf = dd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", dtype=dtypes).head(n = 1000000, npartitions = 48)\nfinal = time()\n\nprint(f\"Lendo o dataframe com dask (1000000 linhas): {(final-inicio):.3f}s\")","metadata":{"execution":{"iopub.status.busy":"2022-06-14T05:16:46.657709Z","iopub.execute_input":"2022-06-14T05:16:46.658759Z","iopub.status.idle":"2022-06-14T05:17:13.71057Z","shell.execute_reply.started":"2022-06-14T05:16:46.658709Z","shell.execute_reply":"2022-06-14T05:17:13.709706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TAREFA 2a: LENDO O DATAFRAME EM PARQUET 1 ##\n\nO dataset foi criado aqui: https://www.kaggle.com/code/odins0n/load-parquet-files-with-low-memory/\n\nAs colunas originais float64 foram reduzidas para float32\n\nO read_parquet do pandas não suporta o argumento nrows, nem dtype","metadata":{}},{"cell_type":"code","source":"#Ao tentar rodar as linhas abaixo, obtenho o erro: \"Your notebook tried to allocate more memory than is available. It has restarted.\"\n#inicio = time()\n\n#pdf = pd.read_parquet('../input/amex-parquet/train_data.parquet')\n#final = time()\n\n#print(f\"Lendo o dataframe com pandas: {(final-inicio):.3f}s\")","metadata":{"execution":{"iopub.status.busy":"2022-06-14T05:17:13.713333Z","iopub.execute_input":"2022-06-14T05:17:13.713963Z","iopub.status.idle":"2022-06-14T05:17:13.717759Z","shell.execute_reply.started":"2022-06-14T05:17:13.713926Z","shell.execute_reply":"2022-06-14T05:17:13.717034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inicio = time()\n\npdf = dd.read_parquet('../input/amex-parquet/train_data.parquet')\nfinal = time()\n\nprint(f\"Lendo o dataframe com dask: {(final-inicio):.3f}s\")","metadata":{"execution":{"iopub.status.busy":"2022-06-14T05:17:13.719121Z","iopub.execute_input":"2022-06-14T05:17:13.719653Z","iopub.status.idle":"2022-06-14T05:17:13.937113Z","shell.execute_reply.started":"2022-06-14T05:17:13.719619Z","shell.execute_reply":"2022-06-14T05:17:13.935747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TAREFA 2b: LENDO O DATAFRAME EM PARQUET 2 ##\n\nO dataset foi criado aqui: https://www.kaggle.com/competitions/amex-default-prediction/discussion/327908\n\nAs colunas originais float64 foram reduzidas para float32\nCustomer_id foi para string\nS_2 foi para datetime\nTodas as categóricas foram para category\n\nO read_parquet do pandas não suporta o argumento nrows, nem dtype","metadata":{}},{"cell_type":"code","source":"#Ao tentar rodar as linhas abaixo, obtenho o erro: \"Your notebook tried to allocate more memory than is available. It has restarted.\"\ninicio = time()\n\npdf = pd.read_parquet('/kaggle/input/american-express-default-prediction-snappy-parquet/train_data.snappy.parquet')\nfinal = time()\n\nprint(f\"Lendo o dataframe com pandas: {(final-inicio):.3f}s\")","metadata":{"execution":{"iopub.status.busy":"2022-06-14T05:17:13.938606Z","iopub.execute_input":"2022-06-14T05:17:13.938979Z","iopub.status.idle":"2022-06-14T05:17:46.306648Z","shell.execute_reply.started":"2022-06-14T05:17:13.938947Z","shell.execute_reply":"2022-06-14T05:17:46.305029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inicio = time()\n\npdf = dd.read_parquet('/kaggle/input/american-express-default-prediction-snappy-parquet/train_data.snappy.parquet')\nfinal = time()\n\nprint(f\"Lendo o dataframe com dask: {(final-inicio):.3f}s\")","metadata":{"execution":{"iopub.status.busy":"2022-06-14T05:17:46.308599Z","iopub.execute_input":"2022-06-14T05:17:46.309424Z","iopub.status.idle":"2022-06-14T05:17:46.660475Z","shell.execute_reply.started":"2022-06-14T05:17:46.309373Z","shell.execute_reply":"2022-06-14T05:17:46.659434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TAREFA 3: LENDO O DATAFRAME EM PICKLE\n\nBaseado nesse notebook: https://www.kaggle.com/code/kingychiu/amex-woe-baseline-with-id-encoded-fp16-dataset\n\nEle transformou as float64 para float32 e usou LabelEncoder em customer_id","metadata":{}},{"cell_type":"code","source":"inicio = time()\n\npdf = pd.read_pickle('../input/ae-credit-id-encoded-dataset-fp32/id_encoded_fp32_train_data.pkl')\nfinal = time()\n\nprint(f\"Lendo o dataframe com pandas: {(final-inicio):.3f}s\")","metadata":{"execution":{"iopub.status.busy":"2022-06-14T05:17:46.662108Z","iopub.execute_input":"2022-06-14T05:17:46.662482Z","iopub.status.idle":"2022-06-14T05:18:18.408649Z","shell.execute_reply.started":"2022-06-14T05:17:46.662446Z","shell.execute_reply":"2022-06-14T05:18:18.407878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#DASK NÃO SUPORTA O FORMATO PICKLE\n#inicio = time()\n\n#pdf = dd.read_pickle('../input/ae-credit-id-encoded-dataset-fp32/id_encoded_fp32_train_data.pkl')\n#final = time()\n\n#print(f\"Lendo o dataframe com dask: {(final-inicio):.3f}s\")","metadata":{"execution":{"iopub.status.busy":"2022-06-14T05:18:18.410222Z","iopub.execute_input":"2022-06-14T05:18:18.411436Z","iopub.status.idle":"2022-06-14T05:18:18.527372Z","shell.execute_reply.started":"2022-06-14T05:18:18.411395Z","shell.execute_reply":"2022-06-14T05:18:18.525745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Vencedor: dask + formato parquet.","metadata":{}}]}