{"cells": [{"cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5", "execution": {"iopub.execute_input": "2024-04-17T10:17:40.130822Z", "iopub.status.busy": "2024-04-17T10:17:40.128943Z", "iopub.status.idle": "2024-04-17T10:18:12.96961Z", "shell.execute_reply": "2024-04-17T10:18:12.967537Z", "shell.execute_reply.started": "2024-04-17T10:17:40.130755Z"}, "trusted": true}, "outputs": [], "source": ["!pip install memory_profiler\n", "!pip install --upgrade distributed"]}, {"cell_type": "code", "execution_count": null, "metadata": {"execution": {"iopub.execute_input": "2024-04-17T10:18:12.974036Z", "iopub.status.busy": "2024-04-17T10:18:12.973442Z", "iopub.status.idle": "2024-04-17T10:18:12.981446Z", "shell.execute_reply": "2024-04-17T10:18:12.980159Z", "shell.execute_reply.started": "2024-04-17T10:18:12.973987Z"}, "trusted": true}, "outputs": [], "source": ["import pathlib\n", "\n", "import dask.dataframe as dd\n", "from dask.diagnostics import ProgressBar\n", "import numpy as np\n", "import pandas as pd"]}, {"cell_type": "code", "execution_count": null, "metadata": {"execution": {"iopub.execute_input": "2024-04-17T10:18:12.983687Z", "iopub.status.busy": "2024-04-17T10:18:12.983144Z", "iopub.status.idle": "2024-04-17T10:18:13.001942Z", "shell.execute_reply": "2024-04-17T10:18:13.000776Z", "shell.execute_reply.started": "2024-04-17T10:18:12.983629Z"}, "trusted": true}, "outputs": [], "source": ["%load_ext memory_profiler"]}, {"cell_type": "code", "execution_count": null, "metadata": {"execution": {"iopub.execute_input": "2024-04-17T10:18:13.004524Z", "iopub.status.busy": "2024-04-17T10:18:13.003936Z", "iopub.status.idle": "2024-04-17T10:18:13.0238Z", "shell.execute_reply": "2024-04-17T10:18:13.022138Z", "shell.execute_reply.started": "2024-04-17T10:18:13.004484Z"}, "trusted": true}, "outputs": [], "source": ["input_path = pathlib.Path(\"/kaggle/input/leash-BELKA\")\n", "train_file = input_path / \"train.parquet\"\n", "test_file = input_path / \"test.parquet\""]}, {"cell_type": "markdown", "metadata": {}, "source": ["# Reading parquet file"]}, {"cell_type": "code", "execution_count": null, "metadata": {"execution": {"iopub.execute_input": "2024-04-17T10:18:13.028055Z", "iopub.status.busy": "2024-04-17T10:18:13.027597Z", "iopub.status.idle": "2024-04-17T10:18:13.959212Z", "shell.execute_reply": "2024-04-17T10:18:13.956047Z", "shell.execute_reply.started": "2024-04-17T10:18:13.028019Z"}, "trusted": true}, "outputs": [], "source": ["%%time\n", "%%memit\n", "ddf = dd.read_parquet(train_file)"]}, {"cell_type": "code", "execution_count": null, "metadata": {"execution": {"iopub.execute_input": "2024-04-17T10:18:13.962267Z", "iopub.status.busy": "2024-04-17T10:18:13.96164Z", "iopub.status.idle": "2024-04-17T10:19:24.987042Z", "shell.execute_reply": "2024-04-17T10:19:24.98422Z", "shell.execute_reply.started": "2024-04-17T10:18:13.962211Z"}, "trusted": true}, "outputs": [], "source": ["%%time\n", "%%memit\n", "with ProgressBar():\n", "    sample_df = ddf.sample(frac=0.01).compute()\n", "\n", "print(sample_df.head())"]}], "metadata": {"kaggle": {"accelerator": "none", "dataSources": [{"databundleVersionId": 8006601, "sourceId": 67356, "sourceType": "competition"}], "dockerImageVersionId": 30684, "isGpuEnabled": false, "isInternetEnabled": true, "language": "python", "sourceType": "notebook"}, "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"codemirror_mode": {"name": "ipython", "version": 3}, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.3"}}, "nbformat": 4, "nbformat_minor": 4}