{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom glob import glob\nimport gc\n\nimport cupy as cp\nimport cudf\nimport cuml\nimport dask_cudf\n\ncp.random.seed(12)\n\nprint('cupy v:', cp.__version__)\nprint('cudf v:', cudf.__version__)\nprint('dask_cudf v:', dask_cudf.__version__)\nprint('cuml v:', cuml.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-16T14:09:38.168440Z","iopub.execute_input":"2022-12-16T14:09:38.169031Z","iopub.status.idle":"2022-12-16T14:09:40.105719Z","shell.execute_reply.started":"2022-12-16T14:09:38.168971Z","shell.execute_reply":"2022-12-16T14:09:40.103618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install dask_cuda","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create a Local Cluster with 2 GPUs for dask_cudf","metadata":{}},{"cell_type":"code","source":"from dask.distributed import Client, wait\nfrom dask_cuda import LocalCUDACluster\n\ncluster = LocalCUDACluster(device_memory_limit=0.9, rmm_pool_size=0.9)\nclient = Client(cluster)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:09:41.380211Z","iopub.execute_input":"2022-12-16T14:09:41.381404Z","iopub.status.idle":"2022-12-16T14:09:46.462349Z","shell.execute_reply.started":"2022-12-16T14:09:41.381359Z","shell.execute_reply":"2022-12-16T14:09:46.461173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load DataFrame","metadata":{}},{"cell_type":"code","source":"ddf = dask_cudf.read_parquet('/kaggle/input/otto-full-optimized-memory-footprint/train.parquet')\nddf = ddf.repartition(npartitions=16).persist()\ngc.collect()\nddf.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:09:46.465120Z","iopub.execute_input":"2022-12-16T14:09:46.465838Z","iopub.status.idle":"2022-12-16T14:09:54.608362Z","shell.execute_reply.started":"2022-12-16T14:09:46.465790Z","shell.execute_reply":"2022-12-16T14:09:54.607193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sort","metadata":{}},{"cell_type":"code","source":"ddf = ddf.sort_values(['session', 'ts'], ignore_index=True)\ngc.collect()\nddf.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:09:54.614597Z","iopub.execute_input":"2022-12-16T14:09:54.618972Z","iopub.status.idle":"2022-12-16T14:10:02.670276Z","shell.execute_reply.started":"2022-12-16T14:09:54.618912Z","shell.execute_reply":"2022-12-16T14:10:02.669205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ddf.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:10:02.675150Z","iopub.execute_input":"2022-12-16T14:10:02.675835Z","iopub.status.idle":"2022-12-16T14:10:02.688162Z","shell.execute_reply.started":"2022-12-16T14:10:02.675789Z","shell.execute_reply":"2022-12-16T14:10:02.686810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Groupbys","metadata":{}},{"cell_type":"code","source":"dt = ddf.groupby(['session'])['aid'].agg('count')\ngc.collect()\ndt.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:10:02.690390Z","iopub.execute_input":"2022-12-16T14:10:02.691744Z","iopub.status.idle":"2022-12-16T14:10:05.104692Z","shell.execute_reply.started":"2022-12-16T14:10:02.691700Z","shell.execute_reply":"2022-12-16T14:10:05.103692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt = ddf.groupby(['session'])['ts'].agg(['min', 'max'])\ngc.collect()\ndt.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:10:05.106221Z","iopub.execute_input":"2022-12-16T14:10:05.106759Z","iopub.status.idle":"2022-12-16T14:10:07.400080Z","shell.execute_reply.started":"2022-12-16T14:10:05.106719Z","shell.execute_reply":"2022-12-16T14:10:07.398729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del dt;\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:10:07.404641Z","iopub.execute_input":"2022-12-16T14:10:07.405023Z","iopub.status.idle":"2022-12-16T14:10:07.614183Z","shell.execute_reply.started":"2022-12-16T14:10:07.404992Z","shell.execute_reply":"2022-12-16T14:10:07.613135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Shifts","metadata":{}},{"cell_type":"code","source":"dt = ddf.groupby('session')['aid'].shift(1)\ndt.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:10:07.617005Z","iopub.execute_input":"2022-12-16T14:10:07.617955Z","iopub.status.idle":"2022-12-16T14:10:09.992826Z","shell.execute_reply.started":"2022-12-16T14:10:07.617916Z","shell.execute_reply":"2022-12-16T14:10:09.991895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt = ddf.groupby('session')['aid'].shift(2)\ndt.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:10:09.994521Z","iopub.execute_input":"2022-12-16T14:10:09.995030Z","iopub.status.idle":"2022-12-16T14:10:12.478297Z","shell.execute_reply.started":"2022-12-16T14:10:09.994993Z","shell.execute_reply":"2022-12-16T14:10:12.477095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt = ddf.groupby('session')['aid'].shift(3)\ndt.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:10:50.982529Z","iopub.execute_input":"2022-12-16T14:10:50.982910Z","iopub.status.idle":"2022-12-16T14:10:53.896647Z","shell.execute_reply.started":"2022-12-16T14:10:50.982877Z","shell.execute_reply":"2022-12-16T14:10:53.895433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}