{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pickle\nimport polars as pl\n\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-30T06:14:59.126823Z","iopub.execute_input":"2023-06-30T06:14:59.127580Z","iopub.status.idle":"2023-06-30T06:14:59.191965Z","shell.execute_reply.started":"2023-06-30T06:14:59.127542Z","shell.execute_reply":"2023-06-30T06:14:59.190983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events = pl.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', n_rows=5_000_000)","metadata":{"execution":{"iopub.status.busy":"2023-06-30T06:14:59.193796Z","iopub.execute_input":"2023-06-30T06:14:59.194297Z","iopub.status.idle":"2023-06-30T06:15:03.659680Z","shell.execute_reply.started":"2023-06-30T06:14:59.194268Z","shell.execute_reply":"2023-06-30T06:15:03.658775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events = events.with_columns(pl.col('fqid').fill_null('null'))","metadata":{"execution":{"iopub.status.busy":"2023-06-30T06:15:03.660934Z","iopub.execute_input":"2023-06-30T06:15:03.661410Z","iopub.status.idle":"2023-06-30T06:15:04.004924Z","shell.execute_reply.started":"2023-06-30T06:15:03.661381Z","shell.execute_reply":"2023-06-30T06:15:04.003985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FQIDS = sorted(events.select(pl.col('fqid').unique())['fqid'].to_list())\n\nFQID_TIME_DELTA_AGGS = {\n    f'room_fqid_time_delta_sum_{fqid}': pl.col('time_delta_sum').filter(pl.col('fqid') == fqid).first()\n    for fqid in FQIDS\n}","metadata":{"execution":{"iopub.status.busy":"2023-06-30T06:15:04.007363Z","iopub.execute_input":"2023-06-30T06:15:04.007982Z","iopub.status.idle":"2023-06-30T06:15:04.114130Z","shell.execute_reply.started":"2023-06-30T06:15:04.007951Z","shell.execute_reply":"2023-06-30T06:15:04.112525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_delta_features(events):\n\n    return (\n        events\n        .with_columns(\n            (pl.col('elapsed_time') - pl.col('elapsed_time').shift(1))\n            .fill_null(0)\n            .over('session_id', 'level_group')\n            .alias('time_delta')\n        )\n        .groupby('session_id', 'fqid')\n        .agg(\n            time_delta_sum=pl.col('time_delta').sum()\n        )\n        .groupby('session_id')\n        .agg(**FQID_TIME_DELTA_AGGS)\n    )\n    ","metadata":{"execution":{"iopub.status.busy":"2023-06-30T06:15:04.116370Z","iopub.execute_input":"2023-06-30T06:15:04.119383Z","iopub.status.idle":"2023-06-30T06:15:04.125133Z","shell.execute_reply.started":"2023-06-30T06:15:04.119340Z","shell.execute_reply":"2023-06-30T06:15:04.124208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SESSION_IDS = sorted(events.select(pl.col('session_id').unique())['session_id'].to_list())\n","metadata":{"execution":{"iopub.status.busy":"2023-06-30T06:15:04.126603Z","iopub.execute_input":"2023-06-30T06:15:04.127228Z","iopub.status.idle":"2023-06-30T06:15:04.270883Z","shell.execute_reply.started":"2023-06-30T06:15:04.127186Z","shell.execute_reply":"2023-06-30T06:15:04.269959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Eager batch computation","metadata":{}},{"cell_type":"code","source":"%%timeit\ntime_delta_features(events)","metadata":{"execution":{"iopub.status.busy":"2023-06-30T06:15:04.275354Z","iopub.execute_input":"2023-06-30T06:15:04.275715Z","iopub.status.idle":"2023-06-30T06:15:13.536535Z","shell.execute_reply.started":"2023-06-30T06:15:04.275687Z","shell.execute_reply":"2023-06-30T06:15:13.535586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lazy batch computation","metadata":{}},{"cell_type":"code","source":"%%timeit\ntime_delta_features(events.lazy()).collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-30T06:15:13.537492Z","iopub.execute_input":"2023-06-30T06:15:13.537781Z","iopub.status.idle":"2023-06-30T06:15:22.917641Z","shell.execute_reply.started":"2023-06-30T06:15:13.537756Z","shell.execute_reply":"2023-06-30T06:15:22.916621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Eager streaming setting","metadata":{}},{"cell_type":"code","source":"for session_id in tqdm(SESSION_IDS):\n    session = events.filter(pl.col('session_id') == session_id)\n    time_delta_features(session)","metadata":{"execution":{"iopub.status.busy":"2023-06-30T06:15:22.918614Z","iopub.execute_input":"2023-06-30T06:15:22.918917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lazy streaming setting","metadata":{}},{"cell_type":"code","source":"for session_id in tqdm(SESSION_IDS):\n    session = events.filter(pl.col('session_id') == session_id)\n    time_delta_features(session.lazy()).collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lazy + hack streaming setting","metadata":{}},{"cell_type":"code","source":"def function_that_returns_global_dataframe_variable(*args, **kwargs):\n    return DATAFRAME\n\nfunction_lazy_frame = pl.LazyFrame._scan_python_function(\n    events.schema,  # Schema of the dataframe\n    pickle.dumps(function_that_returns_global_dataframe_variable),\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lazy_features_hack = time_delta_features(function_lazy_frame)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for session_id in tqdm(SESSION_IDS):\n    session = events.filter(pl.col('session_id') == session_id)\n    DATAFRAME = session\n    lazy_features_hack.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}