{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":38760,"databundleVersionId":4493939,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#!pip install --extra-index-url=https://pypi.nvidia.com polars cudf-polars-cu11\n!pip install polars[gpu] --extra-index-url=https://pypi.nvidia.com \n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-29T14:50:19.663301Z","iopub.execute_input":"2024-09-29T14:50:19.664036Z","iopub.status.idle":"2024-09-29T14:50:33.263249Z","shell.execute_reply.started":"2024-09-29T14:50:19.663989Z","shell.execute_reply":"2024-09-29T14:50:33.262154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl\nfrom pathlib import Path\nimport gc\n\ndef transform_and_save_to_parquet(file, output_dir):\n    df = pl.scan_ndjson(file)  # LazyFrame으로 읽기\n    \n    # 각 세션의 이벤트를 분리하여 2차원 배열 구조로 변환\n    expanded_df = df.explode(\"events\").select(\n        pl.col(\"session\"),\n        pl.col(\"events\").struct.field(\"aid\").alias(\"aid\"),\n        pl.col(\"events\").struct.field(\"ts\").alias(\"ts\"),\n        pl.col(\"events\").struct.field(\"type\").alias(\"type\")\n    )\n    \n    # type을 0, 1, 2, 3으로 치환\n    expanded_df = expanded_df.with_columns(\n        pl.when(pl.col(\"type\") == \"clicks\").then(0)\n        .when(pl.col(\"type\") == \"carts\").then(1)\n        .when(pl.col(\"type\") == \"orders\").then(2)\n        .otherwise(3).alias(\"type\")  # 기본값을 4로 설정\n    )\n    \n    # 데이터 타입 최적화\n    expanded_df = expanded_df.with_columns([\n        pl.col(\"session\").cast(pl.UInt32),\n        pl.col(\"aid\").cast(pl.UInt32),\n        pl.col(\"ts\").cast(pl.Int64),\n        pl.col(\"type\").cast(pl.UInt8)  # 이제 0-4의 값만 허용\n    ])\n    \n    # LazyFrame 병합 후 Parquet 파일로 저장\n    combined_df = expanded_df.collect()  # 데이터 수집\n    combined_df.write_parquet(f\"{output_dir}/transformed.parquet\", compression=\"snappy\")\n    \n    # 메모리 정리\n    del combined_df\n    gc.collect()\n\n# NDJSON 파일을 읽어오기\nfile = Path(\"../input/otto-recommender-system/train.jsonl\")\n\n# Parquet 파일로 저장할 디렉토리 설정\noutput_dir = \"../working\"\nPath(output_dir).mkdir(exist_ok=True)\n\n# Parquet 파일로 저장\ntransform_and_save_to_parquet(file, output_dir)\n\n# 확인을 위한 메시지 출력\nprint(\"Transformation and saving to Parquet completed.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-09-29T14:59:39.325992Z","iopub.execute_input":"2024-09-29T14:59:39.326813Z","iopub.status.idle":"2024-09-29T15:02:47.124408Z","shell.execute_reply.started":"2024-09-29T14:59:39.326768Z","shell.execute_reply":"2024-09-29T15:02:47.123422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gdf = pl.read_parquet('../working/transformed.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-09-29T15:03:05.535086Z","iopub.execute_input":"2024-09-29T15:03:05.535894Z","iopub.status.idle":"2024-09-29T15:03:07.937274Z","shell.execute_reply.started":"2024-09-29T15:03:05.535855Z","shell.execute_reply":"2024-09-29T15:03:07.936487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gdf","metadata":{"execution":{"iopub.status.busy":"2024-09-29T15:03:11.329250Z","iopub.execute_input":"2024-09-29T15:03:11.329643Z","iopub.status.idle":"2024-09-29T15:03:11.347204Z","shell.execute_reply.started":"2024-09-29T15:03:11.329600Z","shell.execute_reply":"2024-09-29T15:03:11.346244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = gdf.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-09-29T15:03:46.203544Z","iopub.execute_input":"2024-09-29T15:03:46.204082Z","iopub.status.idle":"2024-09-29T15:03:48.398732Z","shell.execute_reply.started":"2024-09-29T15:03:46.204044Z","shell.execute_reply":"2024-09-29T15:03:48.397719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-29T15:03:58.719504Z","iopub.execute_input":"2024-09-29T15:03:58.720254Z","iopub.status.idle":"2024-09-29T15:03:58.730440Z","shell.execute_reply.started":"2024-09-29T15:03:58.720213Z","shell.execute_reply":"2024-09-29T15:03:58.729468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}