{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":38760,"databundleVersionId":4493939,"sourceType":"competition"}],"dockerImageVersionId":30762,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!head /kaggle/input/otto-recommender-system/train.jsonl | cut -c -80","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cudf\nimport json\nimport pandas as pd\nimport time\n\n# 메모리 관리 설정\ncudf.set_option(\"default_integer_bitwidth\", 32)\ncudf.set_option(\"default_float_bitwidth\", 32)\n\nid2type = ['clicks', 'carts', 'orders']                                          \ntype2id = {a: i for i, a in enumerate(id2type)}\n\nclass Timer:\n    def __enter__(self):\n        self.start = time.perf_counter()\n        return self\n    \n    def __exit__(self, *args):\n        self.end = time.perf_counter()\n        self.interval = self.end - self.start        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ndef jsonl_to_df(fn, output_dir='output_chunks', chunk_size=100_000, split_size=1000_000):\n    import os\n    sessions = []\n    aids = []\n    tss = []\n    types = []\n    \n    # 각 파일이 저장될 chunk 번호를 저장하는 변수\n    chunk_num = 0\n\n    # 저장할 디렉토리가 없으면 생성\n    if not os.path.exists(output_dir):\n        os.makedirs(output_dir)\n\n    chunks = pd.read_json(fn, lines=True, chunksize=chunk_size)\n\n    # row count tracking\n    total_rows = 0\n    \n    for chunk in chunks:\n        for row_idx, session_data in chunk.iterrows():\n            num_events = len(session_data.events)\n            sessions += ([session_data.session] * num_events)\n            for event in session_data.events:\n                aids.append(event['aid'])\n                tss.append(event['ts'])\n                types.append(type2id[event['type']])\n                \n            total_rows += num_events\n\n            # 일정 크기만큼 데이터가 모이면 Parquet로 저장\n            if total_rows >= split_size:\n                # DataFrame 생성\n                df = pd.DataFrame(data={'session': sessions, 'aid': aids, 'ts': tss, 'type': types})\n                \n                # 메모리 절약을 위해 데이터 타입을 변환\n                df.type = df.type.astype(np.uint8)\n                \n                # Parquet 파일로 저장\n                parquet_file = os.path.join(output_dir, f'chunk_{chunk_num}.parquet')\n                df.to_parquet(parquet_file, index=False)\n                \n                # 다음 chunk 파일을 위해 초기화\n                sessions = []\n                aids = []\n                tss = []\n                types = []\n                total_rows = 0\n                chunk_num += 1\n    \n    # 마지막으로 남은 데이터 저장\n    if sessions:\n        df = pd.DataFrame(data={'session': sessions, 'aid': aids, 'ts': tss, 'type': types})\n        df.type = df.type.astype(np.uint8)\n        parquet_file = os.path.join(output_dir, f'chunk_{chunk_num}.parquet')\n        df.to_parquet(parquet_file, index=False)\n\n    print(f\"Data saved in {output_dir} in {chunk_num + 1} files.\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# jsonl 파일을 Parquet로 분할하여 저장\n%time jsonl_to_df('/kaggle/input/otto-recommender-system/train.jsonl', output_dir='train_chunks', chunk_size=400_000, split_size=4_400_000)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cudf\n\n# 메모리 관리 설정\ncudf.set_option(\"default_integer_bitwidth\", 32)\ncudf.set_option(\"default_float_bitwidth\", 32)\n\ndf = cudf.read_parquet('/kaggle/working/train_chunks/')","metadata":{"execution":{"iopub.status.busy":"2024-09-22T15:53:54.356925Z","iopub.execute_input":"2024-09-22T15:53:54.357667Z","iopub.status.idle":"2024-09-22T15:53:56.340291Z","shell.execute_reply.started":"2024-09-22T15:53:54.357623Z","shell.execute_reply":"2024-09-22T15:53:56.339394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-22T15:58:02.084647Z","iopub.execute_input":"2024-09-22T15:58:02.085342Z","iopub.status.idle":"2024-09-22T15:58:02.104637Z","shell.execute_reply.started":"2024-09-22T15:58:02.085290Z","shell.execute_reply":"2024-09-22T15:58:02.103814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-22T16:02:09.642641Z","iopub.execute_input":"2024-09-22T16:02:09.643413Z","iopub.status.idle":"2024-09-22T16:02:09.650763Z","shell.execute_reply.started":"2024-09-22T16:02:09.643356Z","shell.execute_reply":"2024-09-22T16:02:09.649623Z"},"trusted":true},"execution_count":null,"outputs":[]}]}